/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

Bug Summary

File:	llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
Warning:	line 5968, column 38 1st function call argument is an uninitialized value

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name AMDGPUAsmParser.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/lib/Target/AMDGPU/AsmParser -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/lib/Target/AMDGPU/AsmParser -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/lib/Target/AMDGPU/AsmParser -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/lib/Target/AMDGPU -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/lib/Target/AMDGPU -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/include -I /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/include -D NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/build-llvm/lib/Target/AMDGPU/AsmParser -fdebug-prefix-map=/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e=. -ferror-limit 19 -fvisibility hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-09-04-040900-46481-1 -x c++ /build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

/build/llvm-toolchain-snapshot-14~++20210903100615+fd66b44ec19e/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp

→

1//===- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//

9#include "AMDKernelCodeT.h"
10#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
11#include "MCTargetDesc/AMDGPUTargetStreamer.h"
12#include "SIDefines.h"
13#include "SIInstrInfo.h"
14#include "SIRegisterInfo.h"
15#include "TargetInfo/AMDGPUTargetInfo.h"
16#include "Utils/AMDGPUAsmUtils.h"
17#include "Utils/AMDGPUBaseInfo.h"
18#include "Utils/AMDKernelCodeTUtils.h"
19#include "llvm/ADT/APFloat.h"
20#include "llvm/ADT/SmallBitVector.h"
21#include "llvm/ADT/StringSet.h"
22#include "llvm/ADT/Twine.h"
23#include "llvm/MC/MCAsmInfo.h"
24#include "llvm/MC/MCContext.h"
25#include "llvm/MC/MCExpr.h"
26#include "llvm/MC/MCInst.h"
27#include "llvm/MC/MCParser/MCAsmParser.h"
28#include "llvm/MC/MCParser/MCParsedAsmOperand.h"
29#include "llvm/MC/MCParser/MCTargetAsmParser.h"
30#include "llvm/MC/MCSymbol.h"
31#include "llvm/Support/AMDGPUMetadata.h"
32#include "llvm/Support/AMDHSAKernelDescriptor.h"
33#include "llvm/Support/Casting.h"
34#include "llvm/Support/MachineValueType.h"
35#include "llvm/Support/TargetParser.h"
36#include "llvm/Support/TargetRegistry.h"

38using namespace llvm;
39using namespace llvm::AMDGPU;
40using namespace llvm::amdhsa;

42namespace {

44class AMDGPUAsmParser;

46enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_AGPR, IS_TTMP, IS_SPECIAL };

48//===----------------------------------------------------------------------===//
49// Operand
50//===----------------------------------------------------------------------===//

52class AMDGPUOperand : public MCParsedAsmOperand {
enum KindTy {
  Token,
  Immediate,
  Register,
  Expression
} Kind;

SMLoc StartLoc, EndLoc;
const AMDGPUAsmParser *AsmParser;

63public:
AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
  : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}

using Ptr = std::unique_ptr<AMDGPUOperand>;

struct Modifiers {
  bool Abs = false;
  bool Neg = false;
  bool Sext = false;

  bool hasFPModifiers() const { return Abs || Neg; }
  bool hasIntModifiers() const { return Sext; }
  bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }

  int64_t getFPModifiersOperand() const {
    int64_t Operand = 0;
    Operand |= Abs ? SISrcMods::ABS : 0u;
    Operand |= Neg ? SISrcMods::NEG : 0u;
    return Operand;
  }

  int64_t getIntModifiersOperand() const {
    int64_t Operand = 0;
    Operand |= Sext ? SISrcMods::SEXT : 0u;
    return Operand;
  }

  int64_t getModifiersOperand() const {
    assert(!(hasFPModifiers() && hasIntModifiers())(static_cast<void> (0))
         && "fp and int modifiers should not be used simultaneously")(static_cast<void> (0));
    if (hasFPModifiers()) {
      return getFPModifiersOperand();
    } else if (hasIntModifiers()) {
      return getIntModifiersOperand();
    } else {
      return 0;
    }
  }

  friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
};

enum ImmTy {
  ImmTyNone,
  ImmTyGDS,
  ImmTyLDS,
  ImmTyOffen,
  ImmTyIdxen,
  ImmTyAddr64,
  ImmTyOffset,
  ImmTyInstOffset,
  ImmTyOffset0,
  ImmTyOffset1,
  ImmTyCPol,
  ImmTySWZ,
  ImmTyTFE,
  ImmTyD16,
  ImmTyClampSI,
  ImmTyOModSI,
  ImmTyDPP8,
  ImmTyDppCtrl,
  ImmTyDppRowMask,
  ImmTyDppBankMask,
  ImmTyDppBoundCtrl,
  ImmTyDppFi,
  ImmTySdwaDstSel,
  ImmTySdwaSrc0Sel,
  ImmTySdwaSrc1Sel,
  ImmTySdwaDstUnused,
  ImmTyDMask,
  ImmTyDim,
  ImmTyUNorm,
  ImmTyDA,
  ImmTyR128A16,
  ImmTyA16,
  ImmTyLWE,
  ImmTyExpTgt,
  ImmTyExpCompr,
  ImmTyExpVM,
  ImmTyFORMAT,
  ImmTyHwreg,
  ImmTyOff,
  ImmTySendMsg,
  ImmTyInterpSlot,
  ImmTyInterpAttr,
  ImmTyAttrChan,
  ImmTyOpSel,
  ImmTyOpSelHi,
  ImmTyNegLo,
  ImmTyNegHi,
  ImmTySwizzle,
  ImmTyGprIdxMode,
  ImmTyHigh,
  ImmTyBLGP,
  ImmTyCBSZ,
  ImmTyABID,
  ImmTyEndpgm,
};

enum ImmKindTy {
  ImmKindTyNone,
  ImmKindTyLiteral,
  ImmKindTyConst,
};

169private:
struct TokOp {
  const char *Data;
  unsigned Length;
};

struct ImmOp {
  int64_t Val;
  ImmTy Type;
  bool IsFPImm;
  mutable ImmKindTy Kind;
  Modifiers Mods;
};

struct RegOp {
  unsigned RegNo;
  Modifiers Mods;
};

union {
  TokOp Tok;
  ImmOp Imm;
  RegOp Reg;
  const MCExpr *Expr;
};

195public:
bool isToken() const override {
  if (Kind == Token)
    return true;

  // When parsing operands, we can't always tell if something was meant to be
  // a token, like 'gds', or an expression that references a global variable.
  // In this case, we assume the string is an expression, and if we need to
  // interpret is a token, then we treat the symbol name as the token.
  return isSymbolRefExpr();
}

bool isSymbolRefExpr() const {
  return isExpr() && Expr && isa<MCSymbolRefExpr>(Expr);
}

bool isImm() const override {
  return Kind == Immediate;
}

void setImmKindNone() const {
  assert(isImm())(static_cast<void> (0));
  Imm.Kind = ImmKindTyNone;
}

void setImmKindLiteral() const {
  assert(isImm())(static_cast<void> (0));
  Imm.Kind = ImmKindTyLiteral;
}

void setImmKindConst() const {
  assert(isImm())(static_cast<void> (0));
  Imm.Kind = ImmKindTyConst;
}

bool IsImmKindLiteral() const {
  return isImm() && Imm.Kind == ImmKindTyLiteral;
}

bool isImmKindConst() const {
  return isImm() && Imm.Kind == ImmKindTyConst;
}

bool isInlinableImm(MVT type) const;
bool isLiteralImm(MVT type) const;

bool isRegKind() const {
  return Kind == Register;
}

bool isReg() const override {
  return isRegKind() && !hasModifiers();
}

bool isRegOrImmWithInputMods(unsigned RCID, MVT type) const {
  return isRegClass(RCID) || isInlinableImm(type) || isLiteralImm(type);
}

bool isRegOrImmWithInt16InputMods() const {
  return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::i16);
}

bool isRegOrImmWithInt32InputMods() const {
  return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::i32);
}

bool isRegOrImmWithInt64InputMods() const {
  return isRegOrImmWithInputMods(AMDGPU::VS_64RegClassID, MVT::i64);
}

bool isRegOrImmWithFP16InputMods() const {
  return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::f16);
}

bool isRegOrImmWithFP32InputMods() const {
  return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::f32);
}

bool isRegOrImmWithFP64InputMods() const {
  return isRegOrImmWithInputMods(AMDGPU::VS_64RegClassID, MVT::f64);
}

bool isVReg() const {
  return isRegClass(AMDGPU::VGPR_32RegClassID) ||
         isRegClass(AMDGPU::VReg_64RegClassID) ||
         isRegClass(AMDGPU::VReg_96RegClassID) ||
         isRegClass(AMDGPU::VReg_128RegClassID) ||
         isRegClass(AMDGPU::VReg_160RegClassID) ||
         isRegClass(AMDGPU::VReg_192RegClassID) ||
         isRegClass(AMDGPU::VReg_256RegClassID) ||
         isRegClass(AMDGPU::VReg_512RegClassID) ||
         isRegClass(AMDGPU::VReg_1024RegClassID);
}

bool isVReg32() const {
  return isRegClass(AMDGPU::VGPR_32RegClassID);
}

bool isVReg32OrOff() const {
  return isOff() || isVReg32();
}

bool isNull() const {
  return isRegKind() && getReg() == AMDGPU::SGPR_NULL;
}

bool isVRegWithInputMods() const;

bool isSDWAOperand(MVT type) const;
bool isSDWAFP16Operand() const;
bool isSDWAFP32Operand() const;
bool isSDWAInt16Operand() const;
bool isSDWAInt32Operand() const;

bool isImmTy(ImmTy ImmT) const {
  return isImm() && Imm.Type == ImmT;
}

bool isImmModifier() const {
  return isImm() && Imm.Type != ImmTyNone;
}

bool isClampSI() const { return isImmTy(ImmTyClampSI); }
bool isOModSI() const { return isImmTy(ImmTyOModSI); }
bool isDMask() const { return isImmTy(ImmTyDMask); }
bool isDim() const { return isImmTy(ImmTyDim); }
bool isUNorm() const { return isImmTy(ImmTyUNorm); }
bool isDA() const { return isImmTy(ImmTyDA); }
bool isR128A16() const { return isImmTy(ImmTyR128A16); }
bool isGFX10A16() const { return isImmTy(ImmTyA16); }
bool isLWE() const { return isImmTy(ImmTyLWE); }
bool isOff() const { return isImmTy(ImmTyOff); }
bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
bool isExpVM() const { return isImmTy(ImmTyExpVM); }
bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
bool isOffen() const { return isImmTy(ImmTyOffen); }
bool isIdxen() const { return isImmTy(ImmTyIdxen); }
bool isAddr64() const { return isImmTy(ImmTyAddr64); }
bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<8>(getImm()); }
bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }

bool isFlatOffset() const { return isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset); }
bool isGDS() const { return isImmTy(ImmTyGDS); }
bool isLDS() const { return isImmTy(ImmTyLDS); }
bool isCPol() const { return isImmTy(ImmTyCPol); }
bool isSWZ() const { return isImmTy(ImmTySWZ); }
bool isTFE() const { return isImmTy(ImmTyTFE); }
bool isD16() const { return isImmTy(ImmTyD16); }
bool isFORMAT() const { return isImmTy(ImmTyFORMAT) && isUInt<7>(getImm()); }
bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
bool isFI() const { return isImmTy(ImmTyDppFi); }
bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
bool isOpSel() const { return isImmTy(ImmTyOpSel); }
bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
bool isNegLo() const { return isImmTy(ImmTyNegLo); }
bool isNegHi() const { return isImmTy(ImmTyNegHi); }
bool isHigh() const { return isImmTy(ImmTyHigh); }

bool isMod() const {
  return isClampSI() || isOModSI();
}

bool isRegOrImm() const {
  return isReg() || isImm();
}

bool isRegClass(unsigned RCID) const;

bool isInlineValue() const;

bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
  return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
}

bool isSCSrcB16() const {
  return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
}

bool isSCSrcV2B16() const {
  return isSCSrcB16();
}

bool isSCSrcB32() const {
  return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
}

bool isSCSrcB64() const {
  return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
}

bool isBoolReg() const;

bool isSCSrcF16() const {
  return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
}

bool isSCSrcV2F16() const {
  return isSCSrcF16();
}

bool isSCSrcF32() const {
  return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
}

bool isSCSrcF64() const {
  return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
}

bool isSSrcB32() const {
  return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
}

bool isSSrcB16() const {
  return isSCSrcB16() || isLiteralImm(MVT::i16);
}

bool isSSrcV2B16() const {
  llvm_unreachable("cannot happen")__builtin_unreachable();
  return isSSrcB16();
}

bool isSSrcB64() const {
  // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
  // See isVSrc64().
  return isSCSrcB64() || isLiteralImm(MVT::i64);
}

bool isSSrcF32() const {
  return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
}

bool isSSrcF64() const {
  return isSCSrcB64() || isLiteralImm(MVT::f64);
}

bool isSSrcF16() const {
  return isSCSrcB16() || isLiteralImm(MVT::f16);
}

bool isSSrcV2F16() const {
  llvm_unreachable("cannot happen")__builtin_unreachable();
  return isSSrcF16();
}

bool isSSrcV2FP32() const {
  llvm_unreachable("cannot happen")__builtin_unreachable();
  return isSSrcF32();
}

bool isSCSrcV2FP32() const {
  llvm_unreachable("cannot happen")__builtin_unreachable();
  return isSCSrcF32();
}

bool isSSrcV2INT32() const {
  llvm_unreachable("cannot happen")__builtin_unreachable();
  return isSSrcB32();
}

bool isSCSrcV2INT32() const {
  llvm_unreachable("cannot happen")__builtin_unreachable();
  return isSCSrcB32();
}

bool isSSrcOrLdsB32() const {
  return isRegOrInlineNoMods(AMDGPU::SRegOrLds_32RegClassID, MVT::i32) ||
         isLiteralImm(MVT::i32) || isExpr();
}

bool isVCSrcB32() const {
  return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
}

bool isVCSrcB64() const {
  return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
}

bool isVCSrcB16() const {
  return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
}

bool isVCSrcV2B16() const {
  return isVCSrcB16();
}

bool isVCSrcF32() const {
  return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
}

bool isVCSrcF64() const {
  return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
}

bool isVCSrcF16() const {
  return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
}

bool isVCSrcV2F16() const {
  return isVCSrcF16();
}

bool isVSrcB32() const {
  return isVCSrcF32() || isLiteralImm(MVT::i32) || isExpr();
}

bool isVSrcB64() const {
  return isVCSrcF64() || isLiteralImm(MVT::i64);
}

bool isVSrcB16() const {
  return isVCSrcB16() || isLiteralImm(MVT::i16);
}

bool isVSrcV2B16() const {
  return isVSrcB16() || isLiteralImm(MVT::v2i16);
}

bool isVCSrcV2FP32() const {
  return isVCSrcF64();
}

bool isVSrcV2FP32() const {
  return isVSrcF64() || isLiteralImm(MVT::v2f32);
}

bool isVCSrcV2INT32() const {
  return isVCSrcB64();
}

bool isVSrcV2INT32() const {
  return isVSrcB64() || isLiteralImm(MVT::v2i32);
}

bool isVSrcF32() const {
  return isVCSrcF32() || isLiteralImm(MVT::f32) || isExpr();
}

bool isVSrcF64() const {
  return isVCSrcF64() || isLiteralImm(MVT::f64);
}

bool isVSrcF16() const {
  return isVCSrcF16() || isLiteralImm(MVT::f16);
}

bool isVSrcV2F16() const {
  return isVSrcF16() || isLiteralImm(MVT::v2f16);
}

bool isVISrcB32() const {
  return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i32);
}

bool isVISrcB16() const {
  return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i16);
}

bool isVISrcV2B16() const {
  return isVISrcB16();
}

bool isVISrcF32() const {
  return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f32);
}

bool isVISrcF16() const {
  return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f16);
}

bool isVISrcV2F16() const {
  return isVISrcF16() || isVISrcB32();
}

bool isVISrc_64B64() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i64);
}

bool isVISrc_64F64() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f64);
}

bool isVISrc_64V2FP32() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f32);
}

bool isVISrc_64V2INT32() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i32);
}

bool isVISrc_256B64() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i64);
}

bool isVISrc_256F64() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f64);
}

bool isVISrc_128B16() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i16);
}

bool isVISrc_128V2B16() const {
  return isVISrc_128B16();
}

bool isVISrc_128B32() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i32);
}

bool isVISrc_128F32() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f32);
}

bool isVISrc_256V2FP32() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f32);
}

bool isVISrc_256V2INT32() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i32);
}

bool isVISrc_512B32() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i32);
}

bool isVISrc_512B16() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i16);
}

bool isVISrc_512V2B16() const {
  return isVISrc_512B16();
}

bool isVISrc_512F32() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f32);
}

bool isVISrc_512F16() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f16);
}

bool isVISrc_512V2F16() const {
  return isVISrc_512F16() || isVISrc_512B32();
}

bool isVISrc_1024B32() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i32);
}

bool isVISrc_1024B16() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i16);
}

bool isVISrc_1024V2B16() const {
  return isVISrc_1024B16();
}

bool isVISrc_1024F32() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f32);
}

bool isVISrc_1024F16() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f16);
}

bool isVISrc_1024V2F16() const {
  return isVISrc_1024F16() || isVISrc_1024B32();
}

bool isAISrcB32() const {
  return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i32);
}

bool isAISrcB16() const {
  return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i16);
}

bool isAISrcV2B16() const {
  return isAISrcB16();
}

bool isAISrcF32() const {
  return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f32);
}

bool isAISrcF16() const {
  return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f16);
}

bool isAISrcV2F16() const {
  return isAISrcF16() || isAISrcB32();
}

bool isAISrc_64B64() const {
  return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::i64);
}

bool isAISrc_64F64() const {
  return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::f64);
}

bool isAISrc_128B32() const {
  return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i32);
}

bool isAISrc_128B16() const {
  return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i16);
}

bool isAISrc_128V2B16() const {
  return isAISrc_128B16();
}

bool isAISrc_128F32() const {
  return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f32);
}

bool isAISrc_128F16() const {
  return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f16);
}

bool isAISrc_128V2F16() const {
  return isAISrc_128F16() || isAISrc_128B32();
}

bool isVISrc_128F16() const {
  return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f16);
}

bool isVISrc_128V2F16() const {
  return isVISrc_128F16() || isVISrc_128B32();
}

bool isAISrc_256B64() const {
  return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::i64);
}

bool isAISrc_256F64() const {
  return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::f64);
}

bool isAISrc_512B32() const {
  return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i32);
}

bool isAISrc_512B16() const {
  return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i16);
}

bool isAISrc_512V2B16() const {
  return isAISrc_512B16();
}

bool isAISrc_512F32() const {
  return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f32);
}

bool isAISrc_512F16() const {
  return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f16);
}

bool isAISrc_512V2F16() const {
  return isAISrc_512F16() || isAISrc_512B32();
}

bool isAISrc_1024B32() const {
  return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i32);
}

bool isAISrc_1024B16() const {
  return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i16);
}

bool isAISrc_1024V2B16() const {
  return isAISrc_1024B16();
}

bool isAISrc_1024F32() const {
  return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f32);
}

bool isAISrc_1024F16() const {
  return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f16);
}

bool isAISrc_1024V2F16() const {
  return isAISrc_1024F16() || isAISrc_1024B32();
}

bool isKImmFP32() const {
  return isLiteralImm(MVT::f32);
}

bool isKImmFP16() const {
  return isLiteralImm(MVT::f16);
}

bool isMem() const override {
  return false;
}

bool isExpr() const {
  return Kind == Expression;
}

bool isSoppBrTarget() const {
  return isExpr() || isImm();
}

bool isSWaitCnt() const;
bool isHwreg() const;
bool isSendMsg() const;
bool isSwizzle() const;
bool isSMRDOffset8() const;
bool isSMEMOffset() const;
bool isSMRDLiteralOffset() const;
bool isDPP8() const;
bool isDPPCtrl() const;
bool isBLGP() const;
bool isCBSZ() const;
bool isABID() const;
bool isGPRIdxMode() const;
bool isS16Imm() const;
bool isU16Imm() const;
bool isEndpgm() const;

StringRef getExpressionAsToken() const {
  assert(isExpr())(static_cast<void> (0));
  const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
  return S->getSymbol().getName();
}

StringRef getToken() const {
  assert(isToken())(static_cast<void> (0));

  if (Kind == Expression)
    return getExpressionAsToken();

  return StringRef(Tok.Data, Tok.Length);
}

int64_t getImm() const {
  assert(isImm())(static_cast<void> (0));
  return Imm.Val;
}

void setImm(int64_t Val) {
  assert(isImm())(static_cast<void> (0));
  Imm.Val = Val;
}

ImmTy getImmTy() const {
  assert(isImm())(static_cast<void> (0));
  return Imm.Type;
}

unsigned getReg() const override {
  assert(isRegKind())(static_cast<void> (0));
  return Reg.RegNo;
}

SMLoc getStartLoc() const override {
  return StartLoc;
}

SMLoc getEndLoc() const override {
  return EndLoc;
}

SMRange getLocRange() const {
  return SMRange(StartLoc, EndLoc);
}

Modifiers getModifiers() const {
  assert(isRegKind() || isImmTy(ImmTyNone))(static_cast<void> (0));
  return isRegKind() ? Reg.Mods : Imm.Mods;
}

void setModifiers(Modifiers Mods) {
  assert(isRegKind() || isImmTy(ImmTyNone))(static_cast<void> (0));
  if (isRegKind())
    Reg.Mods = Mods;
  else
    Imm.Mods = Mods;
}

bool hasModifiers() const {
  return getModifiers().hasModifiers();
}

bool hasFPModifiers() const {
  return getModifiers().hasFPModifiers();
}

bool hasIntModifiers() const {
  return getModifiers().hasIntModifiers();
}

uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const;

void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;

void addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const;

template <unsigned Bitwidth>
void addKImmFPOperands(MCInst &Inst, unsigned N) const;

void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
  addKImmFPOperands<16>(Inst, N);
}

void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
  addKImmFPOperands<32>(Inst, N);
}

void addRegOperands(MCInst &Inst, unsigned N) const;

void addBoolRegOperands(MCInst &Inst, unsigned N) const {
  addRegOperands(Inst, N);
}

void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
  if (isRegKind())
    addRegOperands(Inst, N);
  else if (isExpr())
    Inst.addOperand(MCOperand::createExpr(Expr));
  else
    addImmOperands(Inst, N);
}

void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
  Modifiers Mods = getModifiers();
  Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
  if (isRegKind()) {
    addRegOperands(Inst, N);
  } else {
    addImmOperands(Inst, N, false);
  }
}

void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
  assert(!hasIntModifiers())(static_cast<void> (0));
  addRegOrImmWithInputModsOperands(Inst, N);
}

void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
  assert(!hasFPModifiers())(static_cast<void> (0));
  addRegOrImmWithInputModsOperands(Inst, N);
}

void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
  Modifiers Mods = getModifiers();
  Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
  assert(isRegKind())(static_cast<void> (0));
  addRegOperands(Inst, N);
}

void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
  assert(!hasIntModifiers())(static_cast<void> (0));
  addRegWithInputModsOperands(Inst, N);
}

void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
  assert(!hasFPModifiers())(static_cast<void> (0));
  addRegWithInputModsOperands(Inst, N);
}

void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
  if (isImm())
    addImmOperands(Inst, N);
  else {
    assert(isExpr())(static_cast<void> (0));
    Inst.addOperand(MCOperand::createExpr(Expr));
  }
}

static void printImmTy(raw_ostream& OS, ImmTy Type) {
  switch (Type) {
  case ImmTyNone: OS << "None"; break;
  case ImmTyGDS: OS << "GDS"; break;
  case ImmTyLDS: OS << "LDS"; break;
  case ImmTyOffen: OS << "Offen"; break;
  case ImmTyIdxen: OS << "Idxen"; break;
  case ImmTyAddr64: OS << "Addr64"; break;
  case ImmTyOffset: OS << "Offset"; break;
  case ImmTyInstOffset: OS << "InstOffset"; break;
  case ImmTyOffset0: OS << "Offset0"; break;
  case ImmTyOffset1: OS << "Offset1"; break;
  case ImmTyCPol: OS << "CPol"; break;
  case ImmTySWZ: OS << "SWZ"; break;
  case ImmTyTFE: OS << "TFE"; break;
  case ImmTyD16: OS << "D16"; break;
  case ImmTyFORMAT: OS << "FORMAT"; break;
  case ImmTyClampSI: OS << "ClampSI"; break;
  case ImmTyOModSI: OS << "OModSI"; break;
  case ImmTyDPP8: OS << "DPP8"; break;
  case ImmTyDppCtrl: OS << "DppCtrl"; break;
  case ImmTyDppRowMask: OS << "DppRowMask"; break;
  case ImmTyDppBankMask: OS << "DppBankMask"; break;
  case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
  case ImmTyDppFi: OS << "FI"; break;
  case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
  case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
  case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
  case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
  case ImmTyDMask: OS << "DMask"; break;
  case ImmTyDim: OS << "Dim"; break;
  case ImmTyUNorm: OS << "UNorm"; break;
  case ImmTyDA: OS << "DA"; break;
  case ImmTyR128A16: OS << "R128A16"; break;
  case ImmTyA16: OS << "A16"; break;
  case ImmTyLWE: OS << "LWE"; break;
  case ImmTyOff: OS << "Off"; break;
  case ImmTyExpTgt: OS << "ExpTgt"; break;
  case ImmTyExpCompr: OS << "ExpCompr"; break;
  case ImmTyExpVM: OS << "ExpVM"; break;
  case ImmTyHwreg: OS << "Hwreg"; break;
  case ImmTySendMsg: OS << "SendMsg"; break;
  case ImmTyInterpSlot: OS << "InterpSlot"; break;
  case ImmTyInterpAttr: OS << "InterpAttr"; break;
  case ImmTyAttrChan: OS << "AttrChan"; break;
  case ImmTyOpSel: OS << "OpSel"; break;
  case ImmTyOpSelHi: OS << "OpSelHi"; break;
  case ImmTyNegLo: OS << "NegLo"; break;
  case ImmTyNegHi: OS << "NegHi"; break;
  case ImmTySwizzle: OS << "Swizzle"; break;
  case ImmTyGprIdxMode: OS << "GprIdxMode"; break;
  case ImmTyHigh: OS << "High"; break;
  case ImmTyBLGP: OS << "BLGP"; break;
  case ImmTyCBSZ: OS << "CBSZ"; break;
  case ImmTyABID: OS << "ABID"; break;
  case ImmTyEndpgm: OS << "Endpgm"; break;
  }
}

void print(raw_ostream &OS) const override {
  switch (Kind) {
  case Register:
    OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
    break;
  case Immediate:
    OS << '<' << getImm();
    if (getImmTy() != ImmTyNone) {
      OS << " type: "; printImmTy(OS, getImmTy());
    }
    OS << " mods: " << Imm.Mods << '>';
    break;
  case Token:
    OS << '\'' << getToken() << '\'';
    break;
  case Expression:
    OS << "<expr " << *Expr << '>';
    break;
  }
}

static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
                                    int64_t Val, SMLoc Loc,
                                    ImmTy Type = ImmTyNone,
                                    bool IsFPImm = false) {
  auto Op = std::make_unique<AMDGPUOperand>(Immediate, AsmParser);
  Op->Imm.Val = Val;
  Op->Imm.IsFPImm = IsFPImm;
  Op->Imm.Kind = ImmKindTyNone;
  Op->Imm.Type = Type;
  Op->Imm.Mods = Modifiers();
  Op->StartLoc = Loc;
  Op->EndLoc = Loc;
  return Op;
}

static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
                                      StringRef Str, SMLoc Loc,
                                      bool HasExplicitEncodingSize = true) {
  auto Res = std::make_unique<AMDGPUOperand>(Token, AsmParser);
  Res->Tok.Data = Str.data();
  Res->Tok.Length = Str.size();
  Res->StartLoc = Loc;
  Res->EndLoc = Loc;
  return Res;
}

static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
                                    unsigned RegNo, SMLoc S,
                                    SMLoc E) {
  auto Op = std::make_unique<AMDGPUOperand>(Register, AsmParser);
  Op->Reg.RegNo = RegNo;
  Op->Reg.Mods = Modifiers();
  Op->StartLoc = S;
  Op->EndLoc = E;
  return Op;
}

static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
                                     const class MCExpr *Expr, SMLoc S) {
  auto Op = std::make_unique<AMDGPUOperand>(Expression, AsmParser);
  Op->Expr = Expr;
  Op->StartLoc = S;
  Op->EndLoc = S;
  return Op;
}
1105};

1107raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
return OS;
1110}

1112//===----------------------------------------------------------------------===//
1113// AsmParser
1114//===----------------------------------------------------------------------===//

1116// Holds info related to the current kernel, e.g. count of SGPRs used.
1117// Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
1118// .amdgpu_hsa_kernel or at EOF.
1119class KernelScopeInfo {
int SgprIndexUnusedMin = -1;
int VgprIndexUnusedMin = -1;
MCContext *Ctx = nullptr;

void usesSgprAt(int i) {
  if (i >= SgprIndexUnusedMin) {
    SgprIndexUnusedMin = ++i;
    if (Ctx) {
      MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
      Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
    }
  }
}

void usesVgprAt(int i) {
  if (i >= VgprIndexUnusedMin) {
    VgprIndexUnusedMin = ++i;
    if (Ctx) {
      MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
      Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
    }
  }
}

1144public:
KernelScopeInfo() = default;

void initialize(MCContext &Context) {
  Ctx = &Context;
  usesSgprAt(SgprIndexUnusedMin = -1);
  usesVgprAt(VgprIndexUnusedMin = -1);
}

void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
  switch (RegKind) {
    case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
    case IS_AGPR: // fall through
    case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
    default: break;
  }
}
1161};

1163class AMDGPUAsmParser : public MCTargetAsmParser {
MCAsmParser &Parser;

// Number of extra operands parsed after the first optional operand.
// This may be necessary to skip hardcoded mandatory operands.
static const unsigned MAX_OPR_LOOKAHEAD = 8;

unsigned ForcedEncodingSize = 0;
bool ForcedDPP = false;
bool ForcedSDWA = false;
KernelScopeInfo KernelScope;
unsigned CPolSeen;

/// @name Auto-generated Match Functions
/// {

1179#define GET_ASSEMBLER_HEADER
1180#include "AMDGPUGenAsmMatcher.inc"

/// }

1184private:
bool ParseAsAbsoluteExpression(uint32_t &Ret);
bool OutOfRangeError(SMRange Range);
/// Calculate VGPR/SGPR blocks required for given target, reserved
/// registers, and user-specified NextFreeXGPR values.
///
/// \param Features [in] Target features, used for bug corrections.
/// \param VCCUsed [in] Whether VCC special SGPR is reserved.
/// \param FlatScrUsed [in] Whether FLAT_SCRATCH special SGPR is reserved.
/// \param XNACKUsed [in] Whether XNACK_MASK special SGPR is reserved.
/// \param EnableWavefrontSize32 [in] Value of ENABLE_WAVEFRONT_SIZE32 kernel
/// descriptor field, if valid.
/// \param NextFreeVGPR [in] Max VGPR number referenced, plus one.
/// \param VGPRRange [in] Token range, used for VGPR diagnostics.
/// \param NextFreeSGPR [in] Max SGPR number referenced, plus one.
/// \param SGPRRange [in] Token range, used for SGPR diagnostics.
/// \param VGPRBlocks [out] Result VGPR block count.
/// \param SGPRBlocks [out] Result SGPR block count.
bool calculateGPRBlocks(const FeatureBitset &Features, bool VCCUsed,
                        bool FlatScrUsed, bool XNACKUsed,
                        Optional<bool> EnableWavefrontSize32, unsigned NextFreeVGPR,
                        SMRange VGPRRange, unsigned NextFreeSGPR,
                        SMRange SGPRRange, unsigned &VGPRBlocks,
                        unsigned &SGPRBlocks);
bool ParseDirectiveAMDGCNTarget();
bool ParseDirectiveAMDHSAKernel();
bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
bool ParseDirectiveHSACodeObjectVersion();
bool ParseDirectiveHSACodeObjectISA();
bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
bool ParseDirectiveAMDKernelCodeT();
// TODO: Possibly make subtargetHasRegister const.
bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo);
bool ParseDirectiveAMDGPUHsaKernel();

bool ParseDirectiveISAVersion();
bool ParseDirectiveHSAMetadata();
bool ParseDirectivePALMetadataBegin();
bool ParseDirectivePALMetadata();
bool ParseDirectiveAMDGPULDS();

/// Common code to parse out a block of text (typically YAML) between start and
/// end directives.
bool ParseToEndDirective(const char *AssemblerDirectiveBegin,
                         const char *AssemblerDirectiveEnd,
                         std::string &CollectString);

bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
                           RegisterKind RegKind, unsigned Reg1, SMLoc Loc);
bool ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
                         unsigned &RegNum, unsigned &RegWidth,
                         bool RestoreOnFailure = false);
bool ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
                         unsigned &RegNum, unsigned &RegWidth,
                         SmallVectorImpl<AsmToken> &Tokens);
unsigned ParseRegularReg(RegisterKind &RegKind, unsigned &RegNum,
                         unsigned &RegWidth,
                         SmallVectorImpl<AsmToken> &Tokens);
unsigned ParseSpecialReg(RegisterKind &RegKind, unsigned &RegNum,
                         unsigned &RegWidth,
                         SmallVectorImpl<AsmToken> &Tokens);
unsigned ParseRegList(RegisterKind &RegKind, unsigned &RegNum,
                      unsigned &RegWidth, SmallVectorImpl<AsmToken> &Tokens);
bool ParseRegRange(unsigned& Num, unsigned& Width);
unsigned getRegularReg(RegisterKind RegKind,
                       unsigned RegNum,
                       unsigned RegWidth,
                       SMLoc Loc);

bool isRegister();
bool isRegister(const AsmToken &Token, const AsmToken &NextToken) const;
Optional<StringRef> getGprCountSymbolName(RegisterKind RegKind);
void initializeGprCountSymbol(RegisterKind RegKind);
bool updateGprCountSymbols(RegisterKind RegKind, unsigned DwordRegIndex,
                           unsigned RegWidth);
void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
                  bool IsAtomic, bool IsLds = false);
void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
               bool IsGdsHardcoded);

1264public:
enum AMDGPUMatchResultTy {
  Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
};
enum OperandMode {
  OperandMode_Default,
  OperandMode_NSA,
};

using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;

AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
             const MCInstrInfo &MII,
             const MCTargetOptions &Options)
    : MCTargetAsmParser(Options, STI, MII), Parser(_Parser) {
  MCAsmParserExtension::Initialize(Parser);

  if (getFeatureBits().none()) {
    // Set default features.
    copySTI().ToggleFeature("southern-islands");
  }

  setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));

  {
    // TODO: make those pre-defined variables read-only.
    // Currently there is none suitable machinery in the core llvm-mc for this.
    // MCSymbol::isRedefinable is intended for another purpose, and
    // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
    AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(getSTI().getCPU());
    MCContext &Ctx = getContext();
    if (ISA.Major >= 6 && isHsaAbiVersion3Or4(&getSTI())) {
      MCSymbol *Sym =
          Ctx.getOrCreateSymbol(Twine(".amdgcn.gfx_generation_number"));
      Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
      Sym = Ctx.getOrCreateSymbol(Twine(".amdgcn.gfx_generation_minor"));
      Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
      Sym = Ctx.getOrCreateSymbol(Twine(".amdgcn.gfx_generation_stepping"));
      Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
    } else {
      MCSymbol *Sym =
          Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
      Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
      Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
      Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
      Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
      Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
    }
    if (ISA.Major >= 6 && isHsaAbiVersion3Or4(&getSTI())) {
      initializeGprCountSymbol(IS_VGPR);
      initializeGprCountSymbol(IS_SGPR);
    } else
      KernelScope.initialize(getContext());
  }
}

bool hasMIMG_R128() const {
  return AMDGPU::hasMIMG_R128(getSTI());
}

bool hasPackedD16() const {
  return AMDGPU::hasPackedD16(getSTI());
}

bool hasGFX10A16() const {
  return AMDGPU::hasGFX10A16(getSTI());
}

bool hasG16() const { return AMDGPU::hasG16(getSTI()); }

bool isSI() const {
  return AMDGPU::isSI(getSTI());
}

bool isCI() const {
  return AMDGPU::isCI(getSTI());
}

bool isVI() const {
  return AMDGPU::isVI(getSTI());
}

bool isGFX9() const {
  return AMDGPU::isGFX9(getSTI());
}

bool isGFX90A() const {
  return AMDGPU::isGFX90A(getSTI());
}

bool isGFX9Plus() const {
  return AMDGPU::isGFX9Plus(getSTI());
}

bool isGFX10() const {
  return AMDGPU::isGFX10(getSTI());
}

bool isGFX10Plus() const { return AMDGPU::isGFX10Plus(getSTI()); }

bool isGFX10_BEncoding() const {
  return AMDGPU::isGFX10_BEncoding(getSTI());
}

bool hasInv2PiInlineImm() const {
  return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
}

bool hasFlatOffsets() const {
  return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
}

bool hasArchitectedFlatScratch() const {
  return getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch];
}

bool hasSGPR102_SGPR103() const {
  return !isVI() && !isGFX9();
}

bool hasSGPR104_SGPR105() const { return isGFX10Plus(); }

bool hasIntClamp() const {
  return getFeatureBits()[AMDGPU::FeatureIntClamp];
}

AMDGPUTargetStreamer &getTargetStreamer() {
  MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
  return static_cast<AMDGPUTargetStreamer &>(TS);
}

const MCRegisterInfo *getMRI() const {
  // We need this const_cast because for some reason getContext() is not const
  // in MCAsmParser.
  return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
}

const MCInstrInfo *getMII() const {
  return &MII;
}

const FeatureBitset &getFeatureBits() const {
  return getSTI().getFeatureBits();
}

void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }

unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
bool isForcedDPP() const { return ForcedDPP; }
bool isForcedSDWA() const { return ForcedSDWA; }
ArrayRef<unsigned> getMatchedVariants() const;
StringRef getMatchedVariantName() const;

std::unique_ptr<AMDGPUOperand> parseRegister(bool RestoreOnFailure = false);
bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
                   bool RestoreOnFailure);
bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
OperandMatchResultTy tryParseRegister(unsigned &RegNo, SMLoc &StartLoc,
                                      SMLoc &EndLoc) override;
unsigned checkTargetMatchPredicate(MCInst &Inst) override;
unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
                                    unsigned Kind) override;
bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                             OperandVector &Operands, MCStreamer &Out,
                             uint64_t &ErrorInfo,
                             bool MatchingInlineAsm) override;
bool ParseDirective(AsmToken DirectiveID) override;
OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic,
                                  OperandMode Mode = OperandMode_Default);
StringRef parseMnemonicSuffix(StringRef Name);
bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
                      SMLoc NameLoc, OperandVector &Operands) override;
//bool ProcessInstruction(MCInst &Inst);

OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);

OperandMatchResultTy
parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
                   AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
                   bool (*ConvertResult)(int64_t &) = nullptr);

OperandMatchResultTy
parseOperandArrayWithPrefix(const char *Prefix,
                            OperandVector &Operands,
                            AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
                            bool (*ConvertResult)(int64_t&) = nullptr);

OperandMatchResultTy
parseNamedBit(StringRef Name, OperandVector &Operands,
              AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
OperandMatchResultTy parseCPol(OperandVector &Operands);
OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
                                           StringRef &Value,
                                           SMLoc &StringLoc);

bool isModifier();
bool isOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const;
bool isRegOrOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const;
bool isNamedOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const;
bool isOpcodeModifierWithVal(const AsmToken &Token, const AsmToken &NextToken) const;
bool parseSP3NegModifier();
OperandMatchResultTy parseImm(OperandVector &Operands, bool HasSP3AbsModifier = false);
OperandMatchResultTy parseReg(OperandVector &Operands);
OperandMatchResultTy parseRegOrImm(OperandVector &Operands, bool HasSP3AbsMod = false);
OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
OperandMatchResultTy parseDfmtNfmt(int64_t &Format);
OperandMatchResultTy parseUfmt(int64_t &Format);
OperandMatchResultTy parseSymbolicSplitFormat(StringRef FormatStr, SMLoc Loc, int64_t &Format);
OperandMatchResultTy parseSymbolicUnifiedFormat(StringRef FormatStr, SMLoc Loc, int64_t &Format);
OperandMatchResultTy parseFORMAT(OperandVector &Operands);
OperandMatchResultTy parseSymbolicOrNumericFormat(int64_t &Format);
OperandMatchResultTy parseNumericFormat(int64_t &Format);
bool tryParseFmt(const char *Pref, int64_t MaxVal, int64_t &Val);
bool matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt, StringRef FormatStr, SMLoc Loc);

void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
void cvtExp(MCInst &Inst, const OperandVector &Operands);

bool parseCnt(int64_t &IntVal);
OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
OperandMatchResultTy parseHwreg(OperandVector &Operands);

1495private:
struct OperandInfoTy {
  SMLoc Loc;
  int64_t Id;
  bool IsSymbolic = false;
  bool IsDefined = false;

  OperandInfoTy(int64_t Id_) : Id(Id_) {}
};

bool parseSendMsgBody(OperandInfoTy &Msg, OperandInfoTy &Op, OperandInfoTy &Stream);
bool validateSendMsg(const OperandInfoTy &Msg,
                     const OperandInfoTy &Op,
                     const OperandInfoTy &Stream);

bool parseHwregBody(OperandInfoTy &HwReg,
                    OperandInfoTy &Offset,
                    OperandInfoTy &Width);
bool validateHwreg(const OperandInfoTy &HwReg,
                   const OperandInfoTy &Offset,
                   const OperandInfoTy &Width);

SMLoc getFlatOffsetLoc(const OperandVector &Operands) const;
SMLoc getSMEMOffsetLoc(const OperandVector &Operands) const;

SMLoc getOperandLoc(std::function<bool(const AMDGPUOperand&)> Test,
                    const OperandVector &Operands) const;
SMLoc getImmLoc(AMDGPUOperand::ImmTy Type, const OperandVector &Operands) const;
SMLoc getRegLoc(unsigned Reg, const OperandVector &Operands) const;
SMLoc getLitLoc(const OperandVector &Operands) const;
SMLoc getConstLoc(const OperandVector &Operands) const;

bool validateInstruction(const MCInst &Inst, const SMLoc &IDLoc, const OperandVector &Operands);
bool validateFlatOffset(const MCInst &Inst, const OperandVector &Operands);
bool validateSMEMOffset(const MCInst &Inst, const OperandVector &Operands);
bool validateSOPLiteral(const MCInst &Inst) const;
bool validateConstantBusLimitations(const MCInst &Inst, const OperandVector &Operands);
bool validateEarlyClobberLimitations(const MCInst &Inst, const OperandVector &Operands);
bool validateIntClampSupported(const MCInst &Inst);
bool validateMIMGAtomicDMask(const MCInst &Inst);
bool validateMIMGGatherDMask(const MCInst &Inst);
bool validateMovrels(const MCInst &Inst, const OperandVector &Operands);
bool validateMIMGDataSize(const MCInst &Inst);
bool validateMIMGAddrSize(const MCInst &Inst);
bool validateMIMGD16(const MCInst &Inst);
bool validateMIMGDim(const MCInst &Inst);
bool validateMIMGMSAA(const MCInst &Inst);
bool validateOpSel(const MCInst &Inst);
bool validateDPP(const MCInst &Inst, const OperandVector &Operands);
bool validateVccOperand(unsigned Reg) const;
bool validateVOP3Literal(const MCInst &Inst, const OperandVector &Operands);
bool validateMAIAccWrite(const MCInst &Inst, const OperandVector &Operands);
bool validateAGPRLdSt(const MCInst &Inst) const;
bool validateVGPRAlign(const MCInst &Inst) const;
bool validateGWS(const MCInst &Inst, const OperandVector &Operands);
bool validateDivScale(const MCInst &Inst);
bool validateCoherencyBits(const MCInst &Inst, const OperandVector &Operands,
                           const SMLoc &IDLoc);
Optional<StringRef> validateLdsDirect(const MCInst &Inst);
unsigned getConstantBusLimit(unsigned Opcode) const;
bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;

bool isSupportedMnemo(StringRef Mnemo,
                      const FeatureBitset &FBS);
bool isSupportedMnemo(StringRef Mnemo,
                      const FeatureBitset &FBS,
                      ArrayRef<unsigned> Variants);
bool checkUnsupportedInstruction(StringRef Name, const SMLoc &IDLoc);

bool isId(const StringRef Id) const;
bool isId(const AsmToken &Token, const StringRef Id) const;
bool isToken(const AsmToken::TokenKind Kind) const;
bool trySkipId(const StringRef Id);
bool trySkipId(const StringRef Pref, const StringRef Id);
bool trySkipId(const StringRef Id, const AsmToken::TokenKind Kind);
bool trySkipToken(const AsmToken::TokenKind Kind);
bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
bool parseString(StringRef &Val, const StringRef ErrMsg = "expected a string");
bool parseId(StringRef &Val, const StringRef ErrMsg = "");

void peekTokens(MutableArrayRef<AsmToken> Tokens);
AsmToken::TokenKind getTokenKind() const;
bool parseExpr(int64_t &Imm, StringRef Expected = "");
bool parseExpr(OperandVector &Operands);
StringRef getTokenStr() const;
AsmToken peekToken();
AsmToken getToken() const;
SMLoc getLoc() const;
void lex();

1587public:
void onBeginOfFile() override;

OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
OperandMatchResultTy parseOptionalOpr(OperandVector &Operands);

OperandMatchResultTy parseExpTgt(OperandVector &Operands);
OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
OperandMatchResultTy parseBoolReg(OperandVector &Operands);

bool parseSwizzleOperand(int64_t &Op,
                         const unsigned MinVal,
                         const unsigned MaxVal,
                         const StringRef ErrMsg,
                         SMLoc &Loc);
bool parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
                          const unsigned MinVal,
                          const unsigned MaxVal,
                          const StringRef ErrMsg);
OperandMatchResultTy parseSwizzleOp(OperandVector &Operands);
bool parseSwizzleOffset(int64_t &Imm);
bool parseSwizzleMacro(int64_t &Imm);
bool parseSwizzleQuadPerm(int64_t &Imm);
bool parseSwizzleBitmaskPerm(int64_t &Imm);
bool parseSwizzleBroadcast(int64_t &Imm);
bool parseSwizzleSwap(int64_t &Imm);
bool parseSwizzleReverse(int64_t &Imm);

OperandMatchResultTy parseGPRIdxMode(OperandVector &Operands);
int64_t parseGPRIdxMacro();

void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false); }
void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true); }
void cvtMubufLds(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, true); }
void cvtMtbuf(MCInst &Inst, const OperandVector &Operands);

AMDGPUOperand::Ptr defaultCPol() const;

AMDGPUOperand::Ptr defaultSMRDOffset8() const;
AMDGPUOperand::Ptr defaultSMEMOffset() const;
AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
AMDGPUOperand::Ptr defaultFlatOffset() const;

OperandMatchResultTy parseOModOperand(OperandVector &Operands);

void cvtVOP3(MCInst &Inst, const OperandVector &Operands,
             OptionalImmIndexMap &OptionalIdx);
void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands);
void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
void cvtVOP3P(MCInst &Inst, const OperandVector &Operands,
              OptionalImmIndexMap &OptionalIdx);

void cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands);

void cvtMIMG(MCInst &Inst, const OperandVector &Operands,
             bool IsAtomic = false);
void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
void cvtIntersectRay(MCInst &Inst, const OperandVector &Operands);

void cvtSMEMAtomic(MCInst &Inst, const OperandVector &Operands);

bool parseDimId(unsigned &Encoding);
OperandMatchResultTy parseDim(OperandVector &Operands);
OperandMatchResultTy parseDPP8(OperandVector &Operands);
OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
bool isSupportedDPPCtrl(StringRef Ctrl, const OperandVector &Operands);
int64_t parseDPPCtrlSel(StringRef Ctrl);
int64_t parseDPPCtrlPerm();
AMDGPUOperand::Ptr defaultRowMask() const;
AMDGPUOperand::Ptr defaultBankMask() const;
AMDGPUOperand::Ptr defaultBoundCtrl() const;
AMDGPUOperand::Ptr defaultFI() const;
void cvtDPP(MCInst &Inst, const OperandVector &Operands, bool IsDPP8 = false);
void cvtDPP8(MCInst &Inst, const OperandVector &Operands) { cvtDPP(Inst, Operands, true); }

OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
                                  AMDGPUOperand::ImmTy Type);
OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
void cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands);
void cvtSdwaVOP2e(MCInst &Inst, const OperandVector &Operands);
void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
             uint64_t BasicInstType,
             bool SkipDstVcc = false,
             bool SkipSrcVcc = false);

AMDGPUOperand::Ptr defaultBLGP() const;
AMDGPUOperand::Ptr defaultCBSZ() const;
AMDGPUOperand::Ptr defaultABID() const;

OperandMatchResultTy parseEndpgmOp(OperandVector &Operands);
AMDGPUOperand::Ptr defaultEndpgmImmOperands() const;
1685};

1687struct OptionalOperand {
const char *Name;
AMDGPUOperand::ImmTy Type;
bool IsBit;
bool (*ConvertResult)(int64_t&);
1692};

1694} // end anonymous namespace

1696// May be called with integer type with equivalent bitwidth.
1697static const fltSemantics *getFltSemantics(unsigned Size) {
switch (Size) {
case 4:
  return &APFloat::IEEEsingle();
case 8:
  return &APFloat::IEEEdouble();
case 2:
  return &APFloat::IEEEhalf();
default:
  llvm_unreachable("unsupported fp type")__builtin_unreachable();
}
1708}

1710static const fltSemantics *getFltSemantics(MVT VT) {
return getFltSemantics(VT.getSizeInBits() / 8);
1712}

1714static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
switch (OperandType) {
case AMDGPU::OPERAND_REG_IMM_INT32:
case AMDGPU::OPERAND_REG_IMM_FP32:
case AMDGPU::OPERAND_REG_INLINE_C_INT32:
case AMDGPU::OPERAND_REG_INLINE_C_FP32:
case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
case AMDGPU::OPERAND_REG_INLINE_C_V2FP32:
case AMDGPU::OPERAND_REG_IMM_V2FP32:
case AMDGPU::OPERAND_REG_INLINE_C_V2INT32:
case AMDGPU::OPERAND_REG_IMM_V2INT32:
  return &APFloat::IEEEsingle();
case AMDGPU::OPERAND_REG_IMM_INT64:
case AMDGPU::OPERAND_REG_IMM_FP64:
case AMDGPU::OPERAND_REG_INLINE_C_INT64:
case AMDGPU::OPERAND_REG_INLINE_C_FP64:
case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
  return &APFloat::IEEEdouble();
case AMDGPU::OPERAND_REG_IMM_INT16:
case AMDGPU::OPERAND_REG_IMM_FP16:
case AMDGPU::OPERAND_REG_INLINE_C_INT16:
case AMDGPU::OPERAND_REG_INLINE_C_FP16:
case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16:
case AMDGPU::OPERAND_REG_IMM_V2INT16:
case AMDGPU::OPERAND_REG_IMM_V2FP16:
  return &APFloat::IEEEhalf();
default:
  llvm_unreachable("unsupported fp type")__builtin_unreachable();
}
1749}

1751//===----------------------------------------------------------------------===//
1752// Operand
1753//===----------------------------------------------------------------------===//

1755static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
bool Lost;

// Convert literal to single precision
APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
                                             APFloat::rmNearestTiesToEven,
                                             &Lost);
// We allow precision lost but not overflow or underflow
if (Status != APFloat::opOK &&
    Lost &&
    ((Status & APFloat::opOverflow)  != 0 ||
     (Status & APFloat::opUnderflow) != 0)) {
  return false;
}

return true;
1771}

1773static bool isSafeTruncation(int64_t Val, unsigned Size) {
return isUIntN(Size, Val) || isIntN(Size, Val);
1775}

1777static bool isInlineableLiteralOp16(int64_t Val, MVT VT, bool HasInv2Pi) {
if (VT.getScalarType() == MVT::i16) {
  // FP immediate values are broken.
  return isInlinableIntLiteral(Val);
}

// f16/v2f16 operands work correctly for all values.
return AMDGPU::isInlinableLiteral16(Val, HasInv2Pi);
1785}

1787bool AMDGPUOperand::isInlinableImm(MVT type) const {

// This is a hack to enable named inline values like
// shared_base with both 32-bit and 64-bit operands.
// Note that these values are defined as
// 32-bit operands only.
if (isInlineValue()) {
  return true;
}

if (!isImmTy(ImmTyNone)) {
  // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
  return false;
}
// TODO: We should avoid using host float here. It would be better to
// check the float bit values which is what a few other places do.
// We've had bot failures before due to weird NaN support on mips hosts.

APInt Literal(64, Imm.Val);

if (Imm.IsFPImm) { // We got fp literal token
  if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
    return AMDGPU::isInlinableLiteral64(Imm.Val,
                                        AsmParser->hasInv2PiInlineImm());
  }

  APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
  if (!canLosslesslyConvertToFPType(FPLiteral, type))
    return false;

  if (type.getScalarSizeInBits() == 16) {
    return isInlineableLiteralOp16(
      static_cast<int16_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
      type, AsmParser->hasInv2PiInlineImm());
  }

  // Check if single precision literal is inlinable
  return AMDGPU::isInlinableLiteral32(
    static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
    AsmParser->hasInv2PiInlineImm());
}

// We got int literal token.
if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
  return AMDGPU::isInlinableLiteral64(Imm.Val,
                                      AsmParser->hasInv2PiInlineImm());
}

if (!isSafeTruncation(Imm.Val, type.getScalarSizeInBits())) {
  return false;
}

if (type.getScalarSizeInBits() == 16) {
  return isInlineableLiteralOp16(
    static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
    type, AsmParser->hasInv2PiInlineImm());
}

return AMDGPU::isInlinableLiteral32(
  static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
  AsmParser->hasInv2PiInlineImm());
1848}

1850bool AMDGPUOperand::isLiteralImm(MVT type) const {
// Check that this immediate can be added as literal
if (!isImmTy(ImmTyNone)) {
  return false;
}

if (!Imm.IsFPImm) {
  // We got int literal token.

  if (type == MVT::f64 && hasFPModifiers()) {
    // Cannot apply fp modifiers to int literals preserving the same semantics
    // for VOP1/2/C and VOP3 because of integer truncation. To avoid ambiguity,
    // disable these cases.
    return false;
  }

  unsigned Size = type.getSizeInBits();
  if (Size == 64)
    Size = 32;

  // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
  // types.
  return isSafeTruncation(Imm.Val, Size);
}

// We got fp literal token
if (type == MVT::f64) { // Expected 64-bit fp operand
  // We would set low 64-bits of literal to zeroes but we accept this literals
  return true;
}

if (type == MVT::i64) { // Expected 64-bit int operand
  // We don't allow fp literals in 64-bit integer instructions. It is
  // unclear how we should encode them.
  return false;
}

// We allow fp literals with f16x2 operands assuming that the specified
// literal goes into the lower half and the upper half is zero. We also
// require that the literal may be losslesly converted to f16.
MVT ExpectedType = (type == MVT::v2f16)? MVT::f16 :
                   (type == MVT::v2i16)? MVT::i16 :
                   (type == MVT::v2f32)? MVT::f32 : type;

APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
return canLosslesslyConvertToFPType(FPLiteral, ExpectedType);
1896}

1898bool AMDGPUOperand::isRegClass(unsigned RCID) const {
return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1900}

1902bool AMDGPUOperand::isVRegWithInputMods() const {
return isRegClass(AMDGPU::VGPR_32RegClassID) ||
       // GFX90A allows DPP on 64-bit operands.
       (isRegClass(AMDGPU::VReg_64RegClassID) &&
        AsmParser->getFeatureBits()[AMDGPU::Feature64BitDPP]);
1907}

1909bool AMDGPUOperand::isSDWAOperand(MVT type) const {
if (AsmParser->isVI())
  return isVReg32();
else if (AsmParser->isGFX9Plus())
  return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(type);
else
  return false;
1916}

1918bool AMDGPUOperand::isSDWAFP16Operand() const {
return isSDWAOperand(MVT::f16);
1920}

1922bool AMDGPUOperand::isSDWAFP32Operand() const {
return isSDWAOperand(MVT::f32);
1924}

1926bool AMDGPUOperand::isSDWAInt16Operand() const {
return isSDWAOperand(MVT::i16);
1928}

1930bool AMDGPUOperand::isSDWAInt32Operand() const {
return isSDWAOperand(MVT::i32);
1932}

1934bool AMDGPUOperand::isBoolReg() const {
auto FB = AsmParser->getFeatureBits();
return isReg() && ((FB[AMDGPU::FeatureWavefrontSize64] && isSCSrcB64()) ||
                   (FB[AMDGPU::FeatureWavefrontSize32] && isSCSrcB32()));
1938}

1940uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val, unsigned Size) const
1941{
assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers())(static_cast<void> (0));
assert(Size == 2 || Size == 4 || Size == 8)(static_cast<void> (0));

const uint64_t FpSignMask = (1ULL << (Size * 8 - 1));

if (Imm.Mods.Abs) {
  Val &= ~FpSignMask;
}
if (Imm.Mods.Neg) {
  Val ^= FpSignMask;
}

return Val;
1955}

1957void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
                           Inst.getNumOperands())) {
  addLiteralImmOperand(Inst, Imm.Val,
                       ApplyModifiers &
                       isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
} else {
  assert(!isImmTy(ImmTyNone) || !hasModifiers())(static_cast<void> (0));
  Inst.addOperand(MCOperand::createImm(Imm.Val));
  setImmKindNone();
}
1968}

1970void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const {
const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
auto OpNum = Inst.getNumOperands();
// Check that this operand accepts literals
assert(AMDGPU::isSISrcOperand(InstDesc, OpNum))(static_cast<void> (0));

if (ApplyModifiers) {
  assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum))(static_cast<void> (0));
  const unsigned Size = Imm.IsFPImm ? sizeof(double) : getOperandSize(InstDesc, OpNum);
  Val = applyInputFPModifiers(Val, Size);
}

APInt Literal(64, Val);
uint8_t OpTy = InstDesc.OpInfo[OpNum].OperandType;

if (Imm.IsFPImm) { // We got fp literal token
  switch (OpTy) {
  case AMDGPU::OPERAND_REG_IMM_INT64:
  case AMDGPU::OPERAND_REG_IMM_FP64:
  case AMDGPU::OPERAND_REG_INLINE_C_INT64:
  case AMDGPU::OPERAND_REG_INLINE_C_FP64:
  case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
    if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
                                     AsmParser->hasInv2PiInlineImm())) {
      Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
      setImmKindConst();
      return;
    }

    // Non-inlineable
    if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
      // For fp operands we check if low 32 bits are zeros
      if (Literal.getLoBits(32) != 0) {
        const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
        "Can't encode literal as exact 64-bit floating-point operand. "
        "Low 32-bits will be set to zero");
      }

      Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
      setImmKindLiteral();
      return;
    }

    // We don't allow fp literals in 64-bit integer instructions. It is
    // unclear how we should encode them. This case should be checked earlier
    // in predicate methods (isLiteralImm())
    llvm_unreachable("fp literal in 64-bit integer instruction.")__builtin_unreachable();

  case AMDGPU::OPERAND_REG_IMM_INT32:
  case AMDGPU::OPERAND_REG_IMM_FP32:
  case AMDGPU::OPERAND_REG_INLINE_C_INT32:
  case AMDGPU::OPERAND_REG_INLINE_C_FP32:
  case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
  case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
  case AMDGPU::OPERAND_REG_IMM_INT16:
  case AMDGPU::OPERAND_REG_IMM_FP16:
  case AMDGPU::OPERAND_REG_INLINE_C_INT16:
  case AMDGPU::OPERAND_REG_INLINE_C_FP16:
  case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
  case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
  case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
  case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
  case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
  case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16:
  case AMDGPU::OPERAND_REG_IMM_V2INT16:
  case AMDGPU::OPERAND_REG_IMM_V2FP16:
  case AMDGPU::OPERAND_REG_INLINE_C_V2FP32:
  case AMDGPU::OPERAND_REG_IMM_V2FP32:
  case AMDGPU::OPERAND_REG_INLINE_C_V2INT32:
  case AMDGPU::OPERAND_REG_IMM_V2INT32: {
    bool lost;
    APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
    // Convert literal to single precision
    FPLiteral.convert(*getOpFltSemantics(OpTy),
                      APFloat::rmNearestTiesToEven, &lost);
    // We allow precision lost but not overflow or underflow. This should be
    // checked earlier in isLiteralImm()

    uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
    Inst.addOperand(MCOperand::createImm(ImmVal));
    setImmKindLiteral();
    return;
  }
  default:
    llvm_unreachable("invalid operand size")__builtin_unreachable();
  }

  return;
}

// We got int literal token.
// Only sign extend inline immediates.
switch (OpTy) {
case AMDGPU::OPERAND_REG_IMM_INT32:
case AMDGPU::OPERAND_REG_IMM_FP32:
case AMDGPU::OPERAND_REG_INLINE_C_INT32:
case AMDGPU::OPERAND_REG_INLINE_C_FP32:
case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
case AMDGPU::OPERAND_REG_IMM_V2INT16:
case AMDGPU::OPERAND_REG_IMM_V2FP16:
case AMDGPU::OPERAND_REG_IMM_V2FP32:
case AMDGPU::OPERAND_REG_INLINE_C_V2FP32:
case AMDGPU::OPERAND_REG_IMM_V2INT32:
case AMDGPU::OPERAND_REG_INLINE_C_V2INT32:
  if (isSafeTruncation(Val, 32) &&
      AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
                                   AsmParser->hasInv2PiInlineImm())) {
    Inst.addOperand(MCOperand::createImm(Val));
    setImmKindConst();
    return;
  }

  Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
  setImmKindLiteral();
  return;

case AMDGPU::OPERAND_REG_IMM_INT64:
case AMDGPU::OPERAND_REG_IMM_FP64:
case AMDGPU::OPERAND_REG_INLINE_C_INT64:
case AMDGPU::OPERAND_REG_INLINE_C_FP64:
case AMDGPU::OPERAND_REG_INLINE_AC_FP64:
  if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
    Inst.addOperand(MCOperand::createImm(Val));
    setImmKindConst();
    return;
  }

  Inst.addOperand(MCOperand::createImm(Lo_32(Val)));
  setImmKindLiteral();
  return;

case AMDGPU::OPERAND_REG_IMM_INT16:
case AMDGPU::OPERAND_REG_IMM_FP16:
case AMDGPU::OPERAND_REG_INLINE_C_INT16:
case AMDGPU::OPERAND_REG_INLINE_C_FP16:
case AMDGPU::OPERAND_REG_INLINE_AC_INT16:
case AMDGPU::OPERAND_REG_INLINE_AC_FP16:
  if (isSafeTruncation(Val, 16) &&
      AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
                                   AsmParser->hasInv2PiInlineImm())) {
    Inst.addOperand(MCOperand::createImm(Val));
    setImmKindConst();
    return;
  }

  Inst.addOperand(MCOperand::createImm(Val & 0xffff));
  setImmKindLiteral();
  return;

case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
case AMDGPU::OPERAND_REG_INLINE_AC_V2INT16:
case AMDGPU::OPERAND_REG_INLINE_AC_V2FP16: {
  assert(isSafeTruncation(Val, 16))(static_cast<void> (0));
  assert(AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),(static_cast<void> (0))
                                      AsmParser->hasInv2PiInlineImm()))(static_cast<void> (0));

  Inst.addOperand(MCOperand::createImm(Val));
  return;
}
default:
  llvm_unreachable("invalid operand size")__builtin_unreachable();
}
2134}

2136template <unsigned Bitwidth>
2137void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
APInt Literal(64, Imm.Val);
setImmKindNone();

if (!Imm.IsFPImm) {
  // We got int literal token.
  Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
  return;
}

bool Lost;
APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
                  APFloat::rmNearestTiesToEven, &Lost);
Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
2152}

2154void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
2156}

2158static bool isInlineValue(unsigned Reg) {
switch (Reg) {
case AMDGPU::SRC_SHARED_BASE:
case AMDGPU::SRC_SHARED_LIMIT:
case AMDGPU::SRC_PRIVATE_BASE:
case AMDGPU::SRC_PRIVATE_LIMIT:
case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
  return true;
case AMDGPU::SRC_VCCZ:
case AMDGPU::SRC_EXECZ:
case AMDGPU::SRC_SCC:
  return true;
case AMDGPU::SGPR_NULL:
  return true;
default:
  return false;
}
2175}

2177bool AMDGPUOperand::isInlineValue() const {
return isRegKind() && ::isInlineValue(getReg());
2179}

2181//===----------------------------------------------------------------------===//
2182// AsmParser
2183//===----------------------------------------------------------------------===//

2185static int getRegClass(RegisterKind Is, unsigned RegWidth) {
if (Is == IS_VGPR) {
  switch (RegWidth) {
    default: return -1;
    case 1: return AMDGPU::VGPR_32RegClassID;
    case 2: return AMDGPU::VReg_64RegClassID;
    case 3: return AMDGPU::VReg_96RegClassID;
    case 4: return AMDGPU::VReg_128RegClassID;
    case 5: return AMDGPU::VReg_160RegClassID;
    case 6: return AMDGPU::VReg_192RegClassID;
    case 7: return AMDGPU::VReg_224RegClassID;
    case 8: return AMDGPU::VReg_256RegClassID;
    case 16: return AMDGPU::VReg_512RegClassID;
    case 32: return AMDGPU::VReg_1024RegClassID;
  }
} else if (Is == IS_TTMP) {
  switch (RegWidth) {
    default: return -1;
    case 1: return AMDGPU::TTMP_32RegClassID;
    case 2: return AMDGPU::TTMP_64RegClassID;
    case 4: return AMDGPU::TTMP_128RegClassID;
    case 8: return AMDGPU::TTMP_256RegClassID;
    case 16: return AMDGPU::TTMP_512RegClassID;
  }
} else if (Is == IS_SGPR) {
  switch (RegWidth) {
    default: return -1;
    case 1: return AMDGPU::SGPR_32RegClassID;
    case 2: return AMDGPU::SGPR_64RegClassID;
    case 3: return AMDGPU::SGPR_96RegClassID;
    case 4: return AMDGPU::SGPR_128RegClassID;
    case 5: return AMDGPU::SGPR_160RegClassID;
    case 6: return AMDGPU::SGPR_192RegClassID;
    case 7: return AMDGPU::SGPR_224RegClassID;
    case 8: return AMDGPU::SGPR_256RegClassID;
    case 16: return AMDGPU::SGPR_512RegClassID;
  }
} else if (Is == IS_AGPR) {
  switch (RegWidth) {
    default: return -1;
    case 1: return AMDGPU::AGPR_32RegClassID;
    case 2: return AMDGPU::AReg_64RegClassID;
    case 3: return AMDGPU::AReg_96RegClassID;
    case 4: return AMDGPU::AReg_128RegClassID;
    case 5: return AMDGPU::AReg_160RegClassID;
    case 6: return AMDGPU::AReg_192RegClassID;
    case 7: return AMDGPU::AReg_224RegClassID;
    case 8: return AMDGPU::AReg_256RegClassID;
    case 16: return AMDGPU::AReg_512RegClassID;
    case 32: return AMDGPU::AReg_1024RegClassID;
  }
}
return -1;
2238}

2240static unsigned getSpecialRegForName(StringRef RegName) {
return StringSwitch<unsigned>(RegName)
  .Case("exec", AMDGPU::EXEC)
  .Case("vcc", AMDGPU::VCC)
  .Case("flat_scratch", AMDGPU::FLAT_SCR)
  .Case("xnack_mask", AMDGPU::XNACK_MASK)
  .Case("shared_base", AMDGPU::SRC_SHARED_BASE)
  .Case("src_shared_base", AMDGPU::SRC_SHARED_BASE)
  .Case("shared_limit", AMDGPU::SRC_SHARED_LIMIT)
  .Case("src_shared_limit", AMDGPU::SRC_SHARED_LIMIT)
  .Case("private_base", AMDGPU::SRC_PRIVATE_BASE)
  .Case("src_private_base", AMDGPU::SRC_PRIVATE_BASE)
  .Case("private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
  .Case("src_private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
  .Case("pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
  .Case("src_pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
  .Case("lds_direct", AMDGPU::LDS_DIRECT)
  .Case("src_lds_direct", AMDGPU::LDS_DIRECT)
  .Case("m0", AMDGPU::M0)
  .Case("vccz", AMDGPU::SRC_VCCZ)
  .Case("src_vccz", AMDGPU::SRC_VCCZ)
  .Case("execz", AMDGPU::SRC_EXECZ)
  .Case("src_execz", AMDGPU::SRC_EXECZ)
  .Case("scc", AMDGPU::SRC_SCC)
  .Case("src_scc", AMDGPU::SRC_SCC)
  .Case("tba", AMDGPU::TBA)
  .Case("tma", AMDGPU::TMA)
  .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
  .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
  .Case("xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
  .Case("xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
  .Case("vcc_lo", AMDGPU::VCC_LO)
  .Case("vcc_hi", AMDGPU::VCC_HI)
  .Case("exec_lo", AMDGPU::EXEC_LO)
  .Case("exec_hi", AMDGPU::EXEC_HI)
  .Case("tma_lo", AMDGPU::TMA_LO)
  .Case("tma_hi", AMDGPU::TMA_HI)
  .Case("tba_lo", AMDGPU::TBA_LO)
  .Case("tba_hi", AMDGPU::TBA_HI)
  .Case("pc", AMDGPU::PC_REG)
  .Case("null", AMDGPU::SGPR_NULL)
  .Default(AMDGPU::NoRegister);
2282}

2284bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
                                  SMLoc &EndLoc, bool RestoreOnFailure) {
auto R = parseRegister();
if (!R) return true;
assert(R->isReg())(static_cast<void> (0));
RegNo = R->getReg();
StartLoc = R->getStartLoc();
EndLoc = R->getEndLoc();
return false;
2293}

2295bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
                                  SMLoc &EndLoc) {
return ParseRegister(RegNo, StartLoc, EndLoc, /*RestoreOnFailure=*/false);
2298}

2300OperandMatchResultTy AMDGPUAsmParser::tryParseRegister(unsigned &RegNo,
                                                     SMLoc &StartLoc,
                                                     SMLoc &EndLoc) {
bool Result =
    ParseRegister(RegNo, StartLoc, EndLoc, /*RestoreOnFailure=*/true);
bool PendingErrors = getParser().hasPendingError();
getParser().clearPendingErrors();
if (PendingErrors)
  return MatchOperand_ParseFail;
if (Result)
  return MatchOperand_NoMatch;
return MatchOperand_Success;
2312}

2314bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
                                          RegisterKind RegKind, unsigned Reg1,
                                          SMLoc Loc) {
switch (RegKind) {
case IS_SPECIAL:
  if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
    Reg = AMDGPU::EXEC;
    RegWidth = 2;
    return true;
  }
  if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
    Reg = AMDGPU::FLAT_SCR;
    RegWidth = 2;
    return true;
  }
  if (Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
    Reg = AMDGPU::XNACK_MASK;
    RegWidth = 2;
    return true;
  }
  if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
    Reg = AMDGPU::VCC;
    RegWidth = 2;
    return true;
  }
  if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
    Reg = AMDGPU::TBA;
    RegWidth = 2;
    return true;
  }
  if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
    Reg = AMDGPU::TMA;
    RegWidth = 2;
    return true;
  }
  Error(Loc, "register does not fit in the list");
  return false;
case IS_VGPR:
case IS_SGPR:
case IS_AGPR:
case IS_TTMP:
  if (Reg1 != Reg + RegWidth) {
    Error(Loc, "registers in a list must have consecutive indices");
    return false;
  }
  RegWidth++;
  return true;
default:
  llvm_unreachable("unexpected register kind")__builtin_unreachable();
}
2364}

2366struct RegInfo {
StringLiteral Name;
RegisterKind Kind;
2369};

2371static constexpr RegInfo RegularRegisters[] = {
{{"v"},    IS_VGPR},
{{"s"},    IS_SGPR},
{{"ttmp"}, IS_TTMP},
{{"acc"},  IS_AGPR},
{{"a"},    IS_AGPR},
2377};

2379static bool isRegularReg(RegisterKind Kind) {
return Kind == IS_VGPR ||
       Kind == IS_SGPR ||
       Kind == IS_TTMP ||
       Kind == IS_AGPR;
2384}

2386static const RegInfo* getRegularRegInfo(StringRef Str) {
for (const RegInfo &Reg : RegularRegisters)
  if (Str.startswith(Reg.Name))
    return &Reg;
return nullptr;
2391}

2393static bool getRegNum(StringRef Str, unsigned& Num) {
return !Str.getAsInteger(10, Num);
2395}

2397bool
2398AMDGPUAsmParser::isRegister(const AsmToken &Token,
                          const AsmToken &NextToken) const {

// A list of consecutive registers: [s0,s1,s2,s3]
if (Token.is(AsmToken::LBrac))
  return true;

if (!Token.is(AsmToken::Identifier))
  return false;

// A single register like s0 or a range of registers like s[0:1]

StringRef Str = Token.getString();
const RegInfo *Reg = getRegularRegInfo(Str);
if (Reg) {
  StringRef RegName = Reg->Name;
  StringRef RegSuffix = Str.substr(RegName.size());
  if (!RegSuffix.empty()) {
    unsigned Num;
    // A single register with an index: rXX
    if (getRegNum(RegSuffix, Num))
      return true;
  } else {
    // A range of registers: r[XX:YY].
    if (NextToken.is(AsmToken::LBrac))
      return true;
  }
}

return getSpecialRegForName(Str) != AMDGPU::NoRegister;
2428}

2430bool
2431AMDGPUAsmParser::isRegister()
2432{
return isRegister(getToken(), peekToken());
2434}

2436unsigned
2437AMDGPUAsmParser::getRegularReg(RegisterKind RegKind,
                             unsigned RegNum,
                             unsigned RegWidth,
                             SMLoc Loc) {

assert(isRegularReg(RegKind))(static_cast<void> (0));

unsigned AlignSize = 1;
if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
  // SGPR and TTMP registers must be aligned.
  // Max required alignment is 4 dwords.
  AlignSize = std::min(RegWidth, 4u);
}

if (RegNum % AlignSize != 0) {
  Error(Loc, "invalid register alignment");
  return AMDGPU::NoRegister;
}

unsigned RegIdx = RegNum / AlignSize;
int RCID = getRegClass(RegKind, RegWidth);
if (RCID == -1) {
  Error(Loc, "invalid or unsupported register size");
  return AMDGPU::NoRegister;
}

const MCRegisterInfo *TRI = getContext().getRegisterInfo();
const MCRegisterClass RC = TRI->getRegClass(RCID);
if (RegIdx >= RC.getNumRegs()) {
  Error(Loc, "register index is out of range");
  return AMDGPU::NoRegister;
}

return RC.getRegister(RegIdx);
2471}

2473bool
2474AMDGPUAsmParser::ParseRegRange(unsigned& Num, unsigned& Width) {
int64_t RegLo, RegHi;
if (!skipToken(AsmToken::LBrac, "missing register index"))
  return false;

SMLoc FirstIdxLoc = getLoc();
SMLoc SecondIdxLoc;

if (!parseExpr(RegLo))
  return false;

if (trySkipToken(AsmToken::Colon)) {
  SecondIdxLoc = getLoc();
  if (!parseExpr(RegHi))
    return false;
} else {
  RegHi = RegLo;
}

if (!skipToken(AsmToken::RBrac, "expected a closing square bracket"))
  return false;

if (!isUInt<32>(RegLo)) {
  Error(FirstIdxLoc, "invalid register index");
  return false;
}

if (!isUInt<32>(RegHi)) {
  Error(SecondIdxLoc, "invalid register index");
  return false;
}

if (RegLo > RegHi) {
  Error(FirstIdxLoc, "first register index should not exceed second index");
  return false;
}

Num = static_cast<unsigned>(RegLo);
Width = (RegHi - RegLo) + 1;
return true;
2514}

2516unsigned AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind,
                                        unsigned &RegNum, unsigned &RegWidth,
                                        SmallVectorImpl<AsmToken> &Tokens) {
assert(isToken(AsmToken::Identifier))(static_cast<void> (0));
unsigned Reg = getSpecialRegForName(getTokenStr());
if (Reg) {
  RegNum = 0;
  RegWidth = 1;
  RegKind = IS_SPECIAL;
  Tokens.push_back(getToken());
  lex(); // skip register name
}
return Reg;
2529}

2531unsigned AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
                                        unsigned &RegNum, unsigned &RegWidth,
                                        SmallVectorImpl<AsmToken> &Tokens) {
assert(isToken(AsmToken::Identifier))(static_cast<void> (0));
StringRef RegName = getTokenStr();
auto Loc = getLoc();

const RegInfo *RI = getRegularRegInfo(RegName);
if (!RI) {
  Error(Loc, "invalid register name");
  return AMDGPU::NoRegister;
}

Tokens.push_back(getToken());
lex(); // skip register name

RegKind = RI->Kind;
StringRef RegSuffix = RegName.substr(RI->Name.size());
if (!RegSuffix.empty()) {
  // Single 32-bit register: vXX.
  if (!getRegNum(RegSuffix, RegNum)) {
    Error(Loc, "invalid register index");
    return AMDGPU::NoRegister;
  }
  RegWidth = 1;
} else {
  // Range of registers: v[XX:YY]. ":YY" is optional.
  if (!ParseRegRange(RegNum, RegWidth))
    return AMDGPU::NoRegister;
}

return getRegularReg(RegKind, RegNum, RegWidth, Loc);
2563}

2565unsigned AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind, unsigned &RegNum,
                                     unsigned &RegWidth,
                                     SmallVectorImpl<AsmToken> &Tokens) {
unsigned Reg = AMDGPU::NoRegister;
auto ListLoc = getLoc();

if (!skipToken(AsmToken::LBrac,
               "expected a register or a list of registers")) {
  return AMDGPU::NoRegister;
}

// List of consecutive registers, e.g.: [s0,s1,s2,s3]

auto Loc = getLoc();
if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth))
  return AMDGPU::NoRegister;
if (RegWidth != 1) {
  Error(Loc, "expected a single 32-bit register");
  return AMDGPU::NoRegister;
}

for (; trySkipToken(AsmToken::Comma); ) {
  RegisterKind NextRegKind;
  unsigned NextReg, NextRegNum, NextRegWidth;
  Loc = getLoc();

  if (!ParseAMDGPURegister(NextRegKind, NextReg,
                           NextRegNum, NextRegWidth,
                           Tokens)) {
    return AMDGPU::NoRegister;
  }
  if (NextRegWidth != 1) {
    Error(Loc, "expected a single 32-bit register");
    return AMDGPU::NoRegister;
  }
  if (NextRegKind != RegKind) {
    Error(Loc, "registers in a list must be of the same kind");
    return AMDGPU::NoRegister;
  }
  if (!AddNextRegisterToList(Reg, RegWidth, RegKind, NextReg, Loc))
    return AMDGPU::NoRegister;
}

if (!skipToken(AsmToken::RBrac,
               "expected a comma or a closing square bracket")) {
  return AMDGPU::NoRegister;
}

if (isRegularReg(RegKind))
  Reg = getRegularReg(RegKind, RegNum, RegWidth, ListLoc);

return Reg;
2617}

2619bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
                                        unsigned &RegNum, unsigned &RegWidth,
                                        SmallVectorImpl<AsmToken> &Tokens) {
auto Loc = getLoc();
Reg = AMDGPU::NoRegister;

if (isToken(AsmToken::Identifier)) {
  Reg = ParseSpecialReg(RegKind, RegNum, RegWidth, Tokens);
  if (Reg == AMDGPU::NoRegister)
    Reg = ParseRegularReg(RegKind, RegNum, RegWidth, Tokens);
} else {
  Reg = ParseRegList(RegKind, RegNum, RegWidth, Tokens);
}

const MCRegisterInfo *TRI = getContext().getRegisterInfo();
if (Reg == AMDGPU::NoRegister) {
  assert(Parser.hasPendingError())(static_cast<void> (0));
  return false;
}

if (!subtargetHasRegister(*TRI, Reg)) {
  if (Reg == AMDGPU::SGPR_NULL) {
    Error(Loc, "'null' operand is not supported on this GPU");
  } else {
    Error(Loc, "register not available on this GPU");
  }
  return false;
}

return true;
2649}

2651bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
                                        unsigned &RegNum, unsigned &RegWidth,
                                        bool RestoreOnFailure /*=false*/) {
Reg = AMDGPU::NoRegister;

SmallVector<AsmToken, 1> Tokens;
if (ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, Tokens)) {
  if (RestoreOnFailure) {
    while (!Tokens.empty()) {
      getLexer().UnLex(Tokens.pop_back_val());
    }
  }
  return true;
}
return false;
2666}

2668Optional<StringRef>
2669AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) {
switch (RegKind) {
case IS_VGPR:
  return StringRef(".amdgcn.next_free_vgpr");
case IS_SGPR:
  return StringRef(".amdgcn.next_free_sgpr");
default:
  return None;
}
2678}

2680void AMDGPUAsmParser::initializeGprCountSymbol(RegisterKind RegKind) {
auto SymbolName = getGprCountSymbolName(RegKind);
assert(SymbolName && "initializing invalid register kind")(static_cast<void> (0));
MCSymbol *Sym = getContext().getOrCreateSymbol(*SymbolName);
Sym->setVariableValue(MCConstantExpr::create(0, getContext()));
2685}

2687bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind,
                                          unsigned DwordRegIndex,
                                          unsigned RegWidth) {
// Symbols are only defined for GCN targets
if (AMDGPU::getIsaVersion(getSTI().getCPU()).Major < 6)
  return true;

auto SymbolName = getGprCountSymbolName(RegKind);
if (!SymbolName)
  return true;
MCSymbol *Sym = getContext().getOrCreateSymbol(*SymbolName);

int64_t NewMax = DwordRegIndex + RegWidth - 1;
int64_t OldCount;

if (!Sym->isVariable())
  return !Error(getLoc(),
                ".amdgcn.next_free_{v,s}gpr symbols must be variable");
if (!Sym->getVariableValue(false)->evaluateAsAbsolute(OldCount))
  return !Error(
      getLoc(),
      ".amdgcn.next_free_{v,s}gpr symbols must be absolute expressions");

if (OldCount <= NewMax)
  Sym->setVariableValue(MCConstantExpr::create(NewMax + 1, getContext()));

return true;
2714}

2716std::unique_ptr<AMDGPUOperand>
2717AMDGPUAsmParser::parseRegister(bool RestoreOnFailure) {
const auto &Tok = getToken();
SMLoc StartLoc = Tok.getLoc();
SMLoc EndLoc = Tok.getEndLoc();
RegisterKind RegKind;
unsigned Reg, RegNum, RegWidth;

if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) {
  return nullptr;
}
if (isHsaAbiVersion3Or4(&getSTI())) {
  if (!updateGprCountSymbols(RegKind, RegNum, RegWidth))
    return nullptr;
} else
  KernelScope.usesRegister(RegKind, RegNum, RegWidth);
return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc);
2733}

2735OperandMatchResultTy
2736AMDGPUAsmParser::parseImm(OperandVector &Operands, bool HasSP3AbsModifier) {
// TODO: add syntactic sugar for 1/(2*PI)

assert(!isRegister())(static_cast<void> (0));
assert(!isModifier())(static_cast<void> (0));

const auto& Tok = getToken();
const auto& NextTok = peekToken();
bool IsReal = Tok.is(AsmToken::Real);
SMLoc S = getLoc();
bool Negate = false;

if (!IsReal && Tok.is(AsmToken::Minus) && NextTok.is(AsmToken::Real)) {
  lex();
  IsReal = true;
  Negate = true;
}

if (IsReal) {
  // Floating-point expressions are not supported.
  // Can only allow floating-point literals with an
  // optional sign.

  StringRef Num = getTokenStr();
  lex();

  APFloat RealVal(APFloat::IEEEdouble());
  auto roundMode = APFloat::rmNearestTiesToEven;
  if (errorToBool(RealVal.convertFromString(Num, roundMode).takeError())) {
    return MatchOperand_ParseFail;
  }
  if (Negate)
    RealVal.changeSign();

  Operands.push_back(
    AMDGPUOperand::CreateImm(this, RealVal.bitcastToAPInt().getZExtValue(), S,
                             AMDGPUOperand::ImmTyNone, true));

  return MatchOperand_Success;

} else {
  int64_t IntVal;
  const MCExpr *Expr;
  SMLoc S = getLoc();

  if (HasSP3AbsModifier) {
    // This is a workaround for handling expressions
    // as arguments of SP3 'abs' modifier, for example:
    //     |1.0|
    //     |-1|
    //     |1+x|
    // This syntax is not compatible with syntax of standard
    // MC expressions (due to the trailing '|').
    SMLoc EndLoc;
    if (getParser().parsePrimaryExpr(Expr, EndLoc, nullptr))
      return MatchOperand_ParseFail;
  } else {
    if (Parser.parseExpression(Expr))
      return MatchOperand_ParseFail;
  }

  if (Expr->evaluateAsAbsolute(IntVal)) {
    Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
  } else {
    Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
  }

  return MatchOperand_Success;
}

return MatchOperand_NoMatch;
2807}

2809OperandMatchResultTy
2810AMDGPUAsmParser::parseReg(OperandVector &Operands) {
if (!isRegister())
  return MatchOperand_NoMatch;

if (auto R = parseRegister()) {
  assert(R->isReg())(static_cast<void> (0));
  Operands.push_back(std::move(R));
  return MatchOperand_Success;
}
return MatchOperand_ParseFail;
2820}

2822OperandMatchResultTy
2823AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands, bool HasSP3AbsMod) {
auto res = parseReg(Operands);
if (res != MatchOperand_NoMatch) {
  return res;
} else if (isModifier()) {
  return MatchOperand_NoMatch;
} else {
  return parseImm(Operands, HasSP3AbsMod);
}
2832}

2834bool
2835AMDGPUAsmParser::isNamedOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const {
if (Token.is(AsmToken::Identifier) && NextToken.is(AsmToken::LParen)) {
  const auto &str = Token.getString();
  return str == "abs" || str == "neg" || str == "sext";
}
return false;
2841}

2843bool
2844AMDGPUAsmParser::isOpcodeModifierWithVal(const AsmToken &Token, const AsmToken &NextToken) const {
return Token.is(AsmToken::Identifier) && NextToken.is(AsmToken::Colon);
2846}

2848bool
2849AMDGPUAsmParser::isOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const {
return isNamedOperandModifier(Token, NextToken) || Token.is(AsmToken::Pipe);
2851}

2853bool
2854AMDGPUAsmParser::isRegOrOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const {
return isRegister(Token, NextToken) || isOperandModifier(Token, NextToken);
2856}

2858// Check if this is an operand modifier or an opcode modifier
2859// which may look like an expression but it is not. We should
2860// avoid parsing these modifiers as expressions. Currently
2861// recognized sequences are:
2862//   |...|
2863//   abs(...)
2864//   neg(...)
2865//   sext(...)
2866//   -reg
2867//   -|...|
2868//   -abs(...)
2869//   name:...
2870// Note that simple opcode modifiers like 'gds' may be parsed as
2871// expressions; this is a special case. See getExpressionAsToken.
2872//
2873bool
2874AMDGPUAsmParser::isModifier() {

AsmToken Tok = getToken();
AsmToken NextToken[2];
peekTokens(NextToken);

return isOperandModifier(Tok, NextToken[0]) ||
       (Tok.is(AsmToken::Minus) && isRegOrOperandModifier(NextToken[0], NextToken[1])) ||
       isOpcodeModifierWithVal(Tok, NextToken[0]);
2883}

2885// Check if the current token is an SP3 'neg' modifier.
2886// Currently this modifier is allowed in the following context:
2887//
2888// 1. Before a register, e.g. "-v0", "-v[...]" or "-[v0,v1]".
2889// 2. Before an 'abs' modifier: -abs(...)
2890// 3. Before an SP3 'abs' modifier: -|...|
2891//
2892// In all other cases "-" is handled as a part
2893// of an expression that follows the sign.
2894//
2895// Note: When "-" is followed by an integer literal,
2896// this is interpreted as integer negation rather
2897// than a floating-point NEG modifier applied to N.
2898// Beside being contr-intuitive, such use of floating-point
2899// NEG modifier would have resulted in different meaning
2900// of integer literals used with VOP1/2/C and VOP3,
2901// for example:
2902//    v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF
2903//    v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001
2904// Negative fp literals with preceding "-" are
2905// handled likewise for unifomtity
2906//
2907bool
2908AMDGPUAsmParser::parseSP3NegModifier() {

AsmToken NextToken[2];
peekTokens(NextToken);

if (isToken(AsmToken::Minus) &&
    (isRegister(NextToken[0], NextToken[1]) ||
     NextToken[0].is(AsmToken::Pipe) ||
     isId(NextToken[0], "abs"))) {
  lex();
  return true;
}

return false;
2922}

2924OperandMatchResultTy
2925AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
                                            bool AllowImm) {
bool Neg, SP3Neg;
bool Abs, SP3Abs;
SMLoc Loc;

// Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
if (isToken(AsmToken::Minus) && peekToken().is(AsmToken::Minus)) {
  Error(getLoc(), "invalid syntax, expected 'neg' modifier");
  return MatchOperand_ParseFail;
}

SP3Neg = parseSP3NegModifier();

Loc = getLoc();
Neg = trySkipId("neg");
if (Neg && SP3Neg) {
  Error(Loc, "expected register or immediate");
  return MatchOperand_ParseFail;
}
if (Neg && !skipToken(AsmToken::LParen, "expected left paren after neg"))
  return MatchOperand_ParseFail;

Abs = trySkipId("abs");
if (Abs && !skipToken(AsmToken::LParen, "expected left paren after abs"))
  return MatchOperand_ParseFail;

Loc = getLoc();
SP3Abs = trySkipToken(AsmToken::Pipe);
if (Abs && SP3Abs) {
  Error(Loc, "expected register or immediate");
  return MatchOperand_ParseFail;
}

OperandMatchResultTy Res;
if (AllowImm) {
  Res = parseRegOrImm(Operands, SP3Abs);
} else {
  Res = parseReg(Operands);
}
if (Res != MatchOperand_Success) {
  return (SP3Neg || Neg || SP3Abs || Abs)? MatchOperand_ParseFail : Res;
}

if (SP3Abs && !skipToken(AsmToken::Pipe, "expected vertical bar"))
  return MatchOperand_ParseFail;
if (Abs && !skipToken(AsmToken::RParen, "expected closing parentheses"))
  return MatchOperand_ParseFail;
if (Neg && !skipToken(AsmToken::RParen, "expected closing parentheses"))
  return MatchOperand_ParseFail;

AMDGPUOperand::Modifiers Mods;
Mods.Abs = Abs || SP3Abs;
Mods.Neg = Neg || SP3Neg;

if (Mods.hasFPModifiers()) {
  AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
  if (Op.isExpr()) {
    Error(Op.getStartLoc(), "expected an absolute expression");
    return MatchOperand_ParseFail;
  }
  Op.setModifiers(Mods);
}
return MatchOperand_Success;
2989}

2991OperandMatchResultTy
2992AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
                                             bool AllowImm) {
bool Sext = trySkipId("sext");
if (Sext && !skipToken(AsmToken::LParen, "expected left paren after sext"))
  return MatchOperand_ParseFail;

OperandMatchResultTy Res;
if (AllowImm) {
  Res = parseRegOrImm(Operands);
} else {
  Res = parseReg(Operands);
}
if (Res != MatchOperand_Success) {
  return Sext? MatchOperand_ParseFail : Res;
}

if (Sext && !skipToken(AsmToken::RParen, "expected closing parentheses"))
  return MatchOperand_ParseFail;

AMDGPUOperand::Modifiers Mods;
Mods.Sext = Sext;

if (Mods.hasIntModifiers()) {
  AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
  if (Op.isExpr()) {
    Error(Op.getStartLoc(), "expected an absolute expression");
    return MatchOperand_ParseFail;
  }
  Op.setModifiers(Mods);
}

return MatchOperand_Success;
3024}

3026OperandMatchResultTy
3027AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
return parseRegOrImmWithFPInputMods(Operands, false);
3029}

3031OperandMatchResultTy
3032AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
return parseRegOrImmWithIntInputMods(Operands, false);
3034}

3036OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
auto Loc = getLoc();
if (trySkipId("off")) {
  Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Loc,
                                              AMDGPUOperand::ImmTyOff, false));
  return MatchOperand_Success;
}

if (!isRegister())
  return MatchOperand_NoMatch;

std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
if (Reg) {
  Operands.push_back(std::move(Reg));
  return MatchOperand_Success;
}

return MatchOperand_ParseFail;

3055}

3057unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;

if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
    (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
    (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
    (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
  return Match_InvalidOperand;

if ((TSFlags & SIInstrFlags::VOP3) &&
    (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
    getForcedEncodingSize() != 64)
  return Match_PreferE32;

if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
    Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
  // v_mac_f32/16 allow only dst_sel == DWORD;
  auto OpNum =
      AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
  const auto &Op = Inst.getOperand(OpNum);
  if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
    return Match_InvalidOperand;
  }
}

return Match_Success;
3083}

3085static ArrayRef<unsigned> getAllVariants() {
static const unsigned Variants[] = {
  AMDGPUAsmVariants::DEFAULT, AMDGPUAsmVariants::VOP3,
  AMDGPUAsmVariants::SDWA, AMDGPUAsmVariants::SDWA9, AMDGPUAsmVariants::DPP
};

return makeArrayRef(Variants);
3092}

3094// What asm variants we should check
3095ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
if (getForcedEncodingSize() == 32) {
  static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
  return makeArrayRef(Variants);
}

if (isForcedVOP3()) {
  static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
  return makeArrayRef(Variants);
}

if (isForcedSDWA()) {
  static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA,
                                      AMDGPUAsmVariants::SDWA9};
  return makeArrayRef(Variants);
}

if (isForcedDPP()) {
  static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
  return makeArrayRef(Variants);
}

return getAllVariants();
3118}

3120StringRef AMDGPUAsmParser::getMatchedVariantName() const {
if (getForcedEncodingSize() == 32)
  return "e32";

if (isForcedVOP3())
  return "e64";

if (isForcedSDWA())
  return "sdwa";

if (isForcedDPP())
  return "dpp";

return "";
3134}

3136unsigned AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
const unsigned Num = Desc.getNumImplicitUses();
for (unsigned i = 0; i < Num; ++i) {
  unsigned Reg = Desc.ImplicitUses[i];
  switch (Reg) {
  case AMDGPU::FLAT_SCR:
  case AMDGPU::VCC:
  case AMDGPU::VCC_LO:
  case AMDGPU::VCC_HI:
  case AMDGPU::M0:
    return Reg;
  default:
    break;
  }
}
return AMDGPU::NoRegister;
3153}

3155// NB: This code is correct only when used to check constant
3156// bus limitations because GFX7 support no f16 inline constants.
3157// Note that there are no cases when a GFX7 opcode violates
3158// constant bus limitations due to the use of an f16 constant.
3159bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
                                     unsigned OpIdx) const {
const MCInstrDesc &Desc = MII.get(Inst.getOpcode());

if (!AMDGPU::isSISrcOperand(Desc, OpIdx)) {
  return false;
}

const MCOperand &MO = Inst.getOperand(OpIdx);

int64_t Val = MO.getImm();
auto OpSize = AMDGPU::getOperandSize(Desc, OpIdx);

switch (OpSize) { // expected operand size
case 8:
  return AMDGPU::isInlinableLiteral64(Val, hasInv2PiInlineImm());
case 4:
  return AMDGPU::isInlinableLiteral32(Val, hasInv2PiInlineImm());
case 2: {
  const unsigned OperandType = Desc.OpInfo[OpIdx].OperandType;
  if (OperandType == AMDGPU::OPERAND_REG_IMM_INT16 ||
      OperandType == AMDGPU::OPERAND_REG_INLINE_C_INT16 ||
      OperandType == AMDGPU::OPERAND_REG_INLINE_AC_INT16)
    return AMDGPU::isInlinableIntLiteral(Val);

  if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
      OperandType == AMDGPU::OPERAND_REG_INLINE_AC_V2INT16 ||
      OperandType == AMDGPU::OPERAND_REG_IMM_V2INT16)
    return AMDGPU::isInlinableIntLiteralV216(Val);

  if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16 ||
      OperandType == AMDGPU::OPERAND_REG_INLINE_AC_V2FP16 ||
      OperandType == AMDGPU::OPERAND_REG_IMM_V2FP16)
    return AMDGPU::isInlinableLiteralV216(Val, hasInv2PiInlineImm());

  return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm());
}
default:
  llvm_unreachable("invalid operand size")__builtin_unreachable();
}
3199}

3201unsigned AMDGPUAsmParser::getConstantBusLimit(unsigned Opcode) const {
if (!isGFX10Plus())
  return 1;

switch (Opcode) {
// 64-bit shift instructions can use only one scalar value input
case AMDGPU::V_LSHLREV_B64_e64:
case AMDGPU::V_LSHLREV_B64_gfx10:
case AMDGPU::V_LSHRREV_B64_e64:
case AMDGPU::V_LSHRREV_B64_gfx10:
case AMDGPU::V_ASHRREV_I64_e64:
case AMDGPU::V_ASHRREV_I64_gfx10:
case AMDGPU::V_LSHL_B64_e64:
case AMDGPU::V_LSHR_B64_e64:
case AMDGPU::V_ASHR_I64_e64:
  return 1;
default:
  return 2;
}
3220}

3222bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
const MCOperand &MO = Inst.getOperand(OpIdx);
if (MO.isImm()) {
  return !isInlineConstant(Inst, OpIdx);
} else if (MO.isReg()) {
  auto Reg = MO.getReg();
  const MCRegisterInfo *TRI = getContext().getRegisterInfo();
  auto PReg = mc2PseudoReg(Reg);
  return isSGPR(PReg, TRI) && PReg != SGPR_NULL;
} else {
  return true;
}
3234}

3236bool
3237AMDGPUAsmParser::validateConstantBusLimitations(const MCInst &Inst,
                                              const OperandVector &Operands) {
const unsigned Opcode = Inst.getOpcode();
const MCInstrDesc &Desc = MII.get(Opcode);
unsigned LastSGPR = AMDGPU::NoRegister;
unsigned ConstantBusUseCount = 0;
unsigned NumLiterals = 0;
unsigned LiteralSize;

if (Desc.TSFlags &
    (SIInstrFlags::VOPC |
     SIInstrFlags::VOP1 | SIInstrFlags::VOP2 |
     SIInstrFlags::VOP3 | SIInstrFlags::VOP3P |
     SIInstrFlags::SDWA)) {
  // Check special imm operands (used by madmk, etc)
  if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) {
    ++ConstantBusUseCount;
  }

  SmallDenseSet<unsigned> SGPRsUsed;
  unsigned SGPRUsed = findImplicitSGPRReadInVOP(Inst);
  if (SGPRUsed != AMDGPU::NoRegister) {
    SGPRsUsed.insert(SGPRUsed);
    ++ConstantBusUseCount;
  }

  const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
  const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
  const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);

  const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };

  for (int OpIdx : OpIndices) {
    if (OpIdx == -1) break;

    const MCOperand &MO = Inst.getOperand(OpIdx);
    if (usesConstantBus(Inst, OpIdx)) {
      if (MO.isReg()) {
        LastSGPR = mc2PseudoReg(MO.getReg());
        // Pairs of registers with a partial intersections like these
        //   s0, s[0:1]
        //   flat_scratch_lo, flat_scratch
        //   flat_scratch_lo, flat_scratch_hi
        // are theoretically valid but they are disabled anyway.
        // Note that this code mimics SIInstrInfo::verifyInstruction
        if (!SGPRsUsed.count(LastSGPR)) {
          SGPRsUsed.insert(LastSGPR);
          ++ConstantBusUseCount;
        }
      } else { // Expression or a literal

        if (Desc.OpInfo[OpIdx].OperandType == MCOI::OPERAND_IMMEDIATE)
          continue; // special operand like VINTERP attr_chan

        // An instruction may use only one literal.
        // This has been validated on the previous step.
        // See validateVOP3Literal.
        // This literal may be used as more than one operand.
        // If all these operands are of the same size,
        // this literal counts as one scalar value.
        // Otherwise it counts as 2 scalar values.
        // See "GFX10 Shader Programming", section 3.6.2.3.

        unsigned Size = AMDGPU::getOperandSize(Desc, OpIdx);
        if (Size < 4) Size = 4;

        if (NumLiterals == 0) {
          NumLiterals = 1;
          LiteralSize = Size;
        } else if (LiteralSize != Size) {
          NumLiterals = 2;
        }
      }
    }
  }
}
ConstantBusUseCount += NumLiterals;

if (ConstantBusUseCount <= getConstantBusLimit(Opcode))
  return true;

SMLoc LitLoc = getLitLoc(Operands);
SMLoc RegLoc = getRegLoc(LastSGPR, Operands);
SMLoc Loc = (LitLoc.getPointer() < RegLoc.getPointer()) ? RegLoc : LitLoc;
Error(Loc, "invalid operand (violates constant bus restrictions)");
return false;
3323}

3325bool
3326AMDGPUAsmParser::validateEarlyClobberLimitations(const MCInst &Inst,
                                               const OperandVector &Operands) {
const unsigned Opcode = Inst.getOpcode();
const MCInstrDesc &Desc = MII.get(Opcode);

const int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
if (DstIdx == -1 ||
    Desc.getOperandConstraint(DstIdx, MCOI::EARLY_CLOBBER) == -1) {
  return true;
}

const MCRegisterInfo *TRI = getContext().getRegisterInfo();

const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);

assert(DstIdx != -1)(static_cast<void> (0));
const MCOperand &Dst = Inst.getOperand(DstIdx);
assert(Dst.isReg())(static_cast<void> (0));
const unsigned DstReg = mc2PseudoReg(Dst.getReg());

const int SrcIndices[] = { Src0Idx, Src1Idx, Src2Idx };

for (int SrcIdx : SrcIndices) {
  if (SrcIdx == -1) break;
  const MCOperand &Src = Inst.getOperand(SrcIdx);
  if (Src.isReg()) {
    const unsigned SrcReg = mc2PseudoReg(Src.getReg());
    if (isRegIntersect(DstReg, SrcReg, TRI)) {
      Error(getRegLoc(SrcReg, Operands),
        "destination must be different than all sources");
      return false;
    }
  }
}

return true;
3364}

3366bool AMDGPUAsmParser::validateIntClampSupported(const MCInst &Inst) {

const unsigned Opc = Inst.getOpcode();
const MCInstrDesc &Desc = MII.get(Opc);

if ((Desc.TSFlags & SIInstrFlags::IntClamp) != 0 && !hasIntClamp()) {
  int ClampIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp);
  assert(ClampIdx != -1)(static_cast<void> (0));
  return Inst.getOperand(ClampIdx).getImm() == 0;
}

return true;
3378}

3380bool AMDGPUAsmParser::validateMIMGDataSize(const MCInst &Inst) {

const unsigned Opc = Inst.getOpcode();
const MCInstrDesc &Desc = MII.get(Opc);

if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
  return true;

int VDataIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
int TFEIdx   = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::tfe);

assert(VDataIdx != -1)(static_cast<void> (0));

if (DMaskIdx == -1 || TFEIdx == -1) // intersect_ray
  return true;

unsigned VDataSize = AMDGPU::getRegOperandSize(getMRI(), Desc, VDataIdx);
unsigned TFESize = (TFEIdx != -1 && Inst.getOperand(TFEIdx).getImm()) ? 1 : 0;
unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
if (DMask == 0)
  DMask = 1;

unsigned DataSize =
  (Desc.TSFlags & SIInstrFlags::Gather4) ? 4 : countPopulation(DMask);
if (hasPackedD16()) {
  int D16Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::d16);
  if (D16Idx >= 0 && Inst.getOperand(D16Idx).getImm())
    DataSize = (DataSize + 1) / 2;
}

return (VDataSize / 4) == DataSize + TFESize;
3412}

3414bool AMDGPUAsmParser::validateMIMGAddrSize(const MCInst &Inst) {
const unsigned Opc = Inst.getOpcode();
const MCInstrDesc &Desc = MII.get(Opc);

if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0 || !isGFX10Plus())
  return true;

const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc);

const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
    AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);
int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
int SrsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc);
int DimIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dim);
int A16Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::a16);

assert(VAddr0Idx != -1)(static_cast<void> (0));
assert(SrsrcIdx != -1)(static_cast<void> (0));
assert(SrsrcIdx > VAddr0Idx)(static_cast<void> (0));

if (DimIdx == -1)
  return true; // intersect_ray

unsigned Dim = Inst.getOperand(DimIdx).getImm();
const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfoByEncoding(Dim);
bool IsNSA = SrsrcIdx - VAddr0Idx > 1;
unsigned ActualAddrSize =
    IsNSA ? SrsrcIdx - VAddr0Idx
          : AMDGPU::getRegOperandSize(getMRI(), Desc, VAddr0Idx) / 4;
bool IsA16 = (A16Idx != -1 && Inst.getOperand(A16Idx).getImm());

unsigned ExpectedAddrSize =
    AMDGPU::getAddrSizeMIMGOp(BaseOpcode, DimInfo, IsA16, hasG16());

if (!IsNSA) {
  if (ExpectedAddrSize > 8)
    ExpectedAddrSize = 16;

  // Allow oversized 8 VGPR vaddr when only 5/6/7 VGPRs are required.
  // This provides backward compatibility for assembly created
  // before 160b/192b/224b types were directly supported.
  if (ActualAddrSize == 8 && (ExpectedAddrSize >= 5 && ExpectedAddrSize <= 7))
    return true;
}

return ActualAddrSize == ExpectedAddrSize;
3460}

3462bool AMDGPUAsmParser::validateMIMGAtomicDMask(const MCInst &Inst) {

const unsigned Opc = Inst.getOpcode();
const MCInstrDesc &Desc = MII.get(Opc);

if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
  return true;
if (!Desc.mayLoad() || !Desc.mayStore())
  return true; // Not atomic

int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;

// This is an incomplete check because image_atomic_cmpswap
// may only use 0x3 and 0xf while other atomic operations
// may use 0x1 and 0x3. However these limitations are
// verified when we check that dmask matches dst size.
return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
3480}

3482bool AMDGPUAsmParser::validateMIMGGatherDMask(const MCInst &Inst) {

const unsigned Opc = Inst.getOpcode();
const MCInstrDesc &Desc = MII.get(Opc);

if ((Desc.TSFlags & SIInstrFlags::Gather4) == 0)
  return true;

int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;

// GATHER4 instructions use dmask in a different fashion compared to
// other MIMG instructions. The only useful DMASK values are
// 1=red, 2=green, 4=blue, 8=alpha. (e.g. 1 returns
// (red,red,red,red) etc.) The ISA document doesn't mention
// this.
return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8;
3499}

3501bool AMDGPUAsmParser::validateMIMGMSAA(const MCInst &Inst) {
const unsigned Opc = Inst.getOpcode();
const MCInstrDesc &Desc = MII.get(Opc);

if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
  return true;

const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc);
const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
    AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);

if (!BaseOpcode->MSAA)
  return true;

int DimIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dim);
assert(DimIdx != -1)(static_cast<void> (0));

unsigned Dim = Inst.getOperand(DimIdx).getImm();
const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfoByEncoding(Dim);

return DimInfo->MSAA;
3522}

3524static bool IsMovrelsSDWAOpcode(const unsigned Opcode)
3525{
switch (Opcode) {
case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
  return true;
default:
  return false;
}
3534}

3536// movrels* opcodes should only allow VGPRS as src0.
3537// This is specified in .td description for vop1/vop3,
3538// but sdwa is handled differently. See isSDWAOperand.
3539bool AMDGPUAsmParser::validateMovrels(const MCInst &Inst,
                                    const OperandVector &Operands) {

const unsigned Opc = Inst.getOpcode();
const MCInstrDesc &Desc = MII.get(Opc);

if ((Desc.TSFlags & SIInstrFlags::SDWA) == 0 || !IsMovrelsSDWAOpcode(Opc))
  return true;

const int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
assert(Src0Idx != -1)(static_cast<void> (0));

SMLoc ErrLoc;
const MCOperand &Src0 = Inst.getOperand(Src0Idx);
if (Src0.isReg()) {
  auto Reg = mc2PseudoReg(Src0.getReg());
  const MCRegisterInfo *TRI = getContext().getRegisterInfo();
  if (!isSGPR(Reg, TRI))
    return true;
  ErrLoc = getRegLoc(Reg, Operands);
} else {
  ErrLoc = getConstLoc(Operands);
}

Error(ErrLoc, "source operand must be a VGPR");
return false;
3565}

3567bool AMDGPUAsmParser::validateMAIAccWrite(const MCInst &Inst,
                                        const OperandVector &Operands) {

const unsigned Opc = Inst.getOpcode();

if (Opc != AMDGPU::V_ACCVGPR_WRITE_B32_vi)
  return true;

const int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
assert(Src0Idx != -1)(static_cast<void> (0));

const MCOperand &Src0 = Inst.getOperand(Src0Idx);
if (!Src0.isReg())
  return true;

auto Reg = mc2PseudoReg(Src0.getReg());
const MCRegisterInfo *TRI = getContext().getRegisterInfo();
if (isSGPR(Reg, TRI)) {
  Error(getRegLoc(Reg, Operands),
        "source operand must be either a VGPR or an inline constant");
  return false;
}

return true;
3591}

3593bool AMDGPUAsmParser::validateDivScale(const MCInst &Inst) {
switch (Inst.getOpcode()) {
default:
  return true;
case V_DIV_SCALE_F32_gfx6_gfx7:
case V_DIV_SCALE_F32_vi:
case V_DIV_SCALE_F32_gfx10:
case V_DIV_SCALE_F64_gfx6_gfx7:
case V_DIV_SCALE_F64_vi:
case V_DIV_SCALE_F64_gfx10:
  break;
}

// TODO: Check that src0 = src1 or src2.

for (auto Name : {AMDGPU::OpName::src0_modifiers,
                  AMDGPU::OpName::src2_modifiers,
                  AMDGPU::OpName::src2_modifiers}) {
  if (Inst.getOperand(AMDGPU::getNamedOperandIdx(Inst.getOpcode(), Name))
          .getImm() &
      SISrcMods::ABS) {
    return false;
  }
}

return true;
3619}

3621bool AMDGPUAsmParser::validateMIMGD16(const MCInst &Inst) {

const unsigned Opc = Inst.getOpcode();
const MCInstrDesc &Desc = MII.get(Opc);

if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
  return true;

int D16Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::d16);
if (D16Idx >= 0 && Inst.getOperand(D16Idx).getImm()) {
  if (isCI() || isSI())
    return false;
}

return true;
3636}

3638bool AMDGPUAsmParser::validateMIMGDim(const MCInst &Inst) {
const unsigned Opc = Inst.getOpcode();
const MCInstrDesc &Desc = MII.get(Opc);

if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
  return true;

int DimIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dim);
if (DimIdx < 0)
  return true;

long Imm = Inst.getOperand(DimIdx).getImm();
if (Imm < 0 || Imm >= 8)
  return false;

return true;
3654}

3656static bool IsRevOpcode(const unsigned Opcode)
3657{
switch (Opcode) {
case AMDGPU::V_SUBREV_F32_e32:
case AMDGPU::V_SUBREV_F32_e64:
case AMDGPU::V_SUBREV_F32_e32_gfx10:
case AMDGPU::V_SUBREV_F32_e32_gfx6_gfx7:
case AMDGPU::V_SUBREV_F32_e32_vi:
case AMDGPU::V_SUBREV_F32_e64_gfx10:
case AMDGPU::V_SUBREV_F32_e64_gfx6_gfx7:
case AMDGPU::V_SUBREV_F32_e64_vi:

case AMDGPU::V_SUBREV_CO_U32_e32:
case AMDGPU::V_SUBREV_CO_U32_e64:
case AMDGPU::V_SUBREV_I32_e32_gfx6_gfx7:
case AMDGPU::V_SUBREV_I32_e64_gfx6_gfx7:

case AMDGPU::V_SUBBREV_U32_e32:
case AMDGPU::V_SUBBREV_U32_e64:
case AMDGPU::V_SUBBREV_U32_e32_gfx6_gfx7:
case AMDGPU::V_SUBBREV_U32_e32_vi:
case AMDGPU::V_SUBBREV_U32_e64_gfx6_gfx7:
case AMDGPU::V_SUBBREV_U32_e64_vi:

case AMDGPU::V_SUBREV_U32_e32:
case AMDGPU::V_SUBREV_U32_e64:
case AMDGPU::V_SUBREV_U32_e32_gfx9:
case AMDGPU::V_SUBREV_U32_e32_vi:
case AMDGPU::V_SUBREV_U32_e64_gfx9:
case AMDGPU::V_SUBREV_U32_e64_vi:

case AMDGPU::V_SUBREV_F16_e32:
case AMDGPU::V_SUBREV_F16_e64:
case AMDGPU::V_SUBREV_F16_e32_gfx10:
case AMDGPU::V_SUBREV_F16_e32_vi:
case AMDGPU::V_SUBREV_F16_e64_gfx10:
case AMDGPU::V_SUBREV_F16_e64_vi:

case AMDGPU::V_SUBREV_U16_e32:
case AMDGPU::V_SUBREV_U16_e64:
case AMDGPU::V_SUBREV_U16_e32_vi:
case AMDGPU::V_SUBREV_U16_e64_vi:

case AMDGPU::V_SUBREV_CO_U32_e32_gfx9:
case AMDGPU::V_SUBREV_CO_U32_e64_gfx10:
case AMDGPU::V_SUBREV_CO_U32_e64_gfx9:

case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9:
case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9:

case AMDGPU::V_SUBREV_NC_U32_e32_gfx10:
case AMDGPU::V_SUBREV_NC_U32_e64_gfx10:

case AMDGPU::V_SUBREV_CO_CI_U32_e32_gfx10:
case AMDGPU::V_SUBREV_CO_CI_U32_e64_gfx10:

case AMDGPU::V_LSHRREV_B32_e32:
case AMDGPU::V_LSHRREV_B32_e64:
case AMDGPU::V_LSHRREV_B32_e32_gfx6_gfx7:
case AMDGPU::V_LSHRREV_B32_e64_gfx6_gfx7:
case AMDGPU::V_LSHRREV_B32_e32_vi:
case AMDGPU::V_LSHRREV_B32_e64_vi:
case AMDGPU::V_LSHRREV_B32_e32_gfx10:
case AMDGPU::V_LSHRREV_B32_e64_gfx10:

case AMDGPU::V_ASHRREV_I32_e32:
case AMDGPU::V_ASHRREV_I32_e64:
case AMDGPU::V_ASHRREV_I32_e32_gfx10:
case AMDGPU::V_ASHRREV_I32_e32_gfx6_gfx7:
case AMDGPU::V_ASHRREV_I32_e32_vi:
case AMDGPU::V_ASHRREV_I32_e64_gfx10:
case AMDGPU::V_ASHRREV_I32_e64_gfx6_gfx7:
case AMDGPU::V_ASHRREV_I32_e64_vi:

case AMDGPU::V_LSHLREV_B32_e32:
case AMDGPU::V_LSHLREV_B32_e64:
case AMDGPU::V_LSHLREV_B32_e32_gfx10:
case AMDGPU::V_LSHLREV_B32_e32_gfx6_gfx7:
case AMDGPU::V_LSHLREV_B32_e32_vi:
case AMDGPU::V_LSHLREV_B32_e64_gfx10:
case AMDGPU::V_LSHLREV_B32_e64_gfx6_gfx7:
case AMDGPU::V_LSHLREV_B32_e64_vi:

case AMDGPU::V_LSHLREV_B16_e32:
case AMDGPU::V_LSHLREV_B16_e64:
case AMDGPU::V_LSHLREV_B16_e32_vi:
case AMDGPU::V_LSHLREV_B16_e64_vi:
case AMDGPU::V_LSHLREV_B16_gfx10:

case AMDGPU::V_LSHRREV_B16_e32:
case AMDGPU::V_LSHRREV_B16_e64:
case AMDGPU::V_LSHRREV_B16_e32_vi:
case AMDGPU::V_LSHRREV_B16_e64_vi:
case AMDGPU::V_LSHRREV_B16_gfx10:

case AMDGPU::V_ASHRREV_I16_e32:
case AMDGPU::V_ASHRREV_I16_e64:
case AMDGPU::V_ASHRREV_I16_e32_vi:
case AMDGPU::V_ASHRREV_I16_e64_vi:
case AMDGPU::V_ASHRREV_I16_gfx10:

case AMDGPU::V_LSHLREV_B64_e64:
case AMDGPU::V_LSHLREV_B64_gfx10:
case AMDGPU::V_LSHLREV_B64_vi:

case AMDGPU::V_LSHRREV_B64_e64:
case AMDGPU::V_LSHRREV_B64_gfx10:
case AMDGPU::V_LSHRREV_B64_vi:

case AMDGPU::V_ASHRREV_I64_e64:
case AMDGPU::V_ASHRREV_I64_gfx10:
case AMDGPU::V_ASHRREV_I64_vi:

case AMDGPU::V_PK_LSHLREV_B16:
case AMDGPU::V_PK_LSHLREV_B16_gfx10:
case AMDGPU::V_PK_LSHLREV_B16_vi:

case AMDGPU::V_PK_LSHRREV_B16:
case AMDGPU::V_PK_LSHRREV_B16_gfx10:
case AMDGPU::V_PK_LSHRREV_B16_vi:
case AMDGPU::V_PK_ASHRREV_I16:
case AMDGPU::V_PK_ASHRREV_I16_gfx10:
case AMDGPU::V_PK_ASHRREV_I16_vi:
  return true;
default:
  return false;
}
3783}

3785Optional<StringRef> AMDGPUAsmParser::validateLdsDirect(const MCInst &Inst) {

using namespace SIInstrFlags;
const unsigned Opcode = Inst.getOpcode();
const MCInstrDesc &Desc = MII.get(Opcode);

// lds_direct register is defined so that it can be used
// with 9-bit operands only. Ignore encodings which do not accept these.
const auto Enc = VOP1 | VOP2 | VOP3 | VOPC | VOP3P | SIInstrFlags::SDWA;
if ((Desc.TSFlags & Enc) == 0)
  return None;

for (auto SrcName : {OpName::src0, OpName::src1, OpName::src2}) {
  auto SrcIdx = getNamedOperandIdx(Opcode, SrcName);
  if (SrcIdx == -1)
    break;
  const auto &Src = Inst.getOperand(SrcIdx);
  if (Src.isReg() && Src.getReg() == LDS_DIRECT) {

    if (isGFX90A())
      return StringRef("lds_direct is not supported on this GPU");

    if (IsRevOpcode(Opcode) || (Desc.TSFlags & SIInstrFlags::SDWA))
      return StringRef("lds_direct cannot be used with this instruction");

    if (SrcName != OpName::src0)
      return StringRef("lds_direct may be used as src0 only");
  }
}

return None;
3816}

3818SMLoc AMDGPUAsmParser::getFlatOffsetLoc(const OperandVector &Operands) const {
for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
  if (Op.isFlatOffset())
    return Op.getStartLoc();
}
return getLoc();
3825}

3827bool AMDGPUAsmParser::validateFlatOffset(const MCInst &Inst,
                                       const OperandVector &Operands) {
uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
if ((TSFlags & SIInstrFlags::FLAT) == 0)
  return true;

auto Opcode = Inst.getOpcode();
auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
assert(OpNum != -1)(static_cast<void> (0));

const auto &Op = Inst.getOperand(OpNum);
if (!hasFlatOffsets() && Op.getImm() != 0) {
  Error(getFlatOffsetLoc(Operands),
        "flat offset modifier is not supported on this GPU");
  return false;
}

// For FLAT segment the offset must be positive;
// MSB is ignored and forced to zero.
if (TSFlags & (SIInstrFlags::FlatGlobal | SIInstrFlags::FlatScratch)) {
  unsigned OffsetSize = AMDGPU::getNumFlatOffsetBits(getSTI(), true);
  if (!isIntN(OffsetSize, Op.getImm())) {
    Error(getFlatOffsetLoc(Operands),
          Twine("expected a ") + Twine(OffsetSize) + "-bit signed offset");
    return false;
  }
} else {
  unsigned OffsetSize = AMDGPU::getNumFlatOffsetBits(getSTI(), false);
  if (!isUIntN(OffsetSize, Op.getImm())) {
    Error(getFlatOffsetLoc(Operands),
          Twine("expected a ") + Twine(OffsetSize) + "-bit unsigned offset");
    return false;
  }
}

return true;
3863}

3865SMLoc AMDGPUAsmParser::getSMEMOffsetLoc(const OperandVector &Operands) const {
// Start with second operand because SMEM Offset cannot be dst or src0.
for (unsigned i = 2, e = Operands.size(); i != e; ++i) {
  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
  if (Op.isSMEMOffset())
    return Op.getStartLoc();
}
return getLoc();
3873}

3875bool AMDGPUAsmParser::validateSMEMOffset(const MCInst &Inst,
                                       const OperandVector &Operands) {
if (isCI() || isSI())
  return true;

uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
if ((TSFlags & SIInstrFlags::SMRD) == 0)
  return true;

auto Opcode = Inst.getOpcode();
auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
if (OpNum == -1)
  return true;

const auto &Op = Inst.getOperand(OpNum);
if (!Op.isImm())
  return true;

uint64_t Offset = Op.getImm();
bool IsBuffer = AMDGPU::getSMEMIsBuffer(Opcode);
if (AMDGPU::isLegalSMRDEncodedUnsignedOffset(getSTI(), Offset) ||
    AMDGPU::isLegalSMRDEncodedSignedOffset(getSTI(), Offset, IsBuffer))
  return true;

Error(getSMEMOffsetLoc(Operands),
      (isVI() || IsBuffer) ? "expected a 20-bit unsigned offset" :
                             "expected a 21-bit signed offset");

return false;
3904}

3906bool AMDGPUAsmParser::validateSOPLiteral(const MCInst &Inst) const {
unsigned Opcode = Inst.getOpcode();
const MCInstrDesc &Desc = MII.get(Opcode);
if (!(Desc.TSFlags & (SIInstrFlags::SOP2 | SIInstrFlags::SOPC)))
  return true;

const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);

const int OpIndices[] = { Src0Idx, Src1Idx };

unsigned NumExprs = 0;
unsigned NumLiterals = 0;
uint32_t LiteralValue;

for (int OpIdx : OpIndices) {
  if (OpIdx == -1) break;

  const MCOperand &MO = Inst.getOperand(OpIdx);
  // Exclude special imm operands (like that used by s_set_gpr_idx_on)
  if (AMDGPU::isSISrcOperand(Desc, OpIdx)) {
    if (MO.isImm() && !isInlineConstant(Inst, OpIdx)) {
      uint32_t Value = static_cast<uint32_t>(MO.getImm());
      if (NumLiterals == 0 || LiteralValue != Value) {
        LiteralValue = Value;
        ++NumLiterals;
      }
    } else if (MO.isExpr()) {
      ++NumExprs;
    }
  }
}

return NumLiterals + NumExprs <= 1;
3940}

3942bool AMDGPUAsmParser::validateOpSel(const MCInst &Inst) {
const unsigned Opc = Inst.getOpcode();
if (Opc == AMDGPU::V_PERMLANE16_B32_gfx10 ||
    Opc == AMDGPU::V_PERMLANEX16_B32_gfx10) {
  int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
  unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();

  if (OpSel & ~3)
    return false;
}
return true;
3953}

3955bool AMDGPUAsmParser::validateDPP(const MCInst &Inst,
                                const OperandVector &Operands) {
const unsigned Opc = Inst.getOpcode();
int DppCtrlIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dpp_ctrl);
if (DppCtrlIdx < 0)
  return true;
unsigned DppCtrl = Inst.getOperand(DppCtrlIdx).getImm();

if (!AMDGPU::isLegal64BitDPPControl(DppCtrl)) {
  // DPP64 is supported for row_newbcast only.
  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
  if (Src0Idx >= 0 &&
      getMRI()->getSubReg(Inst.getOperand(Src0Idx).getReg(), AMDGPU::sub1)) {
    SMLoc S = getImmLoc(AMDGPUOperand::ImmTyDppCtrl, Operands);
    Error(S, "64 bit dpp only supports row_newbcast");
    return false;
  }
}

return true;
3975}

3977// Check if VCC register matches wavefront size
3978bool AMDGPUAsmParser::validateVccOperand(unsigned Reg) const {
auto FB = getFeatureBits();
return (FB[AMDGPU::FeatureWavefrontSize64] && Reg == AMDGPU::VCC) ||
  (FB[AMDGPU::FeatureWavefrontSize32] && Reg == AMDGPU::VCC_LO);
3982}

3984// VOP3 literal is only allowed in GFX10+ and only one can be used
3985bool AMDGPUAsmParser::validateVOP3Literal(const MCInst &Inst,
                                        const OperandVector &Operands) {
unsigned Opcode = Inst.getOpcode();
const MCInstrDesc &Desc = MII.get(Opcode);
if (!(Desc.TSFlags & (SIInstrFlags::VOP3 | SIInstrFlags::VOP3P)))
  return true;

const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);

const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };

unsigned NumExprs = 0;
unsigned NumLiterals = 0;
uint32_t LiteralValue;

for (int OpIdx : OpIndices) {
  if (OpIdx == -1) break;

  const MCOperand &MO = Inst.getOperand(OpIdx);
  if (!MO.isImm() && !MO.isExpr())
    continue;
  if (!AMDGPU::isSISrcOperand(Desc, OpIdx))
    continue;

  if (OpIdx == Src2Idx && (Desc.TSFlags & SIInstrFlags::IsMAI) &&
      getFeatureBits()[AMDGPU::FeatureMFMAInlineLiteralBug]) {
    Error(getConstLoc(Operands),
          "inline constants are not allowed for this operand");
    return false;
  }

  if (MO.isImm() && !isInlineConstant(Inst, OpIdx)) {
    uint32_t Value = static_cast<uint32_t>(MO.getImm());
    if (NumLiterals == 0 || LiteralValue != Value) {
      LiteralValue = Value;
      ++NumLiterals;
    }
  } else if (MO.isExpr()) {
    ++NumExprs;
  }
}
NumLiterals += NumExprs;

if (!NumLiterals)
  return true;

if (!getFeatureBits()[AMDGPU::FeatureVOP3Literal]) {
  Error(getLitLoc(Operands), "literal operands are not supported");
  return false;
}

if (NumLiterals > 1) {
  Error(getLitLoc(Operands), "only one literal operand is allowed");
  return false;
}

return true;
4044}

4046// Returns -1 if not a register, 0 if VGPR and 1 if AGPR.
4047static int IsAGPROperand(const MCInst &Inst, uint16_t NameIdx,
                       const MCRegisterInfo *MRI) {
int OpIdx = AMDGPU::getNamedOperandIdx(Inst.getOpcode(), NameIdx);
if (OpIdx < 0)
  return -1;

const MCOperand &Op = Inst.getOperand(OpIdx);
if (!Op.isReg())
  return -1;

unsigned Sub = MRI->getSubReg(Op.getReg(), AMDGPU::sub0);
auto Reg = Sub ? Sub : Op.getReg();
const MCRegisterClass &AGPR32 = MRI->getRegClass(AMDGPU::AGPR_32RegClassID);
return AGPR32.contains(Reg) ? 1 : 0;
4061}

4063bool AMDGPUAsmParser::validateAGPRLdSt(const MCInst &Inst) const {
uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
if ((TSFlags & (SIInstrFlags::FLAT | SIInstrFlags::MUBUF |
                SIInstrFlags::MTBUF | SIInstrFlags::MIMG |
                SIInstrFlags::DS)) == 0)
  return true;

uint16_t DataNameIdx = (TSFlags & SIInstrFlags::DS) ? AMDGPU::OpName::data0
                                                    : AMDGPU::OpName::vdata;

const MCRegisterInfo *MRI = getMRI();
int DstAreg = IsAGPROperand(Inst, AMDGPU::OpName::vdst, MRI);
int DataAreg = IsAGPROperand(Inst, DataNameIdx, MRI);

if ((TSFlags & SIInstrFlags::DS) && DataAreg >= 0) {
  int Data2Areg = IsAGPROperand(Inst, AMDGPU::OpName::data1, MRI);
  if (Data2Areg >= 0 && Data2Areg != DataAreg)
    return false;
}

auto FB = getFeatureBits();
if (FB[AMDGPU::FeatureGFX90AInsts]) {
  if (DataAreg < 0 || DstAreg < 0)
    return true;
  return DstAreg == DataAreg;
}

return DstAreg < 1 && DataAreg < 1;
4091}

4093bool AMDGPUAsmParser::validateVGPRAlign(const MCInst &Inst) const {
auto FB = getFeatureBits();
if (!FB[AMDGPU::FeatureGFX90AInsts])
  return true;

const MCRegisterInfo *MRI = getMRI();
const MCRegisterClass &VGPR32 = MRI->getRegClass(AMDGPU::VGPR_32RegClassID);
const MCRegisterClass &AGPR32 = MRI->getRegClass(AMDGPU::AGPR_32RegClassID);
for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) {
  const MCOperand &Op = Inst.getOperand(I);
  if (!Op.isReg())
    continue;

  unsigned Sub = MRI->getSubReg(Op.getReg(), AMDGPU::sub0);
  if (!Sub)
    continue;

  if (VGPR32.contains(Sub) && ((Sub - AMDGPU::VGPR0) & 1))
    return false;
  if (AGPR32.contains(Sub) && ((Sub - AMDGPU::AGPR0) & 1))
    return false;
}

return true;
4117}

4119// gfx90a has an undocumented limitation:
4120// DS_GWS opcodes must use even aligned registers.
4121bool AMDGPUAsmParser::validateGWS(const MCInst &Inst,
                                const OperandVector &Operands) {
if (!getFeatureBits()[AMDGPU::FeatureGFX90AInsts])
  return true;

int Opc = Inst.getOpcode();
if (Opc != AMDGPU::DS_GWS_INIT_vi && Opc != AMDGPU::DS_GWS_BARRIER_vi &&
    Opc != AMDGPU::DS_GWS_SEMA_BR_vi)
  return true;

const MCRegisterInfo *MRI = getMRI();
const MCRegisterClass &VGPR32 = MRI->getRegClass(AMDGPU::VGPR_32RegClassID);
int Data0Pos =
    AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data0);
assert(Data0Pos != -1)(static_cast<void> (0));
auto Reg = Inst.getOperand(Data0Pos).getReg();
auto RegIdx = Reg - (VGPR32.contains(Reg) ? AMDGPU::VGPR0 : AMDGPU::AGPR0);
if (RegIdx & 1) {
  SMLoc RegLoc = getRegLoc(Reg, Operands);
  Error(RegLoc, "vgpr must be even aligned");
  return false;
}

return true;
4145}

4147bool AMDGPUAsmParser::validateCoherencyBits(const MCInst &Inst,
                                          const OperandVector &Operands,
                                          const SMLoc &IDLoc) {
int CPolPos = AMDGPU::getNamedOperandIdx(Inst.getOpcode(),
                                         AMDGPU::OpName::cpol);
if (CPolPos == -1)
  return true;

unsigned CPol = Inst.getOperand(CPolPos).getImm();

uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
if ((TSFlags & (SIInstrFlags::SMRD)) &&
    (CPol & ~(AMDGPU::CPol::GLC | AMDGPU::CPol::DLC))) {
  Error(IDLoc, "invalid cache policy for SMRD instruction");
  return false;
}

if (isGFX90A() && (CPol & CPol::SCC)) {
  SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol, Operands);
  StringRef CStr(S.getPointer());
  S = SMLoc::getFromPointer(&CStr.data()[CStr.find("scc")]);
  Error(S, "scc is not supported on this GPU");
  return false;
}

if (!(TSFlags & (SIInstrFlags::IsAtomicNoRet | SIInstrFlags::IsAtomicRet)))
  return true;

if (TSFlags & SIInstrFlags::IsAtomicRet) {
  if (!(TSFlags & SIInstrFlags::MIMG) && !(CPol & CPol::GLC)) {
    Error(IDLoc, "instruction must use glc");
    return false;
  }
} else {
  if (CPol & CPol::GLC) {
    SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol, Operands);
    StringRef CStr(S.getPointer());
    S = SMLoc::getFromPointer(&CStr.data()[CStr.find("glc")]);
    Error(S, "instruction must not use glc");
    return false;
  }
}

return true;
4191}

4193bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
                                        const SMLoc &IDLoc,
                                        const OperandVector &Operands) {
if (auto ErrMsg = validateLdsDirect(Inst)) {
  Error(getRegLoc(LDS_DIRECT, Operands), *ErrMsg);
  return false;
}
if (!validateSOPLiteral(Inst)) {
  Error(getLitLoc(Operands),
    "only one literal operand is allowed");
  return false;
}
if (!validateVOP3Literal(Inst, Operands)) {
  return false;
}
if (!validateConstantBusLimitations(Inst, Operands)) {
  return false;
}
if (!validateEarlyClobberLimitations(Inst, Operands)) {
  return false;
}
if (!validateIntClampSupported(Inst)) {
  Error(getImmLoc(AMDGPUOperand::ImmTyClampSI, Operands),
    "integer clamping is not supported on this GPU");
  return false;
}
if (!validateOpSel(Inst)) {
  Error(getImmLoc(AMDGPUOperand::ImmTyOpSel, Operands),
    "invalid op_sel operand");
  return false;
}
if (!validateDPP(Inst, Operands)) {
  return false;
}
// For MUBUF/MTBUF d16 is a part of opcode, so there is nothing to validate.
if (!validateMIMGD16(Inst)) {
  Error(getImmLoc(AMDGPUOperand::ImmTyD16, Operands),
    "d16 modifier is not supported on this GPU");
  return false;
}
if (!validateMIMGDim(Inst)) {
  Error(IDLoc, "dim modifier is required on this GPU");
  return false;
}
if (!validateMIMGMSAA(Inst)) {
  Error(getImmLoc(AMDGPUOperand::ImmTyDim, Operands),
        "invalid dim; must be MSAA type");
  return false;
}
if (!validateMIMGDataSize(Inst)) {
  Error(IDLoc,
    "image data size does not match dmask and tfe");
  return false;
}
if (!validateMIMGAddrSize(Inst)) {
  Error(IDLoc,
    "image address size does not match dim and a16");
  return false;
}
if (!validateMIMGAtomicDMask(Inst)) {
  Error(getImmLoc(AMDGPUOperand::ImmTyDMask, Operands),
    "invalid atomic image dmask");
  return false;
}
if (!validateMIMGGatherDMask(Inst)) {
  Error(getImmLoc(AMDGPUOperand::ImmTyDMask, Operands),
    "invalid image_gather dmask: only one bit must be set");
  return false;
}
if (!validateMovrels(Inst, Operands)) {
  return false;
}
if (!validateFlatOffset(Inst, Operands)) {
  return false;
}
if (!validateSMEMOffset(Inst, Operands)) {
  return false;
}
if (!validateMAIAccWrite(Inst, Operands)) {
  return false;
}
if (!validateCoherencyBits(Inst, Operands, IDLoc)) {
  return false;
}

if (!validateAGPRLdSt(Inst)) {
  Error(IDLoc, getFeatureBits()[AMDGPU::FeatureGFX90AInsts]
  ? "invalid register class: data and dst should be all VGPR or AGPR"
  : "invalid register class: agpr loads and stores not supported on this GPU"
  );
  return false;
}
if (!validateVGPRAlign(Inst)) {
  Error(IDLoc,
    "invalid register class: vgpr tuples must be 64 bit aligned");
  return false;
}
if (!validateGWS(Inst, Operands)) {
  return false;
}

if (!validateDivScale(Inst)) {
  Error(IDLoc, "ABS not allowed in VOP3B instructions");
  return false;
}
if (!validateCoherencyBits(Inst, Operands, IDLoc)) {
  return false;
}

return true;
4303}

4305static std::string AMDGPUMnemonicSpellCheck(StringRef S,
                                          const FeatureBitset &FBS,
                                          unsigned VariantID = 0);

4309static bool AMDGPUCheckMnemonic(StringRef Mnemonic,
                              const FeatureBitset &AvailableFeatures,
                              unsigned VariantID);

4313bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
                                     const FeatureBitset &FBS) {
return isSupportedMnemo(Mnemo, FBS, getAllVariants());
4316}

4318bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
                                     const FeatureBitset &FBS,
                                     ArrayRef<unsigned> Variants) {
for (auto Variant : Variants) {
  if (AMDGPUCheckMnemonic(Mnemo, FBS, Variant))
    return true;
}

return false;
4327}

4329bool AMDGPUAsmParser::checkUnsupportedInstruction(StringRef Mnemo,
                                                const SMLoc &IDLoc) {
FeatureBitset FBS = ComputeAvailableFeatures(getSTI().getFeatureBits());

// Check if requested instruction variant is supported.
if (isSupportedMnemo(Mnemo, FBS, getMatchedVariants()))
  return false;

// This instruction is not supported.
// Clear any other pending errors because they are no longer relevant.
getParser().clearPendingErrors();

// Requested instruction variant is not supported.
// Check if any other variants are supported.
StringRef VariantName = getMatchedVariantName();
if (!VariantName.empty() && isSupportedMnemo(Mnemo, FBS)) {
  return Error(IDLoc,
               Twine(VariantName,
                     " variant of this instruction is not supported"));
}

// Finally check if this instruction is supported on any other GPU.
if (isSupportedMnemo(Mnemo, FeatureBitset().set())) {
  return Error(IDLoc, "instruction not supported on this GPU");
}

// Instruction not supported on any GPU. Probably a typo.
std::string Suggestion = AMDGPUMnemonicSpellCheck(Mnemo, FBS);
return Error(IDLoc, "invalid instruction" + Suggestion);
4358}

4360bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                                            OperandVector &Operands,
                                            MCStreamer &Out,
                                            uint64_t &ErrorInfo,
                                            bool MatchingInlineAsm) {
MCInst Inst;
unsigned Result = Match_Success;
for (auto Variant : getMatchedVariants()) {
  uint64_t EI;
  auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
                                Variant);
  // We order match statuses from least to most specific. We use most specific
  // status as resulting
  // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
  if ((R == Match_Success) ||
      (R == Match_PreferE32) ||
      (R == Match_MissingFeature && Result != Match_PreferE32) ||
      (R == Match_InvalidOperand && Result != Match_MissingFeature
                                 && Result != Match_PreferE32) ||
      (R == Match_MnemonicFail   && Result != Match_InvalidOperand
                                 && Result != Match_MissingFeature
                                 && Result != Match_PreferE32)) {
    Result = R;
    ErrorInfo = EI;
  }
  if (R == Match_Success)
    break;
}

if (Result == Match_Success) {
  if (!validateInstruction(Inst, IDLoc, Operands)) {
    return true;
  }
  Inst.setLoc(IDLoc);
  Out.emitInstruction(Inst, getSTI());
  return false;
}

StringRef Mnemo = ((AMDGPUOperand &)*Operands[0]).getToken();
if (checkUnsupportedInstruction(Mnemo, IDLoc)) {
  return true;
}

switch (Result) {
default: break;
case Match_MissingFeature:
  // It has been verified that the specified instruction
  // mnemonic is valid. A match was found but it requires
  // features which are not supported on this GPU.
  return Error(IDLoc, "operands are not valid for this GPU or mode");

case Match_InvalidOperand: {
  SMLoc ErrorLoc = IDLoc;
  if (ErrorInfo != ~0ULL) {
    if (ErrorInfo >= Operands.size()) {
      return Error(IDLoc, "too few operands for instruction");
    }
    ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
    if (ErrorLoc == SMLoc())
      ErrorLoc = IDLoc;
  }
  return Error(ErrorLoc, "invalid operand for instruction");
}

case Match_PreferE32:
  return Error(IDLoc, "internal error: instruction without _e64 suffix "
                      "should be encoded as e32");
case Match_MnemonicFail:
  llvm_unreachable("Invalid instructions should have been handled already")__builtin_unreachable();
}
llvm_unreachable("Implement any new match types added!")__builtin_unreachable();
4431}

4433bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
int64_t Tmp = -1;
if (!isToken(AsmToken::Integer) && !isToken(AsmToken::Identifier)) {
  return true;
}
if (getParser().parseAbsoluteExpression(Tmp)) {
  return true;
}
Ret = static_cast<uint32_t>(Tmp);
return false;
4443}

4445bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
                                             uint32_t &Minor) {
if (ParseAsAbsoluteExpression(Major))
  return TokError("invalid major version");

if (!trySkipToken(AsmToken::Comma))
  return TokError("minor version number required, comma expected");

if (ParseAsAbsoluteExpression(Minor))
  return TokError("invalid minor version");

return false;
4457}

4459bool AMDGPUAsmParser::ParseDirectiveAMDGCNTarget() {
if (getSTI().getTargetTriple().getArch() != Triple::amdgcn)
  return TokError("directive only supported for amdgcn architecture");

std::string TargetIDDirective;
SMLoc TargetStart = getTok().getLoc();
if (getParser().parseEscapedString(TargetIDDirective))
  return true;

SMRange TargetRange = SMRange(TargetStart, getTok().getLoc());
if (getTargetStreamer().getTargetID()->toString() != TargetIDDirective)
  return getParser().Error(TargetRange.Start,
      (Twine(".amdgcn_target directive's target id ") +
       Twine(TargetIDDirective) +
       Twine(" does not match the specified target id ") +
       Twine(getTargetStreamer().getTargetID()->toString())).str());

return false;
4477}

4479bool AMDGPUAsmParser::OutOfRangeError(SMRange Range) {
return Error(Range.Start, "value out of range", Range);
4481}

4483bool AMDGPUAsmParser::calculateGPRBlocks(
  const FeatureBitset &Features, bool VCCUsed, bool FlatScrUsed,
  bool XNACKUsed, Optional<bool> EnableWavefrontSize32, unsigned NextFreeVGPR,
  SMRange VGPRRange, unsigned NextFreeSGPR, SMRange SGPRRange,
  unsigned &VGPRBlocks, unsigned &SGPRBlocks) {
// TODO(scott.linder): These calculations are duplicated from
// AMDGPUAsmPrinter::getSIProgramInfo and could be unified.
IsaVersion Version = getIsaVersion(getSTI().getCPU());

unsigned NumVGPRs = NextFreeVGPR;
unsigned NumSGPRs = NextFreeSGPR;

if (Version.Major >= 10)
  NumSGPRs = 0;
else {
  unsigned MaxAddressableNumSGPRs =
      IsaInfo::getAddressableNumSGPRs(&getSTI());

  if (Version.Major >= 8 && !Features.test(FeatureSGPRInitBug) &&
      NumSGPRs > MaxAddressableNumSGPRs)
    return OutOfRangeError(SGPRRange);

  NumSGPRs +=
      IsaInfo::getNumExtraSGPRs(&getSTI(), VCCUsed, FlatScrUsed, XNACKUsed);

  if ((Version.Major <= 7 || Features.test(FeatureSGPRInitBug)) &&
      NumSGPRs > MaxAddressableNumSGPRs)
    return OutOfRangeError(SGPRRange);

  if (Features.test(FeatureSGPRInitBug))
    NumSGPRs = IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG;
}

VGPRBlocks =
    IsaInfo::getNumVGPRBlocks(&getSTI(), NumVGPRs, EnableWavefrontSize32);
SGPRBlocks = IsaInfo::getNumSGPRBlocks(&getSTI(), NumSGPRs);

return false;
4521}

4523bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
if (getSTI().getTargetTriple().getArch() != Triple::amdgcn)
  return TokError("directive only supported for amdgcn architecture");

if (getSTI().getTargetTriple().getOS() != Triple::AMDHSA)
  return TokError("directive only supported for amdhsa OS");

StringRef KernelName;
if (getParser().parseIdentifier(KernelName))
  return true;

kernel_descriptor_t KD = getDefaultAmdhsaKernelDescriptor(&getSTI());

StringSet<> Seen;

IsaVersion IVersion = getIsaVersion(getSTI().getCPU());

SMRange VGPRRange;
uint64_t NextFreeVGPR = 0;
uint64_t AccumOffset = 0;
SMRange SGPRRange;
uint64_t NextFreeSGPR = 0;
unsigned UserSGPRCount = 0;
bool ReserveVCC = true;
bool ReserveFlatScr = true;
Optional<bool> EnableWavefrontSize32;

while (true) {
  while (trySkipToken(AsmToken::EndOfStatement));

  StringRef ID;
  SMRange IDRange = getTok().getLocRange();
  if (!parseId(ID, "expected .amdhsa_ directive or .end_amdhsa_kernel"))
    return true;

  if (ID == ".end_amdhsa_kernel")
    break;

  if (Seen.find(ID) != Seen.end())
    return TokError(".amdhsa_ directives cannot be repeated");
  Seen.insert(ID);

  SMLoc ValStart = getLoc();
  int64_t IVal;
  if (getParser().parseAbsoluteExpression(IVal))
    return true;
  SMLoc ValEnd = getLoc();
  SMRange ValRange = SMRange(ValStart, ValEnd);

  if (IVal < 0)
    return OutOfRangeError(ValRange);

  uint64_t Val = IVal;

4577#define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE)                           \
if (!isUInt<ENTRY##_WIDTH>(VALUE))                                           \
  return OutOfRangeError(RANGE);                                             \
AMDHSA_BITS_SET(FIELD, ENTRY, VALUE)FIELD &= ~ENTRY; FIELD |= ((VALUE << ENTRY_SHIFT) &
 ENTRY);

  if (ID == ".amdhsa_group_segment_fixed_size") {
    if (!isUInt<sizeof(KD.group_segment_fixed_size) * CHAR_BIT8>(Val))
      return OutOfRangeError(ValRange);
    KD.group_segment_fixed_size = Val;
  } else if (ID == ".amdhsa_private_segment_fixed_size") {
    if (!isUInt<sizeof(KD.private_segment_fixed_size) * CHAR_BIT8>(Val))
      return OutOfRangeError(ValRange);
    KD.private_segment_fixed_size = Val;
  } else if (ID == ".amdhsa_kernarg_size") {
    if (!isUInt<sizeof(KD.kernarg_size) * CHAR_BIT8>(Val))
      return OutOfRangeError(ValRange);
    KD.kernarg_size = Val;
  } else if (ID == ".amdhsa_user_sgpr_private_segment_buffer") {
    if (hasArchitectedFlatScratch())
      return Error(IDRange.Start,
                   "directive is not supported with architected flat scratch",
                   IDRange);
    PARSE_BITS_ENTRY(KD.kernel_code_properties,
                     KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
                     Val, ValRange);
    if (Val)
      UserSGPRCount += 4;
  } else if (ID == ".amdhsa_user_sgpr_dispatch_ptr") {
    PARSE_BITS_ENTRY(KD.kernel_code_properties,
                     KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, Val,
                     ValRange);
    if (Val)
      UserSGPRCount += 2;
  } else if (ID == ".amdhsa_user_sgpr_queue_ptr") {
    PARSE_BITS_ENTRY(KD.kernel_code_properties,
                     KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, Val,
                     ValRange);
    if (Val)
      UserSGPRCount += 2;
  } else if (ID == ".amdhsa_user_sgpr_kernarg_segment_ptr") {
    PARSE_BITS_ENTRY(KD.kernel_code_properties,
                     KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
                     Val, ValRange);
    if (Val)
      UserSGPRCount += 2;
  } else if (ID == ".amdhsa_user_sgpr_dispatch_id") {
    PARSE_BITS_ENTRY(KD.kernel_code_properties,
                     KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, Val,
                     ValRange);
    if (Val)
      UserSGPRCount += 2;
  } else if (ID == ".amdhsa_user_sgpr_flat_scratch_init") {
    if (hasArchitectedFlatScratch())
      return Error(IDRange.Start,
                   "directive is not supported with architected flat scratch",
                   IDRange);
    PARSE_BITS_ENTRY(KD.kernel_code_properties,
                     KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT, Val,
                     ValRange);
    if (Val)
      UserSGPRCount += 2;
  } else if (ID == ".amdhsa_user_sgpr_private_segment_size") {
    PARSE_BITS_ENTRY(KD.kernel_code_properties,
                     KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
                     Val, ValRange);
    if (Val)
      UserSGPRCount += 1;
  } else if (ID == ".amdhsa_wavefront_size32") {
    if (IVersion.Major < 10)
      return Error(IDRange.Start, "directive requires gfx10+", IDRange);
    EnableWavefrontSize32 = Val;
    PARSE_BITS_ENTRY(KD.kernel_code_properties,
                     KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32,
                     Val, ValRange);
  } else if (ID == ".amdhsa_system_sgpr_private_segment_wavefront_offset") {
    if (hasArchitectedFlatScratch())
      return Error(IDRange.Start,
                   "directive is not supported with architected flat scratch",
                   IDRange);
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
                     COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, Val, ValRange);
  } else if (ID == ".amdhsa_enable_private_segment") {
    if (!hasArchitectedFlatScratch())
      return Error(
          IDRange.Start,
          "directive is not supported without architected flat scratch",
          IDRange);
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
                     COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, Val, ValRange);
  } else if (ID == ".amdhsa_system_sgpr_workgroup_id_x") {
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
                     COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, Val,
                     ValRange);
  } else if (ID == ".amdhsa_system_sgpr_workgroup_id_y") {
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
                     COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, Val,
                     ValRange);
  } else if (ID == ".amdhsa_system_sgpr_workgroup_id_z") {
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
                     COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, Val,
                     ValRange);
  } else if (ID == ".amdhsa_system_sgpr_workgroup_info") {
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
                     COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, Val,
                     ValRange);
  } else if (ID == ".amdhsa_system_vgpr_workitem_id") {
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
                     COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, Val,
                     ValRange);
  } else if (ID == ".amdhsa_next_free_vgpr") {
    VGPRRange = ValRange;
    NextFreeVGPR = Val;
  } else if (ID == ".amdhsa_next_free_sgpr") {
    SGPRRange = ValRange;
    NextFreeSGPR = Val;
  } else if (ID == ".amdhsa_accum_offset") {
    if (!isGFX90A())
      return Error(IDRange.Start, "directive requires gfx90a+", IDRange);
    AccumOffset = Val;
  } else if (ID == ".amdhsa_reserve_vcc") {
    if (!isUInt<1>(Val))
      return OutOfRangeError(ValRange);
    ReserveVCC = Val;
  } else if (ID == ".amdhsa_reserve_flat_scratch") {
    if (IVersion.Major < 7)
      return Error(IDRange.Start, "directive requires gfx7+", IDRange);
    if (hasArchitectedFlatScratch())
      return Error(IDRange.Start,
                   "directive is not supported with architected flat scratch",
                   IDRange);
    if (!isUInt<1>(Val))
      return OutOfRangeError(ValRange);
    ReserveFlatScr = Val;
  } else if (ID == ".amdhsa_reserve_xnack_mask") {
    if (IVersion.Major < 8)
      return Error(IDRange.Start, "directive requires gfx8+", IDRange);
    if (!isUInt<1>(Val))
      return OutOfRangeError(ValRange);
    if (Val != getTargetStreamer().getTargetID()->isXnackOnOrAny())
      return getParser().Error(IDRange.Start, ".amdhsa_reserve_xnack_mask does not match target id",
                               IDRange);
  } else if (ID == ".amdhsa_float_round_mode_32") {
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
                     COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, Val, ValRange);
  } else if (ID == ".amdhsa_float_round_mode_16_64") {
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
                     COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, Val, ValRange);
  } else if (ID == ".amdhsa_float_denorm_mode_32") {
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
                     COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, Val, ValRange);
  } else if (ID == ".amdhsa_float_denorm_mode_16_64") {
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
                     COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, Val,
                     ValRange);
  } else if (ID == ".amdhsa_dx10_clamp") {
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
                     COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, Val, ValRange);
  } else if (ID == ".amdhsa_ieee_mode") {
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE,
                     Val, ValRange);
  } else if (ID == ".amdhsa_fp16_overflow") {
    if (IVersion.Major < 9)
      return Error(IDRange.Start, "directive requires gfx9+", IDRange);
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_FP16_OVFL, Val,
                     ValRange);
  } else if (ID == ".amdhsa_tg_split") {
    if (!isGFX90A())
      return Error(IDRange.Start, "directive requires gfx90a+", IDRange);
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_TG_SPLIT, Val,
                     ValRange);
  } else if (ID == ".amdhsa_workgroup_processor_mode") {
    if (IVersion.Major < 10)
      return Error(IDRange.Start, "directive requires gfx10+", IDRange);
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_WGP_MODE, Val,
                     ValRange);
  } else if (ID == ".amdhsa_memory_ordered") {
    if (IVersion.Major < 10)
      return Error(IDRange.Start, "directive requires gfx10+", IDRange);
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_MEM_ORDERED, Val,
                     ValRange);
  } else if (ID == ".amdhsa_forward_progress") {
    if (IVersion.Major < 10)
      return Error(IDRange.Start, "directive requires gfx10+", IDRange);
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_FWD_PROGRESS, Val,
                     ValRange);
  } else if (ID == ".amdhsa_exception_fp_ieee_invalid_op") {
    PARSE_BITS_ENTRY(
        KD.compute_pgm_rsrc2,
        COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION, Val,
        ValRange);
  } else if (ID == ".amdhsa_exception_fp_denorm_src") {
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
                     COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
                     Val, ValRange);
  } else if (ID == ".amdhsa_exception_fp_ieee_div_zero") {
    PARSE_BITS_ENTRY(
        KD.compute_pgm_rsrc2,
        COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO, Val,
        ValRange);
  } else if (ID == ".amdhsa_exception_fp_ieee_overflow") {
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
                     COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
                     Val, ValRange);
  } else if (ID == ".amdhsa_exception_fp_ieee_underflow") {
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
                     COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
                     Val, ValRange);
  } else if (ID == ".amdhsa_exception_fp_ieee_inexact") {
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
                     COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
                     Val, ValRange);
  } else if (ID == ".amdhsa_exception_int_div_zero") {
    PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
                     COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
                     Val, ValRange);
  } else {
    return Error(IDRange.Start, "unknown .amdhsa_kernel directive", IDRange);
  }

4796#undef PARSE_BITS_ENTRY
}

if (Seen.find(".amdhsa_next_free_vgpr") == Seen.end())
  return TokError(".amdhsa_next_free_vgpr directive is required");

if (Seen.find(".amdhsa_next_free_sgpr") == Seen.end())
  return TokError(".amdhsa_next_free_sgpr directive is required");

unsigned VGPRBlocks;
unsigned SGPRBlocks;
if (calculateGPRBlocks(getFeatureBits(), ReserveVCC, ReserveFlatScr,
                       getTargetStreamer().getTargetID()->isXnackOnOrAny(),
                       EnableWavefrontSize32, NextFreeVGPR,
                       VGPRRange, NextFreeSGPR, SGPRRange, VGPRBlocks,
                       SGPRBlocks))
  return true;

if (!isUInt<COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_WIDTH>(
        VGPRBlocks))
  return OutOfRangeError(VGPRRange);
AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,KD.compute_pgm_rsrc1 &= ~COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT
; KD.compute_pgm_rsrc1 |= ((VGPRBlocks << COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT
) & COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT)
                COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, VGPRBlocks)KD.compute_pgm_rsrc1 &= ~COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT
; KD.compute_pgm_rsrc1 |= ((VGPRBlocks << COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_SHIFT
) & COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT);

if (!isUInt<COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_WIDTH>(
        SGPRBlocks))
  return OutOfRangeError(SGPRRange);
AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,KD.compute_pgm_rsrc1 &= ~COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT
; KD.compute_pgm_rsrc1 |= ((SGPRBlocks << COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT
) & COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT)
                COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT,KD.compute_pgm_rsrc1 &= ~COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT
; KD.compute_pgm_rsrc1 |= ((SGPRBlocks << COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT
) & COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT)
                SGPRBlocks)KD.compute_pgm_rsrc1 &= ~COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT
; KD.compute_pgm_rsrc1 |= ((SGPRBlocks << COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_SHIFT
) & COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT);

if (!isUInt<COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_WIDTH>(UserSGPRCount))
  return TokError("too many user SGPRs enabled");
AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_USER_SGPR_COUNT,KD.compute_pgm_rsrc2 &= ~COMPUTE_PGM_RSRC2_USER_SGPR_COUNT
; KD.compute_pgm_rsrc2 |= ((UserSGPRCount << COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_SHIFT
) & COMPUTE_PGM_RSRC2_USER_SGPR_COUNT)
                UserSGPRCount)KD.compute_pgm_rsrc2 &= ~COMPUTE_PGM_RSRC2_USER_SGPR_COUNT
; KD.compute_pgm_rsrc2 |= ((UserSGPRCount << COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_SHIFT
) & COMPUTE_PGM_RSRC2_USER_SGPR_COUNT);

if (isGFX90A()) {
  if (Seen.find(".amdhsa_accum_offset") == Seen.end())
    return TokError(".amdhsa_accum_offset directive is required");
  if (AccumOffset < 4 || AccumOffset > 256 || (AccumOffset & 3))
    return TokError("accum_offset should be in range [4..256] in "
                    "increments of 4");
  if (AccumOffset > alignTo(std::max((uint64_t)1, NextFreeVGPR), 4))
    return TokError("accum_offset exceeds total VGPR allocation");
  AMDHSA_BITS_SET(KD.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET,KD.compute_pgm_rsrc3 &= ~COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET
; KD.compute_pgm_rsrc3 |= (((AccumOffset / 4 - 1) << COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT
) & COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET)
                  (AccumOffset / 4 - 1))KD.compute_pgm_rsrc3 &= ~COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET
; KD.compute_pgm_rsrc3 |= (((AccumOffset / 4 - 1) << COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET_SHIFT
) & COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET);
}

getTargetStreamer().EmitAmdhsaKernelDescriptor(
    getSTI(), KernelName, KD, NextFreeVGPR, NextFreeSGPR, ReserveVCC,
    ReserveFlatScr);
return false;
4848}

4850bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
uint32_t Major;
uint32_t Minor;

if (ParseDirectiveMajorMinor(Major, Minor))
  return true;

getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
return false;
4859}

4861bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
uint32_t Major;
uint32_t Minor;
uint32_t Stepping;
StringRef VendorName;
StringRef ArchName;

// If this directive has no arguments, then use the ISA version for the
// targeted GPU.
if (isToken(AsmToken::EndOfStatement)) {
  AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(getSTI().getCPU());
  getTargetStreamer().EmitDirectiveHSACodeObjectISAV2(ISA.Major, ISA.Minor,
                                                      ISA.Stepping,
                                                      "AMD", "AMDGPU");
  return false;
}

if (ParseDirectiveMajorMinor(Major, Minor))
  return true;

if (!trySkipToken(AsmToken::Comma))
  return TokError("stepping version number required, comma expected");

if (ParseAsAbsoluteExpression(Stepping))
  return TokError("invalid stepping version");

if (!trySkipToken(AsmToken::Comma))
  return TokError("vendor name required, comma expected");

if (!parseString(VendorName, "invalid vendor name"))
  return true;

if (!trySkipToken(AsmToken::Comma))
  return TokError("arch name required, comma expected");

if (!parseString(ArchName, "invalid arch name"))
  return true;

getTargetStreamer().EmitDirectiveHSACodeObjectISAV2(Major, Minor, Stepping,
                                                    VendorName, ArchName);
return false;
4902}

4904bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
                                             amd_kernel_code_t &Header) {
// max_scratch_backing_memory_byte_size is deprecated. Ignore it while parsing
// assembly for backwards compatibility.
if (ID == "max_scratch_backing_memory_byte_size") {
  Parser.eatToEndOfStatement();
  return false;
}

SmallString<40> ErrStr;
raw_svector_ostream Err(ErrStr);
if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
  return TokError(Err.str());
}
Lex();

if (ID == "enable_wavefront_size32") {
  if (Header.code_properties & AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32) {
    if (!isGFX10Plus())
      return TokError("enable_wavefront_size32=1 is only allowed on GFX10+");
    if (!getFeatureBits()[AMDGPU::FeatureWavefrontSize32])
      return TokError("enable_wavefront_size32=1 requires +WavefrontSize32");
  } else {
    if (!getFeatureBits()[AMDGPU::FeatureWavefrontSize64])
      return TokError("enable_wavefront_size32=0 requires +WavefrontSize64");
  }
}

if (ID == "wavefront_size") {
  if (Header.wavefront_size == 5) {
    if (!isGFX10Plus())
      return TokError("wavefront_size=5 is only allowed on GFX10+");
    if (!getFeatureBits()[AMDGPU::FeatureWavefrontSize32])
      return TokError("wavefront_size=5 requires +WavefrontSize32");
  } else if (Header.wavefront_size == 6) {
    if (!getFeatureBits()[AMDGPU::FeatureWavefrontSize64])
      return TokError("wavefront_size=6 requires +WavefrontSize64");
  }
}

if (ID == "enable_wgp_mode") {
  if (G_00B848_WGP_MODE(Header.compute_pgm_resource_registers)(((Header.compute_pgm_resource_registers) >> 29) & 0x1
) &&
      !isGFX10Plus())
    return TokError("enable_wgp_mode=1 is only allowed on GFX10+");
}

if (ID == "enable_mem_ordered") {
  if (G_00B848_MEM_ORDERED(Header.compute_pgm_resource_registers)(((Header.compute_pgm_resource_registers) >> 30) & 0x1
) &&
      !isGFX10Plus())
    return TokError("enable_mem_ordered=1 is only allowed on GFX10+");
}

if (ID == "enable_fwd_progress") {
  if (G_00B848_FWD_PROGRESS(Header.compute_pgm_resource_registers)(((Header.compute_pgm_resource_registers) >> 31) & 0x1
) &&
      !isGFX10Plus())
    return TokError("enable_fwd_progress=1 is only allowed on GFX10+");
}

return false;
4963}

4965bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
amd_kernel_code_t Header;
AMDGPU::initDefaultAMDKernelCodeT(Header, &getSTI());

while (true) {
  // Lex EndOfStatement.  This is in a while loop, because lexing a comment
  // will set the current token to EndOfStatement.
  while(trySkipToken(AsmToken::EndOfStatement));

  StringRef ID;
  if (!parseId(ID, "expected value identifier or .end_amd_kernel_code_t"))
    return true;

  if (ID == ".end_amd_kernel_code_t")
    break;

  if (ParseAMDKernelCodeTValue(ID, Header))
    return true;
}

getTargetStreamer().EmitAMDKernelCodeT(Header);

return false;
4988}

4990bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
StringRef KernelName;
if (!parseId(KernelName, "expected symbol name"))
  return true;

getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
                                         ELF::STT_AMDGPU_HSA_KERNEL);

KernelScope.initialize(getContext());
return false;
5000}

5002bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
if (getSTI().getTargetTriple().getArch() != Triple::amdgcn) {
  return Error(getLoc(),
               ".amd_amdgpu_isa directive is not available on non-amdgcn "
               "architectures");
}

auto TargetIDDirective = getLexer().getTok().getStringContents();
if (getTargetStreamer().getTargetID()->toString() != TargetIDDirective)
  return Error(getParser().getTok().getLoc(), "target id must match options");

getTargetStreamer().EmitISAVersion();
Lex();

return false;
5017}

5019bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
const char *AssemblerDirectiveBegin;
const char *AssemblerDirectiveEnd;
std::tie(AssemblerDirectiveBegin, AssemblerDirectiveEnd) =
    isHsaAbiVersion3Or4(&getSTI())
        ? std::make_tuple(HSAMD::V3::AssemblerDirectiveBegin,
                          HSAMD::V3::AssemblerDirectiveEnd)
        : std::make_tuple(HSAMD::AssemblerDirectiveBegin,
                          HSAMD::AssemblerDirectiveEnd);

if (getSTI().getTargetTriple().getOS() != Triple::AMDHSA) {
  return Error(getLoc(),
               (Twine(AssemblerDirectiveBegin) + Twine(" directive is "
               "not available on non-amdhsa OSes")).str());
}

std::string HSAMetadataString;
if (ParseToEndDirective(AssemblerDirectiveBegin, AssemblerDirectiveEnd,
                        HSAMetadataString))
  return true;

if (isHsaAbiVersion3Or4(&getSTI())) {
  if (!getTargetStreamer().EmitHSAMetadataV3(HSAMetadataString))
    return Error(getLoc(), "invalid HSA metadata");
} else {
  if (!getTargetStreamer().EmitHSAMetadataV2(HSAMetadataString))
    return Error(getLoc(), "invalid HSA metadata");
}

return false;
5049}

5051/// Common code to parse out a block of text (typically YAML) between start and
5052/// end directives.
5053bool AMDGPUAsmParser::ParseToEndDirective(const char *AssemblerDirectiveBegin,
                                        const char *AssemblerDirectiveEnd,
                                        std::string &CollectString) {

raw_string_ostream CollectStream(CollectString);

getLexer().setSkipSpace(false);

bool FoundEnd = false;
while (!isToken(AsmToken::Eof)) {
  while (isToken(AsmToken::Space)) {
    CollectStream << getTokenStr();
    Lex();
  }

  if (trySkipId(AssemblerDirectiveEnd)) {
    FoundEnd = true;
    break;
  }

  CollectStream << Parser.parseStringToEndOfStatement()
                << getContext().getAsmInfo()->getSeparatorString();

  Parser.eatToEndOfStatement();
}

getLexer().setSkipSpace(true);

if (isToken(AsmToken::Eof) && !FoundEnd) {
  return TokError(Twine("expected directive ") +
                  Twine(AssemblerDirectiveEnd) + Twine(" not found"));
}

CollectStream.flush();
return false;
5088}

5090/// Parse the assembler directive for new MsgPack-format PAL metadata.
5091bool AMDGPUAsmParser::ParseDirectivePALMetadataBegin() {
std::string String;
if (ParseToEndDirective(AMDGPU::PALMD::AssemblerDirectiveBegin,
                        AMDGPU::PALMD::AssemblerDirectiveEnd, String))
  return true;

auto PALMetadata = getTargetStreamer().getPALMetadata();
if (!PALMetadata->setFromString(String))
  return Error(getLoc(), "invalid PAL metadata");
return false;
5101}

5103/// Parse the assembler directive for old linear-format PAL metadata.
5104bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
if (getSTI().getTargetTriple().getOS() != Triple::AMDPAL) {
  return Error(getLoc(),
               (Twine(PALMD::AssemblerDirective) + Twine(" directive is "
               "not available on non-amdpal OSes")).str());
}

auto PALMetadata = getTargetStreamer().getPALMetadata();
PALMetadata->setLegacy();
for (;;) {
  uint32_t Key, Value;
  if (ParseAsAbsoluteExpression(Key)) {
    return TokError(Twine("invalid value in ") +
                    Twine(PALMD::AssemblerDirective));
  }
  if (!trySkipToken(AsmToken::Comma)) {
    return TokError(Twine("expected an even number of values in ") +
                    Twine(PALMD::AssemblerDirective));
  }
  if (ParseAsAbsoluteExpression(Value)) {
    return TokError(Twine("invalid value in ") +
                    Twine(PALMD::AssemblerDirective));
  }
  PALMetadata->setRegister(Key, Value);
  if (!trySkipToken(AsmToken::Comma))
    break;
}
return false;
5132}

5134/// ParseDirectiveAMDGPULDS
5135///  ::= .amdgpu_lds identifier ',' size_expression [',' align_expression]
5136bool AMDGPUAsmParser::ParseDirectiveAMDGPULDS() {
if (getParser().checkForValidSection())
  return true;

StringRef Name;
SMLoc NameLoc = getLoc();
if (getParser().parseIdentifier(Name))
  return TokError("expected identifier in directive");

MCSymbol *Symbol = getContext().getOrCreateSymbol(Name);
if (parseToken(AsmToken::Comma, "expected ','"))
  return true;

unsigned LocalMemorySize = AMDGPU::IsaInfo::getLocalMemorySize(&getSTI());

int64_t Size;
SMLoc SizeLoc = getLoc();
if (getParser().parseAbsoluteExpression(Size))
  return true;
if (Size < 0)
  return Error(SizeLoc, "size must be non-negative");
if (Size > LocalMemorySize)
  return Error(SizeLoc, "size is too large");

int64_t Alignment = 4;
if (trySkipToken(AsmToken::Comma)) {
  SMLoc AlignLoc = getLoc();
  if (getParser().parseAbsoluteExpression(Alignment))
    return true;
  if (Alignment < 0 || !isPowerOf2_64(Alignment))
    return Error(AlignLoc, "alignment must be a power of two");

  // Alignment larger than the size of LDS is possible in theory, as long
  // as the linker manages to place to symbol at address 0, but we do want
  // to make sure the alignment fits nicely into a 32-bit integer.
  if (Alignment >= 1u << 31)
    return Error(AlignLoc, "alignment is too large");
}

if (parseToken(AsmToken::EndOfStatement,
               "unexpected token in '.amdgpu_lds' directive"))
  return true;

Symbol->redefineIfPossible();
if (!Symbol->isUndefined())
  return Error(NameLoc, "invalid symbol redefinition");

getTargetStreamer().emitAMDGPULDS(Symbol, Size, Align(Alignment));
return false;
5185}

5187bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
StringRef IDVal = DirectiveID.getString();

if (isHsaAbiVersion3Or4(&getSTI())) {
  if (IDVal == ".amdhsa_kernel")
   return ParseDirectiveAMDHSAKernel();

  // TODO: Restructure/combine with PAL metadata directive.
  if (IDVal == AMDGPU::HSAMD::V3::AssemblerDirectiveBegin)
    return ParseDirectiveHSAMetadata();
} else {
  if (IDVal == ".hsa_code_object_version")
    return ParseDirectiveHSACodeObjectVersion();

  if (IDVal == ".hsa_code_object_isa")
    return ParseDirectiveHSACodeObjectISA();

  if (IDVal == ".amd_kernel_code_t")
    return ParseDirectiveAMDKernelCodeT();

  if (IDVal == ".amdgpu_hsa_kernel")
    return ParseDirectiveAMDGPUHsaKernel();

  if (IDVal == ".amd_amdgpu_isa")
    return ParseDirectiveISAVersion();

  if (IDVal == AMDGPU::HSAMD::AssemblerDirectiveBegin)
    return ParseDirectiveHSAMetadata();
}

if (IDVal == ".amdgcn_target")
  return ParseDirectiveAMDGCNTarget();

if (IDVal == ".amdgpu_lds")
  return ParseDirectiveAMDGPULDS();

if (IDVal == PALMD::AssemblerDirectiveBegin)
  return ParseDirectivePALMetadataBegin();

if (IDVal == PALMD::AssemblerDirective)
  return ParseDirectivePALMetadata();

return true;
5230}

5232bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
                                         unsigned RegNo) {

for (MCRegAliasIterator R(AMDGPU::TTMP12_TTMP13_TTMP14_TTMP15, &MRI, true);
     R.isValid(); ++R) {
  if (*R == RegNo)
    return isGFX9Plus();
}

// GFX10 has 2 more SGPRs 104 and 105.
for (MCRegAliasIterator R(AMDGPU::SGPR104_SGPR105, &MRI, true);
     R.isValid(); ++R) {
  if (*R == RegNo)
    return hasSGPR104_SGPR105();
}

switch (RegNo) {
case AMDGPU::SRC_SHARED_BASE:
case AMDGPU::SRC_SHARED_LIMIT:
case AMDGPU::SRC_PRIVATE_BASE:
case AMDGPU::SRC_PRIVATE_LIMIT:
case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
  return isGFX9Plus();
case AMDGPU::TBA:
case AMDGPU::TBA_LO:
case AMDGPU::TBA_HI:
case AMDGPU::TMA:
case AMDGPU::TMA_LO:
case AMDGPU::TMA_HI:
  return !isGFX9Plus();
case AMDGPU::XNACK_MASK:
case AMDGPU::XNACK_MASK_LO:
case AMDGPU::XNACK_MASK_HI:
  return (isVI() || isGFX9()) && getTargetStreamer().getTargetID()->isXnackSupported();
case AMDGPU::SGPR_NULL:
  return isGFX10Plus();
default:
  break;
}

if (isCI())
  return true;

if (isSI() || isGFX10Plus()) {
  // No flat_scr on SI.
  // On GFX10 flat scratch is not a valid register operand and can only be
  // accessed with s_setreg/s_getreg.
  switch (RegNo) {
  case AMDGPU::FLAT_SCR:
  case AMDGPU::FLAT_SCR_LO:
  case AMDGPU::FLAT_SCR_HI:
    return false;
  default:
    return true;
  }
}

// VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
// SI/CI have.
for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
     R.isValid(); ++R) {
  if (*R == RegNo)
    return hasSGPR102_SGPR103();
}

return true;
5298}

5300OperandMatchResultTy
5301AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic,
                            OperandMode Mode) {
// Try to parse with a custom parser
OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);

// If we successfully parsed the operand or if there as an error parsing,
// we are done.
//
// If we are parsing after we reach EndOfStatement then this means we
// are appending default values to the Operands list.  This is only done
// by custom parser, so we shouldn't continue on to the generic parsing.
if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
    isToken(AsmToken::EndOfStatement))
  return ResTy;

SMLoc RBraceLoc;
SMLoc LBraceLoc = getLoc();
if (Mode == OperandMode_NSA && trySkipToken(AsmToken::LBrac)) {
  unsigned Prefix = Operands.size();

  for (;;) {
    auto Loc = getLoc();
    ResTy = parseReg(Operands);
    if (ResTy == MatchOperand_NoMatch)
      Error(Loc, "expected a register");
    if (ResTy != MatchOperand_Success)
      return MatchOperand_ParseFail;

    RBraceLoc = getLoc();
    if (trySkipToken(AsmToken::RBrac))
      break;

    if (!skipToken(AsmToken::Comma,
                   "expected a comma or a closing square bracket")) {
      return MatchOperand_ParseFail;
    }
  }

  if (Operands.size() - Prefix > 1) {
    Operands.insert(Operands.begin() + Prefix,
                    AMDGPUOperand::CreateToken(this, "[", LBraceLoc));
    Operands.push_back(AMDGPUOperand::CreateToken(this, "]", RBraceLoc));
  }

  return MatchOperand_Success;
}

return parseRegOrImm(Operands);
5349}

5351StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
// Clear any forced encodings from the previous instruction.
setForcedEncodingSize(0);
setForcedDPP(false);
setForcedSDWA(false);

if (Name.endswith("_e64")) {
  setForcedEncodingSize(64);
  return Name.substr(0, Name.size() - 4);
} else if (Name.endswith("_e32")) {
  setForcedEncodingSize(32);
  return Name.substr(0, Name.size() - 4);
} else if (Name.endswith("_dpp")) {
  setForcedDPP(true);
  return Name.substr(0, Name.size() - 4);
} else if (Name.endswith("_sdwa")) {
  setForcedSDWA(true);
  return Name.substr(0, Name.size() - 5);
}
return Name;
5371}

5373bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
                                     StringRef Name,
                                     SMLoc NameLoc, OperandVector &Operands) {
// Add the instruction mnemonic
Name = parseMnemonicSuffix(Name);
Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));

bool IsMIMG = Name.startswith("image_");

while (!trySkipToken(AsmToken::EndOfStatement)) {
  OperandMode Mode = OperandMode_Default;
  if (IsMIMG && isGFX10Plus() && Operands.size() == 2)
    Mode = OperandMode_NSA;
  CPolSeen = 0;
  OperandMatchResultTy Res = parseOperand(Operands, Name, Mode);

  if (Res != MatchOperand_Success) {
    checkUnsupportedInstruction(Name, NameLoc);
    if (!Parser.hasPendingError()) {
      // FIXME: use real operand location rather than the current location.
      StringRef Msg =
        (Res == MatchOperand_ParseFail) ? "failed parsing operand." :
                                          "not a valid operand.";
      Error(getLoc(), Msg);
    }
    while (!trySkipToken(AsmToken::EndOfStatement)) {
      lex();
    }
    return true;
  }

  // Eat the comma or space if there is one.
  trySkipToken(AsmToken::Comma);
}

return false;
5409}

5411//===----------------------------------------------------------------------===//
5412// Utility functions
5413//===----------------------------------------------------------------------===//

5415OperandMatchResultTy
5416AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &IntVal) {

if (!trySkipId(Prefix, AsmToken::Colon))
  return MatchOperand_NoMatch;

return parseExpr(IntVal) ? MatchOperand_Success : MatchOperand_ParseFail;
5422}

5424OperandMatchResultTy
5425AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
                                  AMDGPUOperand::ImmTy ImmTy,
                                  bool (*ConvertResult)(int64_t&)) {
SMLoc S = getLoc();
int64_t Value = 0;

OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
if (Res != MatchOperand_Success)
  return Res;

if (ConvertResult && !ConvertResult(Value)) {
  Error(S, "invalid " + StringRef(Prefix) + " value.");
}

Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
return MatchOperand_Success;
5441}

5443OperandMatchResultTy
5444AMDGPUAsmParser::parseOperandArrayWithPrefix(const char *Prefix,
                                           OperandVector &Operands,
                                           AMDGPUOperand::ImmTy ImmTy,
                                           bool (*ConvertResult)(int64_t&)) {
SMLoc S = getLoc();
if (!trySkipId(Prefix, AsmToken::Colon))
  return MatchOperand_NoMatch;

if (!skipToken(AsmToken::LBrac, "expected a left square bracket"))
  return MatchOperand_ParseFail;

unsigned Val = 0;
const unsigned MaxSize = 4;

// FIXME: How to verify the number of elements matches the number of src
// operands?
for (int I = 0; ; ++I) {
  int64_t Op;
  SMLoc Loc = getLoc();
  if (!parseExpr(Op))
    return MatchOperand_ParseFail;

  if (Op != 0 && Op != 1) {
    Error(Loc, "invalid " + StringRef(Prefix) + " value.");
    return MatchOperand_ParseFail;
  }

  Val |= (Op << I);

  if (trySkipToken(AsmToken::RBrac))
    break;

  if (I + 1 == MaxSize) {
    Error(getLoc(), "expected a closing square bracket");
    return MatchOperand_ParseFail;
  }

  if (!skipToken(AsmToken::Comma, "expected a comma"))
    return MatchOperand_ParseFail;
}

Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S, ImmTy));
return MatchOperand_Success;
5487}

5489OperandMatchResultTy
5490AMDGPUAsmParser::parseNamedBit(StringRef Name, OperandVector &Operands,
                             AMDGPUOperand::ImmTy ImmTy) {
int64_t Bit;
SMLoc S = getLoc();

if (trySkipId(Name)) {
  Bit = 1;
} else if (trySkipId("no", Name)) {
  Bit = 0;
} else {
  return MatchOperand_NoMatch;
}

if (Name == "r128" && !hasMIMG_R128()) {
  Error(S, "r128 modifier is not supported on this GPU");
  return MatchOperand_ParseFail;
}
if (Name == "a16" && !isGFX9() && !hasGFX10A16()) {
  Error(S, "a16 modifier is not supported on this GPU");
  return MatchOperand_ParseFail;
}

if (isGFX9() && ImmTy == AMDGPUOperand::ImmTyA16)
  ImmTy = AMDGPUOperand::ImmTyR128A16;

Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
return MatchOperand_Success;
5517}

5519OperandMatchResultTy
5520AMDGPUAsmParser::parseCPol(OperandVector &Operands) {
unsigned CPolOn = 0;
unsigned CPolOff = 0;
SMLoc S = getLoc();

if (trySkipId("glc"))
  CPolOn = AMDGPU::CPol::GLC;
else if (trySkipId("noglc"))
  CPolOff = AMDGPU::CPol::GLC;
else if (trySkipId("slc"))
  CPolOn = AMDGPU::CPol::SLC;
else if (trySkipId("noslc"))
  CPolOff = AMDGPU::CPol::SLC;
else if (trySkipId("dlc"))
  CPolOn = AMDGPU::CPol::DLC;
else if (trySkipId("nodlc"))
  CPolOff = AMDGPU::CPol::DLC;
else if (trySkipId("scc"))
  CPolOn = AMDGPU::CPol::SCC;
else if (trySkipId("noscc"))
  CPolOff = AMDGPU::CPol::SCC;
else
  return MatchOperand_NoMatch;

if (!isGFX10Plus() && ((CPolOn | CPolOff) & AMDGPU::CPol::DLC)) {
  Error(S, "dlc modifier is not supported on this GPU");
  return MatchOperand_ParseFail;
}

if (!isGFX90A() && ((CPolOn | CPolOff) & AMDGPU::CPol::SCC)) {
  Error(S, "scc modifier is not supported on this GPU");
  return MatchOperand_ParseFail;
}

if (CPolSeen & (CPolOn | CPolOff)) {
  Error(S, "duplicate cache policy modifier");
  return MatchOperand_ParseFail;
}

CPolSeen |= (CPolOn | CPolOff);

for (unsigned I = 1; I != Operands.size(); ++I) {
  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
  if (Op.isCPol()) {
    Op.setImm((Op.getImm() | CPolOn) & ~CPolOff);
    return MatchOperand_Success;
  }
}

Operands.push_back(AMDGPUOperand::CreateImm(this, CPolOn, S,
                                            AMDGPUOperand::ImmTyCPol));

return MatchOperand_Success;
5573}

5575static void addOptionalImmOperand(
MCInst& Inst, const OperandVector& Operands,
AMDGPUAsmParser::OptionalImmIndexMap& OptionalIdx,
AMDGPUOperand::ImmTy ImmT,
int64_t Default = 0) {
auto i = OptionalIdx.find(ImmT);
if (i != OptionalIdx.end()) {
  unsigned Idx = i->second;
  ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
} else {
  Inst.addOperand(MCOperand::createImm(Default));
}
5587}

5589OperandMatchResultTy
5590AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix,
                                     StringRef &Value,
                                     SMLoc &StringLoc) {
if (!trySkipId(Prefix, AsmToken::Colon))
  return MatchOperand_NoMatch;

StringLoc = getLoc();
return parseId(Value, "expected an identifier") ? MatchOperand_Success
                                                : MatchOperand_ParseFail;
5599}

5601//===----------------------------------------------------------------------===//
5602// MTBUF format
5603//===----------------------------------------------------------------------===//

5605bool AMDGPUAsmParser::tryParseFmt(const char *Pref,
                                int64_t MaxVal,
                                int64_t &Fmt) {
int64_t Val;
SMLoc Loc = getLoc();

auto Res = parseIntWithPrefix(Pref, Val);
if (Res == MatchOperand_ParseFail)
  return false;
if (Res == MatchOperand_NoMatch)
  return true;

if (Val < 0 || Val > MaxVal) {
  Error(Loc, Twine("out of range ", StringRef(Pref)));
  return false;
}

Fmt = Val;
return true;
5624}

5626// dfmt and nfmt (in a tbuffer instruction) are parsed as one to allow their
5627// values to live in a joint format operand in the MCInst encoding.
5628OperandMatchResultTy
5629AMDGPUAsmParser::parseDfmtNfmt(int64_t &Format) {
using namespace llvm::AMDGPU::MTBUFFormat;

int64_t Dfmt = DFMT_UNDEF;
int64_t Nfmt = NFMT_UNDEF;

// dfmt and nfmt can appear in either order, and each is optional.
for (int I = 0; I < 2; ++I) {
  if (Dfmt == DFMT_UNDEF && !tryParseFmt("dfmt", DFMT_MAX, Dfmt))
    return MatchOperand_ParseFail;

  if (Nfmt == NFMT_UNDEF && !tryParseFmt("nfmt", NFMT_MAX, Nfmt)) {
    return MatchOperand_ParseFail;
  }
  // Skip optional comma between dfmt/nfmt
  // but guard against 2 commas following each other.
  if ((Dfmt == DFMT_UNDEF) != (Nfmt == NFMT_UNDEF) &&
      !peekToken().is(AsmToken::Comma)) {
    trySkipToken(AsmToken::Comma);
  }
}

if (Dfmt == DFMT_UNDEF && Nfmt == NFMT_UNDEF)
  return MatchOperand_NoMatch;

Dfmt = (Dfmt == DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
Nfmt = (Nfmt == NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;

Format = encodeDfmtNfmt(Dfmt, Nfmt);
return MatchOperand_Success;
5659}

5661OperandMatchResultTy
5662AMDGPUAsmParser::parseUfmt(int64_t &Format) {
using namespace llvm::AMDGPU::MTBUFFormat;

int64_t Fmt = UFMT_UNDEF;

if (!tryParseFmt("format", UFMT_MAX, Fmt))
  return MatchOperand_ParseFail;

if (Fmt == UFMT_UNDEF)
  return MatchOperand_NoMatch;

Format = Fmt;
return MatchOperand_Success;
5675}

5677bool AMDGPUAsmParser::matchDfmtNfmt(int64_t &Dfmt,
                                  int64_t &Nfmt,
                                  StringRef FormatStr,
                                  SMLoc Loc) {
using namespace llvm::AMDGPU::MTBUFFormat;
int64_t Format;

Format = getDfmt(FormatStr);
if (Format != DFMT_UNDEF) {
  Dfmt = Format;
  return true;
}

Format = getNfmt(FormatStr, getSTI());
if (Format != NFMT_UNDEF) {
  Nfmt = Format;
  return true;
}

Error(Loc, "unsupported format");
return false;
5698}

5700OperandMatchResultTy
5701AMDGPUAsmParser::parseSymbolicSplitFormat(StringRef FormatStr,
                                        SMLoc FormatLoc,
                                        int64_t &Format) {
using namespace llvm::AMDGPU::MTBUFFormat;

int64_t Dfmt = DFMT_UNDEF;
int64_t Nfmt = NFMT_UNDEF;
if (!matchDfmtNfmt(Dfmt, Nfmt, FormatStr, FormatLoc))
  return MatchOperand_ParseFail;

if (trySkipToken(AsmToken::Comma)) {
  StringRef Str;
  SMLoc Loc = getLoc();
  if (!parseId(Str, "expected a format string") ||
      !matchDfmtNfmt(Dfmt, Nfmt, Str, Loc)) {
    return MatchOperand_ParseFail;
  }
  if (Dfmt == DFMT_UNDEF) {
    Error(Loc, "duplicate numeric format");
    return MatchOperand_ParseFail;
  } else if (Nfmt == NFMT_UNDEF) {
    Error(Loc, "duplicate data format");
    return MatchOperand_ParseFail;
  }
}

Dfmt = (Dfmt == DFMT_UNDEF) ? DFMT_DEFAULT : Dfmt;
Nfmt = (Nfmt == NFMT_UNDEF) ? NFMT_DEFAULT : Nfmt;

if (isGFX10Plus()) {
  auto Ufmt = convertDfmtNfmt2Ufmt(Dfmt, Nfmt);
  if (Ufmt == UFMT_UNDEF) {
    Error(FormatLoc, "unsupported format");
    return MatchOperand_ParseFail;
  }
  Format = Ufmt;
} else {
  Format = encodeDfmtNfmt(Dfmt, Nfmt);
}

return MatchOperand_Success;
5742}

5744OperandMatchResultTy
5745AMDGPUAsmParser::parseSymbolicUnifiedFormat(StringRef FormatStr,
                                          SMLoc Loc,
                                          int64_t &Format) {
using namespace llvm::AMDGPU::MTBUFFormat;

auto Id = getUnifiedFormat(FormatStr);
if (Id == UFMT_UNDEF)
  return MatchOperand_NoMatch;

if (!isGFX10Plus()) {
  Error(Loc, "unified format is not supported on this GPU");
  return MatchOperand_ParseFail;
}

Format = Id;
return MatchOperand_Success;
5761}

5763OperandMatchResultTy
5764AMDGPUAsmParser::parseNumericFormat(int64_t &Format) {
using namespace llvm::AMDGPU::MTBUFFormat;
SMLoc Loc = getLoc();

if (!parseExpr(Format))
  return MatchOperand_ParseFail;
if (!isValidFormatEncoding(Format, getSTI())) {
  Error(Loc, "out of range format");
  return MatchOperand_ParseFail;
}

return MatchOperand_Success;
5776}

5778OperandMatchResultTy
5779AMDGPUAsmParser::parseSymbolicOrNumericFormat(int64_t &Format) {
using namespace llvm::AMDGPU::MTBUFFormat;

if (!trySkipId("format", AsmToken::Colon))
  return MatchOperand_NoMatch;

if (trySkipToken(AsmToken::LBrac)) {
  StringRef FormatStr;
  SMLoc Loc = getLoc();
  if (!parseId(FormatStr, "expected a format string"))
    return MatchOperand_ParseFail;

  auto Res = parseSymbolicUnifiedFormat(FormatStr, Loc, Format);
  if (Res == MatchOperand_NoMatch)
    Res = parseSymbolicSplitFormat(FormatStr, Loc, Format);
  if (Res != MatchOperand_Success)
    return Res;

  if (!skipToken(AsmToken::RBrac, "expected a closing square bracket"))
    return MatchOperand_ParseFail;

  return MatchOperand_Success;
}

return parseNumericFormat(Format);
5804}

5806OperandMatchResultTy
5807AMDGPUAsmParser::parseFORMAT(OperandVector &Operands) {
using namespace llvm::AMDGPU::MTBUFFormat;

int64_t Format = getDefaultFormatEncoding(getSTI());
OperandMatchResultTy Res;
SMLoc Loc = getLoc();

// Parse legacy format syntax.
Res = isGFX10Plus() ? parseUfmt(Format) : parseDfmtNfmt(Format);
if (Res == MatchOperand_ParseFail)
  return Res;

bool FormatFound = (Res == MatchOperand_Success);

Operands.push_back(
  AMDGPUOperand::CreateImm(this, Format, Loc, AMDGPUOperand::ImmTyFORMAT));

if (FormatFound)
  trySkipToken(AsmToken::Comma);

if (isToken(AsmToken::EndOfStatement)) {
  // We are expecting an soffset operand,
  // but let matcher handle the error.
  return MatchOperand_Success;
}

// Parse soffset.
Res = parseRegOrImm(Operands);
if (Res != MatchOperand_Success)
  return Res;

trySkipToken(AsmToken::Comma);

if (!FormatFound) {
  Res = parseSymbolicOrNumericFormat(Format);
  if (Res == MatchOperand_ParseFail)
    return Res;
  if (Res == MatchOperand_Success) {
    auto Size = Operands.size();
    AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands[Size - 2]);
    assert(Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyFORMAT)(static_cast<void> (0));
    Op.setImm(Format);
  }
  return MatchOperand_Success;
}

if (isId("format") && peekToken().is(AsmToken::Colon)) {
  Error(getLoc(), "duplicate format");
  return MatchOperand_ParseFail;
}
return MatchOperand_Success;
5858}

5860//===----------------------------------------------------------------------===//
5861// ds
5862//===----------------------------------------------------------------------===//

5864void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
                                  const OperandVector &Operands) {
OptionalImmIndexMap OptionalIdx;

for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);

  // Add the register arguments
  if (Op.isReg()) {
    Op.addRegOperands(Inst, 1);
    continue;
  }

  // Handle optional arguments
  OptionalIdx[Op.getImmTy()] = i;
}

addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);

Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
5886}

5888void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
                              bool IsGdsHardcoded) {
OptionalImmIndexMap OptionalIdx;

for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);

  // Add the register arguments
  if (Op.isReg()) {
    Op.addRegOperands(Inst, 1);
    continue;
  }

  if (Op.isToken() && Op.getToken() == "gds") {
    IsGdsHardcoded = true;
    continue;
  }

  // Handle optional arguments
  OptionalIdx[Op.getImmTy()] = i;
}

AMDGPUOperand::ImmTy OffsetType =
  (Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_gfx10 ||
   Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_gfx6_gfx7 ||
   Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_vi) ? AMDGPUOperand::ImmTySwizzle :
                                                    AMDGPUOperand::ImmTyOffset;

addOptionalImmOperand(Inst, Operands, OptionalIdx, OffsetType);

if (!IsGdsHardcoded) {
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
}
Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
5922}

5924void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
OptionalImmIndexMap OptionalIdx;

unsigned OperandIdx[4];
unsigned EnMask = 0;
int SrcIdx = 0;

for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1
Assuming 'i' is not equal to 'e'→
2
←
Loop condition is true.  Entering loop body→
6
←
Assuming 'i' is not equal to 'e'→
7
←
Loop condition is true.  Entering loop body→
11
←
Assuming 'i' is equal to 'e'→
12
←
Loop condition is false. Execution continues on line 5963→
  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);

  // Add the register arguments
  if (Op.isReg()) {
3
←
Assuming the condition is true→
4
←
Taking true branch→
8
←
Assuming the condition is true→
9
←
Taking true branch→
    assert(SrcIdx < 4)(static_cast<void> (0));
    OperandIdx[SrcIdx] = Inst.size();
    Op.addRegOperands(Inst, 1);
    ++SrcIdx;
    continue;
5
←
 Execution continues on line 5931→
10
←
 Execution continues on line 5931→
  }

  if (Op.isOff()) {
    assert(SrcIdx < 4)(static_cast<void> (0));
    OperandIdx[SrcIdx] = Inst.size();
    Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
    ++SrcIdx;
    continue;
  }

  if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
    Op.addImmOperands(Inst, 1);
    continue;
  }

  if (Op.isToken() && Op.getToken() == "done")
    continue;

  // Handle optional arguments
  OptionalIdx[Op.getImmTy()] = i;
}

assert(SrcIdx == 4)(static_cast<void> (0));

bool Compr = false;
if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
13
←
Calling 'operator!='→
16
←
Returning from 'operator!='→
17
←
Taking true branch→
  Compr = true;
  Inst.getOperand(OperandIdx[1]) = Inst.getOperand(OperandIdx[2]);
18
←
1st function call argument is an uninitialized value
  Inst.getOperand(OperandIdx[2]).setReg(AMDGPU::NoRegister);
  Inst.getOperand(OperandIdx[3]).setReg(AMDGPU::NoRegister);
}

for (auto i = 0; i < SrcIdx; ++i) {
  if (Inst.getOperand(OperandIdx[i]).getReg() != AMDGPU::NoRegister) {
    EnMask |= Compr? (0x3 << i * 2) : (0x1 << i);
  }
}

addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);

Inst.addOperand(MCOperand::createImm(EnMask));
5983}

5985//===----------------------------------------------------------------------===//
5986// s_waitcnt
5987//===----------------------------------------------------------------------===//

5989static bool
5990encodeCnt(
const AMDGPU::IsaVersion ISA,
int64_t &IntVal,
int64_t CntVal,
bool Saturate,
unsigned (*encode)(const IsaVersion &Version, unsigned, unsigned),
unsigned (*decode)(const IsaVersion &Version, unsigned))
5997{
bool Failed = false;

IntVal = encode(ISA, IntVal, CntVal);
if (CntVal != decode(ISA, IntVal)) {
  if (Saturate) {
    IntVal = encode(ISA, IntVal, -1);
  } else {
    Failed = true;
  }
}
return Failed;
6009}

6011bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {

SMLoc CntLoc = getLoc();
StringRef CntName = getTokenStr();

if (!skipToken(AsmToken::Identifier, "expected a counter name") ||
    !skipToken(AsmToken::LParen, "expected a left parenthesis"))
  return false;

int64_t CntVal;
SMLoc ValLoc = getLoc();
if (!parseExpr(CntVal))
  return false;

AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(getSTI().getCPU());

bool Failed = true;
bool Sat = CntName.endswith("_sat");

if (CntName == "vmcnt" || CntName == "vmcnt_sat") {
  Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeVmcnt, decodeVmcnt);
} else if (CntName == "expcnt" || CntName == "expcnt_sat") {
  Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeExpcnt, decodeExpcnt);
} else if (CntName == "lgkmcnt" || CntName == "lgkmcnt_sat") {
  Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeLgkmcnt, decodeLgkmcnt);
} else {
  Error(CntLoc, "invalid counter name " + CntName);
  return false;
}

if (Failed) {
  Error(ValLoc, "too large value for " + CntName);
  return false;
}

if (!skipToken(AsmToken::RParen, "expected a closing parenthesis"))
  return false;

if (trySkipToken(AsmToken::Amp) || trySkipToken(AsmToken::Comma)) {
  if (isToken(AsmToken::EndOfStatement)) {
    Error(getLoc(), "expected a counter name");
    return false;
  }
}

return true;
6057}

6059OperandMatchResultTy
6060AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(getSTI().getCPU());
int64_t Waitcnt = getWaitcntBitMask(ISA);
SMLoc S = getLoc();

if (isToken(AsmToken::Identifier) && peekToken().is(AsmToken::LParen)) {
  while (!isToken(AsmToken::EndOfStatement)) {
    if (!parseCnt(Waitcnt))
      return MatchOperand_ParseFail;
  }
} else {
  if (!parseExpr(Waitcnt))
    return MatchOperand_ParseFail;
}

Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
return MatchOperand_Success;
6077}

6079bool
6080AMDGPUOperand::isSWaitCnt() const {
return isImm();
6082}

6084//===----------------------------------------------------------------------===//
6085// hwreg
6086//===----------------------------------------------------------------------===//

6088bool
6089AMDGPUAsmParser::parseHwregBody(OperandInfoTy &HwReg,
                              OperandInfoTy &Offset,
                              OperandInfoTy &Width) {
using namespace llvm::AMDGPU::Hwreg;

// The register may be specified by name or using a numeric code
HwReg.Loc = getLoc();
if (isToken(AsmToken::Identifier) &&
    (HwReg.Id = getHwregId(getTokenStr())) >= 0) {
  HwReg.IsSymbolic = true;
  lex(); // skip register name
} else if (!parseExpr(HwReg.Id, "a register name")) {
  return false;
}

if (trySkipToken(AsmToken::RParen))
  return true;

// parse optional params
if (!skipToken(AsmToken::Comma, "expected a comma or a closing parenthesis"))
  return false;

Offset.Loc = getLoc();
if (!parseExpr(Offset.Id))
  return false;

if (!skipToken(AsmToken::Comma, "expected a comma"))
  return false;

Width.Loc = getLoc();
return parseExpr(Width.Id) &&
       skipToken(AsmToken::RParen, "expected a closing parenthesis");
6121}

6123bool
6124AMDGPUAsmParser::validateHwreg(const OperandInfoTy &HwReg,
                             const OperandInfoTy &Offset,
                             const OperandInfoTy &Width) {

using namespace llvm::AMDGPU::Hwreg;

if (HwReg.IsSymbolic && !isValidHwreg(HwReg.Id, getSTI())) {
  Error(HwReg.Loc,
        "specified hardware register is not supported on this GPU");
  return false;
}
if (!isValidHwreg(HwReg.Id)) {
  Error(HwReg.Loc,
        "invalid code of hardware register: only 6-bit values are legal");
  return false;
}
if (!isValidHwregOffset(Offset.Id)) {
  Error(Offset.Loc, "invalid bit offset: only 5-bit values are legal");
  return false;
}
if (!isValidHwregWidth(Width.Id)) {
  Error(Width.Loc,
        "invalid bitfield width: only values from 1 to 32 are legal");
  return false;
}
return true;
6150}

6152OperandMatchResultTy
6153AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
using namespace llvm::AMDGPU::Hwreg;

int64_t ImmVal = 0;
SMLoc Loc = getLoc();

if (trySkipId("hwreg", AsmToken::LParen)) {
  OperandInfoTy HwReg(ID_UNKNOWN_);
  OperandInfoTy Offset(OFFSET_DEFAULT_);
  OperandInfoTy Width(WIDTH_DEFAULT_);
  if (parseHwregBody(HwReg, Offset, Width) &&
      validateHwreg(HwReg, Offset, Width)) {
    ImmVal = encodeHwreg(HwReg.Id, Offset.Id, Width.Id);
  } else {
    return MatchOperand_ParseFail;
  }
} else if (parseExpr(ImmVal, "a hwreg macro")) {
  if (ImmVal < 0 || !isUInt<16>(ImmVal)) {
    Error(Loc, "invalid immediate: only 16-bit values are legal");
    return MatchOperand_ParseFail;
  }
} else {
  return MatchOperand_ParseFail;
}

Operands.push_back(AMDGPUOperand::CreateImm(this, ImmVal, Loc, AMDGPUOperand::ImmTyHwreg));
return MatchOperand_Success;
6180}

6182bool AMDGPUOperand::isHwreg() const {
return isImmTy(ImmTyHwreg);
6184}

6186//===----------------------------------------------------------------------===//
6187// sendmsg
6188//===----------------------------------------------------------------------===//

6190bool
6191AMDGPUAsmParser::parseSendMsgBody(OperandInfoTy &Msg,
                                OperandInfoTy &Op,
                                OperandInfoTy &Stream) {
using namespace llvm::AMDGPU::SendMsg;

Msg.Loc = getLoc();
if (isToken(AsmToken::Identifier) && (Msg.Id = getMsgId(getTokenStr())) >= 0) {
  Msg.IsSymbolic = true;
  lex(); // skip message name
} else if (!parseExpr(Msg.Id, "a message name")) {
  return false;
}

if (trySkipToken(AsmToken::Comma)) {
  Op.IsDefined = true;
  Op.Loc = getLoc();
  if (isToken(AsmToken::Identifier) &&
      (Op.Id = getMsgOpId(Msg.Id, getTokenStr())) >= 0) {
    lex(); // skip operation name
  } else if (!parseExpr(Op.Id, "an operation name")) {
    return false;
  }

  if (trySkipToken(AsmToken::Comma)) {
    Stream.IsDefined = true;
    Stream.Loc = getLoc();
    if (!parseExpr(Stream.Id))
      return false;
  }
}

return skipToken(AsmToken::RParen, "expected a closing parenthesis");
6223}

6225bool
6226AMDGPUAsmParser::validateSendMsg(const OperandInfoTy &Msg,
                               const OperandInfoTy &Op,
                               const OperandInfoTy &Stream) {
using namespace llvm::AMDGPU::SendMsg;

// Validation strictness depends on whether message is specified
// in a symbolc or in a numeric form. In the latter case
// only encoding possibility is checked.
bool Strict = Msg.IsSymbolic;

if (!isValidMsgId(Msg.Id, getSTI(), Strict)) {
  Error(Msg.Loc, "invalid message id");
  return false;
}
if (Strict && (msgRequiresOp(Msg.Id) != Op.IsDefined)) {
  if (Op.IsDefined) {
    Error(Op.Loc, "message does not support operations");
  } else {
    Error(Msg.Loc, "missing message operation");
  }
  return false;
}
if (!isValidMsgOp(Msg.Id, Op.Id, getSTI(), Strict)) {
  Error(Op.Loc, "invalid operation id");
  return false;
}
if (Strict && !msgSupportsStream(Msg.Id, Op.Id) && Stream.IsDefined) {
  Error(Stream.Loc, "message operation does not support streams");
  return false;
}
if (!isValidMsgStream(Msg.Id, Op.Id, Stream.Id, getSTI(), Strict)) {
  Error(Stream.Loc, "invalid message stream id");
  return false;
}
return true;
6261}

6263OperandMatchResultTy
6264AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
using namespace llvm::AMDGPU::SendMsg;

int64_t ImmVal = 0;
SMLoc Loc = getLoc();

if (trySkipId("sendmsg", AsmToken::LParen)) {
  OperandInfoTy Msg(ID_UNKNOWN_);
  OperandInfoTy Op(OP_NONE_);
  OperandInfoTy Stream(STREAM_ID_NONE_);
  if (parseSendMsgBody(Msg, Op, Stream) &&
      validateSendMsg(Msg, Op, Stream)) {
    ImmVal = encodeMsg(Msg.Id, Op.Id, Stream.Id);
  } else {
    return MatchOperand_ParseFail;
  }
} else if (parseExpr(ImmVal, "a sendmsg macro")) {
  if (ImmVal < 0 || !isUInt<16>(ImmVal)) {
    Error(Loc, "invalid immediate: only 16-bit values are legal");
    return MatchOperand_ParseFail;
  }
} else {
  return MatchOperand_ParseFail;
}

Operands.push_back(AMDGPUOperand::CreateImm(this, ImmVal, Loc, AMDGPUOperand::ImmTySendMsg));
return MatchOperand_Success;
6291}

6293bool AMDGPUOperand::isSendMsg() const {
return isImmTy(ImmTySendMsg);
6295}

6297//===----------------------------------------------------------------------===//
6298// v_interp
6299//===----------------------------------------------------------------------===//

6301OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
StringRef Str;
SMLoc S = getLoc();

if (!parseId(Str))
  return MatchOperand_NoMatch;

int Slot = StringSwitch<int>(Str)
  .Case("p10", 0)
  .Case("p20", 1)
  .Case("p0", 2)
  .Default(-1);

if (Slot == -1) {
  Error(S, "invalid interpolation slot");
  return MatchOperand_ParseFail;
}

Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
                                            AMDGPUOperand::ImmTyInterpSlot));
return MatchOperand_Success;
6322}

6324OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
StringRef Str;
SMLoc S = getLoc();

if (!parseId(Str))
  return MatchOperand_NoMatch;

if (!Str.startswith("attr")) {
  Error(S, "invalid interpolation attribute");
  return MatchOperand_ParseFail;
}

StringRef Chan = Str.take_back(2);
int AttrChan = StringSwitch<int>(Chan)
  .Case(".x", 0)
  .Case(".y", 1)
  .Case(".z", 2)
  .Case(".w", 3)
  .Default(-1);
if (AttrChan == -1) {
  Error(S, "invalid or missing interpolation attribute channel");
  return MatchOperand_ParseFail;
}

Str = Str.drop_back(2).drop_front(4);

uint8_t Attr;
if (Str.getAsInteger(10, Attr)) {
  Error(S, "invalid or missing interpolation attribute number");
  return MatchOperand_ParseFail;
}

if (Attr > 63) {
  Error(S, "out of bounds interpolation attribute number");
  return MatchOperand_ParseFail;
}

SMLoc SChan = SMLoc::getFromPointer(Chan.data());

Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
                                            AMDGPUOperand::ImmTyInterpAttr));
Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
                                            AMDGPUOperand::ImmTyAttrChan));
return MatchOperand_Success;
6368}

6370//===----------------------------------------------------------------------===//
6371// exp
6372//===----------------------------------------------------------------------===//

6374OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
using namespace llvm::AMDGPU::Exp;

StringRef Str;
SMLoc S = getLoc();

if (!parseId(Str))
  return MatchOperand_NoMatch;

unsigned Id = getTgtId(Str);
if (Id == ET_INVALID || !isSupportedTgtId(Id, getSTI())) {
  Error(S, (Id == ET_INVALID) ?
              "invalid exp target" :
              "exp target is not supported on this GPU");
  return MatchOperand_ParseFail;
}

Operands.push_back(AMDGPUOperand::CreateImm(this, Id, S,
                                            AMDGPUOperand::ImmTyExpTgt));
return MatchOperand_Success;
6394}

6396//===----------------------------------------------------------------------===//
6397// parser helpers
6398//===----------------------------------------------------------------------===//

6400bool
6401AMDGPUAsmParser::isId(const AsmToken &Token, const StringRef Id) const {
return Token.is(AsmToken::Identifier) && Token.getString() == Id;
6403}

6405bool
6406AMDGPUAsmParser::isId(const StringRef Id) const {
return isId(getToken(), Id);
6408}

6410bool
6411AMDGPUAsmParser::isToken(const AsmToken::TokenKind Kind) const {
return getTokenKind() == Kind;
6413}

6415bool
6416AMDGPUAsmParser::trySkipId(const StringRef Id) {
if (isId(Id)) {
  lex();
  return true;
}
return false;
6422}

6424bool
6425AMDGPUAsmParser::trySkipId(const StringRef Pref, const StringRef Id) {
if (isToken(AsmToken::Identifier)) {
  StringRef Tok = getTokenStr();
  if (Tok.startswith(Pref) && Tok.drop_front(Pref.size()) == Id) {
    lex();
    return true;
  }
}
return false;
6434}

6436bool
6437AMDGPUAsmParser::trySkipId(const StringRef Id, const AsmToken::TokenKind Kind) {
if (isId(Id) && peekToken().is(Kind)) {
  lex();
  lex();
  return true;
}
return false;
6444}

6446bool
6447AMDGPUAsmParser::trySkipToken(const AsmToken::TokenKind Kind) {
if (isToken(Kind)) {
  lex();
  return true;
}
return false;
6453}

6455bool
6456AMDGPUAsmParser::skipToken(const AsmToken::TokenKind Kind,
                         const StringRef ErrMsg) {
if (!trySkipToken(Kind)) {
  Error(getLoc(), ErrMsg);
  return false;
}
return true;
6463}

6465bool
6466AMDGPUAsmParser::parseExpr(int64_t &Imm, StringRef Expected) {
SMLoc S = getLoc();

const MCExpr *Expr;
if (Parser.parseExpression(Expr))
  return false;

if (Expr->evaluateAsAbsolute(Imm))
  return true;

if (Expected.empty()) {
  Error(S, "expected absolute expression");
} else {
  Error(S, Twine("expected ", Expected) +
           Twine(" or an absolute expression"));
}
return false;
6483}

6485bool
6486AMDGPUAsmParser::parseExpr(OperandVector &Operands) {
SMLoc S = getLoc();

const MCExpr *Expr;
if (Parser.parseExpression(Expr))
  return false;

int64_t IntVal;
if (Expr->evaluateAsAbsolute(IntVal)) {
  Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
} else {
  Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
}
return true;
6500}

6502bool
6503AMDGPUAsmParser::parseString(StringRef &Val, const StringRef ErrMsg) {
if (isToken(AsmToken::String)) {
  Val = getToken().getStringContents();
  lex();
  return true;
} else {
  Error(getLoc(), ErrMsg);
  return false;
}
6512}

6514bool
6515AMDGPUAsmParser::parseId(StringRef &Val, const StringRef ErrMsg) {
if (isToken(AsmToken::Identifier)) {
  Val = getTokenStr();
  lex();
  return true;
} else {
  if (!ErrMsg.empty())
    Error(getLoc(), ErrMsg);
  return false;
}
6525}

6527AsmToken
6528AMDGPUAsmParser::getToken() const {
return Parser.getTok();
6530}

6532AsmToken
6533AMDGPUAsmParser::peekToken() {
return isToken(AsmToken::EndOfStatement) ? getToken() : getLexer().peekTok();
6535}

6537void
6538AMDGPUAsmParser::peekTokens(MutableArrayRef<AsmToken> Tokens) {
auto TokCount = getLexer().peekTokens(Tokens);

for (auto Idx = TokCount; Idx < Tokens.size(); ++Idx)
  Tokens[Idx] = AsmToken(AsmToken::Error, "");
6543}

6545AsmToken::TokenKind
6546AMDGPUAsmParser::getTokenKind() const {
return getLexer().getKind();
6548}

6550SMLoc
6551AMDGPUAsmParser::getLoc() const {
return getToken().getLoc();
6553}

6555StringRef
6556AMDGPUAsmParser::getTokenStr() const {
return getToken().getString();
6558}

6560void
6561AMDGPUAsmParser::lex() {
Parser.Lex();
6563}

6565SMLoc
6566AMDGPUAsmParser::getOperandLoc(std::function<bool(const AMDGPUOperand&)> Test,
                             const OperandVector &Operands) const {
for (unsigned i = Operands.size() - 1; i > 0; --i) {
  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
  if (Test(Op))
    return Op.getStartLoc();
}
return ((AMDGPUOperand &)*Operands[0]).getStartLoc();
6574}

6576SMLoc
6577AMDGPUAsmParser::getImmLoc(AMDGPUOperand::ImmTy Type,
                         const OperandVector &Operands) const {
auto Test = [=](const AMDGPUOperand& Op) { return Op.isImmTy(Type); };
return getOperandLoc(Test, Operands);
6581}

6583SMLoc
6584AMDGPUAsmParser::getRegLoc(unsigned Reg,
                         const OperandVector &Operands) const {
auto Test = [=](const AMDGPUOperand& Op) {
  return Op.isRegKind() && Op.getReg() == Reg;
};
return getOperandLoc(Test, Operands);
6590}

6592SMLoc
6593AMDGPUAsmParser::getLitLoc(const OperandVector &Operands) const {
auto Test = [](const AMDGPUOperand& Op) {
  return Op.IsImmKindLiteral() || Op.isExpr();
};
return getOperandLoc(Test, Operands);
6598}

6600SMLoc
6601AMDGPUAsmParser::getConstLoc(const OperandVector &Operands) const {
auto Test = [](const AMDGPUOperand& Op) {
  return Op.isImmKindConst();
};
return getOperandLoc(Test, Operands);
6606}

6608//===----------------------------------------------------------------------===//
6609// swizzle
6610//===----------------------------------------------------------------------===//

6612LLVM_READNONE__attribute__((__const__))
6613static unsigned
6614encodeBitmaskPerm(const unsigned AndMask,
                const unsigned OrMask,
                const unsigned XorMask) {
using namespace llvm::AMDGPU::Swizzle;

return BITMASK_PERM_ENC |
       (AndMask << BITMASK_AND_SHIFT) |
       (OrMask  << BITMASK_OR_SHIFT)  |
       (XorMask << BITMASK_XOR_SHIFT);
6623}

6625bool
6626AMDGPUAsmParser::parseSwizzleOperand(int64_t &Op,
                                   const unsigned MinVal,
                                   const unsigned MaxVal,
                                   const StringRef ErrMsg,
                                   SMLoc &Loc) {
if (!skipToken(AsmToken::Comma, "expected a comma")) {
  return false;
}
Loc = getLoc();
if (!parseExpr(Op)) {
  return false;
}
if (Op < MinVal || Op > MaxVal) {
  Error(Loc, ErrMsg);
  return false;
}

return true;
6644}

6646bool
6647AMDGPUAsmParser::parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
                                    const unsigned MinVal,
                                    const unsigned MaxVal,
                                    const StringRef ErrMsg) {
SMLoc Loc;
for (unsigned i = 0; i < OpNum; ++i) {
  if (!parseSwizzleOperand(Op[i], MinVal, MaxVal, ErrMsg, Loc))
    return false;
}

return true;
6658}

6660bool
6661AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &Imm) {
using namespace llvm::AMDGPU::Swizzle;

int64_t Lane[LANE_NUM];
if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
                         "expected a 2-bit lane id")) {
  Imm = QUAD_PERM_ENC;
  for (unsigned I = 0; I < LANE_NUM; ++I) {
    Imm |= Lane[I] << (LANE_SHIFT * I);
  }
  return true;
}
return false;
6674}

6676bool
6677AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &Imm) {
using namespace llvm::AMDGPU::Swizzle;

SMLoc Loc;
int64_t GroupSize;
int64_t LaneIdx;

if (!parseSwizzleOperand(GroupSize,
                         2, 32,
                         "group size must be in the interval [2,32]",
                         Loc)) {
  return false;
}
if (!isPowerOf2_64(GroupSize)) {
  Error(Loc, "group size must be a power of two");
  return false;
}
if (parseSwizzleOperand(LaneIdx,
                        0, GroupSize - 1,
                        "lane id must be in the interval [0,group size - 1]",
                        Loc)) {
  Imm = encodeBitmaskPerm(BITMASK_MAX - GroupSize + 1, LaneIdx, 0);
  return true;
}
return false;
6702}

6704bool
6705AMDGPUAsmParser::parseSwizzleReverse(int64_t &Imm) {
using namespace llvm::AMDGPU::Swizzle;

SMLoc Loc;
int64_t GroupSize;

if (!parseSwizzleOperand(GroupSize,
                         2, 32,
                         "group size must be in the interval [2,32]",
                         Loc)) {
  return false;
}
if (!isPowerOf2_64(GroupSize)) {
  Error(Loc, "group size must be a power of two");
  return false;
}

Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize - 1);
return true;
6724}

6726bool
6727AMDGPUAsmParser::parseSwizzleSwap(int64_t &Imm) {
using namespace llvm::AMDGPU::Swizzle;

SMLoc Loc;
int64_t GroupSize;

if (!parseSwizzleOperand(GroupSize,
                         1, 16,
                         "group size must be in the interval [1,16]",
                         Loc)) {
  return false;
}
if (!isPowerOf2_64(GroupSize)) {
  Error(Loc, "group size must be a power of two");
  return false;
}

Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize);
return true;
6746}

6748bool
6749AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &Imm) {
using namespace llvm::AMDGPU::Swizzle;

if (!skipToken(AsmToken::Comma, "expected a comma")) {
  return false;
}

StringRef Ctl;
SMLoc StrLoc = getLoc();
if (!parseString(Ctl)) {
  return false;
}
if (Ctl.size() != BITMASK_WIDTH) {
  Error(StrLoc, "expected a 5-character mask");
  return false;
}

unsigned AndMask = 0;
unsigned OrMask = 0;
unsigned XorMask = 0;

for (size_t i = 0; i < Ctl.size(); ++i) {
  unsigned Mask = 1 << (BITMASK_WIDTH - 1 - i);
  switch(Ctl[i]) {
  default:
    Error(StrLoc, "invalid mask");
    return false;
  case '0':
    break;
  case '1':
    OrMask |= Mask;
    break;
  case 'p':
    AndMask |= Mask;
    break;
  case 'i':
    AndMask |= Mask;
    XorMask |= Mask;
    break;
  }
}

Imm = encodeBitmaskPerm(AndMask, OrMask, XorMask);
return true;
6793}

6795bool
6796AMDGPUAsmParser::parseSwizzleOffset(int64_t &Imm) {

SMLoc OffsetLoc = getLoc();

if (!parseExpr(Imm, "a swizzle macro")) {
  return false;
}
if (!isUInt<16>(Imm)) {
  Error(OffsetLoc, "expected a 16-bit offset");
  return false;
}
return true;
6808}

6810bool
6811AMDGPUAsmParser::parseSwizzleMacro(int64_t &Imm) {
using namespace llvm::AMDGPU::Swizzle;

if (skipToken(AsmToken::LParen, "expected a left parentheses")) {

  SMLoc ModeLoc = getLoc();
  bool Ok = false;

  if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
    Ok = parseSwizzleQuadPerm(Imm);
  } else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
    Ok = parseSwizzleBitmaskPerm(Imm);
  } else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
    Ok = parseSwizzleBroadcast(Imm);
  } else if (trySkipId(IdSymbolic[ID_SWAP])) {
    Ok = parseSwizzleSwap(Imm);
  } else if (trySkipId(IdSymbolic[ID_REVERSE])) {
    Ok = parseSwizzleReverse(Imm);
  } else {
    Error(ModeLoc, "expected a swizzle mode");
  }

  return Ok && skipToken(AsmToken::RParen, "expected a closing parentheses");
}

return false;
6837}

6839OperandMatchResultTy
6840AMDGPUAsmParser::parseSwizzleOp(OperandVector &Operands) {
SMLoc S = getLoc();
int64_t Imm = 0;

if (trySkipId("offset")) {

  bool Ok = false;
  if (skipToken(AsmToken::Colon, "expected a colon")) {
    if (trySkipId("swizzle")) {
      Ok = parseSwizzleMacro(Imm);
    } else {
      Ok = parseSwizzleOffset(Imm);
    }
  }

  Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S, AMDGPUOperand::ImmTySwizzle));

  return Ok? MatchOperand_Success : MatchOperand_ParseFail;
} else {
  // Swizzle "offset" operand is optional.
  // If it is omitted, try parsing other optional operands.
  return parseOptionalOpr(Operands);
}
6863}

6865bool
6866AMDGPUOperand::isSwizzle() const {
return isImmTy(ImmTySwizzle);
6868}

6870//===----------------------------------------------------------------------===//
6871// VGPR Index Mode
6872//===----------------------------------------------------------------------===//

6874int64_t AMDGPUAsmParser::parseGPRIdxMacro() {

using namespace llvm::AMDGPU::VGPRIndexMode;

if (trySkipToken(AsmToken::RParen)) {
  return OFF;
}

int64_t Imm = 0;

while (true) {
  unsigned Mode = 0;
  SMLoc S = getLoc();

  for (unsigned ModeId = ID_MIN; ModeId <= ID_MAX; ++ModeId) {
    if (trySkipId(IdSymbolic[ModeId])) {
      Mode = 1 << ModeId;
      break;
    }
  }

  if (Mode == 0) {
    Error(S, (Imm == 0)?
             "expected a VGPR index mode or a closing parenthesis" :
             "expected a VGPR index mode");
    return UNDEF;
  }

  if (Imm & Mode) {
    Error(S, "duplicate VGPR index mode");
    return UNDEF;
  }
  Imm |= Mode;

  if (trySkipToken(AsmToken::RParen))
    break;
  if (!skipToken(AsmToken::Comma,
                 "expected a comma or a closing parenthesis"))
    return UNDEF;
}

return Imm;
6916}

6918OperandMatchResultTy
6919AMDGPUAsmParser::parseGPRIdxMode(OperandVector &Operands) {

using namespace llvm::AMDGPU::VGPRIndexMode;

int64_t Imm = 0;
SMLoc S = getLoc();

if (trySkipId("gpr_idx", AsmToken::LParen)) {
  Imm = parseGPRIdxMacro();
  if (Imm == UNDEF)
    return MatchOperand_ParseFail;
} else {
  if (getParser().parseAbsoluteExpression(Imm))
    return MatchOperand_ParseFail;
  if (Imm < 0 || !isUInt<4>(Imm)) {
    Error(S, "invalid immediate: only 4-bit values are legal");
    return MatchOperand_ParseFail;
  }
}

Operands.push_back(
    AMDGPUOperand::CreateImm(this, Imm, S, AMDGPUOperand::ImmTyGprIdxMode));
return MatchOperand_Success;
6942}

6944bool AMDGPUOperand::isGPRIdxMode() const {
return isImmTy(ImmTyGprIdxMode);
6946}

6948//===----------------------------------------------------------------------===//
6949// sopp branch targets
6950//===----------------------------------------------------------------------===//

6952OperandMatchResultTy
6953AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {

// Make sure we are not parsing something
// that looks like a label or an expression but is not.
// This will improve error messages.
if (isRegister() || isModifier())
  return MatchOperand_NoMatch;

if (!parseExpr(Operands))
  return MatchOperand_ParseFail;

AMDGPUOperand &Opr = ((AMDGPUOperand &)*Operands[Operands.size() - 1]);
assert(Opr.isImm() || Opr.isExpr())(static_cast<void> (0));
SMLoc Loc = Opr.getStartLoc();

// Currently we do not support arbitrary expressions as branch targets.
// Only labels and absolute expressions are accepted.
if (Opr.isExpr() && !Opr.isSymbolRefExpr()) {
  Error(Loc, "expected an absolute expression or a label");
} else if (Opr.isImm() && !Opr.isS16Imm()) {
  Error(Loc, "expected a 16-bit signed jump offset");
}

return MatchOperand_Success;
6977}

6979//===----------------------------------------------------------------------===//
6980// Boolean holding registers
6981//===----------------------------------------------------------------------===//

6983OperandMatchResultTy
6984AMDGPUAsmParser::parseBoolReg(OperandVector &Operands) {
return parseReg(Operands);
6986}

6988//===----------------------------------------------------------------------===//
6989// mubuf
6990//===----------------------------------------------------------------------===//

6992AMDGPUOperand::Ptr AMDGPUAsmParser::defaultCPol() const {
return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyCPol);
6994}

6996void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
                                 const OperandVector &Operands,
                                 bool IsAtomic,
                                 bool IsLds) {
bool IsLdsOpcode = IsLds;
bool HasLdsModifier = false;
OptionalImmIndexMap OptionalIdx;
unsigned FirstOperandIdx = 1;
bool IsAtomicReturn = false;

if (IsAtomic) {
  for (unsigned i = FirstOperandIdx, e = Operands.size(); i != e; ++i) {
    AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
    if (!Op.isCPol())
      continue;
    IsAtomicReturn = Op.getImm() & AMDGPU::CPol::GLC;
    break;
  }

  if (!IsAtomicReturn) {
    int NewOpc = AMDGPU::getAtomicNoRetOp(Inst.getOpcode());
    if (NewOpc != -1)
      Inst.setOpcode(NewOpc);
  }

  IsAtomicReturn =  MII.get(Inst.getOpcode()).TSFlags &
                    SIInstrFlags::IsAtomicRet;
}

for (unsigned i = FirstOperandIdx, e = Operands.size(); i != e; ++i) {
  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);

  // Add the register arguments
  if (Op.isReg()) {
    Op.addRegOperands(Inst, 1);
    // Insert a tied src for atomic return dst.
    // This cannot be postponed as subsequent calls to
    // addImmOperands rely on correct number of MC operands.
    if (IsAtomicReturn && i == FirstOperandIdx)
      Op.addRegOperands(Inst, 1);
    continue;
  }

  // Handle the case where soffset is an immediate
  if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
    Op.addImmOperands(Inst, 1);
    continue;
  }

  HasLdsModifier |= Op.isLDS();

  // Handle tokens like 'offen' which are sometimes hard-coded into the
  // asm string.  There are no MCInst operands for these.
  if (Op.isToken()) {
    continue;
  }
  assert(Op.isImm())(static_cast<void> (0));

  // Handle optional arguments
  OptionalIdx[Op.getImmTy()] = i;
}

// This is a workaround for an llvm quirk which may result in an
// incorrect instruction selection. Lds and non-lds versions of
// MUBUF instructions are identical except that lds versions
// have mandatory 'lds' modifier. However this modifier follows
// optional modifiers and llvm asm matcher regards this 'lds'
// modifier as an optional one. As a result, an lds version
// of opcode may be selected even if it has no 'lds' modifier.
if (IsLdsOpcode && !HasLdsModifier) {
  int NoLdsOpcode = AMDGPU::getMUBUFNoLdsInst(Inst.getOpcode());
  if (NoLdsOpcode != -1) { // Got lds version - correct it.
    Inst.setOpcode(NoLdsOpcode);
    IsLdsOpcode = false;
  }
}

addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyCPol, 0);

if (!IsLdsOpcode) { // tfe is not legal with lds opcodes
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
}
addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySWZ);
7080}

7082void AMDGPUAsmParser::cvtMtbuf(MCInst &Inst, const OperandVector &Operands) {
OptionalImmIndexMap OptionalIdx;

for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);

  // Add the register arguments
  if (Op.isReg()) {
    Op.addRegOperands(Inst, 1);
    continue;
  }

  // Handle the case where soffset is an immediate
  if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
    Op.addImmOperands(Inst, 1);
    continue;
  }

  // Handle tokens like 'offen' which are sometimes hard-coded into the
  // asm string.  There are no MCInst operands for these.
  if (Op.isToken()) {
    continue;
  }
  assert(Op.isImm())(static_cast<void> (0));

  // Handle optional arguments
  OptionalIdx[Op.getImmTy()] = i;
}

addOptionalImmOperand(Inst, Operands, OptionalIdx,
                      AMDGPUOperand::ImmTyOffset);
addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyFORMAT);
addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyCPol, 0);
addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySWZ);
7117}

7119//===----------------------------------------------------------------------===//
7120// mimg
7121//===----------------------------------------------------------------------===//

7123void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands,
                            bool IsAtomic) {
unsigned I = 1;
const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
  ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
}

if (IsAtomic) {
  // Add src, same as dst
  assert(Desc.getNumDefs() == 1)(static_cast<void> (0));
  ((AMDGPUOperand &)*Operands[I - 1]).addRegOperands(Inst, 1);
}

OptionalImmIndexMap OptionalIdx;

for (unsigned E = Operands.size(); I != E; ++I) {
  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);

  // Add the register arguments
  if (Op.isReg()) {
    Op.addRegOperands(Inst, 1);
  } else if (Op.isImmModifier()) {
    OptionalIdx[Op.getImmTy()] = I;
  } else if (!Op.isToken()) {
    llvm_unreachable("unexpected operand type")__builtin_unreachable();
  }
}

bool IsGFX10Plus = isGFX10Plus();

addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
if (IsGFX10Plus)
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDim, -1);
addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyCPol);
addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128A16);
if (IsGFX10Plus)
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyA16);
if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::tfe) != -1)
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
if (!IsGFX10Plus)
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyD16);
7168}

7170void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
cvtMIMG(Inst, Operands, true);
7172}

7174void AMDGPUAsmParser::cvtSMEMAtomic(MCInst &Inst, const OperandVector &Operands) {
OptionalImmIndexMap OptionalIdx;
bool IsAtomicReturn = false;

for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
  if (!Op.isCPol())
    continue;
  IsAtomicReturn = Op.getImm() & AMDGPU::CPol::GLC;
  break;
}

if (!IsAtomicReturn) {
  int NewOpc = AMDGPU::getAtomicNoRetOp(Inst.getOpcode());
  if (NewOpc != -1)
    Inst.setOpcode(NewOpc);
}

IsAtomicReturn =  MII.get(Inst.getOpcode()).TSFlags &
                  SIInstrFlags::IsAtomicRet;

for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);

  // Add the register arguments
  if (Op.isReg()) {
    Op.addRegOperands(Inst, 1);
    if (IsAtomicReturn && i == 1)
      Op.addRegOperands(Inst, 1);
    continue;
  }

  // Handle the case where soffset is an immediate
  if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
    Op.addImmOperands(Inst, 1);
    continue;
  }

  // Handle tokens like 'offen' which are sometimes hard-coded into the
  // asm string.  There are no MCInst operands for these.
  if (Op.isToken()) {
    continue;
  }
  assert(Op.isImm())(static_cast<void> (0));

  // Handle optional arguments
  OptionalIdx[Op.getImmTy()] = i;
}

if ((int)Inst.getNumOperands() <=
    AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::offset))
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyCPol, 0);
7227}

7229void AMDGPUAsmParser::cvtIntersectRay(MCInst &Inst,
                                    const OperandVector &Operands) {
for (unsigned I = 1; I < Operands.size(); ++I) {
  auto &Operand = (AMDGPUOperand &)*Operands[I];
  if (Operand.isReg())
    Operand.addRegOperands(Inst, 1);
}

Inst.addOperand(MCOperand::createImm(1)); // a16
7238}

7240//===----------------------------------------------------------------------===//
7241// smrd
7242//===----------------------------------------------------------------------===//

7244bool AMDGPUOperand::isSMRDOffset8() const {
return isImm() && isUInt<8>(getImm());
7246}

7248bool AMDGPUOperand::isSMEMOffset() const {
return isImm(); // Offset range is checked later by validator.
7250}

7252bool AMDGPUOperand::isSMRDLiteralOffset() const {
// 32-bit literals are only supported on CI and we only want to use them
// when the offset is > 8-bits.
return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
7256}

7258AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
7260}

7262AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMEMOffset() const {
return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
7264}

7266AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
7268}

7270AMDGPUOperand::Ptr AMDGPUAsmParser::defaultFlatOffset() const {
return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
7272}

7274//===----------------------------------------------------------------------===//
7275// vop3
7276//===----------------------------------------------------------------------===//

7278static bool ConvertOmodMul(int64_t &Mul) {
if (Mul != 1 && Mul != 2 && Mul != 4)
  return false;

Mul >>= 1;
return true;
7284}

7286static bool ConvertOmodDiv(int64_t &Div) {
if (Div == 1) {
  Div = 0;
  return true;
}

if (Div == 2) {
  Div = 3;
  return true;
}

return false;
7298}

7300// Both bound_ctrl:0 and bound_ctrl:1 are encoded as 1.
7301// This is intentional and ensures compatibility with sp3.
7302// See bug 35397 for details.
7303static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
if (BoundCtrl == 0 || BoundCtrl == 1) {
  BoundCtrl = 1;
  return true;
}
return false;
7309}

7311// Note: the order in this table matches the order of operands in AsmString.
7312static const OptionalOperand AMDGPUOptionalOperandTable[] = {
{"offen",   AMDGPUOperand::ImmTyOffen, true, nullptr},
{"idxen",   AMDGPUOperand::ImmTyIdxen, true, nullptr},
{"addr64",  AMDGPUOperand::ImmTyAddr64, true, nullptr},
{"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
{"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
{"gds",     AMDGPUOperand::ImmTyGDS, true, nullptr},
{"lds",     AMDGPUOperand::ImmTyLDS, true, nullptr},
{"offset",  AMDGPUOperand::ImmTyOffset, false, nullptr},
{"inst_offset", AMDGPUOperand::ImmTyInstOffset, false, nullptr},
{"",        AMDGPUOperand::ImmTyCPol, false, nullptr},
{"swz",     AMDGPUOperand::ImmTySWZ, true, nullptr},
{"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
{"d16",     AMDGPUOperand::ImmTyD16, true, nullptr},
{"high",    AMDGPUOperand::ImmTyHigh, true, nullptr},
{"clamp",   AMDGPUOperand::ImmTyClampSI, true, nullptr},
{"omod",    AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
{"unorm",   AMDGPUOperand::ImmTyUNorm, true, nullptr},
{"da",      AMDGPUOperand::ImmTyDA,    true, nullptr},
{"r128",    AMDGPUOperand::ImmTyR128A16,  true, nullptr},
{"a16",     AMDGPUOperand::ImmTyA16,  true, nullptr},
{"lwe",     AMDGPUOperand::ImmTyLWE,   true, nullptr},
{"d16",     AMDGPUOperand::ImmTyD16,   true, nullptr},
{"dmask",   AMDGPUOperand::ImmTyDMask, false, nullptr},
{"dim",     AMDGPUOperand::ImmTyDim,   false, nullptr},
{"row_mask",   AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
{"bank_mask",  AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
{"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
{"fi",         AMDGPUOperand::ImmTyDppFi, false, nullptr},
{"dst_sel",    AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
{"src0_sel",   AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
{"src1_sel",   AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
{"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
{"compr", AMDGPUOperand::ImmTyExpCompr, true, nullptr },
{"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
{"op_sel", AMDGPUOperand::ImmTyOpSel, false, nullptr},
{"op_sel_hi", AMDGPUOperand::ImmTyOpSelHi, false, nullptr},
{"neg_lo", AMDGPUOperand::ImmTyNegLo, false, nullptr},
{"neg_hi", AMDGPUOperand::ImmTyNegHi, false, nullptr},
{"blgp", AMDGPUOperand::ImmTyBLGP, false, nullptr},
{"cbsz", AMDGPUOperand::ImmTyCBSZ, false, nullptr},
{"abid", AMDGPUOperand::ImmTyABID, false, nullptr}
7354};

7356void AMDGPUAsmParser::onBeginOfFile() {
if (!getParser().getStreamer().getTargetStreamer() ||
    getSTI().getTargetTriple().getArch() == Triple::r600)
  return;

if (!getTargetStreamer().getTargetID())
  getTargetStreamer().initializeTargetID(getSTI(), getSTI().getFeatureString());

if (isHsaAbiVersion3Or4(&getSTI()))
  getTargetStreamer().EmitDirectiveAMDGCNTarget();
7366}

7368OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {

OperandMatchResultTy res = parseOptionalOpr(Operands);

// This is a hack to enable hardcoded mandatory operands which follow
// optional operands.
//
// Current design assumes that all operands after the first optional operand
// are also optional. However implementation of some instructions violates
// this rule (see e.g. flat/global atomic which have hardcoded 'glc' operands).
//
// To alleviate this problem, we have to (implicitly) parse extra operands
// to make sure autogenerated parser of custom operands never hit hardcoded
// mandatory operands.

for (unsigned i = 0; i < MAX_OPR_LOOKAHEAD; ++i) {
  if (res != MatchOperand_Success ||
      isToken(AsmToken::EndOfStatement))
    break;

  trySkipToken(AsmToken::Comma);
  res = parseOptionalOpr(Operands);
}

return res;
7393}

7395OperandMatchResultTy AMDGPUAsmParser::parseOptionalOpr(OperandVector &Operands) {
OperandMatchResultTy res;
for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
  // try to parse any optional operand here
  if (Op.IsBit) {
    res = parseNamedBit(Op.Name, Operands, Op.Type);
  } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
    res = parseOModOperand(Operands);
  } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
             Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
             Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
    res = parseSDWASel(Operands, Op.Name, Op.Type);
  } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
    res = parseSDWADstUnused(Operands);
  } else if (Op.Type == AMDGPUOperand::ImmTyOpSel ||
             Op.Type == AMDGPUOperand::ImmTyOpSelHi ||
             Op.Type == AMDGPUOperand::ImmTyNegLo ||
             Op.Type == AMDGPUOperand::ImmTyNegHi) {
    res = parseOperandArrayWithPrefix(Op.Name, Operands, Op.Type,
                                      Op.ConvertResult);
  } else if (Op.Type == AMDGPUOperand::ImmTyDim) {
    res = parseDim(Operands);
  } else if (Op.Type == AMDGPUOperand::ImmTyCPol) {
    res = parseCPol(Operands);
  } else {
    res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
  }
  if (res != MatchOperand_NoMatch) {
    return res;
  }
}
return MatchOperand_NoMatch;
7427}

7429OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
StringRef Name = getTokenStr();
if (Name == "mul") {
  return parseIntWithPrefix("mul", Operands,
                            AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
}

if (Name == "div") {
  return parseIntWithPrefix("div", Operands,
                            AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
}

return MatchOperand_NoMatch;
7442}

7444void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands) {
cvtVOP3P(Inst, Operands);

int Opc = Inst.getOpcode();

int SrcNum;
const int Ops[] = { AMDGPU::OpName::src0,
                    AMDGPU::OpName::src1,
                    AMDGPU::OpName::src2 };
for (SrcNum = 0;
     SrcNum < 3 && AMDGPU::getNamedOperandIdx(Opc, Ops[SrcNum]) != -1;
     ++SrcNum);
assert(SrcNum > 0)(static_cast<void> (0));

int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();

if ((OpSel & (1 << SrcNum)) != 0) {
  int ModIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers);
  uint32_t ModVal = Inst.getOperand(ModIdx).getImm();
  Inst.getOperand(ModIdx).setImm(ModVal | SISrcMods::DST_OP_SEL);
}
7466}

7468static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
    // 1. This operand is input modifiers
return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
    // 2. This is not last operand
    && Desc.NumOperands > (OpNum + 1)
    // 3. Next operand is register class
    && Desc.OpInfo[OpNum + 1].RegClass != -1
    // 4. Next register is not tied to any other operand
    && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
7477}

7479void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands)
7480{
OptionalImmIndexMap OptionalIdx;
unsigned Opc = Inst.getOpcode();

unsigned I = 1;
const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
  ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
}

for (unsigned E = Operands.size(); I != E; ++I) {
  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
  if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
    Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
  } else if (Op.isInterpSlot() ||
             Op.isInterpAttr() ||
             Op.isAttrChan()) {
    Inst.addOperand(MCOperand::createImm(Op.getImm()));
  } else if (Op.isImmModifier()) {
    OptionalIdx[Op.getImmTy()] = I;
  } else {
    llvm_unreachable("unhandled operand type")__builtin_unreachable();
  }
}

if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::high) != -1) {
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyHigh);
}

if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
}

if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
}
7516}

7518void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands,
                            OptionalImmIndexMap &OptionalIdx) {
unsigned Opc = Inst.getOpcode();

unsigned I = 1;
const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
  ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
}

if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers) != -1) {
  // This instruction has src modifiers
  for (unsigned E = Operands.size(); I != E; ++I) {
    AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
    if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
      Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
    } else if (Op.isImmModifier()) {
      OptionalIdx[Op.getImmTy()] = I;
    } else if (Op.isRegOrImm()) {
      Op.addRegOrImmOperands(Inst, 1);
    } else {
      llvm_unreachable("unhandled operand type")__builtin_unreachable();
    }
  }
} else {
  // No src modifiers
  for (unsigned E = Operands.size(); I != E; ++I) {
    AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
    if (Op.isMod()) {
      OptionalIdx[Op.getImmTy()] = I;
    } else {
      Op.addRegOrImmOperands(Inst, 1);
    }
  }
}

if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
}

if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
}

// Special case v_mac_{f16, f32} and v_fmac_{f16, f32} (gfx906/gfx10+):
// it has src2 register operand that is tied to dst operand
// we don't allow modifiers for this operand in assembler so src2_modifiers
// should be 0.
if (Opc == AMDGPU::V_MAC_F32_e64_gfx6_gfx7 ||
    Opc == AMDGPU::V_MAC_F32_e64_gfx10 ||
    Opc == AMDGPU::V_MAC_F32_e64_vi ||
    Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx6_gfx7 ||
    Opc == AMDGPU::V_MAC_LEGACY_F32_e64_gfx10 ||
    Opc == AMDGPU::V_MAC_F16_e64_vi ||
    Opc == AMDGPU::V_FMAC_F64_e64_gfx90a ||
    Opc == AMDGPU::V_FMAC_F32_e64_gfx10 ||
    Opc == AMDGPU::V_FMAC_F32_e64_vi ||
    Opc == AMDGPU::V_FMAC_LEGACY_F32_e64_gfx10 ||
    Opc == AMDGPU::V_FMAC_F16_e64_gfx10) {
  auto it = Inst.begin();
  std::advance(it, AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2_modifiers));
  it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
  ++it;
  // Copy the operand to ensure it's not invalidated when Inst grows.
  Inst.insert(it, MCOperand(Inst.getOperand(0))); // src2 = dst
}
7584}

7586void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
OptionalImmIndexMap OptionalIdx;
cvtVOP3(Inst, Operands, OptionalIdx);
7589}

7591void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands,
                             OptionalImmIndexMap &OptIdx) {
const int Opc = Inst.getOpcode();
const MCInstrDesc &Desc = MII.get(Opc);

const bool IsPacked = (Desc.TSFlags & SIInstrFlags::IsPacked) != 0;

if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in) != -1) {
  assert(!IsPacked)(static_cast<void> (0));
  Inst.addOperand(Inst.getOperand(0));
}

// FIXME: This is messy. Parse the modifiers as if it was a normal VOP3
// instruction, and then figure out where to actually put the modifiers

int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
if (OpSelIdx != -1) {
  addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSel);
}

int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
if (OpSelHiIdx != -1) {
  int DefaultVal = IsPacked ? -1 : 0;
  addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSelHi,
                        DefaultVal);
}

int NegLoIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_lo);
if (NegLoIdx != -1) {
  addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegLo);
  addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegHi);
}

const int Ops[] = { AMDGPU::OpName::src0,
                    AMDGPU::OpName::src1,
                    AMDGPU::OpName::src2 };
const int ModOps[] = { AMDGPU::OpName::src0_modifiers,
                       AMDGPU::OpName::src1_modifiers,
                       AMDGPU::OpName::src2_modifiers };

unsigned OpSel = 0;
unsigned OpSelHi = 0;
unsigned NegLo = 0;
unsigned NegHi = 0;

if (OpSelIdx != -1)
  OpSel = Inst.getOperand(OpSelIdx).getImm();

if (OpSelHiIdx != -1)
  OpSelHi = Inst.getOperand(OpSelHiIdx).getImm();

if (NegLoIdx != -1) {
  int NegHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_hi);
  NegLo = Inst.getOperand(NegLoIdx).getImm();
  NegHi = Inst.getOperand(NegHiIdx).getImm();
}

for (int J = 0; J < 3; ++J) {
  int OpIdx = AMDGPU::getNamedOperandIdx(Opc, Ops[J]);
  if (OpIdx == -1)
    break;

  uint32_t ModVal = 0;

  if ((OpSel & (1 << J)) != 0)
    ModVal |= SISrcMods::OP_SEL_0;

  if ((OpSelHi & (1 << J)) != 0)
    ModVal |= SISrcMods::OP_SEL_1;

  if ((NegLo & (1 << J)) != 0)
    ModVal |= SISrcMods::NEG;

  if ((NegHi & (1 << J)) != 0)
    ModVal |= SISrcMods::NEG_HI;

  int ModIdx = AMDGPU::getNamedOperandIdx(Opc, ModOps[J]);

  Inst.getOperand(ModIdx).setImm(Inst.getOperand(ModIdx).getImm() | ModVal);
}
7671}

7673void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst, const OperandVector &Operands) {
OptionalImmIndexMap OptIdx;
cvtVOP3(Inst, Operands, OptIdx);
cvtVOP3P(Inst, Operands, OptIdx);
7677}

7679//===----------------------------------------------------------------------===//
7680// dpp
7681//===----------------------------------------------------------------------===//

7683bool AMDGPUOperand::isDPP8() const {
return isImmTy(ImmTyDPP8);
7685}

7687bool AMDGPUOperand::isDPPCtrl() const {
using namespace AMDGPU::DPP;

bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
if (result) {
  int64_t Imm = getImm();
  return (Imm >= DppCtrl::QUAD_PERM_FIRST && Imm <= DppCtrl::QUAD_PERM_LAST) ||
         (Imm >= DppCtrl::ROW_SHL_FIRST && Imm <= DppCtrl::ROW_SHL_LAST) ||
         (Imm >= DppCtrl::ROW_SHR_FIRST && Imm <= DppCtrl::ROW_SHR_LAST) ||
         (Imm >= DppCtrl::ROW_ROR_FIRST && Imm <= DppCtrl::ROW_ROR_LAST) ||
         (Imm == DppCtrl::WAVE_SHL1) ||
         (Imm == DppCtrl::WAVE_ROL1) ||
         (Imm == DppCtrl::WAVE_SHR1) ||
         (Imm == DppCtrl::WAVE_ROR1) ||
         (Imm == DppCtrl::ROW_MIRROR) ||
         (Imm == DppCtrl::ROW_HALF_MIRROR) ||
         (Imm == DppCtrl::BCAST15) ||
         (Imm == DppCtrl::BCAST31) ||
         (Imm >= DppCtrl::ROW_SHARE_FIRST && Imm <= DppCtrl::ROW_SHARE_LAST) ||
         (Imm >= DppCtrl::ROW_XMASK_FIRST && Imm <= DppCtrl::ROW_XMASK_LAST);
}
return false;
7709}

7711//===----------------------------------------------------------------------===//
7712// mAI
7713//===----------------------------------------------------------------------===//

7715bool AMDGPUOperand::isBLGP() const {
return isImm() && getImmTy() == ImmTyBLGP && isUInt<3>(getImm());
7717}

7719bool AMDGPUOperand::isCBSZ() const {
return isImm() && getImmTy() == ImmTyCBSZ && isUInt<3>(getImm());
7721}

7723bool AMDGPUOperand::isABID() const {
return isImm() && getImmTy() == ImmTyABID && isUInt<4>(getImm());
7725}

7727bool AMDGPUOperand::isS16Imm() const {
return isImm() && (isInt<16>(getImm()) || isUInt<16>(getImm()));
7729}

7731bool AMDGPUOperand::isU16Imm() const {
return isImm() && isUInt<16>(getImm());
7733}

7735//===----------------------------------------------------------------------===//
7736// dim
7737//===----------------------------------------------------------------------===//

7739bool AMDGPUAsmParser::parseDimId(unsigned &Encoding) {
// We want to allow "dim:1D" etc.,
// but the initial 1 is tokenized as an integer.
std::string Token;
if (isToken(AsmToken::Integer)) {
  SMLoc Loc = getToken().getEndLoc();
  Token = std::string(getTokenStr());
  lex();
  if (getLoc() != Loc)
    return false;
}

StringRef Suffix;
if (!parseId(Suffix))
  return false;
Token += Suffix;

StringRef DimId = Token;
if (DimId.startswith("SQ_RSRC_IMG_"))
  DimId = DimId.drop_front(12);

const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfoByAsmSuffix(DimId);
if (!DimInfo)
  return false;

Encoding = DimInfo->Encoding;
return true;
7766}

7768OperandMatchResultTy AMDGPUAsmParser::parseDim(OperandVector &Operands) {
if (!isGFX10Plus())
  return MatchOperand_NoMatch;

SMLoc S = getLoc();

if (!trySkipId("dim", AsmToken::Colon))
  return MatchOperand_NoMatch;

unsigned Encoding;
SMLoc Loc = getLoc();
if (!parseDimId(Encoding)) {
  Error(Loc, "invalid dim value");
  return MatchOperand_ParseFail;
}

Operands.push_back(AMDGPUOperand::CreateImm(this, Encoding, S,
                                            AMDGPUOperand::ImmTyDim));
return MatchOperand_Success;
7787}

7789//===----------------------------------------------------------------------===//
7790// dpp
7791//===----------------------------------------------------------------------===//

7793OperandMatchResultTy AMDGPUAsmParser::parseDPP8(OperandVector &Operands) {
SMLoc S = getLoc();

if (!isGFX10Plus() || !trySkipId("dpp8", AsmToken::Colon))
  return MatchOperand_NoMatch;

// dpp8:[%d,%d,%d,%d,%d,%d,%d,%d]

int64_t Sels[8];

if (!skipToken(AsmToken::LBrac, "expected an opening square bracket"))
  return MatchOperand_ParseFail;

for (size_t i = 0; i < 8; ++i) {
  if (i > 0 && !skipToken(AsmToken::Comma, "expected a comma"))
    return MatchOperand_ParseFail;

  SMLoc Loc = getLoc();
  if (getParser().parseAbsoluteExpression(Sels[i]))
    return MatchOperand_ParseFail;
  if (0 > Sels[i] || 7 < Sels[i]) {
    Error(Loc, "expected a 3-bit value");
    return MatchOperand_ParseFail;
  }
}

if (!skipToken(AsmToken::RBrac, "expected a closing square bracket"))
  return MatchOperand_ParseFail;

unsigned DPP8 = 0;
for (size_t i = 0; i < 8; ++i)
  DPP8 |= (Sels[i] << (i * 3));

Operands.push_back(AMDGPUOperand::CreateImm(this, DPP8, S, AMDGPUOperand::ImmTyDPP8));
return MatchOperand_Success;
7828}

7830bool
7831AMDGPUAsmParser::isSupportedDPPCtrl(StringRef Ctrl,
                                  const OperandVector &Operands) {
if (Ctrl == "row_newbcast")
  return isGFX90A();

if (Ctrl == "row_share" ||
    Ctrl == "row_xmask")
  return isGFX10Plus();

if (Ctrl == "wave_shl" ||
    Ctrl == "wave_shr" ||
    Ctrl == "wave_rol" ||
    Ctrl == "wave_ror" ||
    Ctrl == "row_bcast")
  return isVI() || isGFX9();

return Ctrl == "row_mirror" ||
       Ctrl == "row_half_mirror" ||
       Ctrl == "quad_perm" ||
       Ctrl == "row_shl" ||
       Ctrl == "row_shr" ||
       Ctrl == "row_ror";
7853}

7855int64_t
7856AMDGPUAsmParser::parseDPPCtrlPerm() {
// quad_perm:[%d,%d,%d,%d]

if (!skipToken(AsmToken::LBrac, "expected an opening square bracket"))
  return -1;

int64_t Val = 0;
for (int i = 0; i < 4; ++i) {
  if (i > 0 && !skipToken(AsmToken::Comma, "expected a comma"))
    return -1;

  int64_t Temp;
  SMLoc Loc = getLoc();
  if (getParser().parseAbsoluteExpression(Temp))
    return -1;
  if (Temp < 0 || Temp > 3) {
    Error(Loc, "expected a 2-bit value");
    return -1;
  }

  Val += (Temp << i * 2);
}

if (!skipToken(AsmToken::RBrac, "expected a closing square bracket"))
  return -1;

return Val;
7883}

7885int64_t
7886AMDGPUAsmParser::parseDPPCtrlSel(StringRef Ctrl) {
using namespace AMDGPU::DPP;

// sel:%d

int64_t Val;
SMLoc Loc = getLoc();

if (getParser().parseAbsoluteExpression(Val))
  return -1;

struct DppCtrlCheck {
  int64_t Ctrl;
  int Lo;
  int Hi;
};

DppCtrlCheck Check = StringSwitch<DppCtrlCheck>(Ctrl)
  .Case("wave_shl",  {DppCtrl::WAVE_SHL1,       1,  1})
  .Case("wave_rol",  {DppCtrl::WAVE_ROL1,       1,  1})
  .Case("wave_shr",  {DppCtrl::WAVE_SHR1,       1,  1})
  .Case("wave_ror",  {DppCtrl::WAVE_ROR1,       1,  1})
  .Case("row_shl",   {DppCtrl::ROW_SHL0,        1, 15})
  .Case("row_shr",   {DppCtrl::ROW_SHR0,        1, 15})
  .Case("row_ror",   {DppCtrl::ROW_ROR0,        1, 15})
  .Case("row_share", {DppCtrl::ROW_SHARE_FIRST, 0, 15})
  .Case("row_xmask", {DppCtrl::ROW_XMASK_FIRST, 0, 15})
  .Case("row_newbcast", {DppCtrl::ROW_NEWBCAST_FIRST, 0, 15})
  .Default({-1, 0, 0});

bool Valid;
if (Check.Ctrl == -1) {
  Valid = (Ctrl == "row_bcast" && (Val == 15 || Val == 31));
  Val = (Val == 15)? DppCtrl::BCAST15 : DppCtrl::BCAST31;
} else {
  Valid = Check.Lo <= Val && Val <= Check.Hi;
  Val = (Check.Lo == Check.Hi) ? Check.Ctrl : (Check.Ctrl | Val);
}

if (!Valid) {
  Error(Loc, Twine("invalid ", Ctrl) + Twine(" value"));
  return -1;
}

return Val;
7931}

7933OperandMatchResultTy
7934AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
using namespace AMDGPU::DPP;

if (!isToken(AsmToken::Identifier) ||
    !isSupportedDPPCtrl(getTokenStr(), Operands))
  return MatchOperand_NoMatch;

SMLoc S = getLoc();
int64_t Val = -1;
StringRef Ctrl;

parseId(Ctrl);

if (Ctrl == "row_mirror") {
  Val = DppCtrl::ROW_MIRROR;
} else if (Ctrl == "row_half_mirror") {
  Val = DppCtrl::ROW_HALF_MIRROR;
} else {
  if (skipToken(AsmToken::Colon, "expected a colon")) {
    if (Ctrl == "quad_perm") {
      Val = parseDPPCtrlPerm();
    } else {
      Val = parseDPPCtrlSel(Ctrl);
    }
  }
}

if (Val == -1)
  return MatchOperand_ParseFail;

Operands.push_back(
  AMDGPUOperand::CreateImm(this, Val, S, AMDGPUOperand::ImmTyDppCtrl));
return MatchOperand_Success;
7967}

7969AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppRowMask);
7971}

7973AMDGPUOperand::Ptr AMDGPUAsmParser::defaultEndpgmImmOperands() const {
return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyEndpgm);
7975}

7977AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBankMask() const {
return AMDGPUOperand::CreateImm(this, 0xf, SMLoc(), AMDGPUOperand::ImmTyDppBankMask);
7979}

7981AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBoundCtrl() const {
return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppBoundCtrl);
7983}

7985AMDGPUOperand::Ptr AMDGPUAsmParser::defaultFI() const {
return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDppFi);
7987}

7989void AMDGPUAsmParser::cvtDPP(MCInst &Inst, const OperandVector &Operands, bool IsDPP8) {
OptionalImmIndexMap OptionalIdx;

unsigned Opc = Inst.getOpcode();
bool HasModifiers =
    AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers) != -1;
unsigned I = 1;
const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
  ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
}

int Fi = 0;
for (unsigned E = Operands.size(); I != E; ++I) {
  auto TiedTo = Desc.getOperandConstraint(Inst.getNumOperands(),
                                          MCOI::TIED_TO);
  if (TiedTo != -1) {
    assert((unsigned)TiedTo < Inst.getNumOperands())(static_cast<void> (0));
    // handle tied old or src2 for MAC instructions
    Inst.addOperand(Inst.getOperand(TiedTo));
  }
  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
  // Add the register arguments
  if (Op.isReg() && validateVccOperand(Op.getReg())) {
    // VOP2b (v_add_u32, v_sub_u32 ...) dpp use "vcc" token.
    // Skip it.
    continue;
  }

  if (IsDPP8) {
    if (Op.isDPP8()) {
      Op.addImmOperands(Inst, 1);
    } else if (HasModifiers &&
               isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
      Op.addRegWithFPInputModsOperands(Inst, 2);
    } else if (Op.isFI()) {
      Fi = Op.getImm();
    } else if (Op.isReg()) {
      Op.addRegOperands(Inst, 1);
    } else {
      llvm_unreachable("Invalid operand type")__builtin_unreachable();
    }
  } else {
    if (HasModifiers &&
        isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
      Op.addRegWithFPInputModsOperands(Inst, 2);
    } else if (Op.isReg()) {
      Op.addRegOperands(Inst, 1);
    } else if (Op.isDPPCtrl()) {
      Op.addImmOperands(Inst, 1);
    } else if (Op.isImm()) {
      // Handle optional arguments
      OptionalIdx[Op.getImmTy()] = I;
    } else {
      llvm_unreachable("Invalid operand type")__builtin_unreachable();
    }
  }
}

if (IsDPP8) {
  using namespace llvm::AMDGPU::DPP;
  Inst.addOperand(MCOperand::createImm(Fi? DPP8_FI_1 : DPP8_FI_0));
} else {
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppRowMask, 0xf);
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBankMask, 0xf);
  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppBoundCtrl);
  if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::fi) != -1) {
    addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDppFi);
  }
}
8059}

8061//===----------------------------------------------------------------------===//
8062// sdwa
8063//===----------------------------------------------------------------------===//

8065OperandMatchResultTy
8066AMDGPUAsmParser::parseSDWASel(OperandVector &Operands, StringRef Prefix,
                            AMDGPUOperand::ImmTy Type) {
using namespace llvm::AMDGPU::SDWA;

SMLoc S = getLoc();
StringRef Value;
OperandMatchResultTy res;

SMLoc StringLoc;
res = parseStringWithPrefix(Prefix, Value, StringLoc);
if (res != MatchOperand_Success) {
  return res;
}

int64_t Int;
Int = StringSwitch<int64_t>(Value)
      .Case("BYTE_0", SdwaSel::BYTE_0)
      .Case("BYTE_1", SdwaSel::BYTE_1)
      .Case("BYTE_2", SdwaSel::BYTE_2)
      .Case("BYTE_3", SdwaSel::BYTE_3)
      .Case("WORD_0", SdwaSel::WORD_0)
      .Case("WORD_1", SdwaSel::WORD_1)
      .Case("DWORD", SdwaSel::DWORD)
      .Default(0xffffffff);

if (Int == 0xffffffff) {
  Error(StringLoc, "invalid " + Twine(Prefix) + " value");
  return MatchOperand_ParseFail;
}

Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, Type));
return MatchOperand_Success;
8098}

8100OperandMatchResultTy
8101AMDGPUAsmParser::parseSDWADstUnused(OperandVector &Operands) {
using namespace llvm::AMDGPU::SDWA;

SMLoc S = getLoc();
StringRef Value;
OperandMatchResultTy res;

SMLoc StringLoc;
res = parseStringWithPrefix("dst_unused", Value, StringLoc);
if (res != MatchOperand_Success) {
  return res;
}

int64_t Int;
Int = StringSwitch<int64_t>(Value)
      .Case("UNUSED_PAD", DstUnused::UNUSED_PAD)
      .Case("UNUSED_SEXT", DstUnused::UNUSED_SEXT)
      .Case("UNUSED_PRESERVE", DstUnused::UNUSED_PRESERVE)
      .Default(0xffffffff);

if (Int == 0xffffffff) {
  Error(StringLoc, "invalid dst_unused value");
  return MatchOperand_ParseFail;
}

Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTySdwaDstUnused));
return MatchOperand_Success;
8128}

8130void AMDGPUAsmParser::cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands) {
cvtSDWA(Inst, Operands, SIInstrFlags::VOP1);
8132}

8134void AMDGPUAsmParser::cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands) {
cvtSDWA(Inst, Operands, SIInstrFlags::VOP2);
8136}

8138void AMDGPUAsmParser::cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands) {
cvtSDWA(Inst, Operands, SIInstrFlags::VOP2, true, true);
8140}

8142void AMDGPUAsmParser::cvtSdwaVOP2e(MCInst &Inst, const OperandVector &Operands) {
cvtSDWA(Inst, Operands, SIInstrFlags::VOP2, false, true);
8144}

8146void AMDGPUAsmParser::cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands) {
cvtSDWA(Inst, Operands, SIInstrFlags::VOPC, isVI());
8148}

8150void AMDGPUAsmParser::cvtSDWA(MCInst &Inst, const OperandVector &Operands,
                            uint64_t BasicInstType,
                            bool SkipDstVcc,
                            bool SkipSrcVcc) {
using namespace llvm::AMDGPU::SDWA;

OptionalImmIndexMap OptionalIdx;
bool SkipVcc = SkipDstVcc || SkipSrcVcc;
bool SkippedVcc = false;

unsigned I = 1;
const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
  ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
}

for (unsigned E = Operands.size(); I != E; ++I) {
  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
  if (SkipVcc && !SkippedVcc && Op.isReg() &&
      (Op.getReg() == AMDGPU::VCC || Op.getReg() == AMDGPU::VCC_LO)) {
    // VOP2b (v_add_u32, v_sub_u32 ...) sdwa use "vcc" token as dst.
    // Skip it if it's 2nd (e.g. v_add_i32_sdwa v1, vcc, v2, v3)
    // or 4th (v_addc_u32_sdwa v1, vcc, v2, v3, vcc) operand.
    // Skip VCC only if we didn't skip it on previous iteration.
    // Note that src0 and src1 occupy 2 slots each because of modifiers.
    if (BasicInstType == SIInstrFlags::VOP2 &&
        ((SkipDstVcc && Inst.getNumOperands() == 1) ||
         (SkipSrcVcc && Inst.getNumOperands() == 5))) {
      SkippedVcc = true;
      continue;
    } else if (BasicInstType == SIInstrFlags::VOPC &&
               Inst.getNumOperands() == 0) {
      SkippedVcc = true;
      continue;
    }
  }
  if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
    Op.addRegOrImmWithInputModsOperands(Inst, 2);
  } else if (Op.isImm()) {
    // Handle optional arguments
    OptionalIdx[Op.getImmTy()] = I;
  } else {
    llvm_unreachable("Invalid operand type")__builtin_unreachable();
  }
  SkippedVcc = false;
}

if (Inst.getOpcode() != AMDGPU::V_NOP_sdwa_gfx10 &&
    Inst.getOpcode() != AMDGPU::V_NOP_sdwa_gfx9 &&
    Inst.getOpcode() != AMDGPU::V_NOP_sdwa_vi) {
  // v_nop_sdwa_sdwa_vi/gfx9 has no optional sdwa arguments
  switch (BasicInstType) {
  case SIInstrFlags::VOP1:
    addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
    if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
      addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
    }
    addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
    addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
    addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
    break;

  case SIInstrFlags::VOP2:
    addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
    if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::omod) != -1) {
      addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI, 0);
    }
    addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstSel, SdwaSel::DWORD);
    addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaDstUnused, DstUnused::UNUSED_PRESERVE);
    addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
    addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
    break;

  case SIInstrFlags::VOPC:
    if (AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::clamp) != -1)
      addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI, 0);
    addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc0Sel, SdwaSel::DWORD);
    addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySdwaSrc1Sel, SdwaSel::DWORD);
    break;

  default:
    llvm_unreachable("Invalid instruction type. Only VOP1, VOP2 and VOPC allowed")__builtin_unreachable();
  }
}

// special case v_mac_{f16, f32}:
// it has src2 register operand that is tied to dst operand
if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
    Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi)  {
  auto it = Inst.begin();
  std::advance(
    it, AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::src2));
  Inst.insert(it, Inst.getOperand(0)); // src2 = dst
}
8244}

8246//===----------------------------------------------------------------------===//
8247// mAI
8248//===----------------------------------------------------------------------===//

8250AMDGPUOperand::Ptr AMDGPUAsmParser::defaultBLGP() const {
return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyBLGP);
8252}

8254AMDGPUOperand::Ptr AMDGPUAsmParser::defaultCBSZ() const {
return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyCBSZ);
8256}

8258AMDGPUOperand::Ptr AMDGPUAsmParser::defaultABID() const {
return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyABID);
8260}

8262/// Force static initialization.
8263extern "C" LLVM_EXTERNAL_VISIBILITY__attribute__ ((visibility("default"))) void LLVMInitializeAMDGPUAsmParser() {
RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget());
RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget());
8266}

8268#define GET_REGISTER_MATCHER
8269#define GET_MATCHER_IMPLEMENTATION
8270#define GET_MNEMONIC_SPELL_CHECKER
8271#define GET_MNEMONIC_CHECKER
8272#include "AMDGPUGenAsmMatcher.inc"

8274// This fuction should be defined after auto-generated include so that we have
8275// MatchClassKind enum defined
8276unsigned AMDGPUAsmParser::validateTargetOperandClass(MCParsedAsmOperand &Op,
                                                   unsigned Kind) {
// Tokens like "glc" would be parsed as immediate operands in ParseOperand().
// But MatchInstructionImpl() expects to meet token and fails to validate
// operand. This method checks if we are given immediate operand but expect to
// get corresponding token.
AMDGPUOperand &Operand = (AMDGPUOperand&)Op;
switch (Kind) {
case MCK_addr64:
  return Operand.isAddr64() ? Match_Success : Match_InvalidOperand;
case MCK_gds:
  return Operand.isGDS() ? Match_Success : Match_InvalidOperand;
case MCK_lds:
  return Operand.isLDS() ? Match_Success : Match_InvalidOperand;
case MCK_idxen:
  return Operand.isIdxen() ? Match_Success : Match_InvalidOperand;
case MCK_offen:
  return Operand.isOffen() ? Match_Success : Match_InvalidOperand;
case MCK_SSrcB32:
  // When operands have expression values, they will return true for isToken,
  // because it is not possible to distinguish between a token and an
  // expression at parse time. MatchInstructionImpl() will always try to
  // match an operand as a token, when isToken returns true, and when the
  // name of the expression is not a valid token, the match will fail,
  // so we need to handle it here.
  return Operand.isSSrcB32() ? Match_Success : Match_InvalidOperand;
case MCK_SSrcF32:
  return Operand.isSSrcF32() ? Match_Success : Match_InvalidOperand;
case MCK_SoppBrTarget:
  return Operand.isSoppBrTarget() ? Match_Success : Match_InvalidOperand;
case MCK_VReg32OrOff:
  return Operand.isVReg32OrOff() ? Match_Success : Match_InvalidOperand;
case MCK_InterpSlot:
  return Operand.isInterpSlot() ? Match_Success : Match_InvalidOperand;
case MCK_Attr:
  return Operand.isInterpAttr() ? Match_Success : Match_InvalidOperand;
case MCK_AttrChan:
  return Operand.isAttrChan() ? Match_Success : Match_InvalidOperand;
case MCK_ImmSMEMOffset:
  return Operand.isSMEMOffset() ? Match_Success : Match_InvalidOperand;
case MCK_SReg_64:
case MCK_SReg_64_XEXEC:
  // Null is defined as a 32-bit register but
  // it should also be enabled with 64-bit operands.
  // The following code enables it for SReg_64 operands
  // used as source and destination. Remaining source
  // operands are handled in isInlinableImm.
  return Operand.isNull() ? Match_Success : Match_InvalidOperand;
default:
  return Match_InvalidOperand;
}
8327}

8329//===----------------------------------------------------------------------===//
8330// endpgm
8331//===----------------------------------------------------------------------===//

8333OperandMatchResultTy AMDGPUAsmParser::parseEndpgmOp(OperandVector &Operands) {
SMLoc S = getLoc();
int64_t Imm = 0;

if (!parseExpr(Imm)) {
  // The operand is optional, if not present default to 0
  Imm = 0;
}

if (!isUInt<16>(Imm)) {
  Error(S, "expected a 16-bit value");
  return MatchOperand_ParseFail;
}

Operands.push_back(
    AMDGPUOperand::CreateImm(this, Imm, S, AMDGPUOperand::ImmTyEndpgm));
return MatchOperand_Success;
8350}

8352bool AMDGPUOperand::isEndpgm() const { return isImmTy(ImmTyEndpgm); }

←

/usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/bits/stl_tree.h

1// RB tree implementation -*- C++ -*-

3// Copyright (C) 2001-2020 Free Software Foundation, Inc.
4//
5// This file is part of the GNU ISO C++ Library.  This library is free
6// software; you can redistribute it and/or modify it under the
7// terms of the GNU General Public License as published by the
8// Free Software Foundation; either version 3, or (at your option)
9// any later version.

11// This library is distributed in the hope that it will be useful,
12// but WITHOUT ANY WARRANTY; without even the implied warranty of
13// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14// GNU General Public License for more details.

16// Under Section 7 of GPL version 3, you are granted additional
17// permissions described in the GCC Runtime Library Exception, version
18// 3.1, as published by the Free Software Foundation.

20// You should have received a copy of the GNU General Public License and
21// a copy of the GCC Runtime Library Exception along with this program;
22// see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
23// <http://www.gnu.org/licenses/>.

25/*
*
* Copyright (c) 1996,1997
* Silicon Graphics Computer Systems, Inc.
*
* Permission to use, copy, modify, distribute and sell this software
* and its documentation for any purpose is hereby granted without fee,
* provided that the above copyright notice appear in all copies and
* that both that copyright notice and this permission notice appear
* in supporting documentation.  Silicon Graphics makes no
* representations about the suitability of this software for any
* purpose.  It is provided "as is" without express or implied warranty.
*
*
* Copyright (c) 1994
* Hewlett-Packard Company
*
* Permission to use, copy, modify, distribute and sell this software
* and its documentation for any purpose is hereby granted without fee,
* provided that the above copyright notice appear in all copies and
* that both that copyright notice and this permission notice appear
* in supporting documentation.  Hewlett-Packard Company makes no
* representations about the suitability of this software for any
* purpose.  It is provided "as is" without express or implied warranty.
*
*
*/

53/** @file bits/stl_tree.h
*  This is an internal header file, included by other library headers.
*  Do not attempt to use it directly. @headername{map,set}
*/

58#ifndef _STL_TREE_H1
59#define _STL_TREE_H1 1

61#pragma GCC system_header

63#include <bits/stl_algobase.h>
64#include <bits/allocator.h>
65#include <bits/stl_function.h>
66#include <bits/cpp_type_traits.h>
67#include <ext/alloc_traits.h>
68#if __cplusplus201402L >= 201103L
69# include <ext/aligned_buffer.h>
70#endif
71#if __cplusplus201402L > 201402L
72# include <bits/node_handle.h>
73#endif

75namespace std _GLIBCXX_VISIBILITY(default)__attribute__ ((__visibility__ ("default")))
76{
77_GLIBCXX_BEGIN_NAMESPACE_VERSION

79#if __cplusplus201402L > 201103L
80# define __cpp_lib_generic_associative_lookup201304 201304
81#endif

// Red-black tree class, designed for use in implementing STL
// associative containers (set, multiset, map, and multimap). The
// insertion and deletion algorithms are based on those in Cormen,
// Leiserson, and Rivest, Introduction to Algorithms (MIT Press,
// 1990), except that
//
// (1) the header cell is maintained with links not only to the root
// but also to the leftmost node of the tree, to enable constant
// time begin(), and to the rightmost node of the tree, to enable
// linear time performance when used with the generic set algorithms
// (set_union, etc.)
//
// (2) when a node being deleted has two children its successor node
// is relinked into its place, rather than copied, so that the only
// iterators invalidated are those referring to the deleted node.

enum _Rb_tree_color { _S_red = false, _S_black = true };

struct _Rb_tree_node_base
{
  typedef _Rb_tree_node_base* _Base_ptr;
  typedef const _Rb_tree_node_base* _Const_Base_ptr;

  _Rb_tree_color	_M_color;
  _Base_ptr		_M_parent;
  _Base_ptr		_M_left;
  _Base_ptr		_M_right;

  static _Base_ptr
  _S_minimum(_Base_ptr __x) _GLIBCXX_NOEXCEPTnoexcept
  {
    while (__x->_M_left != 0) __x = __x->_M_left;
    return __x;
  }

  static _Const_Base_ptr
  _S_minimum(_Const_Base_ptr __x) _GLIBCXX_NOEXCEPTnoexcept
  {
    while (__x->_M_left != 0) __x = __x->_M_left;
    return __x;
  }

  static _Base_ptr
  _S_maximum(_Base_ptr __x) _GLIBCXX_NOEXCEPTnoexcept
  {
    while (__x->_M_right != 0) __x = __x->_M_right;
    return __x;
  }

  static _Const_Base_ptr
  _S_maximum(_Const_Base_ptr __x) _GLIBCXX_NOEXCEPTnoexcept
  {
    while (__x->_M_right != 0) __x = __x->_M_right;
    return __x;
  }
};

// Helper type offering value initialization guarantee on the compare functor.
template<typename _Key_compare>
  struct _Rb_tree_key_compare
  {
    _Key_compare		_M_key_compare;

    _Rb_tree_key_compare()
    _GLIBCXX_NOEXCEPT_IF(noexcept(is_nothrow_default_constructible<_Key_compare>
::value)
is_nothrow_default_constructible<_Key_compare>::value)noexcept(is_nothrow_default_constructible<_Key_compare>
::value)
    : _M_key_compare()
    { }

    _Rb_tree_key_compare(const _Key_compare& __comp)
    : _M_key_compare(__comp)
    { }

156#if __cplusplus201402L >= 201103L
    // Copy constructor added for consistency with C++98 mode.
    _Rb_tree_key_compare(const _Rb_tree_key_compare&) = default;

    _Rb_tree_key_compare(_Rb_tree_key_compare&& __x)
noexcept(is_nothrow_copy_constructible<_Key_compare>::value)
    : _M_key_compare(__x._M_key_compare)
    { }
164#endif
  };

// Helper type to manage default initialization of node count and header.
struct _Rb_tree_header
{
  _Rb_tree_node_base	_M_header;
  size_t		_M_node_count; // Keeps track of size of tree.

  _Rb_tree_header() _GLIBCXX_NOEXCEPTnoexcept
  {
    _M_header._M_color = _S_red;
    _M_reset();
  }

179#if __cplusplus201402L >= 201103L
  _Rb_tree_header(_Rb_tree_header&& __x) noexcept
  {
    if (__x._M_header._M_parent != nullptr)
_M_move_data(__x);
    else
{
 _M_header._M_color = _S_red;
 _M_reset();
}
  }
190#endif

  void
  _M_move_data(_Rb_tree_header& __from)
  {
    _M_header._M_color = __from._M_header._M_color;
    _M_header._M_parent = __from._M_header._M_parent;
    _M_header._M_left = __from._M_header._M_left;
    _M_header._M_right = __from._M_header._M_right;
    _M_header._M_parent->_M_parent = &_M_header;
    _M_node_count = __from._M_node_count;

    __from._M_reset();
  }

  void
  _M_reset()
  {
    _M_header._M_parent = 0;
    _M_header._M_left = &_M_header;
    _M_header._M_right = &_M_header;
    _M_node_count = 0;
  }
};

template<typename _Val>
  struct _Rb_tree_node : public _Rb_tree_node_base
  {
    typedef _Rb_tree_node<_Val>* _Link_type;

220#if __cplusplus201402L < 201103L
    _Val _M_value_field;

    _Val*
    _M_valptr()
    { return std::__addressof(_M_value_field); }

    const _Val*
    _M_valptr() const
    { return std::__addressof(_M_value_field); }
230#else
    __gnu_cxx::__aligned_membuf<_Val> _M_storage;

    _Val*
    _M_valptr()
    { return _M_storage._M_ptr(); }

    const _Val*
    _M_valptr() const
    { return _M_storage._M_ptr(); }
240#endif
  };

_GLIBCXX_PURE__attribute__ ((__pure__)) _Rb_tree_node_base*
_Rb_tree_increment(_Rb_tree_node_base* __x) throw ();

_GLIBCXX_PURE__attribute__ ((__pure__)) const _Rb_tree_node_base*
_Rb_tree_increment(const _Rb_tree_node_base* __x) throw ();

_GLIBCXX_PURE__attribute__ ((__pure__)) _Rb_tree_node_base*
_Rb_tree_decrement(_Rb_tree_node_base* __x) throw ();

_GLIBCXX_PURE__attribute__ ((__pure__)) const _Rb_tree_node_base*
_Rb_tree_decrement(const _Rb_tree_node_base* __x) throw ();

template<typename _Tp>
  struct _Rb_tree_iterator
  {
    typedef _Tp  value_type;
    typedef _Tp& reference;
    typedef _Tp* pointer;

    typedef bidirectional_iterator_tag iterator_category;
    typedef ptrdiff_t			 difference_type;

    typedef _Rb_tree_iterator<_Tp>		_Self;
    typedef _Rb_tree_node_base::_Base_ptr	_Base_ptr;
    typedef _Rb_tree_node<_Tp>*		_Link_type;

    _Rb_tree_iterator() _GLIBCXX_NOEXCEPTnoexcept
    : _M_node() { }

    explicit
    _Rb_tree_iterator(_Base_ptr __x) _GLIBCXX_NOEXCEPTnoexcept
    : _M_node(__x) { }

    reference
    operator*() const _GLIBCXX_NOEXCEPTnoexcept
    { return *static_cast<_Link_type>(_M_node)->_M_valptr(); }

    pointer
    operator->() const _GLIBCXX_NOEXCEPTnoexcept
    { return static_cast<_Link_type> (_M_node)->_M_valptr(); }

    _Self&
    operator++() _GLIBCXX_NOEXCEPTnoexcept
    {
_M_node = _Rb_tree_increment(_M_node);
return *this;
    }

    _Self
    operator++(int) _GLIBCXX_NOEXCEPTnoexcept
    {
_Self __tmp = *this;
_M_node = _Rb_tree_increment(_M_node);
return __tmp;
    }

    _Self&
    operator--() _GLIBCXX_NOEXCEPTnoexcept
    {
_M_node = _Rb_tree_decrement(_M_node);
return *this;
    }

    _Self
    operator--(int) _GLIBCXX_NOEXCEPTnoexcept
    {
_Self __tmp = *this;
_M_node = _Rb_tree_decrement(_M_node);
return __tmp;
    }

    friend bool
    operator==(const _Self& __x, const _Self& __y) _GLIBCXX_NOEXCEPTnoexcept
    { return __x._M_node == __y._M_node; }

318#if ! __cpp_lib_three_way_comparison
    friend bool
    operator!=(const _Self& __x, const _Self& __y) _GLIBCXX_NOEXCEPTnoexcept
    { return __x._M_node != __y._M_node; }
14
←
Assuming '__x._M_node' is not equal to '__y._M_node'→
15
←
Returning the value 1, which participates in a condition later→
322#endif

    _Base_ptr _M_node;
};

template<typename _Tp>
  struct _Rb_tree_const_iterator
  {
    typedef _Tp	 value_type;
    typedef const _Tp& reference;
    typedef const _Tp* pointer;

    typedef _Rb_tree_iterator<_Tp> iterator;

    typedef bidirectional_iterator_tag iterator_category;
    typedef ptrdiff_t			 difference_type;

    typedef _Rb_tree_const_iterator<_Tp>		_Self;
    typedef _Rb_tree_node_base::_Const_Base_ptr	_Base_ptr;
    typedef const _Rb_tree_node<_Tp>*			_Link_type;

    _Rb_tree_const_iterator() _GLIBCXX_NOEXCEPTnoexcept
    : _M_node() { }

    explicit
    _Rb_tree_const_iterator(_Base_ptr __x) _GLIBCXX_NOEXCEPTnoexcept
    : _M_node(__x) { }

    _Rb_tree_const_iterator(const iterator& __it) _GLIBCXX_NOEXCEPTnoexcept
    : _M_node(__it._M_node) { }

    iterator
    _M_const_cast() const _GLIBCXX_NOEXCEPTnoexcept
    { return iterator(const_cast<typename iterator::_Base_ptr>(_M_node)); }

    reference
    operator*() const _GLIBCXX_NOEXCEPTnoexcept
    { return *static_cast<_Link_type>(_M_node)->_M_valptr(); }

    pointer
    operator->() const _GLIBCXX_NOEXCEPTnoexcept
    { return static_cast<_Link_type>(_M_node)->_M_valptr(); }

    _Self&
    operator++() _GLIBCXX_NOEXCEPTnoexcept
    {
_M_node = _Rb_tree_increment(_M_node);
return *this;
    }

    _Self
    operator++(int) _GLIBCXX_NOEXCEPTnoexcept
    {
_Self __tmp = *this;
_M_node = _Rb_tree_increment(_M_node);
return __tmp;
    }

    _Self&
    operator--() _GLIBCXX_NOEXCEPTnoexcept
    {
_M_node = _Rb_tree_decrement(_M_node);
return *this;
    }

    _Self
    operator--(int) _GLIBCXX_NOEXCEPTnoexcept
    {
_Self __tmp = *this;
_M_node = _Rb_tree_decrement(_M_node);
return __tmp;
    }

    friend bool
    operator==(const _Self& __x, const _Self& __y) _GLIBCXX_NOEXCEPTnoexcept
    { return __x._M_node == __y._M_node; }

399#if ! __cpp_lib_three_way_comparison
    friend bool
    operator!=(const _Self& __x, const _Self& __y) _GLIBCXX_NOEXCEPTnoexcept
    { return __x._M_node != __y._M_node; }
403#endif

    _Base_ptr _M_node;
  };

void
_Rb_tree_insert_and_rebalance(const bool __insert_left,
		_Rb_tree_node_base* __x,
		_Rb_tree_node_base* __p,
		_Rb_tree_node_base& __header) throw ();

_Rb_tree_node_base*
_Rb_tree_rebalance_for_erase(_Rb_tree_node_base* const __z,
	       _Rb_tree_node_base& __header) throw ();

418#if __cplusplus201402L >= 201402L
template<typename _Cmp, typename _SfinaeType, typename = __void_t<>>
  struct __has_is_transparent
  { };

template<typename _Cmp, typename _SfinaeType>
  struct __has_is_transparent<_Cmp, _SfinaeType,
		__void_t<typename _Cmp::is_transparent>>
  { typedef void type; };

template<typename _Cmp, typename _SfinaeType>
  using __has_is_transparent_t
    = typename __has_is_transparent<_Cmp, _SfinaeType>::type;
431#endif

433#if __cplusplus201402L > 201402L
template<typename _Tree1, typename _Cmp2>
  struct _Rb_tree_merge_helper { };
436#endif

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc = allocator<_Val> >
  class _Rb_tree
  {
    typedef typename __gnu_cxx::__alloc_traits<_Alloc>::template
rebind<_Rb_tree_node<_Val> >::other _Node_allocator;

    typedef __gnu_cxx::__alloc_traits<_Node_allocator> _Alloc_traits;

  protected:
    typedef _Rb_tree_node_base* 		_Base_ptr;
    typedef const _Rb_tree_node_base* 	_Const_Base_ptr;
    typedef _Rb_tree_node<_Val>* 		_Link_type;
    typedef const _Rb_tree_node<_Val>*	_Const_Link_type;

  private:
    // Functor recycling a pool of nodes and using allocation once the pool
    // is empty.
    struct _Reuse_or_alloc_node
    {
_Reuse_or_alloc_node(_Rb_tree& __t)
: _M_root(__t._M_root()), _M_nodes(__t._M_rightmost()), _M_t(__t)
{
 if (_M_root)
   {
     _M_root->_M_parent = 0;

     if (_M_nodes->_M_left)
_M_nodes = _M_nodes->_M_left;
   }
 else
   _M_nodes = 0;
}

472#if __cplusplus201402L >= 201103L
_Reuse_or_alloc_node(const _Reuse_or_alloc_node&) = delete;
474#endif

~_Reuse_or_alloc_node()
{ _M_t._M_erase(static_cast<_Link_type>(_M_root)); }

template<typename _Arg>
 _Link_type
481#if __cplusplus201402L < 201103L
 operator()(const _Arg& __arg)
483#else
 operator()(_Arg&& __arg)
485#endif
 {
   _Link_type __node = static_cast<_Link_type>(_M_extract());
   if (__node)
     {
_M_t._M_destroy_node(__node);
_M_t._M_construct_node(__node, _GLIBCXX_FORWARD(_Arg, __arg)std::forward<_Arg>(__arg));
return __node;
     }

   return _M_t._M_create_node(_GLIBCXX_FORWARD(_Arg, __arg)std::forward<_Arg>(__arg));
 }

    private:
_Base_ptr
_M_extract()
{
 if (!_M_nodes)
   return _M_nodes;

 _Base_ptr __node = _M_nodes;
 _M_nodes = _M_nodes->_M_parent;
 if (_M_nodes)
   {
     if (_M_nodes->_M_right == __node)
{
  _M_nodes->_M_right = 0;

  if (_M_nodes->_M_left)
    {
      _M_nodes = _M_nodes->_M_left;

      while (_M_nodes->_M_right)
	_M_nodes = _M_nodes->_M_right;

      if (_M_nodes->_M_left)
	_M_nodes = _M_nodes->_M_left;
    }
}
     else // __node is on the left.
_M_nodes->_M_left = 0;
   }
 else
   _M_root = 0;

 return __node;
}

_Base_ptr _M_root;
_Base_ptr _M_nodes;
_Rb_tree& _M_t;
    };

    // Functor similar to the previous one but without any pool of nodes to
    // recycle.
    struct _Alloc_node
    {
_Alloc_node(_Rb_tree& __t)
: _M_t(__t) { }

template<typename _Arg>
 _Link_type
547#if __cplusplus201402L < 201103L
 operator()(const _Arg& __arg) const
549#else
 operator()(_Arg&& __arg) const
551#endif
 { return _M_t._M_create_node(_GLIBCXX_FORWARD(_Arg, __arg)std::forward<_Arg>(__arg)); }

    private:
_Rb_tree& _M_t;
    };

  public:
    typedef _Key 				key_type;
    typedef _Val 				value_type;
    typedef value_type* 			pointer;
    typedef const value_type* 		const_pointer;
    typedef value_type& 			reference;
    typedef const value_type& 		const_reference;
    typedef size_t 				size_type;
    typedef ptrdiff_t 			difference_type;
    typedef _Alloc 				allocator_type;

    _Node_allocator&
    _M_get_Node_allocator() _GLIBCXX_NOEXCEPTnoexcept
    { return this->_M_impl; }

    const _Node_allocator&
    _M_get_Node_allocator() const _GLIBCXX_NOEXCEPTnoexcept
    { return this->_M_impl; }

    allocator_type
    get_allocator() const _GLIBCXX_NOEXCEPTnoexcept
    { return allocator_type(_M_get_Node_allocator()); }

  protected:
    _Link_type
    _M_get_node()
    { return _Alloc_traits::allocate(_M_get_Node_allocator(), 1); }

    void
    _M_put_node(_Link_type __p) _GLIBCXX_NOEXCEPTnoexcept
    { _Alloc_traits::deallocate(_M_get_Node_allocator(), __p, 1); }

590#if __cplusplus201402L < 201103L
    void
    _M_construct_node(_Link_type __node, const value_type& __x)
    {
__tryif (true)
 { get_allocator().construct(__node->_M_valptr(), __x); }
__catch(...)if (false)
 {
   _M_put_node(__node);
   __throw_exception_again;
 }
    }

    _Link_type
    _M_create_node(const value_type& __x)
    {
_Link_type __tmp = _M_get_node();
_M_construct_node(__tmp, __x);
return __tmp;
    }
610#else
    template<typename... _Args>
void
_M_construct_node(_Link_type __node, _Args&&... __args)
{
 __tryif (true)
   {
     ::new(__node) _Rb_tree_node<_Val>;
     _Alloc_traits::construct(_M_get_Node_allocator(),
		       __node->_M_valptr(),
		       std::forward<_Args>(__args)...);
   }
 __catch(...)if (false)
   {
     __node->~_Rb_tree_node<_Val>();
     _M_put_node(__node);
     __throw_exception_again;
   }
}

    template<typename... _Args>
_Link_type
_M_create_node(_Args&&... __args)
{
 _Link_type __tmp = _M_get_node();
 _M_construct_node(__tmp, std::forward<_Args>(__args)...);
 return __tmp;
}
638#endif

    void
    _M_destroy_node(_Link_type __p) _GLIBCXX_NOEXCEPTnoexcept
    {
643#if __cplusplus201402L < 201103L
get_allocator().destroy(__p->_M_valptr());
645#else
_Alloc_traits::destroy(_M_get_Node_allocator(), __p->_M_valptr());
__p->~_Rb_tree_node<_Val>();
648#endif
    }

    void
    _M_drop_node(_Link_type __p) _GLIBCXX_NOEXCEPTnoexcept
    {
_M_destroy_node(__p);
_M_put_node(__p);
    }

    template<typename _NodeGen>
_Link_type
_M_clone_node(_Const_Link_type __x, _NodeGen& __node_gen)
{
 _Link_type __tmp = __node_gen(*__x->_M_valptr());
 __tmp->_M_color = __x->_M_color;
 __tmp->_M_left = 0;
 __tmp->_M_right = 0;
 return __tmp;
}

  protected:
670#if _GLIBCXX_INLINE_VERSION0
    template<typename _Key_compare>
672#else
    // Unused _Is_pod_comparator is kept as it is part of mangled name.
    template<typename _Key_compare,
      bool /* _Is_pod_comparator */ = __is_pod(_Key_compare)>
676#endif
struct _Rb_tree_impl
: public _Node_allocator
, public _Rb_tree_key_compare<_Key_compare>
, public _Rb_tree_header
{
 typedef _Rb_tree_key_compare<_Key_compare> _Base_key_compare;

 _Rb_tree_impl()
   _GLIBCXX_NOEXCEPT_IF(noexcept(is_nothrow_default_constructible<_Node_allocator>
::value && is_nothrow_default_constructible<_Base_key_compare
>::value)
is_nothrow_default_constructible<_Node_allocator>::valuenoexcept(is_nothrow_default_constructible<_Node_allocator>
::value && is_nothrow_default_constructible<_Base_key_compare
>::value)
&& is_nothrow_default_constructible<_Base_key_compare>::value )noexcept(is_nothrow_default_constructible<_Node_allocator>
::value && is_nothrow_default_constructible<_Base_key_compare
>::value)
 : _Node_allocator()
 { }

 _Rb_tree_impl(const _Rb_tree_impl& __x)
 : _Node_allocator(_Alloc_traits::_S_select_on_copy(__x))
 , _Base_key_compare(__x._M_key_compare)
 { }

696#if __cplusplus201402L < 201103L
 _Rb_tree_impl(const _Key_compare& __comp, const _Node_allocator& __a)
 : _Node_allocator(__a), _Base_key_compare(__comp)
 { }
700#else
 _Rb_tree_impl(_Rb_tree_impl&&) = default;

 explicit
 _Rb_tree_impl(_Node_allocator&& __a)
 : _Node_allocator(std::move(__a))
 { }

 _Rb_tree_impl(_Rb_tree_impl&& __x, _Node_allocator&& __a)
 : _Node_allocator(std::move(__a)),
   _Base_key_compare(std::move(__x)),
   _Rb_tree_header(std::move(__x))
 { }

 _Rb_tree_impl(const _Key_compare& __comp, _Node_allocator&& __a)
 : _Node_allocator(std::move(__a)), _Base_key_compare(__comp)
 { }
717#endif
};

    _Rb_tree_impl<_Compare> _M_impl;

  protected:
    _Base_ptr&
    _M_root() _GLIBCXX_NOEXCEPTnoexcept
    { return this->_M_impl._M_header._M_parent; }

    _Const_Base_ptr
    _M_root() const _GLIBCXX_NOEXCEPTnoexcept
    { return this->_M_impl._M_header._M_parent; }

    _Base_ptr&
    _M_leftmost() _GLIBCXX_NOEXCEPTnoexcept
    { return this->_M_impl._M_header._M_left; }

    _Const_Base_ptr
    _M_leftmost() const _GLIBCXX_NOEXCEPTnoexcept
    { return this->_M_impl._M_header._M_left; }

    _Base_ptr&
    _M_rightmost() _GLIBCXX_NOEXCEPTnoexcept
    { return this->_M_impl._M_header._M_right; }

    _Const_Base_ptr
    _M_rightmost() const _GLIBCXX_NOEXCEPTnoexcept
    { return this->_M_impl._M_header._M_right; }

    _Link_type
    _M_begin() _GLIBCXX_NOEXCEPTnoexcept
    { return static_cast<_Link_type>(this->_M_impl._M_header._M_parent); }

    _Const_Link_type
    _M_begin() const _GLIBCXX_NOEXCEPTnoexcept
    {
return static_cast<_Const_Link_type>
 (this->_M_impl._M_header._M_parent);
    }

    _Base_ptr
    _M_end() _GLIBCXX_NOEXCEPTnoexcept
    { return &this->_M_impl._M_header; }

    _Const_Base_ptr
    _M_end() const _GLIBCXX_NOEXCEPTnoexcept
    { return &this->_M_impl._M_header; }

    static const _Key&
    _S_key(_Const_Link_type __x)
    {
769#if __cplusplus201402L >= 201103L
// If we're asking for the key we're presumably using the comparison
// object, and so this is a good place to sanity check it.
static_assert(__is_invocable<_Compare&, const _Key&, const _Key&>{},
      "comparison object must be invocable "
      "with two arguments of key type");
775# if __cplusplus201402L >= 201703L
// _GLIBCXX_RESOLVE_LIB_DEFECTS
// 2542. Missing const requirements for associative containers
if constexpr (__is_invocable<_Compare&, const _Key&, const _Key&>{})
 static_assert(
     is_invocable_v<const _Compare&, const _Key&, const _Key&>,
     "comparison object must be invocable as const");
782# endif // C++17
783#endif // C++11

return _KeyOfValue()(*__x->_M_valptr());
    }

    static _Link_type
    _S_left(_Base_ptr __x) _GLIBCXX_NOEXCEPTnoexcept
    { return static_cast<_Link_type>(__x->_M_left); }

    static _Const_Link_type
    _S_left(_Const_Base_ptr __x) _GLIBCXX_NOEXCEPTnoexcept
    { return static_cast<_Const_Link_type>(__x->_M_left); }

    static _Link_type
    _S_right(_Base_ptr __x) _GLIBCXX_NOEXCEPTnoexcept
    { return static_cast<_Link_type>(__x->_M_right); }

    static _Const_Link_type
    _S_right(_Const_Base_ptr __x) _GLIBCXX_NOEXCEPTnoexcept
    { return static_cast<_Const_Link_type>(__x->_M_right); }

    static const _Key&
    _S_key(_Const_Base_ptr __x)
    { return _S_key(static_cast<_Const_Link_type>(__x)); }

    static _Base_ptr
    _S_minimum(_Base_ptr __x) _GLIBCXX_NOEXCEPTnoexcept
    { return _Rb_tree_node_base::_S_minimum(__x); }

    static _Const_Base_ptr
    _S_minimum(_Const_Base_ptr __x) _GLIBCXX_NOEXCEPTnoexcept
    { return _Rb_tree_node_base::_S_minimum(__x); }

    static _Base_ptr
    _S_maximum(_Base_ptr __x) _GLIBCXX_NOEXCEPTnoexcept
    { return _Rb_tree_node_base::_S_maximum(__x); }

    static _Const_Base_ptr
    _S_maximum(_Const_Base_ptr __x) _GLIBCXX_NOEXCEPTnoexcept
    { return _Rb_tree_node_base::_S_maximum(__x); }

  public:
    typedef _Rb_tree_iterator<value_type>       iterator;
    typedef _Rb_tree_const_iterator<value_type> const_iterator;

    typedef std::reverse_iterator<iterator>       reverse_iterator;
    typedef std::reverse_iterator<const_iterator> const_reverse_iterator;

831#if __cplusplus201402L > 201402L
    using node_type = _Node_handle<_Key, _Val, _Node_allocator>;
    using insert_return_type = _Node_insert_return<
conditional_t<is_same_v<_Key, _Val>, const_iterator, iterator>,
node_type>;
836#endif

    pair<_Base_ptr, _Base_ptr>
    _M_get_insert_unique_pos(const key_type& __k);

    pair<_Base_ptr, _Base_ptr>
    _M_get_insert_equal_pos(const key_type& __k);

    pair<_Base_ptr, _Base_ptr>
    _M_get_insert_hint_unique_pos(const_iterator __pos,
		    const key_type& __k);

    pair<_Base_ptr, _Base_ptr>
    _M_get_insert_hint_equal_pos(const_iterator __pos,
		   const key_type& __k);

  private:
853#if __cplusplus201402L >= 201103L
    template<typename _Arg, typename _NodeGen>
iterator
_M_insert_(_Base_ptr __x, _Base_ptr __y, _Arg&& __v, _NodeGen&);

    iterator
    _M_insert_node(_Base_ptr __x, _Base_ptr __y, _Link_type __z);

    template<typename _Arg>
iterator
_M_insert_lower(_Base_ptr __y, _Arg&& __v);

    template<typename _Arg>
iterator
_M_insert_equal_lower(_Arg&& __x);

    iterator
    _M_insert_lower_node(_Base_ptr __p, _Link_type __z);

    iterator
    _M_insert_equal_lower_node(_Link_type __z);
874#else
    template<typename _NodeGen>
iterator
_M_insert_(_Base_ptr __x, _Base_ptr __y,
   const value_type& __v, _NodeGen&);

    // _GLIBCXX_RESOLVE_LIB_DEFECTS
    // 233. Insertion hints in associative containers.
    iterator
    _M_insert_lower(_Base_ptr __y, const value_type& __v);

    iterator
    _M_insert_equal_lower(const value_type& __x);
887#endif

    template<typename _NodeGen>
_Link_type
_M_copy(_Const_Link_type __x, _Base_ptr __p, _NodeGen&);

    template<typename _NodeGen>
_Link_type
_M_copy(const _Rb_tree& __x, _NodeGen& __gen)
{
 _Link_type __root = _M_copy(__x._M_begin(), _M_end(), __gen);
 _M_leftmost() = _S_minimum(__root);
 _M_rightmost() = _S_maximum(__root);
 _M_impl._M_node_count = __x._M_impl._M_node_count;
 return __root;
}

    _Link_type
    _M_copy(const _Rb_tree& __x)
    {
_Alloc_node __an(*this);
return _M_copy(__x, __an);
    }

    void
    _M_erase(_Link_type __x);

    iterator
    _M_lower_bound(_Link_type __x, _Base_ptr __y,
     const _Key& __k);

    const_iterator
    _M_lower_bound(_Const_Link_type __x, _Const_Base_ptr __y,
     const _Key& __k) const;

    iterator
    _M_upper_bound(_Link_type __x, _Base_ptr __y,
     const _Key& __k);

    const_iterator
    _M_upper_bound(_Const_Link_type __x, _Const_Base_ptr __y,
     const _Key& __k) const;

  public:
    // allocation/deallocation
932#if __cplusplus201402L < 201103L
    _Rb_tree() { }
934#else
    _Rb_tree() = default;
936#endif

    _Rb_tree(const _Compare& __comp,
      const allocator_type& __a = allocator_type())
    : _M_impl(__comp, _Node_allocator(__a)) { }

    _Rb_tree(const _Rb_tree& __x)
    : _M_impl(__x._M_impl)
    {
if (__x._M_root() != 0)
 _M_root() = _M_copy(__x);
    }

949#if __cplusplus201402L >= 201103L
    _Rb_tree(const allocator_type& __a)
    : _M_impl(_Node_allocator(__a))
    { }

    _Rb_tree(const _Rb_tree& __x, const allocator_type& __a)
    : _M_impl(__x._M_impl._M_key_compare, _Node_allocator(__a))
    {
if (__x._M_root() != nullptr)
 _M_root() = _M_copy(__x);
    }

    _Rb_tree(_Rb_tree&&) = default;

    _Rb_tree(_Rb_tree&& __x, const allocator_type& __a)
    : _Rb_tree(std::move(__x), _Node_allocator(__a))
    { }

  private:
    _Rb_tree(_Rb_tree&& __x, _Node_allocator&& __a, true_type)
    noexcept(is_nothrow_default_constructible<_Compare>::value)
    : _M_impl(std::move(__x._M_impl), std::move(__a))
    { }

    _Rb_tree(_Rb_tree&& __x, _Node_allocator&& __a, false_type)
    : _M_impl(__x._M_impl._M_key_compare, std::move(__a))
    {
if (__x._M_root() != nullptr)
 _M_move_data(__x, false_type{});
    }

  public:
    _Rb_tree(_Rb_tree&& __x, _Node_allocator&& __a)
    noexcept( noexcept(
_Rb_tree(std::declval<_Rb_tree&&>(), std::declval<_Node_allocator&&>(),
 std::declval<typename _Alloc_traits::is_always_equal>())) )
    : _Rb_tree(std::move(__x), std::move(__a),
 typename _Alloc_traits::is_always_equal{})
    { }
988#endif

    ~_Rb_tree() _GLIBCXX_NOEXCEPTnoexcept
    { _M_erase(_M_begin()); }

    _Rb_tree&
    operator=(const _Rb_tree& __x);

    // Accessors.
    _Compare
    key_comp() const
    { return _M_impl._M_key_compare; }

    iterator
    begin() _GLIBCXX_NOEXCEPTnoexcept
    { return iterator(this->_M_impl._M_header._M_left); }

    const_iterator
    begin() const _GLIBCXX_NOEXCEPTnoexcept
    { return const_iterator(this->_M_impl._M_header._M_left); }

    iterator
    end() _GLIBCXX_NOEXCEPTnoexcept
    { return iterator(&this->_M_impl._M_header); }

    const_iterator
    end() const _GLIBCXX_NOEXCEPTnoexcept
    { return const_iterator(&this->_M_impl._M_header); }

    reverse_iterator
    rbegin() _GLIBCXX_NOEXCEPTnoexcept
    { return reverse_iterator(end()); }

    const_reverse_iterator
    rbegin() const _GLIBCXX_NOEXCEPTnoexcept
    { return const_reverse_iterator(end()); }

    reverse_iterator
    rend() _GLIBCXX_NOEXCEPTnoexcept
    { return reverse_iterator(begin()); }

    const_reverse_iterator
    rend() const _GLIBCXX_NOEXCEPTnoexcept
    { return const_reverse_iterator(begin()); }

    _GLIBCXX_NODISCARD bool
    empty() const _GLIBCXX_NOEXCEPTnoexcept
    { return _M_impl._M_node_count == 0; }

    size_type
    size() const _GLIBCXX_NOEXCEPTnoexcept
    { return _M_impl._M_node_count; }

    size_type
    max_size() const _GLIBCXX_NOEXCEPTnoexcept
    { return _Alloc_traits::max_size(_M_get_Node_allocator()); }

    void
    swap(_Rb_tree& __t)
    _GLIBCXX_NOEXCEPT_IF(__is_nothrow_swappable<_Compare>::value)noexcept(__is_nothrow_swappable<_Compare>::value);

    // Insert/erase.
1050#if __cplusplus201402L >= 201103L
    template<typename _Arg>
pair<iterator, bool>
_M_insert_unique(_Arg&& __x);

    template<typename _Arg>
iterator
_M_insert_equal(_Arg&& __x);

    template<typename _Arg, typename _NodeGen>
iterator
_M_insert_unique_(const_iterator __pos, _Arg&& __x, _NodeGen&);

    template<typename _Arg>
iterator
_M_insert_unique_(const_iterator __pos, _Arg&& __x)
{
 _Alloc_node __an(*this);
 return _M_insert_unique_(__pos, std::forward<_Arg>(__x), __an);
}

    template<typename _Arg, typename _NodeGen>
iterator
_M_insert_equal_(const_iterator __pos, _Arg&& __x, _NodeGen&);

    template<typename _Arg>
iterator
_M_insert_equal_(const_iterator __pos, _Arg&& __x)
{
 _Alloc_node __an(*this);
 return _M_insert_equal_(__pos, std::forward<_Arg>(__x), __an);
}

    template<typename... _Args>
pair<iterator, bool>
_M_emplace_unique(_Args&&... __args);

    template<typename... _Args>
iterator
_M_emplace_equal(_Args&&... __args);

    template<typename... _Args>
iterator
_M_emplace_hint_unique(const_iterator __pos, _Args&&... __args);

    template<typename... _Args>
iterator
_M_emplace_hint_equal(const_iterator __pos, _Args&&... __args);

    template<typename _Iter>
using __same_value_type
 = is_same<value_type, typename iterator_traits<_Iter>::value_type>;

    template<typename _InputIterator>
__enable_if_t<__same_value_type<_InputIterator>::value>
_M_insert_range_unique(_InputIterator __first, _InputIterator __last)
{
 _Alloc_node __an(*this);
 for (; __first != __last; ++__first)
   _M_insert_unique_(end(), *__first, __an);
}

    template<typename _InputIterator>
__enable_if_t<!__same_value_type<_InputIterator>::value>
_M_insert_range_unique(_InputIterator __first, _InputIterator __last)
{
 for (; __first != __last; ++__first)
   _M_emplace_unique(*__first);
}

    template<typename _InputIterator>
__enable_if_t<__same_value_type<_InputIterator>::value>
_M_insert_range_equal(_InputIterator __first, _InputIterator __last)
{
 _Alloc_node __an(*this);
 for (; __first != __last; ++__first)
   _M_insert_equal_(end(), *__first, __an);
}

    template<typename _InputIterator>
__enable_if_t<!__same_value_type<_InputIterator>::value>
_M_insert_range_equal(_InputIterator __first, _InputIterator __last)
{
 _Alloc_node __an(*this);
 for (; __first != __last; ++__first)
   _M_emplace_equal(*__first);
}
1137#else
    pair<iterator, bool>
    _M_insert_unique(const value_type& __x);

    iterator
    _M_insert_equal(const value_type& __x);

    template<typename _NodeGen>
iterator
_M_insert_unique_(const_iterator __pos, const value_type& __x,
	  _NodeGen&);

    iterator
    _M_insert_unique_(const_iterator __pos, const value_type& __x)
    {
_Alloc_node __an(*this);
return _M_insert_unique_(__pos, __x, __an);
    }

    template<typename _NodeGen>
iterator
_M_insert_equal_(const_iterator __pos, const value_type& __x,
	 _NodeGen&);
    iterator
    _M_insert_equal_(const_iterator __pos, const value_type& __x)
    {
_Alloc_node __an(*this);
return _M_insert_equal_(__pos, __x, __an);
    }

    template<typename _InputIterator>
void
_M_insert_range_unique(_InputIterator __first, _InputIterator __last)
{
 _Alloc_node __an(*this);
 for (; __first != __last; ++__first)
   _M_insert_unique_(end(), *__first, __an);
}

    template<typename _InputIterator>
void
_M_insert_range_equal(_InputIterator __first, _InputIterator __last)
{
 _Alloc_node __an(*this);
 for (; __first != __last; ++__first)
   _M_insert_equal_(end(), *__first, __an);
}
1184#endif

  private:
    void
    _M_erase_aux(const_iterator __position);

    void
    _M_erase_aux(const_iterator __first, const_iterator __last);

  public:
1194#if __cplusplus201402L >= 201103L
    // _GLIBCXX_RESOLVE_LIB_DEFECTS
    // DR 130. Associative erase should return an iterator.
    _GLIBCXX_ABI_TAG_CXX11__attribute ((__abi_tag__ ("cxx11")))
    iterator
    erase(const_iterator __position)
    {
__glibcxx_assert(__position != end());
const_iterator __result = __position;
++__result;
_M_erase_aux(__position);
return __result._M_const_cast();
    }

    // LWG 2059.
    _GLIBCXX_ABI_TAG_CXX11__attribute ((__abi_tag__ ("cxx11")))
    iterator
    erase(iterator __position)
    {
__glibcxx_assert(__position != end());
iterator __result = __position;
++__result;
_M_erase_aux(__position);
return __result;
    }
1219#else
    void
    erase(iterator __position)
    {
__glibcxx_assert(__position != end());
_M_erase_aux(__position);
    }

    void
    erase(const_iterator __position)
    {
__glibcxx_assert(__position != end());
_M_erase_aux(__position);
    }
1233#endif

    size_type
    erase(const key_type& __x);

1238#if __cplusplus201402L >= 201103L
    // _GLIBCXX_RESOLVE_LIB_DEFECTS
    // DR 130. Associative erase should return an iterator.
    _GLIBCXX_ABI_TAG_CXX11__attribute ((__abi_tag__ ("cxx11")))
    iterator
    erase(const_iterator __first, const_iterator __last)
    {
_M_erase_aux(__first, __last);
return __last._M_const_cast();
    }
1248#else
    void
    erase(iterator __first, iterator __last)
    { _M_erase_aux(__first, __last); }

    void
    erase(const_iterator __first, const_iterator __last)
    { _M_erase_aux(__first, __last); }
1256#endif

    void
    clear() _GLIBCXX_NOEXCEPTnoexcept
    {
_M_erase(_M_begin());
_M_impl._M_reset();
    }

    // Set operations.
    iterator
    find(const key_type& __k);

    const_iterator
    find(const key_type& __k) const;

    size_type
    count(const key_type& __k) const;

    iterator
    lower_bound(const key_type& __k)
    { return _M_lower_bound(_M_begin(), _M_end(), __k); }

    const_iterator
    lower_bound(const key_type& __k) const
    { return _M_lower_bound(_M_begin(), _M_end(), __k); }

    iterator
    upper_bound(const key_type& __k)
    { return _M_upper_bound(_M_begin(), _M_end(), __k); }

    const_iterator
    upper_bound(const key_type& __k) const
    { return _M_upper_bound(_M_begin(), _M_end(), __k); }

    pair<iterator, iterator>
    equal_range(const key_type& __k);

    pair<const_iterator, const_iterator>
    equal_range(const key_type& __k) const;

1297#if __cplusplus201402L >= 201402L
    template<typename _Kt,
      typename _Req = __has_is_transparent_t<_Compare, _Kt>>
iterator
_M_find_tr(const _Kt& __k)
{
 const _Rb_tree* __const_this = this;
 return __const_this->_M_find_tr(__k)._M_const_cast();
}

    template<typename _Kt,
      typename _Req = __has_is_transparent_t<_Compare, _Kt>>
const_iterator
_M_find_tr(const _Kt& __k) const
{
 auto __j = _M_lower_bound_tr(__k);
 if (__j != end() && _M_impl._M_key_compare(__k, _S_key(__j._M_node)))
   __j = end();
 return __j;
}

    template<typename _Kt,
      typename _Req = __has_is_transparent_t<_Compare, _Kt>>
size_type
_M_count_tr(const _Kt& __k) const
{
 auto __p = _M_equal_range_tr(__k);
 return std::distance(__p.first, __p.second);
}

    template<typename _Kt,
      typename _Req = __has_is_transparent_t<_Compare, _Kt>>
iterator
_M_lower_bound_tr(const _Kt& __k)
{
 const _Rb_tree* __const_this = this;
 return __const_this->_M_lower_bound_tr(__k)._M_const_cast();
}

    template<typename _Kt,
      typename _Req = __has_is_transparent_t<_Compare, _Kt>>
const_iterator
_M_lower_bound_tr(const _Kt& __k) const
{
 auto __x = _M_begin();
 auto __y = _M_end();
 while (__x != 0)
   if (!_M_impl._M_key_compare(_S_key(__x), __k))
     {
__y = __x;
__x = _S_left(__x);
     }
   else
     __x = _S_right(__x);
 return const_iterator(__y);
}

    template<typename _Kt,
      typename _Req = __has_is_transparent_t<_Compare, _Kt>>
iterator
_M_upper_bound_tr(const _Kt& __k)
{
 const _Rb_tree* __const_this = this;
 return __const_this->_M_upper_bound_tr(__k)._M_const_cast();
}

    template<typename _Kt,
      typename _Req = __has_is_transparent_t<_Compare, _Kt>>
const_iterator
_M_upper_bound_tr(const _Kt& __k) const
{
 auto __x = _M_begin();
 auto __y = _M_end();
 while (__x != 0)
   if (_M_impl._M_key_compare(__k, _S_key(__x)))
     {
__y = __x;
__x = _S_left(__x);
     }
   else
     __x = _S_right(__x);
 return const_iterator(__y);
}

    template<typename _Kt,
      typename _Req = __has_is_transparent_t<_Compare, _Kt>>
pair<iterator, iterator>
_M_equal_range_tr(const _Kt& __k)
{
 const _Rb_tree* __const_this = this;
 auto __ret = __const_this->_M_equal_range_tr(__k);
 return { __ret.first._M_const_cast(), __ret.second._M_const_cast() };
}

    template<typename _Kt,
      typename _Req = __has_is_transparent_t<_Compare, _Kt>>
pair<const_iterator, const_iterator>
_M_equal_range_tr(const _Kt& __k) const
{
 auto __low = _M_lower_bound_tr(__k);
 auto __high = __low;
 auto& __cmp = _M_impl._M_key_compare;
 while (__high != end() && !__cmp(__k, _S_key(__high._M_node)))
   ++__high;
 return { __low, __high };
}
1403#endif

    // Debugging.
    bool
    __rb_verify() const;

1409#if __cplusplus201402L >= 201103L
    _Rb_tree&
    operator=(_Rb_tree&&)
    noexcept(_Alloc_traits::_S_nothrow_move()
      && is_nothrow_move_assignable<_Compare>::value);

    template<typename _Iterator>
void
_M_assign_unique(_Iterator, _Iterator);

    template<typename _Iterator>
void
_M_assign_equal(_Iterator, _Iterator);

  private:
    // Move elements from container with equal allocator.
    void
    _M_move_data(_Rb_tree& __x, true_type)
    { _M_impl._M_move_data(__x._M_impl); }

    // Move elements from container with possibly non-equal allocator,
    // which might result in a copy not a move.
    void
    _M_move_data(_Rb_tree&, false_type);

    // Move assignment from container with equal allocator.
    void
    _M_move_assign(_Rb_tree&, true_type);

    // Move assignment from container with possibly non-equal allocator,
    // which might result in a copy not a move.
    void
    _M_move_assign(_Rb_tree&, false_type);
1442#endif

1444#if __cplusplus201402L > 201402L
  public:
    /// Re-insert an extracted node.
    insert_return_type
    _M_reinsert_node_unique(node_type&& __nh)
    {
insert_return_type __ret;
if (__nh.empty())
 __ret.position = end();
else
 {
   __glibcxx_assert(_M_get_Node_allocator() == *__nh._M_alloc);

   auto __res = _M_get_insert_unique_pos(__nh._M_key());
   if (__res.second)
     {
__ret.position
  = _M_insert_node(__res.first, __res.second, __nh._M_ptr);
__nh._M_ptr = nullptr;
__ret.inserted = true;
     }
   else
     {
__ret.node = std::move(__nh);
__ret.position = iterator(__res.first);
__ret.inserted = false;
     }
 }
return __ret;
    }

    /// Re-insert an extracted node.
    iterator
    _M_reinsert_node_equal(node_type&& __nh)
    {
iterator __ret;
if (__nh.empty())
 __ret = end();
else
 {
   __glibcxx_assert(_M_get_Node_allocator() == *__nh._M_alloc);
   auto __res = _M_get_insert_equal_pos(__nh._M_key());
   if (__res.second)
     __ret = _M_insert_node(__res.first, __res.second, __nh._M_ptr);
   else
     __ret = _M_insert_equal_lower_node(__nh._M_ptr);
   __nh._M_ptr = nullptr;
 }
return __ret;
    }

    /// Re-insert an extracted node.
    iterator
    _M_reinsert_node_hint_unique(const_iterator __hint, node_type&& __nh)
    {
iterator __ret;
if (__nh.empty())
 __ret = end();
else
 {
   __glibcxx_assert(_M_get_Node_allocator() == *__nh._M_alloc);
   auto __res = _M_get_insert_hint_unique_pos(__hint, __nh._M_key());
   if (__res.second)
     {
__ret = _M_insert_node(__res.first, __res.second, __nh._M_ptr);
__nh._M_ptr = nullptr;
     }
   else
     __ret = iterator(__res.first);
 }
return __ret;
    }

    /// Re-insert an extracted node.
    iterator
    _M_reinsert_node_hint_equal(const_iterator __hint, node_type&& __nh)
    {
iterator __ret;
if (__nh.empty())
 __ret = end();
else
 {
   __glibcxx_assert(_M_get_Node_allocator() == *__nh._M_alloc);
   auto __res = _M_get_insert_hint_equal_pos(__hint, __nh._M_key());
   if (__res.second)
     __ret = _M_insert_node(__res.first, __res.second, __nh._M_ptr);
   else
     __ret = _M_insert_equal_lower_node(__nh._M_ptr);
   __nh._M_ptr = nullptr;
 }
return __ret;
    }

    /// Extract a node.
    node_type
    extract(const_iterator __pos)
    {
auto __ptr = _Rb_tree_rebalance_for_erase(
   __pos._M_const_cast()._M_node, _M_impl._M_header);
--_M_impl._M_node_count;
return { static_cast<_Link_type>(__ptr), _M_get_Node_allocator() };
    }

    /// Extract a node.
    node_type
    extract(const key_type& __k)
    {
node_type __nh;
auto __pos = find(__k);
if (__pos != end())
 __nh = extract(const_iterator(__pos));
return __nh;
    }

    template<typename _Compare2>
using _Compatible_tree
 = _Rb_tree<_Key, _Val, _KeyOfValue, _Compare2, _Alloc>;

    template<typename, typename>
friend class _Rb_tree_merge_helper;

    /// Merge from a compatible container into one with unique keys.
    template<typename _Compare2>
void
_M_merge_unique(_Compatible_tree<_Compare2>& __src) noexcept
{
 using _Merge_helper = _Rb_tree_merge_helper<_Rb_tree, _Compare2>;
 for (auto __i = __src.begin(), __end = __src.end(); __i != __end;)
   {
     auto __pos = __i++;
     auto __res = _M_get_insert_unique_pos(_KeyOfValue()(*__pos));
     if (__res.second)
{
  auto& __src_impl = _Merge_helper::_S_get_impl(__src);
  auto __ptr = _Rb_tree_rebalance_for_erase(
      __pos._M_node, __src_impl._M_header);
  --__src_impl._M_node_count;
  _M_insert_node(__res.first, __res.second,
		 static_cast<_Link_type>(__ptr));
}
   }
}

    /// Merge from a compatible container into one with equivalent keys.
    template<typename _Compare2>
void
_M_merge_equal(_Compatible_tree<_Compare2>& __src) noexcept
{
 using _Merge_helper = _Rb_tree_merge_helper<_Rb_tree, _Compare2>;
 for (auto __i = __src.begin(), __end = __src.end(); __i != __end;)
   {
     auto __pos = __i++;
     auto __res = _M_get_insert_equal_pos(_KeyOfValue()(*__pos));
     if (__res.second)
{
  auto& __src_impl = _Merge_helper::_S_get_impl(__src);
  auto __ptr = _Rb_tree_rebalance_for_erase(
      __pos._M_node, __src_impl._M_header);
  --__src_impl._M_node_count;
  _M_insert_node(__res.first, __res.second,
		 static_cast<_Link_type>(__ptr));
}
   }
}
1608#endif // C++17

    friend bool
    operator==(const _Rb_tree& __x, const _Rb_tree& __y)
    {
return __x.size() == __y.size()
 && std::equal(__x.begin(), __x.end(), __y.begin());
    }

1617#if __cpp_lib_three_way_comparison
    friend auto
    operator<=>(const _Rb_tree& __x, const _Rb_tree& __y)
    {
if constexpr (requires { typename __detail::__synth3way_t<_Val>; })
 return std::lexicographical_compare_three_way(__x.begin(), __x.end(),
					__y.begin(), __y.end(),
					__detail::__synth3way);
    }
1626#else
    friend bool
    operator<(const _Rb_tree& __x, const _Rb_tree& __y)
    {
return std::lexicographical_compare(__x.begin(), __x.end(),
			    __y.begin(), __y.end());
    }

    friend bool _GLIBCXX_DEPRECATED__attribute__ ((__deprecated__))
    operator!=(const _Rb_tree& __x, const _Rb_tree& __y)
    { return !(__x == __y); }

    friend bool _GLIBCXX_DEPRECATED__attribute__ ((__deprecated__))
    operator>(const _Rb_tree& __x, const _Rb_tree& __y)
    { return __y < __x; }

    friend bool _GLIBCXX_DEPRECATED__attribute__ ((__deprecated__))
    operator<=(const _Rb_tree& __x, const _Rb_tree& __y)
    { return !(__y < __x); }

    friend bool _GLIBCXX_DEPRECATED__attribute__ ((__deprecated__))
    operator>=(const _Rb_tree& __x, const _Rb_tree& __y)
    { return !(__x < __y); }
1649#endif
  };

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  inline void
  swap(_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>& __x,
_Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>& __y)
  { __x.swap(__y); }

1659#if __cplusplus201402L >= 201103L
template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  void
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  _M_move_data(_Rb_tree& __x, false_type)
  {
    if (_M_get_Node_allocator() == __x._M_get_Node_allocator())
_M_move_data(__x, true_type());
    else
{
 _Alloc_node __an(*this);
 auto __lbd =
   [&__an](const value_type& __cval)
   {
     auto& __val = const_cast<value_type&>(__cval);
     return __an(std::move_if_noexcept(__val));
   };
 _M_root() = _M_copy(__x, __lbd);
}
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  inline void
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  _M_move_assign(_Rb_tree& __x, true_type)
  {
    clear();
    if (__x._M_root() != nullptr)
_M_move_data(__x, true_type());
    std::__alloc_on_move(_M_get_Node_allocator(),
	   __x._M_get_Node_allocator());
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  void
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  _M_move_assign(_Rb_tree& __x, false_type)
  {
    if (_M_get_Node_allocator() == __x._M_get_Node_allocator())
return _M_move_assign(__x, true_type{});

    // Try to move each node reusing existing nodes and copying __x nodes
    // structure.
    _Reuse_or_alloc_node __roan(*this);
    _M_impl._M_reset();
    if (__x._M_root() != nullptr)
{
 auto __lbd =
   [&__roan](const value_type& __cval)
   {
     auto& __val = const_cast<value_type&>(__cval);
     return __roan(std::move(__val));
   };
 _M_root() = _M_copy(__x, __lbd);
 __x.clear();
}
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  inline _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>&
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  operator=(_Rb_tree&& __x)
  noexcept(_Alloc_traits::_S_nothrow_move()
    && is_nothrow_move_assignable<_Compare>::value)
  {
    _M_impl._M_key_compare = std::move(__x._M_impl._M_key_compare);
    _M_move_assign(__x, __bool_constant<_Alloc_traits::_S_nothrow_move()>());
    return *this;
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  template<typename _Iterator>
    void
    _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
    _M_assign_unique(_Iterator __first, _Iterator __last)
    {
_Reuse_or_alloc_node __roan(*this);
_M_impl._M_reset();
for (; __first != __last; ++__first)
 _M_insert_unique_(end(), *__first, __roan);
    }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  template<typename _Iterator>
    void
    _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
    _M_assign_equal(_Iterator __first, _Iterator __last)
    {
_Reuse_or_alloc_node __roan(*this);
_M_impl._M_reset();
for (; __first != __last; ++__first)
 _M_insert_equal_(end(), *__first, __roan);
    }
1758#endif

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>&
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  operator=(const _Rb_tree& __x)
  {
    if (this != &__x)
{
 // Note that _Key may be a constant type.
1769#if __cplusplus201402L >= 201103L
 if (_Alloc_traits::_S_propagate_on_copy_assign())
   {
     auto& __this_alloc = this->_M_get_Node_allocator();
     auto& __that_alloc = __x._M_get_Node_allocator();
     if (!_Alloc_traits::_S_always_equal()
  && __this_alloc != __that_alloc)
{
  // Replacement allocator cannot free existing storage, we need
  // to erase nodes first.
  clear();
  std::__alloc_on_copy(__this_alloc, __that_alloc);
}
   }
1783#endif

 _Reuse_or_alloc_node __roan(*this);
 _M_impl._M_reset();
 _M_impl._M_key_compare = __x._M_impl._M_key_compare;
 if (__x._M_root() != 0)
   _M_root() = _M_copy(__x, __roan);
}

    return *this;
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
1797#if __cplusplus201402L >= 201103L
  template<typename _Arg, typename _NodeGen>
1799#else
  template<typename _NodeGen>
1801#endif
    typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::iterator
    _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
    _M_insert_(_Base_ptr __x, _Base_ptr __p,
1805#if __cplusplus201402L >= 201103L
 _Arg&& __v,
1807#else
 const _Val& __v,
1809#endif
 _NodeGen& __node_gen)
    {
bool __insert_left = (__x != 0 || __p == _M_end()
	      || _M_impl._M_key_compare(_KeyOfValue()(__v),
					_S_key(__p)));

_Link_type __z = __node_gen(_GLIBCXX_FORWARD(_Arg, __v)std::forward<_Arg>(__v));

_Rb_tree_insert_and_rebalance(__insert_left, __z, __p,
		      this->_M_impl._M_header);
++_M_impl._M_node_count;
return iterator(__z);
    }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
1826#if __cplusplus201402L >= 201103L
  template<typename _Arg>
1828#endif
  typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::iterator
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
1831#if __cplusplus201402L >= 201103L
  _M_insert_lower(_Base_ptr __p, _Arg&& __v)
1833#else
  _M_insert_lower(_Base_ptr __p, const _Val& __v)
1835#endif
  {
    bool __insert_left = (__p == _M_end()
	    || !_M_impl._M_key_compare(_S_key(__p),
				       _KeyOfValue()(__v)));

    _Link_type __z = _M_create_node(_GLIBCXX_FORWARD(_Arg, __v)std::forward<_Arg>(__v));

    _Rb_tree_insert_and_rebalance(__insert_left, __z, __p,
		    this->_M_impl._M_header);
    ++_M_impl._M_node_count;
    return iterator(__z);
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
1851#if __cplusplus201402L >= 201103L
  template<typename _Arg>
1853#endif
  typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::iterator
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
1856#if __cplusplus201402L >= 201103L
  _M_insert_equal_lower(_Arg&& __v)
1858#else
  _M_insert_equal_lower(const _Val& __v)
1860#endif
  {
    _Link_type __x = _M_begin();
    _Base_ptr __y = _M_end();
    while (__x != 0)
{
 __y = __x;
 __x = !_M_impl._M_key_compare(_S_key(__x), _KeyOfValue()(__v)) ?
_S_left(__x) : _S_right(__x);
}
    return _M_insert_lower(__y, _GLIBCXX_FORWARD(_Arg, __v)std::forward<_Arg>(__v));
  }

template<typename _Key, typename _Val, typename _KoV,
  typename _Compare, typename _Alloc>
  template<typename _NodeGen>
    typename _Rb_tree<_Key, _Val, _KoV, _Compare, _Alloc>::_Link_type
    _Rb_tree<_Key, _Val, _KoV, _Compare, _Alloc>::
    _M_copy(_Const_Link_type __x, _Base_ptr __p, _NodeGen& __node_gen)
    {
// Structural copy. __x and __p must be non-null.
_Link_type __top = _M_clone_node(__x, __node_gen);
__top->_M_parent = __p;

__tryif (true)
 {
   if (__x->_M_right)
     __top->_M_right = _M_copy(_S_right(__x), __top, __node_gen);
   __p = __top;
   __x = _S_left(__x);

   while (__x != 0)
     {
_Link_type __y = _M_clone_node(__x, __node_gen);
__p->_M_left = __y;
__y->_M_parent = __p;
if (__x->_M_right)
  __y->_M_right = _M_copy(_S_right(__x), __y, __node_gen);
__p = __y;
__x = _S_left(__x);
     }
 }
__catch(...)if (false)
 {
   _M_erase(__top);
   __throw_exception_again;
 }
return __top;
    }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  void
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  _M_erase(_Link_type __x)
  {
    // Erase without rebalancing.
    while (__x != 0)
{
 _M_erase(_S_right(__x));
 _Link_type __y = _S_left(__x);
 _M_drop_node(__x);
 __x = __y;
}
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  typename _Rb_tree<_Key, _Val, _KeyOfValue,
      _Compare, _Alloc>::iterator
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  _M_lower_bound(_Link_type __x, _Base_ptr __y,
   const _Key& __k)
  {
    while (__x != 0)
if (!_M_impl._M_key_compare(_S_key(__x), __k))
 __y = __x, __x = _S_left(__x);
else
 __x = _S_right(__x);
    return iterator(__y);
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  typename _Rb_tree<_Key, _Val, _KeyOfValue,
      _Compare, _Alloc>::const_iterator
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  _M_lower_bound(_Const_Link_type __x, _Const_Base_ptr __y,
   const _Key& __k) const
  {
    while (__x != 0)
if (!_M_impl._M_key_compare(_S_key(__x), __k))
 __y = __x, __x = _S_left(__x);
else
 __x = _S_right(__x);
    return const_iterator(__y);
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  typename _Rb_tree<_Key, _Val, _KeyOfValue,
      _Compare, _Alloc>::iterator
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  _M_upper_bound(_Link_type __x, _Base_ptr __y,
   const _Key& __k)
  {
    while (__x != 0)
if (_M_impl._M_key_compare(__k, _S_key(__x)))
 __y = __x, __x = _S_left(__x);
else
 __x = _S_right(__x);
    return iterator(__y);
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  typename _Rb_tree<_Key, _Val, _KeyOfValue,
      _Compare, _Alloc>::const_iterator
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  _M_upper_bound(_Const_Link_type __x, _Const_Base_ptr __y,
   const _Key& __k) const
  {
    while (__x != 0)
if (_M_impl._M_key_compare(__k, _S_key(__x)))
 __y = __x, __x = _S_left(__x);
else
 __x = _S_right(__x);
    return const_iterator(__y);
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  pair<typename _Rb_tree<_Key, _Val, _KeyOfValue,
	   _Compare, _Alloc>::iterator,
typename _Rb_tree<_Key, _Val, _KeyOfValue,
	   _Compare, _Alloc>::iterator>
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  equal_range(const _Key& __k)
  {
    _Link_type __x = _M_begin();
    _Base_ptr __y = _M_end();
    while (__x != 0)
{
 if (_M_impl._M_key_compare(_S_key(__x), __k))
   __x = _S_right(__x);
 else if (_M_impl._M_key_compare(__k, _S_key(__x)))
   __y = __x, __x = _S_left(__x);
 else
   {
     _Link_type __xu(__x);
     _Base_ptr __yu(__y);
     __y = __x, __x = _S_left(__x);
     __xu = _S_right(__xu);
     return pair<iterator,
	  iterator>(_M_lower_bound(__x, __y, __k),
		    _M_upper_bound(__xu, __yu, __k));
   }
}
    return pair<iterator, iterator>(iterator(__y),
		      iterator(__y));
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  pair<typename _Rb_tree<_Key, _Val, _KeyOfValue,
	   _Compare, _Alloc>::const_iterator,
typename _Rb_tree<_Key, _Val, _KeyOfValue,
	   _Compare, _Alloc>::const_iterator>
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  equal_range(const _Key& __k) const
  {
    _Const_Link_type __x = _M_begin();
    _Const_Base_ptr __y = _M_end();
    while (__x != 0)
{
 if (_M_impl._M_key_compare(_S_key(__x), __k))
   __x = _S_right(__x);
 else if (_M_impl._M_key_compare(__k, _S_key(__x)))
   __y = __x, __x = _S_left(__x);
 else
   {
     _Const_Link_type __xu(__x);
     _Const_Base_ptr __yu(__y);
     __y = __x, __x = _S_left(__x);
     __xu = _S_right(__xu);
     return pair<const_iterator,
	  const_iterator>(_M_lower_bound(__x, __y, __k),
			  _M_upper_bound(__xu, __yu, __k));
   }
}
    return pair<const_iterator, const_iterator>(const_iterator(__y),
				  const_iterator(__y));
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  void
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  swap(_Rb_tree& __t)
  _GLIBCXX_NOEXCEPT_IF(__is_nothrow_swappable<_Compare>::value)noexcept(__is_nothrow_swappable<_Compare>::value)
  {
    if (_M_root() == 0)
{
 if (__t._M_root() != 0)
   _M_impl._M_move_data(__t._M_impl);
}
    else if (__t._M_root() == 0)
__t._M_impl._M_move_data(_M_impl);
    else
{
 std::swap(_M_root(),__t._M_root());
 std::swap(_M_leftmost(),__t._M_leftmost());
 std::swap(_M_rightmost(),__t._M_rightmost());

 _M_root()->_M_parent = _M_end();
 __t._M_root()->_M_parent = __t._M_end();
 std::swap(this->_M_impl._M_node_count, __t._M_impl._M_node_count);
}
    // No need to swap header's color as it does not change.
    std::swap(this->_M_impl._M_key_compare, __t._M_impl._M_key_compare);

    _Alloc_traits::_S_on_swap(_M_get_Node_allocator(),
		__t._M_get_Node_allocator());
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  pair<typename _Rb_tree<_Key, _Val, _KeyOfValue,
	   _Compare, _Alloc>::_Base_ptr,
typename _Rb_tree<_Key, _Val, _KeyOfValue,
	   _Compare, _Alloc>::_Base_ptr>
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  _M_get_insert_unique_pos(const key_type& __k)
  {
    typedef pair<_Base_ptr, _Base_ptr> _Res;
    _Link_type __x = _M_begin();
    _Base_ptr __y = _M_end();
    bool __comp = true;
    while (__x != 0)
{
 __y = __x;
 __comp = _M_impl._M_key_compare(__k, _S_key(__x));
 __x = __comp ? _S_left(__x) : _S_right(__x);
}
    iterator __j = iterator(__y);
    if (__comp)
{
 if (__j == begin())
   return _Res(__x, __y);
 else
   --__j;
}
    if (_M_impl._M_key_compare(_S_key(__j._M_node), __k))
return _Res(__x, __y);
    return _Res(__j._M_node, 0);
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  pair<typename _Rb_tree<_Key, _Val, _KeyOfValue,
	   _Compare, _Alloc>::_Base_ptr,
typename _Rb_tree<_Key, _Val, _KeyOfValue,
	   _Compare, _Alloc>::_Base_ptr>
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  _M_get_insert_equal_pos(const key_type& __k)
  {
    typedef pair<_Base_ptr, _Base_ptr> _Res;
    _Link_type __x = _M_begin();
    _Base_ptr __y = _M_end();
    while (__x != 0)
{
 __y = __x;
 __x = _M_impl._M_key_compare(__k, _S_key(__x)) ?
_S_left(__x) : _S_right(__x);
}
    return _Res(__x, __y);
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
2140#if __cplusplus201402L >= 201103L
  template<typename _Arg>
2142#endif
  pair<typename _Rb_tree<_Key, _Val, _KeyOfValue,
	   _Compare, _Alloc>::iterator, bool>
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
2146#if __cplusplus201402L >= 201103L
  _M_insert_unique(_Arg&& __v)
2148#else
  _M_insert_unique(const _Val& __v)
2150#endif
  {
    typedef pair<iterator, bool> _Res;
    pair<_Base_ptr, _Base_ptr> __res
= _M_get_insert_unique_pos(_KeyOfValue()(__v));

    if (__res.second)
{
 _Alloc_node __an(*this);
 return _Res(_M_insert_(__res.first, __res.second,
		 _GLIBCXX_FORWARD(_Arg, __v)std::forward<_Arg>(__v), __an),
      true);
}

    return _Res(iterator(__res.first), false);
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
2169#if __cplusplus201402L >= 201103L
  template<typename _Arg>
2171#endif
  typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::iterator
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
2174#if __cplusplus201402L >= 201103L
  _M_insert_equal(_Arg&& __v)
2176#else
  _M_insert_equal(const _Val& __v)
2178#endif
  {
    pair<_Base_ptr, _Base_ptr> __res
= _M_get_insert_equal_pos(_KeyOfValue()(__v));
    _Alloc_node __an(*this);
    return _M_insert_(__res.first, __res.second,
	_GLIBCXX_FORWARD(_Arg, __v)std::forward<_Arg>(__v), __an);
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  pair<typename _Rb_tree<_Key, _Val, _KeyOfValue,
	   _Compare, _Alloc>::_Base_ptr,
typename _Rb_tree<_Key, _Val, _KeyOfValue,
	   _Compare, _Alloc>::_Base_ptr>
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  _M_get_insert_hint_unique_pos(const_iterator __position,
		  const key_type& __k)
  {
    iterator __pos = __position._M_const_cast();
    typedef pair<_Base_ptr, _Base_ptr> _Res;

    // end()
    if (__pos._M_node == _M_end())
{
 if (size() > 0
     && _M_impl._M_key_compare(_S_key(_M_rightmost()), __k))
   return _Res(0, _M_rightmost());
 else
   return _M_get_insert_unique_pos(__k);
}
    else if (_M_impl._M_key_compare(__k, _S_key(__pos._M_node)))
{
 // First, try before...
 iterator __before = __pos;
 if (__pos._M_node == _M_leftmost()) // begin()
   return _Res(_M_leftmost(), _M_leftmost());
 else if (_M_impl._M_key_compare(_S_key((--__before)._M_node), __k))
   {
     if (_S_right(__before._M_node) == 0)
return _Res(0, __before._M_node);
     else
return _Res(__pos._M_node, __pos._M_node);
   }
 else
   return _M_get_insert_unique_pos(__k);
}
    else if (_M_impl._M_key_compare(_S_key(__pos._M_node), __k))
{
 // ... then try after.
 iterator __after = __pos;
 if (__pos._M_node == _M_rightmost())
   return _Res(0, _M_rightmost());
 else if (_M_impl._M_key_compare(__k, _S_key((++__after)._M_node)))
   {
     if (_S_right(__pos._M_node) == 0)
return _Res(0, __pos._M_node);
     else
return _Res(__after._M_node, __after._M_node);
   }
 else
   return _M_get_insert_unique_pos(__k);
}
    else
// Equivalent keys.
return _Res(__pos._M_node, 0);
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
2248#if __cplusplus201402L >= 201103L
  template<typename _Arg, typename _NodeGen>
2250#else
  template<typename _NodeGen>
2252#endif
    typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::iterator
    _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
    _M_insert_unique_(const_iterator __position,
2256#if __cplusplus201402L >= 201103L
	_Arg&& __v,
2258#else
	const _Val& __v,
2260#endif
	_NodeGen& __node_gen)
  {
    pair<_Base_ptr, _Base_ptr> __res
= _M_get_insert_hint_unique_pos(__position, _KeyOfValue()(__v));

    if (__res.second)
return _M_insert_(__res.first, __res.second,
	  _GLIBCXX_FORWARD(_Arg, __v)std::forward<_Arg>(__v),
	  __node_gen);
    return iterator(__res.first);
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  pair<typename _Rb_tree<_Key, _Val, _KeyOfValue,
	   _Compare, _Alloc>::_Base_ptr,
typename _Rb_tree<_Key, _Val, _KeyOfValue,
	   _Compare, _Alloc>::_Base_ptr>
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  _M_get_insert_hint_equal_pos(const_iterator __position, const key_type& __k)
  {
    iterator __pos = __position._M_const_cast();
    typedef pair<_Base_ptr, _Base_ptr> _Res;

    // end()
    if (__pos._M_node == _M_end())
{
 if (size() > 0
     && !_M_impl._M_key_compare(__k, _S_key(_M_rightmost())))
   return _Res(0, _M_rightmost());
 else
   return _M_get_insert_equal_pos(__k);
}
    else if (!_M_impl._M_key_compare(_S_key(__pos._M_node), __k))
{
 // First, try before...
 iterator __before = __pos;
 if (__pos._M_node == _M_leftmost()) // begin()
   return _Res(_M_leftmost(), _M_leftmost());
 else if (!_M_impl._M_key_compare(__k, _S_key((--__before)._M_node)))
   {
     if (_S_right(__before._M_node) == 0)
return _Res(0, __before._M_node);
     else
return _Res(__pos._M_node, __pos._M_node);
   }
 else
   return _M_get_insert_equal_pos(__k);
}
    else
{
 // ... then try after.
 iterator __after = __pos;
 if (__pos._M_node == _M_rightmost())
   return _Res(0, _M_rightmost());
 else if (!_M_impl._M_key_compare(_S_key((++__after)._M_node), __k))
   {
     if (_S_right(__pos._M_node) == 0)
return _Res(0, __pos._M_node);
     else
return _Res(__after._M_node, __after._M_node);
   }
 else
   return _Res(0, 0);
}
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
2330#if __cplusplus201402L >= 201103L
  template<typename _Arg, typename _NodeGen>
2332#else
  template<typename _NodeGen>
2334#endif
    typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::iterator
    _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
    _M_insert_equal_(const_iterator __position,
2338#if __cplusplus201402L >= 201103L
       _Arg&& __v,
2340#else
       const _Val& __v,
2342#endif
       _NodeGen& __node_gen)
    {
pair<_Base_ptr, _Base_ptr> __res
 = _M_get_insert_hint_equal_pos(__position, _KeyOfValue()(__v));

if (__res.second)
 return _M_insert_(__res.first, __res.second,
	    _GLIBCXX_FORWARD(_Arg, __v)std::forward<_Arg>(__v),
	    __node_gen);

return _M_insert_equal_lower(_GLIBCXX_FORWARD(_Arg, __v)std::forward<_Arg>(__v));
    }

2356#if __cplusplus201402L >= 201103L
template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::iterator
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  _M_insert_node(_Base_ptr __x, _Base_ptr __p, _Link_type __z)
  {
    bool __insert_left = (__x != 0 || __p == _M_end()
	    || _M_impl._M_key_compare(_S_key(__z),
				      _S_key(__p)));

    _Rb_tree_insert_and_rebalance(__insert_left, __z, __p,
		    this->_M_impl._M_header);
    ++_M_impl._M_node_count;
    return iterator(__z);
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::iterator
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  _M_insert_lower_node(_Base_ptr __p, _Link_type __z)
  {
    bool __insert_left = (__p == _M_end()
	    || !_M_impl._M_key_compare(_S_key(__p),
				       _S_key(__z)));

    _Rb_tree_insert_and_rebalance(__insert_left, __z, __p,
		    this->_M_impl._M_header);
    ++_M_impl._M_node_count;
    return iterator(__z);
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::iterator
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  _M_insert_equal_lower_node(_Link_type __z)
  {
    _Link_type __x = _M_begin();
    _Base_ptr __y = _M_end();
    while (__x != 0)
{
 __y = __x;
 __x = !_M_impl._M_key_compare(_S_key(__x), _S_key(__z)) ?
_S_left(__x) : _S_right(__x);
}
    return _M_insert_lower_node(__y, __z);
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  template<typename... _Args>
    pair<typename _Rb_tree<_Key, _Val, _KeyOfValue,
	     _Compare, _Alloc>::iterator, bool>
    _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
    _M_emplace_unique(_Args&&... __args)
    {
_Link_type __z = _M_create_node(std::forward<_Args>(__args)...);

__tryif (true)
 {
   typedef pair<iterator, bool> _Res;
   auto __res = _M_get_insert_unique_pos(_S_key(__z));
   if (__res.second)
     return _Res(_M_insert_node(__res.first, __res.second, __z), true);

   _M_drop_node(__z);
   return _Res(iterator(__res.first), false);
 }
__catch(...)if (false)
 {
   _M_drop_node(__z);
   __throw_exception_again;
 }
    }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  template<typename... _Args>
    typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::iterator
    _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
    _M_emplace_equal(_Args&&... __args)
    {
_Link_type __z = _M_create_node(std::forward<_Args>(__args)...);

__tryif (true)
 {
   auto __res = _M_get_insert_equal_pos(_S_key(__z));
   return _M_insert_node(__res.first, __res.second, __z);
 }
__catch(...)if (false)
 {
   _M_drop_node(__z);
   __throw_exception_again;
 }
    }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  template<typename... _Args>
    typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::iterator
    _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
    _M_emplace_hint_unique(const_iterator __pos, _Args&&... __args)
    {
_Link_type __z = _M_create_node(std::forward<_Args>(__args)...);

__tryif (true)
 {
   auto __res = _M_get_insert_hint_unique_pos(__pos, _S_key(__z));

   if (__res.second)
     return _M_insert_node(__res.first, __res.second, __z);

   _M_drop_node(__z);
   return iterator(__res.first);
 }
__catch(...)if (false)
 {
   _M_drop_node(__z);
   __throw_exception_again;
 }
    }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  template<typename... _Args>
    typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::iterator
    _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
    _M_emplace_hint_equal(const_iterator __pos, _Args&&... __args)
    {
_Link_type __z = _M_create_node(std::forward<_Args>(__args)...);

__tryif (true)
 {
   auto __res = _M_get_insert_hint_equal_pos(__pos, _S_key(__z));

   if (__res.second)
     return _M_insert_node(__res.first, __res.second, __z);

   return _M_insert_equal_lower_node(__z);
 }
__catch(...)if (false)
 {
   _M_drop_node(__z);
   __throw_exception_again;
 }
    }
2504#endif


template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  void
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  _M_erase_aux(const_iterator __position)
  {
    _Link_type __y =
static_cast<_Link_type>(_Rb_tree_rebalance_for_erase
		(const_cast<_Base_ptr>(__position._M_node),
		 this->_M_impl._M_header));
    _M_drop_node(__y);
    --_M_impl._M_node_count;
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  void
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  _M_erase_aux(const_iterator __first, const_iterator __last)
  {
    if (__first == begin() && __last == end())
clear();
    else
while (__first != __last)
 _M_erase_aux(__first++);
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::size_type
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  erase(const _Key& __x)
  {
    pair<iterator, iterator> __p = equal_range(__x);
    const size_type __old_size = size();
    _M_erase_aux(__p.first, __p.second);
    return __old_size - size();
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  typename _Rb_tree<_Key, _Val, _KeyOfValue,
      _Compare, _Alloc>::iterator
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  find(const _Key& __k)
  {
    iterator __j = _M_lower_bound(_M_begin(), _M_end(), __k);
    return (__j == end()
     || _M_impl._M_key_compare(__k,
			_S_key(__j._M_node))) ? end() : __j;
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  typename _Rb_tree<_Key, _Val, _KeyOfValue,
      _Compare, _Alloc>::const_iterator
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  find(const _Key& __k) const
  {
    const_iterator __j = _M_lower_bound(_M_begin(), _M_end(), __k);
    return (__j == end()
     || _M_impl._M_key_compare(__k,
			_S_key(__j._M_node))) ? end() : __j;
  }

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  typename _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::size_type
  _Rb_tree<_Key, _Val, _KeyOfValue, _Compare, _Alloc>::
  count(const _Key& __k) const
  {
    pair<const_iterator, const_iterator> __p = equal_range(__k);
    const size_type __n = std::distance(__p.first, __p.second);
    return __n;
  }

_GLIBCXX_PURE__attribute__ ((__pure__)) unsigned int
_Rb_tree_black_count(const _Rb_tree_node_base* __node,
       const _Rb_tree_node_base* __root) throw ();

template<typename _Key, typename _Val, typename _KeyOfValue,
  typename _Compare, typename _Alloc>
  bool
  _Rb_tree<_Key,_Val,_KeyOfValue,_Compare,_Alloc>::__rb_verify() const
  {
    if (_M_impl._M_node_count == 0 || begin() == end())
return _M_impl._M_node_count == 0 && begin() == end()
      && this->_M_impl._M_header._M_left == _M_end()
      && this->_M_impl._M_header._M_right == _M_end();

    unsigned int __len = _Rb_tree_black_count(_M_leftmost(), _M_root());
    for (const_iterator __it = begin(); __it != end(); ++__it)
{
 _Const_Link_type __x = static_cast<_Const_Link_type>(__it._M_node);
 _Const_Link_type __L = _S_left(__x);
 _Const_Link_type __R = _S_right(__x);

 if (__x->_M_color == _S_red)
   if ((__L && __L->_M_color == _S_red)
|| (__R && __R->_M_color == _S_red))
     return false;

 if (__L && _M_impl._M_key_compare(_S_key(__x), _S_key(__L)))
   return false;
 if (__R && _M_impl._M_key_compare(_S_key(__R), _S_key(__x)))
   return false;

 if (!__L && !__R && _Rb_tree_black_count(__x, _M_root()) != __len)
   return false;
}

    if (_M_leftmost() != _Rb_tree_node_base::_S_minimum(_M_root()))
return false;
    if (_M_rightmost() != _Rb_tree_node_base::_S_maximum(_M_root()))
return false;
    return true;
  }

2625#if __cplusplus201402L > 201402L
// Allow access to internals of compatible _Rb_tree specializations.
template<typename _Key, typename _Val, typename _Sel, typename _Cmp1,
  typename _Alloc, typename _Cmp2>
  struct _Rb_tree_merge_helper<_Rb_tree<_Key, _Val, _Sel, _Cmp1, _Alloc>,
		 _Cmp2>
  {
  private:
    friend class _Rb_tree<_Key, _Val, _Sel, _Cmp1, _Alloc>;

    static auto&
    _S_get_impl(_Rb_tree<_Key, _Val, _Sel, _Cmp2, _Alloc>& __tree)
    { return __tree._M_impl; }
  };
2639#endif // C++17

2641_GLIBCXX_END_NAMESPACE_VERSION
2642} // namespace

2644#endif