/build/source/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp

Bug Summary

File:	build/source/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
Warning:	line 4209, column 24 2nd function call argument is an uninitialized value
Annotated Source Code

Press '?' to see keyboard shortcuts
Show analyzer invocation
clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name SPIRVOps.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-16/lib/clang/16 -I tools/mlir/lib/Dialect/SPIRV/IR -I /build/source/mlir/lib/Dialect/SPIRV/IR -I include -I /build/source/llvm/include -I /build/source/mlir/include -I tools/mlir/include -D MLIR_CUDA_CONVERSIONS_ENABLED=1 -D MLIR_ROCM_CONVERSIONS_ENABLED=1 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-16/lib/clang/16/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/source/= -fcoverage-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/source/= -source-date-epoch 1671487667 -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-12-20-010714-16201-1 -x c++ /build/source/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp
1//===- SPIRVOps.cpp - MLIR SPIR-V operations ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the operations in the SPIR-V dialect.
10//
11//===----------------------------------------------------------------------===//

13#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"

15#include "mlir/Dialect/SPIRV/IR/ParserUtils.h"
16#include "mlir/Dialect/SPIRV/IR/SPIRVAttributes.h"
17#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
18#include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h"
19#include "mlir/Dialect/SPIRV/IR/SPIRVOpTraits.h"
20#include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
21#include "mlir/Dialect/SPIRV/IR/TargetAndABI.h"
22#include "mlir/IR/Builders.h"
23#include "mlir/IR/BuiltinTypes.h"
24#include "mlir/IR/FunctionImplementation.h"
25#include "mlir/IR/OpDefinition.h"
26#include "mlir/IR/OpImplementation.h"
27#include "mlir/IR/Operation.h"
28#include "mlir/IR/TypeUtilities.h"
29#include "mlir/Interfaces/CallInterfaces.h"
30#include "llvm/ADT/APFloat.h"
31#include "llvm/ADT/APInt.h"
32#include "llvm/ADT/ArrayRef.h"
33#include "llvm/ADT/STLExtras.h"
34#include "llvm/ADT/StringExtras.h"
35#include "llvm/Support/FormatVariadic.h"
36#include <cassert>
37#include <numeric>

39using namespace mlir;

41// TODO: generate these strings using ODS.
42constexpr char kAlignmentAttrName[] = "alignment";
43constexpr char kBranchWeightAttrName[] = "branch_weights";
44constexpr char kCallee[] = "callee";
45constexpr char kClusterSize[] = "cluster_size";
46constexpr char kControl[] = "control";
47constexpr char kDefaultValueAttrName[] = "default_value";
48constexpr char kEqualSemanticsAttrName[] = "equal_semantics";
49constexpr char kExecutionScopeAttrName[] = "execution_scope";
50constexpr char kFnNameAttrName[] = "fn";
51constexpr char kGroupOperationAttrName[] = "group_operation";
52constexpr char kIndicesAttrName[] = "indices";
53constexpr char kInitializerAttrName[] = "initializer";
54constexpr char kInterfaceAttrName[] = "interface";
55constexpr char kMemoryAccessAttrName[] = "memory_access";
56constexpr char kMemoryScopeAttrName[] = "memory_scope";
57constexpr char kPackedVectorFormatAttrName[] = "format";
58constexpr char kSemanticsAttrName[] = "semantics";
59constexpr char kSourceAlignmentAttrName[] = "source_alignment";
60constexpr char kSourceMemoryAccessAttrName[] = "source_memory_access";
61constexpr char kSpecIdAttrName[] = "spec_id";
62constexpr char kTypeAttrName[] = "type";
63constexpr char kUnequalSemanticsAttrName[] = "unequal_semantics";
64constexpr char kValueAttrName[] = "value";
65constexpr char kValuesAttrName[] = "values";
66constexpr char kCompositeSpecConstituentsName[] = "constituents";

68//===----------------------------------------------------------------------===//
69// Common utility functions
70//===----------------------------------------------------------------------===//

72static ParseResult parseOneResultSameOperandTypeOp(OpAsmParser &parser,
                                                 OperationState &result) {
SmallVector<OpAsmParser::UnresolvedOperand, 2> ops;
Type type;
// If the operand list is in-between parentheses, then we have a generic form.
// (see the fallback in `printOneResultOp`).
SMLoc loc = parser.getCurrentLocation();
if (!parser.parseOptionalLParen()) {
  if (parser.parseOperandList(ops) || parser.parseRParen() ||
      parser.parseOptionalAttrDict(result.attributes) ||
      parser.parseColon() || parser.parseType(type))
    return failure();
  auto fnType = type.dyn_cast<FunctionType>();
  if (!fnType) {
    parser.emitError(loc, "expected function type");
    return failure();
  }
  if (parser.resolveOperands(ops, fnType.getInputs(), loc, result.operands))
    return failure();
  result.addTypes(fnType.getResults());
  return success();
}
return failure(parser.parseOperandList(ops) ||
               parser.parseOptionalAttrDict(result.attributes) ||
               parser.parseColonType(type) ||
               parser.resolveOperands(ops, type, result.operands) ||
               parser.addTypeToList(type, result.types));
99}

101static void printOneResultOp(Operation *op, OpAsmPrinter &p) {
assert(op->getNumResults() == 1 && "op should have one result")(static_cast <bool> (op->getNumResults() == 1 &&
 "op should have one result") ? void (0) : __assert_fail ("op->getNumResults() == 1 && \"op should have one result\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 102, __extension__
 __PRETTY_FUNCTION__));

// If not all the operand and result types are the same, just use the
// generic assembly form to avoid omitting information in printing.
auto resultType = op->getResult(0).getType();
if (llvm::any_of(op->getOperandTypes(),
                 [&](Type type) { return type != resultType; })) {
  p.printGenericOp(op, /*printOpName=*/false);
  return;
}

p << ' ';
p.printOperands(op->getOperands());
p.printOptionalAttrDict(op->getAttrs());
// Now we can output only one type for all operands and the result.
p << " : " << resultType;
118}

120/// Returns true if the given op is a function-like op or nested in a
121/// function-like op without a module-like op in the middle.
122static bool isNestedInFunctionOpInterface(Operation *op) {
if (!op)
  return false;
if (op->hasTrait<OpTrait::SymbolTable>())
  return false;
if (isa<FunctionOpInterface>(op))
  return true;
return isNestedInFunctionOpInterface(op->getParentOp());
130}

132/// Returns true if the given op is an module-like op that maintains a symbol
133/// table.
134static bool isDirectInModuleLikeOp(Operation *op) {
return op && op->hasTrait<OpTrait::SymbolTable>();
136}

138static LogicalResult extractValueFromConstOp(Operation *op, int32_t &value) {
auto constOp = dyn_cast_or_null<spirv::ConstantOp>(op);
if (!constOp) {
  return failure();
}
auto valueAttr = constOp.getValue();
auto integerValueAttr = valueAttr.dyn_cast<IntegerAttr>();
if (!integerValueAttr) {
  return failure();
}

if (integerValueAttr.getType().isSignlessInteger())
  value = integerValueAttr.getInt();
else
  value = integerValueAttr.getSInt();

return success();
155}

157template <typename Ty>
158static ArrayAttr
159getStrArrayAttrForEnumList(Builder &builder, ArrayRef<Ty> enumValues,
                         function_ref<StringRef(Ty)> stringifyFn) {
if (enumValues.empty()) {
  return nullptr;
}
SmallVector<StringRef, 1> enumValStrs;
enumValStrs.reserve(enumValues.size());
for (auto val : enumValues) {
  enumValStrs.emplace_back(stringifyFn(val));
}
return builder.getStrArrayAttr(enumValStrs);
170}

172/// Parses the next string attribute in `parser` as an enumerant of the given
173/// `EnumClass`.
174template <typename EnumClass>
175static ParseResult
176parseEnumStrAttr(EnumClass &value, OpAsmParser &parser,
               StringRef attrName = spirv::attributeName<EnumClass>()) {
Attribute attrVal;
NamedAttrList attr;
auto loc = parser.getCurrentLocation();
if (parser.parseAttribute(attrVal, parser.getBuilder().getNoneType(),
3
←
Taking false branch→
                          attrName, attr))
  return failure();
if (!attrVal.isa<StringAttr>())
4
←
Taking true branch→
  return parser.emitError(loc, "expected ")
5
←
Returning without writing to 'value'→
         << attrName << " attribute specified as string";
auto attrOptional =
    spirv::symbolizeEnum<EnumClass>(attrVal.cast<StringAttr>().getValue());
if (!attrOptional)
  return parser.emitError(loc, "invalid ")
         << attrName << " attribute specification: " << attrVal;
value = *attrOptional;
return success();
194}

196/// Parses the next string attribute in `parser` as an enumerant of the given
197/// `EnumClass` and inserts the enumerant into `state` as an 32-bit integer
198/// attribute with the enum class's name as attribute name.
199template <typename EnumAttrClass,
        typename EnumClass = typename EnumAttrClass::ValueType>
201static ParseResult
202parseEnumStrAttr(EnumClass &value, OpAsmParser &parser, OperationState &state,
               StringRef attrName = spirv::attributeName<EnumClass>()) {
if (parseEnumStrAttr(value, parser))
  return failure();
state.addAttribute(attrName,
                   parser.getBuilder().getAttr<EnumAttrClass>(value));
return success();
209}

211/// Parses the next keyword in `parser` as an enumerant of the given `EnumClass`
212/// and inserts the enumerant into `state` as an 32-bit integer attribute with
213/// the enum class's name as attribute name.
214template <typename EnumAttrClass,
        typename EnumClass = typename EnumAttrClass::ValueType>
216static ParseResult
217parseEnumKeywordAttr(EnumClass &value, OpAsmParser &parser,
                   OperationState &state,
                   StringRef attrName = spirv::attributeName<EnumClass>()) {
if (parseEnumKeywordAttr(value, parser))
  return failure();
state.addAttribute(attrName,
                   parser.getBuilder().getAttr<EnumAttrClass>(value));
return success();
225}

227/// Parses Function, Selection and Loop control attributes. If no control is
228/// specified, "None" is used as a default.
229template <typename EnumAttrClass, typename EnumClass>
230static ParseResult
231parseControlAttribute(OpAsmParser &parser, OperationState &state,
                    StringRef attrName = spirv::attributeName<EnumClass>()) {
if (succeeded(parser.parseOptionalKeyword(kControl))) {
  EnumClass control;
  if (parser.parseLParen() ||
      parseEnumKeywordAttr<EnumAttrClass>(control, parser, state) ||
      parser.parseRParen())
    return failure();
  return success();
}
// Set control to "None" otherwise.
Builder builder = parser.getBuilder();
state.addAttribute(attrName,
                   builder.getAttr<EnumAttrClass>(static_cast<EnumClass>(0)));
return success();
246}

248/// Parses optional memory access attributes attached to a memory access
249/// operand/pointer. Specifically, parses the following syntax:
250///     (`[` memory-access `]`)?
251/// where:
252///     memory-access ::= `"None"` | `"Volatile"` | `"Aligned", `
253///         integer-literal | `"NonTemporal"`
254static ParseResult parseMemoryAccessAttributes(OpAsmParser &parser,
                                             OperationState &state) {
// Parse an optional list of attributes staring with '['
if (parser.parseOptionalLSquare()) {
  // Nothing to do
  return success();
}

spirv::MemoryAccess memoryAccessAttr;
if (parseEnumStrAttr<spirv::MemoryAccessAttr>(memoryAccessAttr, parser, state,
                                              kMemoryAccessAttrName))
  return failure();

if (spirv::bitEnumContainsAll(memoryAccessAttr,
                              spirv::MemoryAccess::Aligned)) {
  // Parse integer attribute for alignment.
  Attribute alignmentAttr;
  Type i32Type = parser.getBuilder().getIntegerType(32);
  if (parser.parseComma() ||
      parser.parseAttribute(alignmentAttr, i32Type, kAlignmentAttrName,
                            state.attributes)) {
    return failure();
  }
}
return parser.parseRSquare();
279}

281// TODO Make sure to merge this and the previous function into one template
282// parameterized by memory access attribute name and alignment. Doing so now
283// results in VS2017 in producing an internal error (at the call site) that's
284// not detailed enough to understand what is happening.
285static ParseResult parseSourceMemoryAccessAttributes(OpAsmParser &parser,
                                                   OperationState &state) {
// Parse an optional list of attributes staring with '['
if (parser.parseOptionalLSquare()) {
  // Nothing to do
  return success();
}

spirv::MemoryAccess memoryAccessAttr;
if (parseEnumStrAttr<spirv::MemoryAccessAttr>(memoryAccessAttr, parser, state,
                                              kSourceMemoryAccessAttrName))
  return failure();

if (spirv::bitEnumContainsAll(memoryAccessAttr,
                              spirv::MemoryAccess::Aligned)) {
  // Parse integer attribute for alignment.
  Attribute alignmentAttr;
  Type i32Type = parser.getBuilder().getIntegerType(32);
  if (parser.parseComma() ||
      parser.parseAttribute(alignmentAttr, i32Type, kSourceAlignmentAttrName,
                            state.attributes)) {
    return failure();
  }
}
return parser.parseRSquare();
310}

312template <typename MemoryOpTy>
313static void printMemoryAccessAttribute(
  MemoryOpTy memoryOp, OpAsmPrinter &printer,
  SmallVectorImpl<StringRef> &elidedAttrs,
  std::optional<spirv::MemoryAccess> memoryAccessAtrrValue = std::nullopt,
  std::optional<uint32_t> alignmentAttrValue = std::nullopt) {
// Print optional memory access attribute.
if (auto memAccess = (memoryAccessAtrrValue ? memoryAccessAtrrValue
                                            : memoryOp.getMemoryAccess())) {
  elidedAttrs.push_back(kMemoryAccessAttrName);

  printer << " [\"" << stringifyMemoryAccess(*memAccess) << "\"";

  if (spirv::bitEnumContainsAll(*memAccess, spirv::MemoryAccess::Aligned)) {
    // Print integer alignment attribute.
    if (auto alignment = (alignmentAttrValue ? alignmentAttrValue
                                             : memoryOp.getAlignment())) {
      elidedAttrs.push_back(kAlignmentAttrName);
      printer << ", " << *alignment;
    }
  }
  printer << "]";
}
elidedAttrs.push_back(spirv::attributeName<spirv::StorageClass>());
336}

338// TODO Make sure to merge this and the previous function into one template
339// parameterized by memory access attribute name and alignment. Doing so now
340// results in VS2017 in producing an internal error (at the call site) that's
341// not detailed enough to understand what is happening.
342template <typename MemoryOpTy>
343static void printSourceMemoryAccessAttribute(
  MemoryOpTy memoryOp, OpAsmPrinter &printer,
  SmallVectorImpl<StringRef> &elidedAttrs,
  std::optional<spirv::MemoryAccess> memoryAccessAtrrValue = std::nullopt,
  std::optional<uint32_t> alignmentAttrValue = std::nullopt) {

printer << ", ";

// Print optional memory access attribute.
if (auto memAccess = (memoryAccessAtrrValue ? memoryAccessAtrrValue
                                            : memoryOp.getMemoryAccess())) {
  elidedAttrs.push_back(kSourceMemoryAccessAttrName);

  printer << " [\"" << stringifyMemoryAccess(*memAccess) << "\"";

  if (spirv::bitEnumContainsAll(*memAccess, spirv::MemoryAccess::Aligned)) {
    // Print integer alignment attribute.
    if (auto alignment = (alignmentAttrValue ? alignmentAttrValue
                                             : memoryOp.getAlignment())) {
      elidedAttrs.push_back(kSourceAlignmentAttrName);
      printer << ", " << *alignment;
    }
  }
  printer << "]";
}
elidedAttrs.push_back(spirv::attributeName<spirv::StorageClass>());
369}

371static ParseResult parseImageOperands(OpAsmParser &parser,
                                    spirv::ImageOperandsAttr &attr) {
// Expect image operands
if (parser.parseOptionalLSquare())
  return success();

spirv::ImageOperands imageOperands;
if (parseEnumStrAttr(imageOperands, parser))
  return failure();

attr = spirv::ImageOperandsAttr::get(parser.getContext(), imageOperands);

return parser.parseRSquare();
384}

386static void printImageOperands(OpAsmPrinter &printer, Operation *imageOp,
                             spirv::ImageOperandsAttr attr) {
if (attr) {
  auto strImageOperands = stringifyImageOperands(attr.getValue());
  printer << "[\"" << strImageOperands << "\"]";
}
392}

394template <typename Op>
395static LogicalResult verifyImageOperands(Op imageOp,
                                       spirv::ImageOperandsAttr attr,
                                       Operation::operand_range operands) {
if (!attr) {
  if (operands.empty())
    return success();

  return imageOp.emitError("the Image Operands should encode what operands "
                           "follow, as per Image Operands");
}

// TODO: Add the validation rules for the following Image Operands.
spirv::ImageOperands noSupportOperands =
    spirv::ImageOperands::Bias | spirv::ImageOperands::Lod |
    spirv::ImageOperands::Grad | spirv::ImageOperands::ConstOffset |
    spirv::ImageOperands::Offset | spirv::ImageOperands::ConstOffsets |
    spirv::ImageOperands::Sample | spirv::ImageOperands::MinLod |
    spirv::ImageOperands::MakeTexelAvailable |
    spirv::ImageOperands::MakeTexelVisible |
    spirv::ImageOperands::SignExtend | spirv::ImageOperands::ZeroExtend;

if (spirv::bitEnumContainsAll(attr.getValue(), noSupportOperands))
  llvm_unreachable("unimplemented operands of Image Operands")::llvm::llvm_unreachable_internal("unimplemented operands of Image Operands"
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 417);

return success();
420}

422static LogicalResult verifyCastOp(Operation *op,
                                bool requireSameBitWidth = true,
                                bool skipBitWidthCheck = false) {
// Some CastOps have no limit on bit widths for result and operand type.
if (skipBitWidthCheck)
  return success();

Type operandType = op->getOperand(0).getType();
Type resultType = op->getResult(0).getType();

// ODS checks that result type and operand type have the same shape.
if (auto vectorType = operandType.dyn_cast<VectorType>()) {
  operandType = vectorType.getElementType();
  resultType = resultType.cast<VectorType>().getElementType();
}

if (auto coopMatrixType =
        operandType.dyn_cast<spirv::CooperativeMatrixNVType>()) {
  operandType = coopMatrixType.getElementType();
  resultType =
      resultType.cast<spirv::CooperativeMatrixNVType>().getElementType();
}

if (auto jointMatrixType =
        operandType.dyn_cast<spirv::JointMatrixINTELType>()) {
  operandType = jointMatrixType.getElementType();
  resultType =
      resultType.cast<spirv::JointMatrixINTELType>().getElementType();
}

auto operandTypeBitWidth = operandType.getIntOrFloatBitWidth();
auto resultTypeBitWidth = resultType.getIntOrFloatBitWidth();
auto isSameBitWidth = operandTypeBitWidth == resultTypeBitWidth;

if (requireSameBitWidth) {
  if (!isSameBitWidth) {
    return op->emitOpError(
               "expected the same bit widths for operand type and result "
               "type, but provided ")
           << operandType << " and " << resultType;
  }
  return success();
}

if (isSameBitWidth) {
  return op->emitOpError(
             "expected the different bit widths for operand type and result "
             "type, but provided ")
         << operandType << " and " << resultType;
}
return success();
473}

475template <typename MemoryOpTy>
476static LogicalResult verifyMemoryAccessAttribute(MemoryOpTy memoryOp) {
// ODS checks for attributes values. Just need to verify that if the
// memory-access attribute is Aligned, then the alignment attribute must be
// present.
auto *op = memoryOp.getOperation();
auto memAccessAttr = op->getAttr(kMemoryAccessAttrName);
if (!memAccessAttr) {
  // Alignment attribute shouldn't be present if memory access attribute is
  // not present.
  if (op->getAttr(kAlignmentAttrName)) {
    return memoryOp.emitOpError(
        "invalid alignment specification without aligned memory access "
        "specification");
  }
  return success();
}

auto memAccess = memAccessAttr.template cast<spirv::MemoryAccessAttr>();

if (!memAccess) {
  return memoryOp.emitOpError("invalid memory access specifier: ")
         << memAccessAttr;
}

if (spirv::bitEnumContainsAll(memAccess.getValue(),
                              spirv::MemoryAccess::Aligned)) {
  if (!op->getAttr(kAlignmentAttrName)) {
    return memoryOp.emitOpError("missing alignment value");
  }
} else {
  if (op->getAttr(kAlignmentAttrName)) {
    return memoryOp.emitOpError(
        "invalid alignment specification with non-aligned memory access "
        "specification");
  }
}
return success();
513}

515// TODO Make sure to merge this and the previous function into one template
516// parameterized by memory access attribute name and alignment. Doing so now
517// results in VS2017 in producing an internal error (at the call site) that's
518// not detailed enough to understand what is happening.
519template <typename MemoryOpTy>
520static LogicalResult verifySourceMemoryAccessAttribute(MemoryOpTy memoryOp) {
// ODS checks for attributes values. Just need to verify that if the
// memory-access attribute is Aligned, then the alignment attribute must be
// present.
auto *op = memoryOp.getOperation();
auto memAccessAttr = op->getAttr(kSourceMemoryAccessAttrName);
if (!memAccessAttr) {
  // Alignment attribute shouldn't be present if memory access attribute is
  // not present.
  if (op->getAttr(kSourceAlignmentAttrName)) {
    return memoryOp.emitOpError(
        "invalid alignment specification without aligned memory access "
        "specification");
  }
  return success();
}

auto memAccess = memAccessAttr.template cast<spirv::MemoryAccessAttr>();

if (!memAccess) {
  return memoryOp.emitOpError("invalid memory access specifier: ")
         << memAccess;
}

if (spirv::bitEnumContainsAll(memAccess.getValue(),
                              spirv::MemoryAccess::Aligned)) {
  if (!op->getAttr(kSourceAlignmentAttrName)) {
    return memoryOp.emitOpError("missing alignment value");
  }
} else {
  if (op->getAttr(kSourceAlignmentAttrName)) {
    return memoryOp.emitOpError(
        "invalid alignment specification with non-aligned memory access "
        "specification");
  }
}
return success();
557}

559static LogicalResult
560verifyMemorySemantics(Operation *op, spirv::MemorySemantics memorySemantics) {
// According to the SPIR-V specification:
// "Despite being a mask and allowing multiple bits to be combined, it is
// invalid for more than one of these four bits to be set: Acquire, Release,
// AcquireRelease, or SequentiallyConsistent. Requesting both Acquire and
// Release semantics is done by setting the AcquireRelease bit, not by setting
// two bits."
auto atMostOneInSet = spirv::MemorySemantics::Acquire |
                      spirv::MemorySemantics::Release |
                      spirv::MemorySemantics::AcquireRelease |
                      spirv::MemorySemantics::SequentiallyConsistent;

auto bitCount = llvm::countPopulation(
    static_cast<uint32_t>(memorySemantics & atMostOneInSet));
if (bitCount > 1) {
  return op->emitError(
      "expected at most one of these four memory constraints "
      "to be set: `Acquire`, `Release`,"
      "`AcquireRelease` or `SequentiallyConsistent`");
}
return success();
581}

583template <typename LoadStoreOpTy>
584static LogicalResult verifyLoadStorePtrAndValTypes(LoadStoreOpTy op, Value ptr,
                                                 Value val) {
// ODS already checks ptr is spirv::PointerType. Just check that the pointee
// type of the pointer and the type of the value are the same
//
// TODO: Check that the value type satisfies restrictions of
// SPIR-V OpLoad/OpStore operations
if (val.getType() !=
    ptr.getType().cast<spirv::PointerType>().getPointeeType()) {
  return op.emitOpError("mismatch in result type and pointer type");
}
return success();
596}

598template <typename BlockReadWriteOpTy>
599static LogicalResult verifyBlockReadWritePtrAndValTypes(BlockReadWriteOpTy op,
                                                      Value ptr, Value val) {
auto valType = val.getType();
if (auto valVecTy = valType.dyn_cast<VectorType>())
  valType = valVecTy.getElementType();

if (valType != ptr.getType().cast<spirv::PointerType>().getPointeeType()) {
  return op.emitOpError("mismatch in result type and pointer type");
}
return success();
609}

611static ParseResult parseVariableDecorations(OpAsmParser &parser,
                                          OperationState &state) {
auto builtInName = llvm::convertToSnakeFromCamelCase(
    stringifyDecoration(spirv::Decoration::BuiltIn));
if (succeeded(parser.parseOptionalKeyword("bind"))) {
  Attribute set, binding;
  // Parse optional descriptor binding
  auto descriptorSetName = llvm::convertToSnakeFromCamelCase(
      stringifyDecoration(spirv::Decoration::DescriptorSet));
  auto bindingName = llvm::convertToSnakeFromCamelCase(
      stringifyDecoration(spirv::Decoration::Binding));
  Type i32Type = parser.getBuilder().getIntegerType(32);
  if (parser.parseLParen() ||
      parser.parseAttribute(set, i32Type, descriptorSetName,
                            state.attributes) ||
      parser.parseComma() ||
      parser.parseAttribute(binding, i32Type, bindingName,
                            state.attributes) ||
      parser.parseRParen()) {
    return failure();
  }
} else if (succeeded(parser.parseOptionalKeyword(builtInName))) {
  StringAttr builtIn;
  if (parser.parseLParen() ||
      parser.parseAttribute(builtIn, builtInName, state.attributes) ||
      parser.parseRParen()) {
    return failure();
  }
}

// Parse other attributes
if (parser.parseOptionalAttrDict(state.attributes))
  return failure();

return success();
646}

648static void printVariableDecorations(Operation *op, OpAsmPrinter &printer,
                                   SmallVectorImpl<StringRef> &elidedAttrs) {
// Print optional descriptor binding
auto descriptorSetName = llvm::convertToSnakeFromCamelCase(
    stringifyDecoration(spirv::Decoration::DescriptorSet));
auto bindingName = llvm::convertToSnakeFromCamelCase(
    stringifyDecoration(spirv::Decoration::Binding));
auto descriptorSet = op->getAttrOfType<IntegerAttr>(descriptorSetName);
auto binding = op->getAttrOfType<IntegerAttr>(bindingName);
if (descriptorSet && binding) {
  elidedAttrs.push_back(descriptorSetName);
  elidedAttrs.push_back(bindingName);
  printer << " bind(" << descriptorSet.getInt() << ", " << binding.getInt()
          << ")";
}

// Print BuiltIn attribute if present
auto builtInName = llvm::convertToSnakeFromCamelCase(
    stringifyDecoration(spirv::Decoration::BuiltIn));
if (auto builtin = op->getAttrOfType<StringAttr>(builtInName)) {
  printer << " " << builtInName << "(\"" << builtin.getValue() << "\")";
  elidedAttrs.push_back(builtInName);
}

printer.printOptionalAttrDict(op->getAttrs(), elidedAttrs);
673}

675// Get bit width of types.
676static unsigned getBitWidth(Type type) {
if (type.isa<spirv::PointerType>()) {
  // Just return 64 bits for pointer types for now.
  // TODO: Make sure not caller relies on the actual pointer width value.
  return 64;
}

if (type.isIntOrFloat())
  return type.getIntOrFloatBitWidth();

if (auto vectorType = type.dyn_cast<VectorType>()) {
  assert(vectorType.getElementType().isIntOrFloat())(static_cast <bool> (vectorType.getElementType().isIntOrFloat
()) ? void (0) : __assert_fail ("vectorType.getElementType().isIntOrFloat()"
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 687, __extension__
 __PRETTY_FUNCTION__));
  return vectorType.getNumElements() *
         vectorType.getElementType().getIntOrFloatBitWidth();
}
llvm_unreachable("unhandled bit width computation for type")::llvm::llvm_unreachable_internal("unhandled bit width computation for type"
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 691);
692}

694/// Walks the given type hierarchy with the given indices, potentially down
695/// to component granularity, to select an element type. Returns null type and
696/// emits errors with the given loc on failure.
697static Type
698getElementType(Type type, ArrayRef<int32_t> indices,
             function_ref<InFlightDiagnostic(StringRef)> emitErrorFn) {
if (indices.empty()) {
  emitErrorFn("expected at least one index for spirv.CompositeExtract");
  return nullptr;
}

for (auto index : indices) {
  if (auto cType = type.dyn_cast<spirv::CompositeType>()) {
    if (cType.hasCompileTimeKnownNumElements() &&
        (index < 0 ||
         static_cast<uint64_t>(index) >= cType.getNumElements())) {
      emitErrorFn("index ") << index << " out of bounds for " << type;
      return nullptr;
    }
    type = cType.getElementType(index);
  } else {
    emitErrorFn("cannot extract from non-composite type ")
        << type << " with index " << index;
    return nullptr;
  }
}
return type;
721}

723static Type
724getElementType(Type type, Attribute indices,
             function_ref<InFlightDiagnostic(StringRef)> emitErrorFn) {
auto indicesArrayAttr = indices.dyn_cast<ArrayAttr>();
if (!indicesArrayAttr) {
  emitErrorFn("expected a 32-bit integer array attribute for 'indices'");
  return nullptr;
}
if (indicesArrayAttr.empty()) {
  emitErrorFn("expected at least one index for spirv.CompositeExtract");
  return nullptr;
}

SmallVector<int32_t, 2> indexVals;
for (auto indexAttr : indicesArrayAttr) {
  auto indexIntAttr = indexAttr.dyn_cast<IntegerAttr>();
  if (!indexIntAttr) {
    emitErrorFn("expected an 32-bit integer for index, but found '")
        << indexAttr << "'";
    return nullptr;
  }
  indexVals.push_back(indexIntAttr.getInt());
}
return getElementType(type, indexVals, emitErrorFn);
747}

749static Type getElementType(Type type, Attribute indices, Location loc) {
auto errorFn = [&](StringRef err) -> InFlightDiagnostic {
  return ::mlir::emitError(loc, err);
};
return getElementType(type, indices, errorFn);
754}

756static Type getElementType(Type type, Attribute indices, OpAsmParser &parser,
                         SMLoc loc) {
auto errorFn = [&](StringRef err) -> InFlightDiagnostic {
  return parser.emitError(loc, err);
};
return getElementType(type, indices, errorFn);
762}

764/// Returns true if the given `block` only contains one `spirv.mlir.merge` op.
765static inline bool isMergeBlock(Block &block) {
return !block.empty() && std::next(block.begin()) == block.end() &&
       isa<spirv::MergeOp>(block.front());
768}

770template <typename ExtendedBinaryOp>
771static LogicalResult verifyArithmeticExtendedBinaryOp(ExtendedBinaryOp op) {
auto resultType = op.getType().template cast<spirv::StructType>();
if (resultType.getNumElements() != 2)
  return op.emitOpError("expected result struct type containing two members");

if (!llvm::all_equal({op.getOperand1().getType(), op.getOperand2().getType(),
                      resultType.getElementType(0),
                      resultType.getElementType(1)}))
  return op.emitOpError(
      "expected all operand types and struct member types are the same");

return success();
783}

785static ParseResult parseArithmeticExtendedBinaryOp(OpAsmParser &parser,
                                                 OperationState &result) {
SmallVector<OpAsmParser::UnresolvedOperand, 2> operands;
if (parser.parseOptionalAttrDict(result.attributes) ||
    parser.parseOperandList(operands) || parser.parseColon())
  return failure();

Type resultType;
SMLoc loc = parser.getCurrentLocation();
if (parser.parseType(resultType))
  return failure();

auto structType = resultType.dyn_cast<spirv::StructType>();
if (!structType || structType.getNumElements() != 2)
  return parser.emitError(loc, "expected spirv.struct type with two members");

SmallVector<Type, 2> operandTypes(2, structType.getElementType(0));
if (parser.resolveOperands(operands, operandTypes, loc, result.operands))
  return failure();

result.addTypes(resultType);
return success();
807}

809static void printArithmeticExtendedBinaryOp(Operation *op,
                                          OpAsmPrinter &printer) {
printer << ' ';
printer.printOptionalAttrDict(op->getAttrs());
printer.printOperands(op->getOperands());
printer << " : " << op->getResultTypes().front();
815}

817//===----------------------------------------------------------------------===//
818// Common parsers and printers
819//===----------------------------------------------------------------------===//

821// Parses an atomic update op. If the update op does not take a value (like
822// AtomicIIncrement) `hasValue` must be false.
823static ParseResult parseAtomicUpdateOp(OpAsmParser &parser,
                                     OperationState &state, bool hasValue) {
spirv::Scope scope;
spirv::MemorySemantics memoryScope;
SmallVector<OpAsmParser::UnresolvedOperand, 2> operandInfo;
OpAsmParser::UnresolvedOperand ptrInfo, valueInfo;
Type type;
SMLoc loc;
if (parseEnumStrAttr<spirv::ScopeAttr>(scope, parser, state,
                                       kMemoryScopeAttrName) ||
    parseEnumStrAttr<spirv::MemorySemanticsAttr>(memoryScope, parser, state,
                                                 kSemanticsAttrName) ||
    parser.parseOperandList(operandInfo, (hasValue ? 2 : 1)) ||
    parser.getCurrentLocation(&loc) || parser.parseColonType(type))
  return failure();

auto ptrType = type.dyn_cast<spirv::PointerType>();
if (!ptrType)
  return parser.emitError(loc, "expected pointer type");

SmallVector<Type, 2> operandTypes;
operandTypes.push_back(ptrType);
if (hasValue)
  operandTypes.push_back(ptrType.getPointeeType());
if (parser.resolveOperands(operandInfo, operandTypes, parser.getNameLoc(),
                           state.operands))
  return failure();
return parser.addTypeToList(ptrType.getPointeeType(), state.types);
851}

853// Prints an atomic update op.
854static void printAtomicUpdateOp(Operation *op, OpAsmPrinter &printer) {
printer << " \"";
auto scopeAttr = op->getAttrOfType<spirv::ScopeAttr>(kMemoryScopeAttrName);
printer << spirv::stringifyScope(scopeAttr.getValue()) << "\" \"";
auto memorySemanticsAttr =
    op->getAttrOfType<spirv::MemorySemanticsAttr>(kSemanticsAttrName);
printer << spirv::stringifyMemorySemantics(memorySemanticsAttr.getValue())
        << "\" " << op->getOperands() << " : " << op->getOperand(0).getType();
862}

864template <typename T>
865static StringRef stringifyTypeName();

867template <>
868StringRef stringifyTypeName<IntegerType>() {
return "integer";
870}

872template <>
873StringRef stringifyTypeName<FloatType>() {
return "float";
875}

877// Verifies an atomic update op.
878template <typename ExpectedElementType>
879static LogicalResult verifyAtomicUpdateOp(Operation *op) {
auto ptrType = op->getOperand(0).getType().cast<spirv::PointerType>();
auto elementType = ptrType.getPointeeType();
if (!elementType.isa<ExpectedElementType>())
  return op->emitOpError() << "pointer operand must point to an "
                           << stringifyTypeName<ExpectedElementType>()
                           << " value, found " << elementType;

if (op->getNumOperands() > 1) {
  auto valueType = op->getOperand(1).getType();
  if (valueType != elementType)
    return op->emitOpError("expected value to have the same type as the "
                           "pointer operand's pointee type ")
           << elementType << ", but found " << valueType;
}
auto memorySemantics =
    op->getAttrOfType<spirv::MemorySemanticsAttr>(kSemanticsAttrName)
        .getValue();
if (failed(verifyMemorySemantics(op, memorySemantics))) {
  return failure();
}
return success();
901}

903static ParseResult parseGroupNonUniformArithmeticOp(OpAsmParser &parser,
                                                  OperationState &state) {
spirv::Scope executionScope;
spirv::GroupOperation groupOperation;
OpAsmParser::UnresolvedOperand valueInfo;
if (parseEnumStrAttr<spirv::ScopeAttr>(executionScope, parser, state,
                                       kExecutionScopeAttrName) ||
    parseEnumStrAttr<spirv::GroupOperationAttr>(groupOperation, parser, state,
                                                kGroupOperationAttrName) ||
    parser.parseOperand(valueInfo))
  return failure();

std::optional<OpAsmParser::UnresolvedOperand> clusterSizeInfo;
if (succeeded(parser.parseOptionalKeyword(kClusterSize))) {
  clusterSizeInfo = OpAsmParser::UnresolvedOperand();
  if (parser.parseLParen() || parser.parseOperand(*clusterSizeInfo) ||
      parser.parseRParen())
    return failure();
}

Type resultType;
if (parser.parseColonType(resultType))
  return failure();

if (parser.resolveOperand(valueInfo, resultType, state.operands))
  return failure();

if (clusterSizeInfo) {
  Type i32Type = parser.getBuilder().getIntegerType(32);
  if (parser.resolveOperand(*clusterSizeInfo, i32Type, state.operands))
    return failure();
}

return parser.addTypeToList(resultType, state.types);
937}

939static void printGroupNonUniformArithmeticOp(Operation *groupOp,
                                           OpAsmPrinter &printer) {
printer
    << " \""
    << stringifyScope(
           groupOp->getAttrOfType<spirv::ScopeAttr>(kExecutionScopeAttrName)
               .getValue())
    << "\" \""
    << stringifyGroupOperation(groupOp
                                   ->getAttrOfType<spirv::GroupOperationAttr>(
                                       kGroupOperationAttrName)
                                   .getValue())
    << "\" " << groupOp->getOperand(0);

if (groupOp->getNumOperands() > 1)
  printer << " " << kClusterSize << '(' << groupOp->getOperand(1) << ')';
printer << " : " << groupOp->getResult(0).getType();
956}

958static LogicalResult verifyGroupNonUniformArithmeticOp(Operation *groupOp) {
spirv::Scope scope =
    groupOp->getAttrOfType<spirv::ScopeAttr>(kExecutionScopeAttrName)
        .getValue();
if (scope != spirv::Scope::Workgroup && scope != spirv::Scope::Subgroup)
  return groupOp->emitOpError(
      "execution scope must be 'Workgroup' or 'Subgroup'");

spirv::GroupOperation operation =
    groupOp->getAttrOfType<spirv::GroupOperationAttr>(kGroupOperationAttrName)
        .getValue();
if (operation == spirv::GroupOperation::ClusteredReduce &&
    groupOp->getNumOperands() == 1)
  return groupOp->emitOpError("cluster size operand must be provided for "
                              "'ClusteredReduce' group operation");
if (groupOp->getNumOperands() > 1) {
  Operation *sizeOp = groupOp->getOperand(1).getDefiningOp();
  int32_t clusterSize = 0;

  // TODO: support specialization constant here.
  if (failed(extractValueFromConstOp(sizeOp, clusterSize)))
    return groupOp->emitOpError(
        "cluster size operand must come from a constant op");

  if (!llvm::isPowerOf2_32(clusterSize))
    return groupOp->emitOpError(
        "cluster size operand must be a power of two");
}
return success();
987}

989/// Result of a logical op must be a scalar or vector of boolean type.
990static Type getUnaryOpResultType(Type operandType) {
Builder builder(operandType.getContext());
Type resultType = builder.getIntegerType(1);
if (auto vecType = operandType.dyn_cast<VectorType>())
  return VectorType::get(vecType.getNumElements(), resultType);
return resultType;
996}

998static LogicalResult verifyShiftOp(Operation *op) {
if (op->getOperand(0).getType() != op->getResult(0).getType()) {
  return op->emitError("expected the same type for the first operand and "
                       "result, but provided ")
         << op->getOperand(0).getType() << " and "
         << op->getResult(0).getType();
}
return success();
1006}

1008static void buildLogicalBinaryOp(OpBuilder &builder, OperationState &state,
                               Value lhs, Value rhs) {
assert(lhs.getType() == rhs.getType())(static_cast <bool> (lhs.getType() == rhs.getType()) ? void
 (0) : __assert_fail ("lhs.getType() == rhs.getType()", "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp"
, 1010, __extension__ __PRETTY_FUNCTION__));

Type boolType = builder.getI1Type();
if (auto vecType = lhs.getType().dyn_cast<VectorType>())
  boolType = VectorType::get(vecType.getShape(), boolType);
state.addTypes(boolType);

state.addOperands({lhs, rhs});
1018}

1020static void buildLogicalUnaryOp(OpBuilder &builder, OperationState &state,
                              Value value) {
Type boolType = builder.getI1Type();
if (auto vecType = value.getType().dyn_cast<VectorType>())
  boolType = VectorType::get(vecType.getShape(), boolType);
state.addTypes(boolType);

state.addOperands(value);
1028}

1030//===----------------------------------------------------------------------===//
1031// spirv.AccessChainOp
1032//===----------------------------------------------------------------------===//

1034static Type getElementPtrType(Type type, ValueRange indices, Location baseLoc) {
auto ptrType = type.dyn_cast<spirv::PointerType>();
if (!ptrType) {
  emitError(baseLoc, "'spirv.AccessChain' op expected a pointer "
                     "to composite type, but provided ")
      << type;
  return nullptr;
}

auto resultType = ptrType.getPointeeType();
auto resultStorageClass = ptrType.getStorageClass();
int32_t index = 0;

for (auto indexSSA : indices) {
  auto cType = resultType.dyn_cast<spirv::CompositeType>();
  if (!cType) {
    emitError(
        baseLoc,
        "'spirv.AccessChain' op cannot extract from non-composite type ")
        << resultType << " with index " << index;
    return nullptr;
  }
  index = 0;
  if (resultType.isa<spirv::StructType>()) {
    Operation *op = indexSSA.getDefiningOp();
    if (!op) {
      emitError(baseLoc, "'spirv.AccessChain' op index must be an "
                         "integer spirv.Constant to access "
                         "element of spirv.struct");
      return nullptr;
    }

    // TODO: this should be relaxed to allow
    // integer literals of other bitwidths.
    if (failed(extractValueFromConstOp(op, index))) {
      emitError(
          baseLoc,
          "'spirv.AccessChain' index must be an integer spirv.Constant to "
          "access element of spirv.struct, but provided ")
          << op->getName();
      return nullptr;
    }
    if (index < 0 || static_cast<uint64_t>(index) >= cType.getNumElements()) {
      emitError(baseLoc, "'spirv.AccessChain' op index ")
          << index << " out of bounds for " << resultType;
      return nullptr;
    }
  }
  resultType = cType.getElementType(index);
}
return spirv::PointerType::get(resultType, resultStorageClass);
1085}

1087void spirv::AccessChainOp::build(OpBuilder &builder, OperationState &state,
                               Value basePtr, ValueRange indices) {
auto type = getElementPtrType(basePtr.getType(), indices, state.location);
assert(type && "Unable to deduce return type based on basePtr and indices")(static_cast <bool> (type && "Unable to deduce return type based on basePtr and indices"
) ? void (0) : __assert_fail ("type && \"Unable to deduce return type based on basePtr and indices\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 1090, __extension__
 __PRETTY_FUNCTION__));
build(builder, state, type, basePtr, indices);
1092}

1094ParseResult spirv::AccessChainOp::parse(OpAsmParser &parser,
                                      OperationState &result) {
OpAsmParser::UnresolvedOperand ptrInfo;
SmallVector<OpAsmParser::UnresolvedOperand, 4> indicesInfo;
Type type;
auto loc = parser.getCurrentLocation();
SmallVector<Type, 4> indicesTypes;

if (parser.parseOperand(ptrInfo) ||
    parser.parseOperandList(indicesInfo, OpAsmParser::Delimiter::Square) ||
    parser.parseColonType(type) ||
    parser.resolveOperand(ptrInfo, type, result.operands)) {
  return failure();
}

// Check that the provided indices list is not empty before parsing their
// type list.
if (indicesInfo.empty()) {
  return mlir::emitError(result.location,
                         "'spirv.AccessChain' op expected at "
                         "least one index ");
}

if (parser.parseComma() || parser.parseTypeList(indicesTypes))
  return failure();

// Check that the indices types list is not empty and that it has a one-to-one
// mapping to the provided indices.
if (indicesTypes.size() != indicesInfo.size()) {
  return mlir::emitError(
      result.location, "'spirv.AccessChain' op indices types' count must be "
                       "equal to indices info count");
}

if (parser.resolveOperands(indicesInfo, indicesTypes, loc, result.operands))
  return failure();

auto resultType = getElementPtrType(
    type, llvm::makeArrayRef(result.operands).drop_front(), result.location);
if (!resultType) {
  return failure();
}

result.addTypes(resultType);
return success();
1139}

1141template <typename Op>
1142static void printAccessChain(Op op, ValueRange indices, OpAsmPrinter &printer) {
printer << ' ' << op.getBasePtr() << '[' << indices
        << "] : " << op.getBasePtr().getType() << ", " << indices.getTypes();
1145}

1147void spirv::AccessChainOp::print(OpAsmPrinter &printer) {
printAccessChain(*this, getIndices(), printer);
1149}

1151template <typename Op>
1152static LogicalResult verifyAccessChain(Op accessChainOp, ValueRange indices) {
auto resultType = getElementPtrType(accessChainOp.getBasePtr().getType(),
                                    indices, accessChainOp.getLoc());
if (!resultType)
  return failure();

auto providedResultType =
    accessChainOp.getType().template dyn_cast<spirv::PointerType>();
if (!providedResultType)
  return accessChainOp.emitOpError(
             "result type must be a pointer, but provided")
         << providedResultType;

if (resultType != providedResultType)
  return accessChainOp.emitOpError("invalid result type: expected ")
         << resultType << ", but provided " << providedResultType;

return success();
1170}

1172LogicalResult spirv::AccessChainOp::verify() {
return verifyAccessChain(*this, getIndices());
1174}

1176//===----------------------------------------------------------------------===//
1177// spirv.mlir.addressof
1178//===----------------------------------------------------------------------===//

1180void spirv::AddressOfOp::build(OpBuilder &builder, OperationState &state,
                             spirv::GlobalVariableOp var) {
build(builder, state, var.getType(), SymbolRefAttr::get(var));
1183}

1185LogicalResult spirv::AddressOfOp::verify() {
auto varOp = dyn_cast_or_null<spirv::GlobalVariableOp>(
    SymbolTable::lookupNearestSymbolFrom((*this)->getParentOp(),
                                         getVariableAttr()));
if (!varOp) {
  return emitOpError("expected spirv.GlobalVariable symbol");
}
if (getPointer().getType() != varOp.getType()) {
  return emitOpError(
      "result type mismatch with the referenced global variable's type");
}
return success();
1197}

1199template <typename T>
1200static void printAtomicCompareExchangeImpl(T atomOp, OpAsmPrinter &printer) {
printer << " \"" << stringifyScope(atomOp.getMemoryScope()) << "\" \""
        << stringifyMemorySemantics(atomOp.getEqualSemantics()) << "\" \""
        << stringifyMemorySemantics(atomOp.getUnequalSemantics()) << "\" "
        << atomOp.getOperands() << " : " << atomOp.getPointer().getType();
1205}

1207static ParseResult parseAtomicCompareExchangeImpl(OpAsmParser &parser,
                                                OperationState &state) {
spirv::Scope memoryScope;
spirv::MemorySemantics equalSemantics, unequalSemantics;
SmallVector<OpAsmParser::UnresolvedOperand, 3> operandInfo;
Type type;
if (parseEnumStrAttr<spirv::ScopeAttr>(memoryScope, parser, state,
                                       kMemoryScopeAttrName) ||
    parseEnumStrAttr<spirv::MemorySemanticsAttr>(
        equalSemantics, parser, state, kEqualSemanticsAttrName) ||
    parseEnumStrAttr<spirv::MemorySemanticsAttr>(
        unequalSemantics, parser, state, kUnequalSemanticsAttrName) ||
    parser.parseOperandList(operandInfo, 3))
  return failure();

auto loc = parser.getCurrentLocation();
if (parser.parseColonType(type))
  return failure();

auto ptrType = type.dyn_cast<spirv::PointerType>();
if (!ptrType)
  return parser.emitError(loc, "expected pointer type");

if (parser.resolveOperands(
        operandInfo,
        {ptrType, ptrType.getPointeeType(), ptrType.getPointeeType()},
        parser.getNameLoc(), state.operands))
  return failure();

return parser.addTypeToList(ptrType.getPointeeType(), state.types);
1237}

1239template <typename T>
1240static LogicalResult verifyAtomicCompareExchangeImpl(T atomOp) {
// According to the spec:
// "The type of Value must be the same as Result Type. The type of the value
// pointed to by Pointer must be the same as Result Type. This type must also
// match the type of Comparator."
if (atomOp.getType() != atomOp.getValue().getType())
  return atomOp.emitOpError("value operand must have the same type as the op "
                            "result, but found ")
         << atomOp.getValue().getType() << " vs " << atomOp.getType();

if (atomOp.getType() != atomOp.getComparator().getType())
  return atomOp.emitOpError(
             "comparator operand must have the same type as the op "
             "result, but found ")
         << atomOp.getComparator().getType() << " vs " << atomOp.getType();

Type pointeeType = atomOp.getPointer()
                       .getType()
                       .template cast<spirv::PointerType>()
                       .getPointeeType();
if (atomOp.getType() != pointeeType)
  return atomOp.emitOpError(
             "pointer operand's pointee type must have the same "
             "as the op result type, but found ")
         << pointeeType << " vs " << atomOp.getType();

// TODO: Unequal cannot be set to Release or Acquire and Release.
// In addition, Unequal cannot be set to a stronger memory-order then Equal.

return success();
1270}

1272//===----------------------------------------------------------------------===//
1273// spirv.AtomicAndOp
1274//===----------------------------------------------------------------------===//

1276LogicalResult spirv::AtomicAndOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1278}

1280ParseResult spirv::AtomicAndOp::parse(OpAsmParser &parser,
                                    OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1283}
1284void spirv::AtomicAndOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1286}

1288//===----------------------------------------------------------------------===//
1289// spirv.AtomicCompareExchangeOp
1290//===----------------------------------------------------------------------===//

1292LogicalResult spirv::AtomicCompareExchangeOp::verify() {
return ::verifyAtomicCompareExchangeImpl(*this);
1294}

1296ParseResult spirv::AtomicCompareExchangeOp::parse(OpAsmParser &parser,
                                                OperationState &result) {
return ::parseAtomicCompareExchangeImpl(parser, result);
1299}
1300void spirv::AtomicCompareExchangeOp::print(OpAsmPrinter &p) {
::printAtomicCompareExchangeImpl(*this, p);
1302}

1304//===----------------------------------------------------------------------===//
1305// spirv.AtomicCompareExchangeWeakOp
1306//===----------------------------------------------------------------------===//

1308LogicalResult spirv::AtomicCompareExchangeWeakOp::verify() {
return ::verifyAtomicCompareExchangeImpl(*this);
1310}

1312ParseResult spirv::AtomicCompareExchangeWeakOp::parse(OpAsmParser &parser,
                                                    OperationState &result) {
return ::parseAtomicCompareExchangeImpl(parser, result);
1315}
1316void spirv::AtomicCompareExchangeWeakOp::print(OpAsmPrinter &p) {
::printAtomicCompareExchangeImpl(*this, p);
1318}

1320//===----------------------------------------------------------------------===//
1321// spirv.AtomicExchange
1322//===----------------------------------------------------------------------===//

1324void spirv::AtomicExchangeOp::print(OpAsmPrinter &printer) {
printer << " \"" << stringifyScope(getMemoryScope()) << "\" \""
        << stringifyMemorySemantics(getSemantics()) << "\" " << getOperands()
        << " : " << getPointer().getType();
1328}

1330ParseResult spirv::AtomicExchangeOp::parse(OpAsmParser &parser,
                                         OperationState &result) {
spirv::Scope memoryScope;
spirv::MemorySemantics semantics;
SmallVector<OpAsmParser::UnresolvedOperand, 2> operandInfo;
Type type;
if (parseEnumStrAttr<spirv::ScopeAttr>(memoryScope, parser, result,
                                       kMemoryScopeAttrName) ||
    parseEnumStrAttr<spirv::MemorySemanticsAttr>(semantics, parser, result,
                                                 kSemanticsAttrName) ||
    parser.parseOperandList(operandInfo, 2))
  return failure();

auto loc = parser.getCurrentLocation();
if (parser.parseColonType(type))
  return failure();

auto ptrType = type.dyn_cast<spirv::PointerType>();
if (!ptrType)
  return parser.emitError(loc, "expected pointer type");

if (parser.resolveOperands(operandInfo, {ptrType, ptrType.getPointeeType()},
                           parser.getNameLoc(), result.operands))
  return failure();

return parser.addTypeToList(ptrType.getPointeeType(), result.types);
1356}

1358LogicalResult spirv::AtomicExchangeOp::verify() {
if (getType() != getValue().getType())
  return emitOpError("value operand must have the same type as the op "
                     "result, but found ")
         << getValue().getType() << " vs " << getType();

Type pointeeType =
    getPointer().getType().cast<spirv::PointerType>().getPointeeType();
if (getType() != pointeeType)
  return emitOpError("pointer operand's pointee type must have the same "
                     "as the op result type, but found ")
         << pointeeType << " vs " << getType();

return success();
1372}

1374//===----------------------------------------------------------------------===//
1375// spirv.AtomicIAddOp
1376//===----------------------------------------------------------------------===//

1378LogicalResult spirv::AtomicIAddOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1380}

1382ParseResult spirv::AtomicIAddOp::parse(OpAsmParser &parser,
                                     OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1385}
1386void spirv::AtomicIAddOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1388}

1390//===----------------------------------------------------------------------===//
1391// spirv.EXT.AtomicFAddOp
1392//===----------------------------------------------------------------------===//

1394LogicalResult spirv::EXTAtomicFAddOp::verify() {
return ::verifyAtomicUpdateOp<FloatType>(getOperation());
1396}

1398ParseResult spirv::EXTAtomicFAddOp::parse(OpAsmParser &parser,
                                        OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1401}
1402void spirv::EXTAtomicFAddOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1404}

1406//===----------------------------------------------------------------------===//
1407// spirv.AtomicIDecrementOp
1408//===----------------------------------------------------------------------===//

1410LogicalResult spirv::AtomicIDecrementOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1412}

1414ParseResult spirv::AtomicIDecrementOp::parse(OpAsmParser &parser,
                                           OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, false);
1417}
1418void spirv::AtomicIDecrementOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1420}

1422//===----------------------------------------------------------------------===//
1423// spirv.AtomicIIncrementOp
1424//===----------------------------------------------------------------------===//

1426LogicalResult spirv::AtomicIIncrementOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1428}

1430ParseResult spirv::AtomicIIncrementOp::parse(OpAsmParser &parser,
                                           OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, false);
1433}
1434void spirv::AtomicIIncrementOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1436}

1438//===----------------------------------------------------------------------===//
1439// spirv.AtomicISubOp
1440//===----------------------------------------------------------------------===//

1442LogicalResult spirv::AtomicISubOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1444}

1446ParseResult spirv::AtomicISubOp::parse(OpAsmParser &parser,
                                     OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1449}
1450void spirv::AtomicISubOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1452}

1454//===----------------------------------------------------------------------===//
1455// spirv.AtomicOrOp
1456//===----------------------------------------------------------------------===//

1458LogicalResult spirv::AtomicOrOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1460}

1462ParseResult spirv::AtomicOrOp::parse(OpAsmParser &parser,
                                   OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1465}
1466void spirv::AtomicOrOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1468}

1470//===----------------------------------------------------------------------===//
1471// spirv.AtomicSMaxOp
1472//===----------------------------------------------------------------------===//

1474LogicalResult spirv::AtomicSMaxOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1476}

1478ParseResult spirv::AtomicSMaxOp::parse(OpAsmParser &parser,
                                     OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1481}
1482void spirv::AtomicSMaxOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1484}

1486//===----------------------------------------------------------------------===//
1487// spirv.AtomicSMinOp
1488//===----------------------------------------------------------------------===//

1490LogicalResult spirv::AtomicSMinOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1492}

1494ParseResult spirv::AtomicSMinOp::parse(OpAsmParser &parser,
                                     OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1497}
1498void spirv::AtomicSMinOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1500}

1502//===----------------------------------------------------------------------===//
1503// spirv.AtomicUMaxOp
1504//===----------------------------------------------------------------------===//

1506LogicalResult spirv::AtomicUMaxOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1508}

1510ParseResult spirv::AtomicUMaxOp::parse(OpAsmParser &parser,
                                     OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1513}
1514void spirv::AtomicUMaxOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1516}

1518//===----------------------------------------------------------------------===//
1519// spirv.AtomicUMinOp
1520//===----------------------------------------------------------------------===//

1522LogicalResult spirv::AtomicUMinOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1524}

1526ParseResult spirv::AtomicUMinOp::parse(OpAsmParser &parser,
                                     OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1529}
1530void spirv::AtomicUMinOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1532}

1534//===----------------------------------------------------------------------===//
1535// spirv.AtomicXorOp
1536//===----------------------------------------------------------------------===//

1538LogicalResult spirv::AtomicXorOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1540}

1542ParseResult spirv::AtomicXorOp::parse(OpAsmParser &parser,
                                    OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1545}
1546void spirv::AtomicXorOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1548}

1550//===----------------------------------------------------------------------===//
1551// spirv.BitcastOp
1552//===----------------------------------------------------------------------===//

1554LogicalResult spirv::BitcastOp::verify() {
// TODO: The SPIR-V spec validation rules are different for different
// versions.
auto operandType = getOperand().getType();
auto resultType = getResult().getType();
if (operandType == resultType) {
  return emitError("result type must be different from operand type");
}
if (operandType.isa<spirv::PointerType>() &&
    !resultType.isa<spirv::PointerType>()) {
  return emitError(
      "unhandled bit cast conversion from pointer type to non-pointer type");
}
if (!operandType.isa<spirv::PointerType>() &&
    resultType.isa<spirv::PointerType>()) {
  return emitError(
      "unhandled bit cast conversion from non-pointer type to pointer type");
}
auto operandBitWidth = getBitWidth(operandType);
auto resultBitWidth = getBitWidth(resultType);
if (operandBitWidth != resultBitWidth) {
  return emitOpError("mismatch in result type bitwidth ")
         << resultBitWidth << " and operand type bitwidth "
         << operandBitWidth;
}
return success();
1580}

1582//===----------------------------------------------------------------------===//
1583// spirv.PtrCastToGenericOp
1584//===----------------------------------------------------------------------===//

1586LogicalResult spirv::PtrCastToGenericOp::verify() {
auto operandType = getPointer().getType().cast<spirv::PointerType>();
auto resultType = getResult().getType().cast<spirv::PointerType>();

spirv::StorageClass operandStorage = operandType.getStorageClass();
if (operandStorage != spirv::StorageClass::Workgroup &&
    operandStorage != spirv::StorageClass::CrossWorkgroup &&
    operandStorage != spirv::StorageClass::Function)
  return emitError("pointer must point to the Workgroup, CrossWorkgroup"
                   ", or Function Storage Class");

spirv::StorageClass resultStorage = resultType.getStorageClass();
if (resultStorage != spirv::StorageClass::Generic)
  return emitError("result type must be of storage class Generic");

Type operandPointeeType = operandType.getPointeeType();
Type resultPointeeType = resultType.getPointeeType();
if (operandPointeeType != resultPointeeType)
  return emitOpError("pointer operand's pointee type must have the same "
                     "as the op result type, but found ")
         << operandPointeeType << " vs " << resultPointeeType;
return success();
1608}

1610//===----------------------------------------------------------------------===//
1611// spirv.GenericCastToPtrOp
1612//===----------------------------------------------------------------------===//

1614LogicalResult spirv::GenericCastToPtrOp::verify() {
auto operandType = getPointer().getType().cast<spirv::PointerType>();
auto resultType = getResult().getType().cast<spirv::PointerType>();

spirv::StorageClass operandStorage = operandType.getStorageClass();
if (operandStorage != spirv::StorageClass::Generic)
  return emitError("pointer type must be of storage class Generic");

spirv::StorageClass resultStorage = resultType.getStorageClass();
if (resultStorage != spirv::StorageClass::Workgroup &&
    resultStorage != spirv::StorageClass::CrossWorkgroup &&
    resultStorage != spirv::StorageClass::Function)
  return emitError("result must point to the Workgroup, CrossWorkgroup, "
                   "or Function Storage Class");

Type operandPointeeType = operandType.getPointeeType();
Type resultPointeeType = resultType.getPointeeType();
if (operandPointeeType != resultPointeeType)
  return emitOpError("pointer operand's pointee type must have the same "
                     "as the op result type, but found ")
         << operandPointeeType << " vs " << resultPointeeType;
return success();
1636}

1638//===----------------------------------------------------------------------===//
1639// spirv.GenericCastToPtrExplicitOp
1640//===----------------------------------------------------------------------===//

1642LogicalResult spirv::GenericCastToPtrExplicitOp::verify() {
auto operandType = getPointer().getType().cast<spirv::PointerType>();
auto resultType = getResult().getType().cast<spirv::PointerType>();

spirv::StorageClass operandStorage = operandType.getStorageClass();
if (operandStorage != spirv::StorageClass::Generic)
  return emitError("pointer type must be of storage class Generic");

spirv::StorageClass resultStorage = resultType.getStorageClass();
if (resultStorage != spirv::StorageClass::Workgroup &&
    resultStorage != spirv::StorageClass::CrossWorkgroup &&
    resultStorage != spirv::StorageClass::Function)
  return emitError("result must point to the Workgroup, CrossWorkgroup, "
                   "or Function Storage Class");

Type operandPointeeType = operandType.getPointeeType();
Type resultPointeeType = resultType.getPointeeType();
if (operandPointeeType != resultPointeeType)
  return emitOpError("pointer operand's pointee type must have the same "
                     "as the op result type, but found ")
         << operandPointeeType << " vs " << resultPointeeType;
return success();
1664}

1666//===----------------------------------------------------------------------===//
1667// spirv.BranchOp
1668//===----------------------------------------------------------------------===//

1670SuccessorOperands spirv::BranchOp::getSuccessorOperands(unsigned index) {
assert(index == 0 && "invalid successor index")(static_cast <bool> (index == 0 && "invalid successor index"
) ? void (0) : __assert_fail ("index == 0 && \"invalid successor index\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 1671, __extension__
 __PRETTY_FUNCTION__));
return SuccessorOperands(0, getTargetOperandsMutable());
1673}

1675//===----------------------------------------------------------------------===//
1676// spirv.BranchConditionalOp
1677//===----------------------------------------------------------------------===//

1679SuccessorOperands
1680spirv::BranchConditionalOp::getSuccessorOperands(unsigned index) {
assert(index < 2 && "invalid successor index")(static_cast <bool> (index < 2 && "invalid successor index"
) ? void (0) : __assert_fail ("index < 2 && \"invalid successor index\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 1681, __extension__
 __PRETTY_FUNCTION__));
return SuccessorOperands(index == kTrueIndex
                             ? getTrueTargetOperandsMutable()
                             : getFalseTargetOperandsMutable());
1685}

1687ParseResult spirv::BranchConditionalOp::parse(OpAsmParser &parser,
                                            OperationState &result) {
auto &builder = parser.getBuilder();
OpAsmParser::UnresolvedOperand condInfo;
Block *dest;

// Parse the condition.
Type boolTy = builder.getI1Type();
if (parser.parseOperand(condInfo) ||
    parser.resolveOperand(condInfo, boolTy, result.operands))
  return failure();

// Parse the optional branch weights.
if (succeeded(parser.parseOptionalLSquare())) {
  IntegerAttr trueWeight, falseWeight;
  NamedAttrList weights;

  auto i32Type = builder.getIntegerType(32);
  if (parser.parseAttribute(trueWeight, i32Type, "weight", weights) ||
      parser.parseComma() ||
      parser.parseAttribute(falseWeight, i32Type, "weight", weights) ||
      parser.parseRSquare())
    return failure();

  result.addAttribute(kBranchWeightAttrName,
                      builder.getArrayAttr({trueWeight, falseWeight}));
}

// Parse the true branch.
SmallVector<Value, 4> trueOperands;
if (parser.parseComma() ||
    parser.parseSuccessorAndUseList(dest, trueOperands))
  return failure();
result.addSuccessors(dest);
result.addOperands(trueOperands);

// Parse the false branch.
SmallVector<Value, 4> falseOperands;
if (parser.parseComma() ||
    parser.parseSuccessorAndUseList(dest, falseOperands))
  return failure();
result.addSuccessors(dest);
result.addOperands(falseOperands);
result.addAttribute(spirv::BranchConditionalOp::getOperandSegmentSizeAttr(),
                    builder.getDenseI32ArrayAttr(
                        {1, static_cast<int32_t>(trueOperands.size()),
                         static_cast<int32_t>(falseOperands.size())}));

return success();
1736}

1738void spirv::BranchConditionalOp::print(OpAsmPrinter &printer) {
printer << ' ' << getCondition();

if (auto weights = getBranchWeights()) {
  printer << " [";
  llvm::interleaveComma(weights->getValue(), printer, [&](Attribute a) {
    printer << a.cast<IntegerAttr>().getInt();
  });
  printer << "]";
}

printer << ", ";
printer.printSuccessorAndUseList(getTrueBlock(), getTrueBlockArguments());
printer << ", ";
printer.printSuccessorAndUseList(getFalseBlock(), getFalseBlockArguments());
1753}

1755LogicalResult spirv::BranchConditionalOp::verify() {
if (auto weights = getBranchWeights()) {
  if (weights->getValue().size() != 2) {
    return emitOpError("must have exactly two branch weights");
  }
  if (llvm::all_of(*weights, [](Attribute attr) {
        return attr.cast<IntegerAttr>().getValue().isNullValue();
      }))
    return emitOpError("branch weights cannot both be zero");
}

return success();
1767}

1769//===----------------------------------------------------------------------===//
1770// spirv.CompositeConstruct
1771//===----------------------------------------------------------------------===//

1773LogicalResult spirv::CompositeConstructOp::verify() {
auto cType = getType().cast<spirv::CompositeType>();
operand_range constituents = this->getConstituents();

if (auto coopType = cType.dyn_cast<spirv::CooperativeMatrixNVType>()) {
  if (constituents.size() != 1)
    return emitOpError("has incorrect number of operands: expected ")
           << "1, but provided " << constituents.size();
  if (coopType.getElementType() != constituents.front().getType())
    return emitOpError("operand type mismatch: expected operand type ")
           << coopType.getElementType() << ", but provided "
           << constituents.front().getType();
  return success();
}

if (auto jointType = cType.dyn_cast<spirv::JointMatrixINTELType>()) {
  if (constituents.size() != 1)
    return emitOpError("has incorrect number of operands: expected ")
           << "1, but provided " << constituents.size();
  if (jointType.getElementType() != constituents.front().getType())
    return emitOpError("operand type mismatch: expected operand type ")
           << jointType.getElementType() << ", but provided "
           << constituents.front().getType();
  return success();
}

if (constituents.size() == cType.getNumElements()) {
  for (auto index : llvm::seq<uint32_t>(0, constituents.size())) {
    if (constituents[index].getType() != cType.getElementType(index)) {
      return emitOpError("operand type mismatch: expected operand type ")
             << cType.getElementType(index) << ", but provided "
             << constituents[index].getType();
    }
  }
  return success();
}

// If not constructing a cooperative matrix type, then we must be constructing
// a vector type.
auto resultType = cType.dyn_cast<VectorType>();
if (!resultType)
  return emitOpError(
      "expected to return a vector or cooperative matrix when the number of "
      "constituents is less than what the result needs");

SmallVector<unsigned> sizes;
for (Value component : constituents) {
  if (!component.getType().isa<VectorType>() &&
      !component.getType().isIntOrFloat())
    return emitOpError("operand type mismatch: expected operand to have "
                       "a scalar or vector type, but provided ")
           << component.getType();

  Type elementType = component.getType();
  if (auto vectorType = component.getType().dyn_cast<VectorType>()) {
    sizes.push_back(vectorType.getNumElements());
    elementType = vectorType.getElementType();
  } else {
    sizes.push_back(1);
  }

  if (elementType != resultType.getElementType())
    return emitOpError("operand element type mismatch: expected to be ")
           << resultType.getElementType() << ", but provided " << elementType;
}
unsigned totalCount = std::accumulate(sizes.begin(), sizes.end(), 0);
if (totalCount != cType.getNumElements())
  return emitOpError("has incorrect number of operands: expected ")
         << cType.getNumElements() << ", but provided " << totalCount;
return success();
1843}

1845//===----------------------------------------------------------------------===//
1846// spirv.CompositeExtractOp
1847//===----------------------------------------------------------------------===//

1849void spirv::CompositeExtractOp::build(OpBuilder &builder, OperationState &state,
                                    Value composite,
                                    ArrayRef<int32_t> indices) {
auto indexAttr = builder.getI32ArrayAttr(indices);
auto elementType =
    getElementType(composite.getType(), indexAttr, state.location);
if (!elementType) {
  return;
}
build(builder, state, elementType, composite, indexAttr);
1859}

1861ParseResult spirv::CompositeExtractOp::parse(OpAsmParser &parser,
                                           OperationState &result) {
OpAsmParser::UnresolvedOperand compositeInfo;
Attribute indicesAttr;
Type compositeType;
SMLoc attrLocation;

if (parser.parseOperand(compositeInfo) ||
    parser.getCurrentLocation(&attrLocation) ||
    parser.parseAttribute(indicesAttr, kIndicesAttrName, result.attributes) ||
    parser.parseColonType(compositeType) ||
    parser.resolveOperand(compositeInfo, compositeType, result.operands)) {
  return failure();
}

Type resultType =
    getElementType(compositeType, indicesAttr, parser, attrLocation);
if (!resultType) {
  return failure();
}
result.addTypes(resultType);
return success();
1883}

1885void spirv::CompositeExtractOp::print(OpAsmPrinter &printer) {
printer << ' ' << getComposite() << getIndices() << " : "
        << getComposite().getType();
1888}

1890LogicalResult spirv::CompositeExtractOp::verify() {
auto indicesArrayAttr = getIndices().dyn_cast<ArrayAttr>();
auto resultType =
    getElementType(getComposite().getType(), indicesArrayAttr, getLoc());
if (!resultType)
  return failure();

if (resultType != getType()) {
  return emitOpError("invalid result type: expected ")
         << resultType << " but provided " << getType();
}

return success();
1903}

1905//===----------------------------------------------------------------------===//
1906// spirv.CompositeInsert
1907//===----------------------------------------------------------------------===//

1909void spirv::CompositeInsertOp::build(OpBuilder &builder, OperationState &state,
                                   Value object, Value composite,
                                   ArrayRef<int32_t> indices) {
auto indexAttr = builder.getI32ArrayAttr(indices);
build(builder, state, composite.getType(), object, composite, indexAttr);
1914}

1916ParseResult spirv::CompositeInsertOp::parse(OpAsmParser &parser,
                                          OperationState &result) {
SmallVector<OpAsmParser::UnresolvedOperand, 2> operands;
Type objectType, compositeType;
Attribute indicesAttr;
auto loc = parser.getCurrentLocation();

return failure(
    parser.parseOperandList(operands, 2) ||
    parser.parseAttribute(indicesAttr, kIndicesAttrName, result.attributes) ||
    parser.parseColonType(objectType) ||
    parser.parseKeywordType("into", compositeType) ||
    parser.resolveOperands(operands, {objectType, compositeType}, loc,
                           result.operands) ||
    parser.addTypesToList(compositeType, result.types));
1931}

1933LogicalResult spirv::CompositeInsertOp::verify() {
auto indicesArrayAttr = getIndices().dyn_cast<ArrayAttr>();
auto objectType =
    getElementType(getComposite().getType(), indicesArrayAttr, getLoc());
if (!objectType)
  return failure();

if (objectType != getObject().getType()) {
  return emitOpError("object operand type should be ")
         << objectType << ", but found " << getObject().getType();
}

if (getComposite().getType() != getType()) {
  return emitOpError("result type should be the same as "
                     "the composite type, but found ")
         << getComposite().getType() << " vs " << getType();
}

return success();
1952}

1954void spirv::CompositeInsertOp::print(OpAsmPrinter &printer) {
printer << " " << getObject() << ", " << getComposite() << getIndices()
        << " : " << getObject().getType() << " into "
        << getComposite().getType();
1958}

1960//===----------------------------------------------------------------------===//
1961// spirv.Constant
1962//===----------------------------------------------------------------------===//

1964ParseResult spirv::ConstantOp::parse(OpAsmParser &parser,
                                   OperationState &result) {
Attribute value;
if (parser.parseAttribute(value, kValueAttrName, result.attributes))
  return failure();

Type type = NoneType::get(parser.getContext());
if (auto typedAttr = value.dyn_cast<TypedAttr>())
  type = typedAttr.getType();
if (type.isa<NoneType, TensorType>()) {
  if (parser.parseColonType(type))
    return failure();
}

return parser.addTypeToList(type, result.types);
1979}

1981void spirv::ConstantOp::print(OpAsmPrinter &printer) {
printer << ' ' << getValue();
if (getType().isa<spirv::ArrayType>())
  printer << " : " << getType();
1985}

1987static LogicalResult verifyConstantType(spirv::ConstantOp op, Attribute value,
                                      Type opType) {
if (value.isa<IntegerAttr, FloatAttr>()) {
  auto valueType = value.cast<TypedAttr>().getType();
  if (valueType != opType)
    return op.emitOpError("result type (")
           << opType << ") does not match value type (" << valueType << ")";
  return success();
}
if (value.isa<DenseIntOrFPElementsAttr, SparseElementsAttr>()) {
  auto valueType = value.cast<TypedAttr>().getType();
  if (valueType == opType)
    return success();
  auto arrayType = opType.dyn_cast<spirv::ArrayType>();
  auto shapedType = valueType.dyn_cast<ShapedType>();
  if (!arrayType)
    return op.emitOpError("result or element type (")
           << opType << ") does not match value type (" << valueType
           << "), must be the same or spirv.array";

  int numElements = arrayType.getNumElements();
  auto opElemType = arrayType.getElementType();
  while (auto t = opElemType.dyn_cast<spirv::ArrayType>()) {
    numElements *= t.getNumElements();
    opElemType = t.getElementType();
  }
  if (!opElemType.isIntOrFloat())
    return op.emitOpError("only support nested array result type");

  auto valueElemType = shapedType.getElementType();
  if (valueElemType != opElemType) {
    return op.emitOpError("result element type (")
           << opElemType << ") does not match value element type ("
           << valueElemType << ")";
  }

  if (numElements != shapedType.getNumElements()) {
    return op.emitOpError("result number of elements (")
           << numElements << ") does not match value number of elements ("
           << shapedType.getNumElements() << ")";
  }
  return success();
}
if (auto arrayAttr = value.dyn_cast<ArrayAttr>()) {
  auto arrayType = opType.dyn_cast<spirv::ArrayType>();
  if (!arrayType)
    return op.emitOpError(
        "must have spirv.array result type for array value");
  Type elemType = arrayType.getElementType();
  for (Attribute element : arrayAttr.getValue()) {
    // Verify array elements recursively.
    if (failed(verifyConstantType(op, element, elemType)))
      return failure();
  }
  return success();
}
return op.emitOpError("cannot have attribute: ") << value;
2044}

2046LogicalResult spirv::ConstantOp::verify() {
// ODS already generates checks to make sure the result type is valid. We just
// need to additionally check that the value's attribute type is consistent
// with the result type.
return verifyConstantType(*this, getValueAttr(), getType());
2051}

2053bool spirv::ConstantOp::isBuildableWith(Type type) {
// Must be valid SPIR-V type first.
if (!type.isa<spirv::SPIRVType>())
  return false;

if (isa<SPIRVDialect>(type.getDialect())) {
  // TODO: support constant struct
  return type.isa<spirv::ArrayType>();
}

return true;
2064}

2066spirv::ConstantOp spirv::ConstantOp::getZero(Type type, Location loc,
                                           OpBuilder &builder) {
if (auto intType = type.dyn_cast<IntegerType>()) {
  unsigned width = intType.getWidth();
  if (width == 1)
    return builder.create<spirv::ConstantOp>(loc, type,
                                             builder.getBoolAttr(false));
  return builder.create<spirv::ConstantOp>(
      loc, type, builder.getIntegerAttr(type, APInt(width, 0)));
}
if (auto floatType = type.dyn_cast<FloatType>()) {
  return builder.create<spirv::ConstantOp>(
      loc, type, builder.getFloatAttr(floatType, 0.0));
}
if (auto vectorType = type.dyn_cast<VectorType>()) {
  Type elemType = vectorType.getElementType();
  if (elemType.isa<IntegerType>()) {
    return builder.create<spirv::ConstantOp>(
        loc, type,
        DenseElementsAttr::get(vectorType,
                               IntegerAttr::get(elemType, 0).getValue()));
  }
  if (elemType.isa<FloatType>()) {
    return builder.create<spirv::ConstantOp>(
        loc, type,
        DenseFPElementsAttr::get(vectorType,
                                 FloatAttr::get(elemType, 0.0).getValue()));
  }
}

llvm_unreachable("unimplemented types for ConstantOp::getZero()")::llvm::llvm_unreachable_internal("unimplemented types for ConstantOp::getZero()"
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 2096);
2097}

2099spirv::ConstantOp spirv::ConstantOp::getOne(Type type, Location loc,
                                          OpBuilder &builder) {
if (auto intType = type.dyn_cast<IntegerType>()) {
  unsigned width = intType.getWidth();
  if (width == 1)
    return builder.create<spirv::ConstantOp>(loc, type,
                                             builder.getBoolAttr(true));
  return builder.create<spirv::ConstantOp>(
      loc, type, builder.getIntegerAttr(type, APInt(width, 1)));
}
if (auto floatType = type.dyn_cast<FloatType>()) {
  return builder.create<spirv::ConstantOp>(
      loc, type, builder.getFloatAttr(floatType, 1.0));
}
if (auto vectorType = type.dyn_cast<VectorType>()) {
  Type elemType = vectorType.getElementType();
  if (elemType.isa<IntegerType>()) {
    return builder.create<spirv::ConstantOp>(
        loc, type,
        DenseElementsAttr::get(vectorType,
                               IntegerAttr::get(elemType, 1).getValue()));
  }
  if (elemType.isa<FloatType>()) {
    return builder.create<spirv::ConstantOp>(
        loc, type,
        DenseFPElementsAttr::get(vectorType,
                                 FloatAttr::get(elemType, 1.0).getValue()));
  }
}

llvm_unreachable("unimplemented types for ConstantOp::getOne()")::llvm::llvm_unreachable_internal("unimplemented types for ConstantOp::getOne()"
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 2129);
2130}

2132void mlir::spirv::ConstantOp::getAsmResultNames(
  llvm::function_ref<void(mlir::Value, llvm::StringRef)> setNameFn) {
Type type = getType();

SmallString<32> specialNameBuffer;
llvm::raw_svector_ostream specialName(specialNameBuffer);
specialName << "cst";

IntegerType intTy = type.dyn_cast<IntegerType>();

if (IntegerAttr intCst = getValue().dyn_cast<IntegerAttr>()) {
  if (intTy && intTy.getWidth() == 1) {
    return setNameFn(getResult(), (intCst.getInt() ? "true" : "false"));
  }

  if (intTy.isSignless()) {
    specialName << intCst.getInt();
  } else {
    specialName << intCst.getSInt();
  }
}

if (intTy || type.isa<FloatType>()) {
  specialName << '_' << type;
}

if (auto vecType = type.dyn_cast<VectorType>()) {
  specialName << "_vec_";
  specialName << vecType.getDimSize(0);

  Type elementType = vecType.getElementType();

  if (elementType.isa<IntegerType>() || elementType.isa<FloatType>()) {
    specialName << "x" << elementType;
  }
}

setNameFn(getResult(), specialName.str());
2170}

2172void mlir::spirv::AddressOfOp::getAsmResultNames(
  llvm::function_ref<void(mlir::Value, llvm::StringRef)> setNameFn) {
SmallString<32> specialNameBuffer;
llvm::raw_svector_ostream specialName(specialNameBuffer);
specialName << getVariable() << "_addr";
setNameFn(getResult(), specialName.str());
2178}

2180//===----------------------------------------------------------------------===//
2181// spirv.ControlBarrierOp
2182//===----------------------------------------------------------------------===//

2184LogicalResult spirv::ControlBarrierOp::verify() {
return verifyMemorySemantics(getOperation(), getMemorySemantics());
2186}

2188//===----------------------------------------------------------------------===//
2189// spirv.ConvertFToSOp
2190//===----------------------------------------------------------------------===//

2192LogicalResult spirv::ConvertFToSOp::verify() {
return verifyCastOp(*this, /*requireSameBitWidth=*/false,
                    /*skipBitWidthCheck=*/true);
2195}

2197//===----------------------------------------------------------------------===//
2198// spirv.ConvertFToUOp
2199//===----------------------------------------------------------------------===//

2201LogicalResult spirv::ConvertFToUOp::verify() {
return verifyCastOp(*this, /*requireSameBitWidth=*/false,
                    /*skipBitWidthCheck=*/true);
2204}

2206//===----------------------------------------------------------------------===//
2207// spirv.ConvertSToFOp
2208//===----------------------------------------------------------------------===//

2210LogicalResult spirv::ConvertSToFOp::verify() {
return verifyCastOp(*this, /*requireSameBitWidth=*/false,
                    /*skipBitWidthCheck=*/true);
2213}

2215//===----------------------------------------------------------------------===//
2216// spirv.ConvertUToFOp
2217//===----------------------------------------------------------------------===//

2219LogicalResult spirv::ConvertUToFOp::verify() {
return verifyCastOp(*this, /*requireSameBitWidth=*/false,
                    /*skipBitWidthCheck=*/true);
2222}

2224//===----------------------------------------------------------------------===//
2225// spirv.EntryPoint
2226//===----------------------------------------------------------------------===//

2228void spirv::EntryPointOp::build(OpBuilder &builder, OperationState &state,
                              spirv::ExecutionModel executionModel,
                              spirv::FuncOp function,
                              ArrayRef<Attribute> interfaceVars) {
build(builder, state,
      spirv::ExecutionModelAttr::get(builder.getContext(), executionModel),
      SymbolRefAttr::get(function), builder.getArrayAttr(interfaceVars));
2235}

2237ParseResult spirv::EntryPointOp::parse(OpAsmParser &parser,
                                     OperationState &result) {
spirv::ExecutionModel execModel;
SmallVector<OpAsmParser::UnresolvedOperand, 0> identifiers;
SmallVector<Type, 0> idTypes;
SmallVector<Attribute, 4> interfaceVars;

FlatSymbolRefAttr fn;
if (parseEnumStrAttr<spirv::ExecutionModelAttr>(execModel, parser, result) ||
    parser.parseAttribute(fn, Type(), kFnNameAttrName, result.attributes)) {
  return failure();
}

if (!parser.parseOptionalComma()) {
  // Parse the interface variables
  if (parser.parseCommaSeparatedList([&]() -> ParseResult {
        // The name of the interface variable attribute isnt important
        FlatSymbolRefAttr var;
        NamedAttrList attrs;
        if (parser.parseAttribute(var, Type(), "var_symbol", attrs))
          return failure();
        interfaceVars.push_back(var);
        return success();
      }))
    return failure();
}
result.addAttribute(kInterfaceAttrName,
                    parser.getBuilder().getArrayAttr(interfaceVars));
return success();
2266}

2268void spirv::EntryPointOp::print(OpAsmPrinter &printer) {
printer << " \"" << stringifyExecutionModel(getExecutionModel()) << "\" ";
printer.printSymbolName(getFn());
auto interfaceVars = getInterface().getValue();
if (!interfaceVars.empty()) {
  printer << ", ";
  llvm::interleaveComma(interfaceVars, printer);
}
2276}

2278LogicalResult spirv::EntryPointOp::verify() {
// Checks for fn and interface symbol reference are done in spirv::ModuleOp
// verification.
return success();
2282}

2284//===----------------------------------------------------------------------===//
2285// spirv.ExecutionMode
2286//===----------------------------------------------------------------------===//

2288void spirv::ExecutionModeOp::build(OpBuilder &builder, OperationState &state,
                                 spirv::FuncOp function,
                                 spirv::ExecutionMode executionMode,
                                 ArrayRef<int32_t> params) {
build(builder, state, SymbolRefAttr::get(function),
      spirv::ExecutionModeAttr::get(builder.getContext(), executionMode),
      builder.getI32ArrayAttr(params));
2295}

2297ParseResult spirv::ExecutionModeOp::parse(OpAsmParser &parser,
                                        OperationState &result) {
spirv::ExecutionMode execMode;
Attribute fn;
if (parser.parseAttribute(fn, kFnNameAttrName, result.attributes) ||
    parseEnumStrAttr<spirv::ExecutionModeAttr>(execMode, parser, result)) {
  return failure();
}

SmallVector<int32_t, 4> values;
Type i32Type = parser.getBuilder().getIntegerType(32);
while (!parser.parseOptionalComma()) {
  NamedAttrList attr;
  Attribute value;
  if (parser.parseAttribute(value, i32Type, "value", attr)) {
    return failure();
  }
  values.push_back(value.cast<IntegerAttr>().getInt());
}
result.addAttribute(kValuesAttrName,
                    parser.getBuilder().getI32ArrayAttr(values));
return success();
2319}

2321void spirv::ExecutionModeOp::print(OpAsmPrinter &printer) {
printer << " ";
printer.printSymbolName(getFn());
printer << " \"" << stringifyExecutionMode(getExecutionMode()) << "\"";
auto values = this->getValues();
if (values.empty())
  return;
printer << ", ";
llvm::interleaveComma(values, printer, [&](Attribute a) {
  printer << a.cast<IntegerAttr>().getInt();
});
2332}

2334//===----------------------------------------------------------------------===//
2335// spirv.FConvertOp
2336//===----------------------------------------------------------------------===//

2338LogicalResult spirv::FConvertOp::verify() {
return verifyCastOp(*this, /*requireSameBitWidth=*/false);
2340}

2342//===----------------------------------------------------------------------===//
2343// spirv.SConvertOp
2344//===----------------------------------------------------------------------===//

2346LogicalResult spirv::SConvertOp::verify() {
return verifyCastOp(*this, /*requireSameBitWidth=*/false);
2348}

2350//===----------------------------------------------------------------------===//
2351// spirv.UConvertOp
2352//===----------------------------------------------------------------------===//

2354LogicalResult spirv::UConvertOp::verify() {
return verifyCastOp(*this, /*requireSameBitWidth=*/false);
2356}

2358//===----------------------------------------------------------------------===//
2359// spirv.func
2360//===----------------------------------------------------------------------===//

2362ParseResult spirv::FuncOp::parse(OpAsmParser &parser, OperationState &result) {
SmallVector<OpAsmParser::Argument> entryArgs;
SmallVector<DictionaryAttr> resultAttrs;
SmallVector<Type> resultTypes;
auto &builder = parser.getBuilder();

// Parse the name as a symbol.
StringAttr nameAttr;
if (parser.parseSymbolName(nameAttr, SymbolTable::getSymbolAttrName(),
                           result.attributes))
  return failure();

// Parse the function signature.
bool isVariadic = false;
if (function_interface_impl::parseFunctionSignature(
        parser, /*allowVariadic=*/false, entryArgs, isVariadic, resultTypes,
        resultAttrs))
  return failure();

SmallVector<Type> argTypes;
for (auto &arg : entryArgs)
  argTypes.push_back(arg.type);
auto fnType = builder.getFunctionType(argTypes, resultTypes);
result.addAttribute(getFunctionTypeAttrName(result.name),
                    TypeAttr::get(fnType));

// Parse the optional function control keyword.
spirv::FunctionControl fnControl;
if (parseEnumStrAttr<spirv::FunctionControlAttr>(fnControl, parser, result))
  return failure();

// If additional attributes are present, parse them.
if (parser.parseOptionalAttrDictWithKeyword(result.attributes))
  return failure();

// Add the attributes to the function arguments.
assert(resultAttrs.size() == resultTypes.size())(static_cast <bool> (resultAttrs.size() == resultTypes.
size()) ? void (0) : __assert_fail ("resultAttrs.size() == resultTypes.size()"
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 2398, __extension__
 __PRETTY_FUNCTION__));
function_interface_impl::addArgAndResultAttrs(
    builder, result, entryArgs, resultAttrs, getArgAttrsAttrName(result.name),
    getResAttrsAttrName(result.name));

// Parse the optional function body.
auto *body = result.addRegion();
OptionalParseResult parseResult =
    parser.parseOptionalRegion(*body, entryArgs);
return failure(parseResult.has_value() && failed(*parseResult));
2408}

2410void spirv::FuncOp::print(OpAsmPrinter &printer) {
// Print function name, signature, and control.
printer << " ";
printer.printSymbolName(getSymName());
auto fnType = getFunctionType();
function_interface_impl::printFunctionSignature(
    printer, *this, fnType.getInputs(),
    /*isVariadic=*/false, fnType.getResults());
printer << " \"" << spirv::stringifyFunctionControl(getFunctionControl())
        << "\"";
function_interface_impl::printFunctionAttributes(
    printer, *this,
    {spirv::attributeName<spirv::FunctionControl>(),
     getFunctionTypeAttrName(), getArgAttrsAttrName(), getResAttrsAttrName(),
     getFunctionControlAttrName()});

// Print the body if this is not an external function.
Region &body = this->getBody();
if (!body.empty()) {
  printer << ' ';
  printer.printRegion(body, /*printEntryBlockArgs=*/false,
                      /*printBlockTerminators=*/true);
}
2433}

2435LogicalResult spirv::FuncOp::verifyType() {
if (getFunctionType().getNumResults() > 1)
  return emitOpError("cannot have more than one result");
return success();
2439}

2441LogicalResult spirv::FuncOp::verifyBody() {
FunctionType fnType = getFunctionType();

auto walkResult = walk([fnType](Operation *op) -> WalkResult {
  if (auto retOp = dyn_cast<spirv::ReturnOp>(op)) {
    if (fnType.getNumResults() != 0)
      return retOp.emitOpError("cannot be used in functions returning value");
  } else if (auto retOp = dyn_cast<spirv::ReturnValueOp>(op)) {
    if (fnType.getNumResults() != 1)
      return retOp.emitOpError(
                 "returns 1 value but enclosing function requires ")
             << fnType.getNumResults() << " results";

    auto retOperandType = retOp.getValue().getType();
    auto fnResultType = fnType.getResult(0);
    if (retOperandType != fnResultType)
      return retOp.emitOpError(" return value's type (")
             << retOperandType << ") mismatch with function's result type ("
             << fnResultType << ")";
  }
  return WalkResult::advance();
});

// TODO: verify other bits like linkage type.

return failure(walkResult.wasInterrupted());
2467}

2469void spirv::FuncOp::build(OpBuilder &builder, OperationState &state,
                        StringRef name, FunctionType type,
                        spirv::FunctionControl control,
                        ArrayRef<NamedAttribute> attrs) {
state.addAttribute(SymbolTable::getSymbolAttrName(),
                   builder.getStringAttr(name));
state.addAttribute(getFunctionTypeAttrName(state.name), TypeAttr::get(type));
state.addAttribute(spirv::attributeName<spirv::FunctionControl>(),
                   builder.getAttr<spirv::FunctionControlAttr>(control));
state.attributes.append(attrs.begin(), attrs.end());
state.addRegion();
2480}

2482// CallableOpInterface
2483Region *spirv::FuncOp::getCallableRegion() {
return isExternal() ? nullptr : &getBody();
2485}

2487// CallableOpInterface
2488ArrayRef<Type> spirv::FuncOp::getCallableResults() {
return getFunctionType().getResults();
2490}

2492//===----------------------------------------------------------------------===//
2493// spirv.FunctionCall
2494//===----------------------------------------------------------------------===//

2496LogicalResult spirv::FunctionCallOp::verify() {
auto fnName = getCalleeAttr();

auto funcOp = dyn_cast_or_null<spirv::FuncOp>(
    SymbolTable::lookupNearestSymbolFrom((*this)->getParentOp(), fnName));
if (!funcOp) {
  return emitOpError("callee function '")
         << fnName.getValue() << "' not found in nearest symbol table";
}

auto functionType = funcOp.getFunctionType();

if (getNumResults() > 1) {
  return emitOpError(
             "expected callee function to have 0 or 1 result, but provided ")
         << getNumResults();
}

if (functionType.getNumInputs() != getNumOperands()) {
  return emitOpError("has incorrect number of operands for callee: expected ")
         << functionType.getNumInputs() << ", but provided "
         << getNumOperands();
}

for (uint32_t i = 0, e = functionType.getNumInputs(); i != e; ++i) {
  if (getOperand(i).getType() != functionType.getInput(i)) {
    return emitOpError("operand type mismatch: expected operand type ")
           << functionType.getInput(i) << ", but provided "
           << getOperand(i).getType() << " for operand number " << i;
  }
}

if (functionType.getNumResults() != getNumResults()) {
  return emitOpError(
             "has incorrect number of results has for callee: expected ")
         << functionType.getNumResults() << ", but provided "
         << getNumResults();
}

if (getNumResults() &&
    (getResult(0).getType() != functionType.getResult(0))) {
  return emitOpError("result type mismatch: expected ")
         << functionType.getResult(0) << ", but provided "
         << getResult(0).getType();
}

return success();
2543}

2545CallInterfaceCallable spirv::FunctionCallOp::getCallableForCallee() {
return (*this)->getAttrOfType<SymbolRefAttr>(kCallee);
2547}

2549Operation::operand_range spirv::FunctionCallOp::getArgOperands() {
return getArguments();
2551}

2553//===----------------------------------------------------------------------===//
2554// spirv.GLFClampOp
2555//===----------------------------------------------------------------------===//

2557ParseResult spirv::GLFClampOp::parse(OpAsmParser &parser,
                                   OperationState &result) {
return parseOneResultSameOperandTypeOp(parser, result);
2560}
2561void spirv::GLFClampOp::print(OpAsmPrinter &p) { printOneResultOp(*this, p); }

2563//===----------------------------------------------------------------------===//
2564// spirv.GLUClampOp
2565//===----------------------------------------------------------------------===//

2567ParseResult spirv::GLUClampOp::parse(OpAsmParser &parser,
                                   OperationState &result) {
return parseOneResultSameOperandTypeOp(parser, result);
2570}
2571void spirv::GLUClampOp::print(OpAsmPrinter &p) { printOneResultOp(*this, p); }

2573//===----------------------------------------------------------------------===//
2574// spirv.GLSClampOp
2575//===----------------------------------------------------------------------===//

2577ParseResult spirv::GLSClampOp::parse(OpAsmParser &parser,
                                   OperationState &result) {
return parseOneResultSameOperandTypeOp(parser, result);
2580}
2581void spirv::GLSClampOp::print(OpAsmPrinter &p) { printOneResultOp(*this, p); }

2583//===----------------------------------------------------------------------===//
2584// spirv.GLFmaOp
2585//===----------------------------------------------------------------------===//

2587ParseResult spirv::GLFmaOp::parse(OpAsmParser &parser, OperationState &result) {
return parseOneResultSameOperandTypeOp(parser, result);
2589}
2590void spirv::GLFmaOp::print(OpAsmPrinter &p) { printOneResultOp(*this, p); }

2592//===----------------------------------------------------------------------===//
2593// spirv.GlobalVariable
2594//===----------------------------------------------------------------------===//

2596void spirv::GlobalVariableOp::build(OpBuilder &builder, OperationState &state,
                                  Type type, StringRef name,
                                  unsigned descriptorSet, unsigned binding) {
build(builder, state, TypeAttr::get(type), builder.getStringAttr(name));
state.addAttribute(
    spirv::SPIRVDialect::getAttributeName(spirv::Decoration::DescriptorSet),
    builder.getI32IntegerAttr(descriptorSet));
state.addAttribute(
    spirv::SPIRVDialect::getAttributeName(spirv::Decoration::Binding),
    builder.getI32IntegerAttr(binding));
2606}

2608void spirv::GlobalVariableOp::build(OpBuilder &builder, OperationState &state,
                                  Type type, StringRef name,
                                  spirv::BuiltIn builtin) {
build(builder, state, TypeAttr::get(type), builder.getStringAttr(name));
state.addAttribute(
    spirv::SPIRVDialect::getAttributeName(spirv::Decoration::BuiltIn),
    builder.getStringAttr(spirv::stringifyBuiltIn(builtin)));
2615}

2617ParseResult spirv::GlobalVariableOp::parse(OpAsmParser &parser,
                                         OperationState &result) {
// Parse variable name.
StringAttr nameAttr;
if (parser.parseSymbolName(nameAttr, SymbolTable::getSymbolAttrName(),
                           result.attributes)) {
  return failure();
}

// Parse optional initializer
if (succeeded(parser.parseOptionalKeyword(kInitializerAttrName))) {
  FlatSymbolRefAttr initSymbol;
  if (parser.parseLParen() ||
      parser.parseAttribute(initSymbol, Type(), kInitializerAttrName,
                            result.attributes) ||
      parser.parseRParen())
    return failure();
}

if (parseVariableDecorations(parser, result)) {
  return failure();
}

Type type;
auto loc = parser.getCurrentLocation();
if (parser.parseColonType(type)) {
  return failure();
}
if (!type.isa<spirv::PointerType>()) {
  return parser.emitError(loc, "expected spirv.ptr type");
}
result.addAttribute(kTypeAttrName, TypeAttr::get(type));

return success();
2651}

2653void spirv::GlobalVariableOp::print(OpAsmPrinter &printer) {
SmallVector<StringRef, 4> elidedAttrs{
    spirv::attributeName<spirv::StorageClass>()};

// Print variable name.
printer << ' ';
printer.printSymbolName(getSymName());
elidedAttrs.push_back(SymbolTable::getSymbolAttrName());

// Print optional initializer
if (auto initializer = this->getInitializer()) {
  printer << " " << kInitializerAttrName << '(';
  printer.printSymbolName(*initializer);
  printer << ')';
  elidedAttrs.push_back(kInitializerAttrName);
}

elidedAttrs.push_back(kTypeAttrName);
printVariableDecorations(*this, printer, elidedAttrs);
printer << " : " << getType();
2673}

2675LogicalResult spirv::GlobalVariableOp::verify() {
if (!getType().isa<spirv::PointerType>())
  return emitOpError("result must be of a !spv.ptr type");

// SPIR-V spec: "Storage Class is the Storage Class of the memory holding the
// object. It cannot be Generic. It must be the same as the Storage Class
// operand of the Result Type."
// Also, Function storage class is reserved by spirv.Variable.
auto storageClass = this->storageClass();
if (storageClass == spirv::StorageClass::Generic ||
    storageClass == spirv::StorageClass::Function) {
  return emitOpError("storage class cannot be '")
         << stringifyStorageClass(storageClass) << "'";
}

if (auto init =
        (*this)->getAttrOfType<FlatSymbolRefAttr>(kInitializerAttrName)) {
  Operation *initOp = SymbolTable::lookupNearestSymbolFrom(
      (*this)->getParentOp(), init.getAttr());
  // TODO: Currently only variable initialization with specialization
  // constants and other variables is supported. They could be normal
  // constants in the module scope as well.
  if (!initOp ||
      !isa<spirv::GlobalVariableOp, spirv::SpecConstantOp>(initOp)) {
    return emitOpError("initializer must be result of a "
                       "spirv.SpecConstant or spirv.GlobalVariable op");
  }
}

return success();
2705}

2707//===----------------------------------------------------------------------===//
2708// spirv.GroupBroadcast
2709//===----------------------------------------------------------------------===//

2711LogicalResult spirv::GroupBroadcastOp::verify() {
spirv::Scope scope = getExecutionScope();
if (scope != spirv::Scope::Workgroup && scope != spirv::Scope::Subgroup)
  return emitOpError("execution scope must be 'Workgroup' or 'Subgroup'");

if (auto localIdTy = getLocalid().getType().dyn_cast<VectorType>())
  if (localIdTy.getNumElements() != 2 && localIdTy.getNumElements() != 3)
    return emitOpError("localid is a vector and can be with only "
                       " 2 or 3 components, actual number is ")
           << localIdTy.getNumElements();

return success();
2723}

2725//===----------------------------------------------------------------------===//
2726// spirv.GroupNonUniformBallotOp
2727//===----------------------------------------------------------------------===//

2729LogicalResult spirv::GroupNonUniformBallotOp::verify() {
spirv::Scope scope = getExecutionScope();
if (scope != spirv::Scope::Workgroup && scope != spirv::Scope::Subgroup)
  return emitOpError("execution scope must be 'Workgroup' or 'Subgroup'");

return success();
2735}

2737//===----------------------------------------------------------------------===//
2738// spirv.GroupNonUniformBroadcast
2739//===----------------------------------------------------------------------===//

2741LogicalResult spirv::GroupNonUniformBroadcastOp::verify() {
spirv::Scope scope = getExecutionScope();
if (scope != spirv::Scope::Workgroup && scope != spirv::Scope::Subgroup)
  return emitOpError("execution scope must be 'Workgroup' or 'Subgroup'");

// SPIR-V spec: "Before version 1.5, Id must come from a
// constant instruction.
auto targetEnv = spirv::getDefaultTargetEnv(getContext());
if (auto spirvModule = (*this)->getParentOfType<spirv::ModuleOp>())
  targetEnv = spirv::lookupTargetEnvOrDefault(spirvModule);

if (targetEnv.getVersion() < spirv::Version::V_1_5) {
  auto *idOp = getId().getDefiningOp();
  if (!idOp || !isa<spirv::ConstantOp,           // for normal constant
                    spirv::ReferenceOfOp>(idOp)) // for spec constant
    return emitOpError("id must be the result of a constant op");
}

return success();
2760}

2762//===----------------------------------------------------------------------===//
2763// spirv.GroupNonUniformShuffle*
2764//===----------------------------------------------------------------------===//

2766template <typename OpTy>
2767static LogicalResult verifyGroupNonUniformShuffleOp(OpTy op) {
spirv::Scope scope = op.getExecutionScope();
if (scope != spirv::Scope::Workgroup && scope != spirv::Scope::Subgroup)
  return op.emitOpError("execution scope must be 'Workgroup' or 'Subgroup'");

if (op.getOperands().back().getType().isSignedInteger())
  return op.emitOpError("second operand must be a singless/unsigned integer");

return success();
2776}

2778LogicalResult spirv::GroupNonUniformShuffleOp::verify() {
return verifyGroupNonUniformShuffleOp(*this);
2780}
2781LogicalResult spirv::GroupNonUniformShuffleDownOp::verify() {
return verifyGroupNonUniformShuffleOp(*this);
2783}
2784LogicalResult spirv::GroupNonUniformShuffleUpOp::verify() {
return verifyGroupNonUniformShuffleOp(*this);
2786}
2787LogicalResult spirv::GroupNonUniformShuffleXorOp::verify() {
return verifyGroupNonUniformShuffleOp(*this);
2789}

2791//===----------------------------------------------------------------------===//
2792// spirv.INTEL.SubgroupBlockRead
2793//===----------------------------------------------------------------------===//

2795ParseResult spirv::INTELSubgroupBlockReadOp::parse(OpAsmParser &parser,
                                                 OperationState &result) {
// Parse the storage class specification
spirv::StorageClass storageClass;
OpAsmParser::UnresolvedOperand ptrInfo;
Type elementType;
if (parseEnumStrAttr(storageClass, parser) || parser.parseOperand(ptrInfo) ||
    parser.parseColon() || parser.parseType(elementType)) {
  return failure();
}

auto ptrType = spirv::PointerType::get(elementType, storageClass);
if (auto valVecTy = elementType.dyn_cast<VectorType>())
  ptrType = spirv::PointerType::get(valVecTy.getElementType(), storageClass);

if (parser.resolveOperand(ptrInfo, ptrType, result.operands)) {
  return failure();
}

result.addTypes(elementType);
return success();
2816}

2818void spirv::INTELSubgroupBlockReadOp::print(OpAsmPrinter &printer) {
printer << " " << getPtr() << " : " << getType();
2820}

2822LogicalResult spirv::INTELSubgroupBlockReadOp::verify() {
if (failed(verifyBlockReadWritePtrAndValTypes(*this, getPtr(), getValue())))
  return failure();

return success();
2827}

2829//===----------------------------------------------------------------------===//
2830// spirv.INTEL.SubgroupBlockWrite
2831//===----------------------------------------------------------------------===//

2833ParseResult spirv::INTELSubgroupBlockWriteOp::parse(OpAsmParser &parser,
                                                  OperationState &result) {
// Parse the storage class specification
spirv::StorageClass storageClass;
SmallVector<OpAsmParser::UnresolvedOperand, 2> operandInfo;
auto loc = parser.getCurrentLocation();
Type elementType;
if (parseEnumStrAttr(storageClass, parser) ||
    parser.parseOperandList(operandInfo, 2) || parser.parseColon() ||
    parser.parseType(elementType)) {
  return failure();
}

auto ptrType = spirv::PointerType::get(elementType, storageClass);
if (auto valVecTy = elementType.dyn_cast<VectorType>())
  ptrType = spirv::PointerType::get(valVecTy.getElementType(), storageClass);

if (parser.resolveOperands(operandInfo, {ptrType, elementType}, loc,
                           result.operands)) {
  return failure();
}
return success();
2855}

2857void spirv::INTELSubgroupBlockWriteOp::print(OpAsmPrinter &printer) {
printer << " " << getPtr() << ", " << getValue() << " : "
        << getValue().getType();
2860}

2862LogicalResult spirv::INTELSubgroupBlockWriteOp::verify() {
if (failed(verifyBlockReadWritePtrAndValTypes(*this, getPtr(), getValue())))
  return failure();

return success();
2867}

2869//===----------------------------------------------------------------------===//
2870// spirv.GroupNonUniformElectOp
2871//===----------------------------------------------------------------------===//

2873LogicalResult spirv::GroupNonUniformElectOp::verify() {
spirv::Scope scope = getExecutionScope();
if (scope != spirv::Scope::Workgroup && scope != spirv::Scope::Subgroup)
  return emitOpError("execution scope must be 'Workgroup' or 'Subgroup'");

return success();
2879}

2881//===----------------------------------------------------------------------===//
2882// spirv.GroupNonUniformFAddOp
2883//===----------------------------------------------------------------------===//

2885LogicalResult spirv::GroupNonUniformFAddOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
2887}

2889ParseResult spirv::GroupNonUniformFAddOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
2892}
2893void spirv::GroupNonUniformFAddOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
2895}

2897//===----------------------------------------------------------------------===//
2898// spirv.GroupNonUniformFMaxOp
2899//===----------------------------------------------------------------------===//

2901LogicalResult spirv::GroupNonUniformFMaxOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
2903}

2905ParseResult spirv::GroupNonUniformFMaxOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
2908}
2909void spirv::GroupNonUniformFMaxOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
2911}

2913//===----------------------------------------------------------------------===//
2914// spirv.GroupNonUniformFMinOp
2915//===----------------------------------------------------------------------===//

2917LogicalResult spirv::GroupNonUniformFMinOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
2919}

2921ParseResult spirv::GroupNonUniformFMinOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
2924}
2925void spirv::GroupNonUniformFMinOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
2927}

2929//===----------------------------------------------------------------------===//
2930// spirv.GroupNonUniformFMulOp
2931//===----------------------------------------------------------------------===//

2933LogicalResult spirv::GroupNonUniformFMulOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
2935}

2937ParseResult spirv::GroupNonUniformFMulOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
2940}
2941void spirv::GroupNonUniformFMulOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
2943}

2945//===----------------------------------------------------------------------===//
2946// spirv.GroupNonUniformIAddOp
2947//===----------------------------------------------------------------------===//

2949LogicalResult spirv::GroupNonUniformIAddOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
2951}

2953ParseResult spirv::GroupNonUniformIAddOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
2956}
2957void spirv::GroupNonUniformIAddOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
2959}

2961//===----------------------------------------------------------------------===//
2962// spirv.GroupNonUniformIMulOp
2963//===----------------------------------------------------------------------===//

2965LogicalResult spirv::GroupNonUniformIMulOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
2967}

2969ParseResult spirv::GroupNonUniformIMulOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
2972}
2973void spirv::GroupNonUniformIMulOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
2975}

2977//===----------------------------------------------------------------------===//
2978// spirv.GroupNonUniformSMaxOp
2979//===----------------------------------------------------------------------===//

2981LogicalResult spirv::GroupNonUniformSMaxOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
2983}

2985ParseResult spirv::GroupNonUniformSMaxOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
2988}
2989void spirv::GroupNonUniformSMaxOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
2991}

2993//===----------------------------------------------------------------------===//
2994// spirv.GroupNonUniformSMinOp
2995//===----------------------------------------------------------------------===//

2997LogicalResult spirv::GroupNonUniformSMinOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
2999}

3001ParseResult spirv::GroupNonUniformSMinOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
3004}
3005void spirv::GroupNonUniformSMinOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
3007}

3009//===----------------------------------------------------------------------===//
3010// spirv.GroupNonUniformUMaxOp
3011//===----------------------------------------------------------------------===//

3013LogicalResult spirv::GroupNonUniformUMaxOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
3015}

3017ParseResult spirv::GroupNonUniformUMaxOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
3020}
3021void spirv::GroupNonUniformUMaxOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
3023}

3025//===----------------------------------------------------------------------===//
3026// spirv.GroupNonUniformUMinOp
3027//===----------------------------------------------------------------------===//

3029LogicalResult spirv::GroupNonUniformUMinOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
3031}

3033ParseResult spirv::GroupNonUniformUMinOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
3036}
3037void spirv::GroupNonUniformUMinOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
3039}

3041//===----------------------------------------------------------------------===//
3042// spirv.IAddCarryOp
3043//===----------------------------------------------------------------------===//

3045LogicalResult spirv::IAddCarryOp::verify() {
return ::verifyArithmeticExtendedBinaryOp(*this);
3047}

3049ParseResult spirv::IAddCarryOp::parse(OpAsmParser &parser,
                                    OperationState &result) {
return ::parseArithmeticExtendedBinaryOp(parser, result);
3052}

3054void spirv::IAddCarryOp::print(OpAsmPrinter &printer) {
::printArithmeticExtendedBinaryOp(*this, printer);
3056}

3058//===----------------------------------------------------------------------===//
3059// spirv.ISubBorrowOp
3060//===----------------------------------------------------------------------===//

3062LogicalResult spirv::ISubBorrowOp::verify() {
return ::verifyArithmeticExtendedBinaryOp(*this);
3064}

3066ParseResult spirv::ISubBorrowOp::parse(OpAsmParser &parser,
                                     OperationState &result) {
return ::parseArithmeticExtendedBinaryOp(parser, result);
3069}

3071void spirv::ISubBorrowOp::print(OpAsmPrinter &printer) {
::printArithmeticExtendedBinaryOp(*this, printer);
3073}

3075//===----------------------------------------------------------------------===//
3076// spirv.SMulExtended
3077//===----------------------------------------------------------------------===//

3079LogicalResult spirv::SMulExtendedOp::verify() {
return ::verifyArithmeticExtendedBinaryOp(*this);
3081}

3083ParseResult spirv::SMulExtendedOp::parse(OpAsmParser &parser,
                                       OperationState &result) {
return ::parseArithmeticExtendedBinaryOp(parser, result);
3086}

3088void spirv::SMulExtendedOp::print(OpAsmPrinter &printer) {
::printArithmeticExtendedBinaryOp(*this, printer);
3090}

3092//===----------------------------------------------------------------------===//
3093// spirv.UMulExtended
3094//===----------------------------------------------------------------------===//

3096LogicalResult spirv::UMulExtendedOp::verify() {
return ::verifyArithmeticExtendedBinaryOp(*this);
3098}

3100ParseResult spirv::UMulExtendedOp::parse(OpAsmParser &parser,
                                       OperationState &result) {
return ::parseArithmeticExtendedBinaryOp(parser, result);
3103}

3105void spirv::UMulExtendedOp::print(OpAsmPrinter &printer) {
::printArithmeticExtendedBinaryOp(*this, printer);
3107}

3109//===----------------------------------------------------------------------===//
3110// spirv.LoadOp
3111//===----------------------------------------------------------------------===//

3113void spirv::LoadOp::build(OpBuilder &builder, OperationState &state,
                        Value basePtr, MemoryAccessAttr memoryAccess,
                        IntegerAttr alignment) {
auto ptrType = basePtr.getType().cast<spirv::PointerType>();
build(builder, state, ptrType.getPointeeType(), basePtr, memoryAccess,
      alignment);
3119}

3121ParseResult spirv::LoadOp::parse(OpAsmParser &parser, OperationState &result) {
// Parse the storage class specification
spirv::StorageClass storageClass;
OpAsmParser::UnresolvedOperand ptrInfo;
Type elementType;
if (parseEnumStrAttr(storageClass, parser) || parser.parseOperand(ptrInfo) ||
    parseMemoryAccessAttributes(parser, result) ||
    parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() ||
    parser.parseType(elementType)) {
  return failure();
}

auto ptrType = spirv::PointerType::get(elementType, storageClass);
if (parser.resolveOperand(ptrInfo, ptrType, result.operands)) {
  return failure();
}

result.addTypes(elementType);
return success();
3140}

3142void spirv::LoadOp::print(OpAsmPrinter &printer) {
SmallVector<StringRef, 4> elidedAttrs;
StringRef sc = stringifyStorageClass(
    getPtr().getType().cast<spirv::PointerType>().getStorageClass());
printer << " \"" << sc << "\" " << getPtr();

printMemoryAccessAttribute(*this, printer, elidedAttrs);

printer.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
printer << " : " << getType();
3152}

3154LogicalResult spirv::LoadOp::verify() {
// SPIR-V spec : "Result Type is the type of the loaded object. It must be a
// type with fixed size; i.e., it cannot be, nor include, any
// OpTypeRuntimeArray types."
if (failed(verifyLoadStorePtrAndValTypes(*this, getPtr(), getValue()))) {
  return failure();
}
return verifyMemoryAccessAttribute(*this);
3162}

3164//===----------------------------------------------------------------------===//
3165// spirv.mlir.loop
3166//===----------------------------------------------------------------------===//

3168void spirv::LoopOp::build(OpBuilder &builder, OperationState &state) {
state.addAttribute("loop_control", builder.getAttr<spirv::LoopControlAttr>(
                                       spirv::LoopControl::None));
state.addRegion();
3172}

3174ParseResult spirv::LoopOp::parse(OpAsmParser &parser, OperationState &result) {
if (parseControlAttribute<spirv::LoopControlAttr, spirv::LoopControl>(parser,
                                                                      result))
  return failure();
return parser.parseRegion(*result.addRegion(), /*arguments=*/{});
3179}

3181void spirv::LoopOp::print(OpAsmPrinter &printer) {
auto control = getLoopControl();
if (control != spirv::LoopControl::None)
  printer << " control(" << spirv::stringifyLoopControl(control) << ")";
printer << ' ';
printer.printRegion(getRegion(), /*printEntryBlockArgs=*/false,
                    /*printBlockTerminators=*/true);
3188}

3190/// Returns true if the given `srcBlock` contains only one `spirv.Branch` to the
3191/// given `dstBlock`.
3192static inline bool hasOneBranchOpTo(Block &srcBlock, Block &dstBlock) {
// Check that there is only one op in the `srcBlock`.
if (!llvm::hasSingleElement(srcBlock))
  return false;

auto branchOp = dyn_cast<spirv::BranchOp>(srcBlock.back());
return branchOp && branchOp.getSuccessor() == &dstBlock;
3199}

3201LogicalResult spirv::LoopOp::verifyRegions() {
auto *op = getOperation();

// We need to verify that the blocks follow the following layout:
//
//                     +-------------+
//                     | entry block |
//                     +-------------+
//                            |
//                            v
//                     +-------------+
//                     | loop header | <-----+
//                     +-------------+       |
//                                           |
//                           ...             |
//                          \ | /            |
//                            v              |
//                    +---------------+      |
//                    | loop continue | -----+
//                    +---------------+
//
//                           ...
//                          \ | /
//                            v
//                     +-------------+
//                     | merge block |
//                     +-------------+

auto &region = op->getRegion(0);
// Allow empty region as a degenerated case, which can come from
// optimizations.
if (region.empty())
  return success();

// The last block is the merge block.
Block &merge = region.back();
if (!isMergeBlock(merge))
  return emitOpError("last block must be the merge block with only one "
                     "'spirv.mlir.merge' op");

if (std::next(region.begin()) == region.end())
  return emitOpError(
      "must have an entry block branching to the loop header block");
// The first block is the entry block.
Block &entry = region.front();

if (std::next(region.begin(), 2) == region.end())
  return emitOpError(
      "must have a loop header block branched from the entry block");
// The second block is the loop header block.
Block &header = *std::next(region.begin(), 1);

if (!hasOneBranchOpTo(entry, header))
  return emitOpError(
      "entry block must only have one 'spirv.Branch' op to the second block");

if (std::next(region.begin(), 3) == region.end())
  return emitOpError(
      "requires a loop continue block branching to the loop header block");
// The second to last block is the loop continue block.
Block &cont = *std::prev(region.end(), 2);

// Make sure that we have a branch from the loop continue block to the loop
// header block.
if (llvm::none_of(
        llvm::seq<unsigned>(0, cont.getNumSuccessors()),
        [&](unsigned index) { return cont.getSuccessor(index) == &header; }))
  return emitOpError("second to last block must be the loop continue "
                     "block that branches to the loop header block");

// Make sure that no other blocks (except the entry and loop continue block)
// branches to the loop header block.
for (auto &block : llvm::make_range(std::next(region.begin(), 2),
                                    std::prev(region.end(), 2))) {
  for (auto i : llvm::seq<unsigned>(0, block.getNumSuccessors())) {
    if (block.getSuccessor(i) == &header) {
      return emitOpError("can only have the entry and loop continue "
                         "block branching to the loop header block");
    }
  }
}

return success();
3284}

3286Block *spirv::LoopOp::getEntryBlock() {
assert(!getBody().empty() && "op region should not be empty!")(static_cast <bool> (!getBody().empty() && "op region should not be empty!"
) ? void (0) : __assert_fail ("!getBody().empty() && \"op region should not be empty!\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 3287, __extension__
 __PRETTY_FUNCTION__));
return &getBody().front();
3289}

3291Block *spirv::LoopOp::getHeaderBlock() {
assert(!getBody().empty() && "op region should not be empty!")(static_cast <bool> (!getBody().empty() && "op region should not be empty!"
) ? void (0) : __assert_fail ("!getBody().empty() && \"op region should not be empty!\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 3292, __extension__
 __PRETTY_FUNCTION__));
// The second block is the loop header block.
return &*std::next(getBody().begin());
3295}

3297Block *spirv::LoopOp::getContinueBlock() {
assert(!getBody().empty() && "op region should not be empty!")(static_cast <bool> (!getBody().empty() && "op region should not be empty!"
) ? void (0) : __assert_fail ("!getBody().empty() && \"op region should not be empty!\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 3298, __extension__
 __PRETTY_FUNCTION__));
// The second to last block is the loop continue block.
return &*std::prev(getBody().end(), 2);
3301}

3303Block *spirv::LoopOp::getMergeBlock() {
assert(!getBody().empty() && "op region should not be empty!")(static_cast <bool> (!getBody().empty() && "op region should not be empty!"
) ? void (0) : __assert_fail ("!getBody().empty() && \"op region should not be empty!\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 3304, __extension__
 __PRETTY_FUNCTION__));
// The last block is the loop merge block.
return &getBody().back();
3307}

3309void spirv::LoopOp::addEntryAndMergeBlock() {
assert(getBody().empty() && "entry and merge block already exist")(static_cast <bool> (getBody().empty() && "entry and merge block already exist"
) ? void (0) : __assert_fail ("getBody().empty() && \"entry and merge block already exist\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 3310, __extension__
 __PRETTY_FUNCTION__));
getBody().push_back(new Block());
auto *mergeBlock = new Block();
getBody().push_back(mergeBlock);
OpBuilder builder = OpBuilder::atBlockEnd(mergeBlock);

// Add a spirv.mlir.merge op into the merge block.
builder.create<spirv::MergeOp>(getLoc());
3318}

3320//===----------------------------------------------------------------------===//
3321// spirv.MemoryBarrierOp
3322//===----------------------------------------------------------------------===//

3324LogicalResult spirv::MemoryBarrierOp::verify() {
return verifyMemorySemantics(getOperation(), getMemorySemantics());
3326}

3328//===----------------------------------------------------------------------===//
3329// spirv.mlir.merge
3330//===----------------------------------------------------------------------===//

3332LogicalResult spirv::MergeOp::verify() {
auto *parentOp = (*this)->getParentOp();
if (!parentOp || !isa<spirv::SelectionOp, spirv::LoopOp>(parentOp))
  return emitOpError(
      "expected parent op to be 'spirv.mlir.selection' or 'spirv.mlir.loop'");

// TODO: This check should be done in `verifyRegions` of parent op.
Block &parentLastBlock = (*this)->getParentRegion()->back();
if (getOperation() != parentLastBlock.getTerminator())
  return emitOpError("can only be used in the last block of "
                     "'spirv.mlir.selection' or 'spirv.mlir.loop'");
return success();
3344}

3346//===----------------------------------------------------------------------===//
3347// spirv.module
3348//===----------------------------------------------------------------------===//

3350void spirv::ModuleOp::build(OpBuilder &builder, OperationState &state,
                          std::optional<StringRef> name) {
OpBuilder::InsertionGuard guard(builder);
builder.createBlock(state.addRegion());
if (name) {
  state.attributes.append(mlir::SymbolTable::getSymbolAttrName(),
                          builder.getStringAttr(*name));
}
3358}

3360void spirv::ModuleOp::build(OpBuilder &builder, OperationState &state,
                          spirv::AddressingModel addressingModel,
                          spirv::MemoryModel memoryModel,
                          std::optional<VerCapExtAttr> vceTriple,
                          std::optional<StringRef> name) {
state.addAttribute(
    "addressing_model",
    builder.getAttr<spirv::AddressingModelAttr>(addressingModel));
state.addAttribute("memory_model",
                   builder.getAttr<spirv::MemoryModelAttr>(memoryModel));
OpBuilder::InsertionGuard guard(builder);
builder.createBlock(state.addRegion());
if (vceTriple)
  state.addAttribute(getVCETripleAttrName(), *vceTriple);
if (name)
  state.addAttribute(mlir::SymbolTable::getSymbolAttrName(),
                     builder.getStringAttr(*name));
3377}

3379ParseResult spirv::ModuleOp::parse(OpAsmParser &parser,
                                 OperationState &result) {
Region *body = result.addRegion();

// If the name is present, parse it.
StringAttr nameAttr;
(void)parser.parseOptionalSymbolName(
    nameAttr, mlir::SymbolTable::getSymbolAttrName(), result.attributes);

// Parse attributes
spirv::AddressingModel addrModel;
spirv::MemoryModel memoryModel;
if (::parseEnumKeywordAttr<spirv::AddressingModelAttr>(addrModel, parser,
                                                       result) ||
    ::parseEnumKeywordAttr<spirv::MemoryModelAttr>(memoryModel, parser,
                                                   result))
  return failure();

if (succeeded(parser.parseOptionalKeyword("requires"))) {
  spirv::VerCapExtAttr vceTriple;
  if (parser.parseAttribute(vceTriple,
                            spirv::ModuleOp::getVCETripleAttrName(),
                            result.attributes))
    return failure();
}

if (parser.parseOptionalAttrDictWithKeyword(result.attributes) ||
    parser.parseRegion(*body, /*arguments=*/{}))
  return failure();

// Make sure we have at least one block.
if (body->empty())
  body->push_back(new Block());

return success();
3414}

3416void spirv::ModuleOp::print(OpAsmPrinter &printer) {
if (std::optional<StringRef> name = getName()) {
  printer << ' ';
  printer.printSymbolName(*name);
}

SmallVector<StringRef, 2> elidedAttrs;

printer << " " << spirv::stringifyAddressingModel(getAddressingModel()) << " "
        << spirv::stringifyMemoryModel(getMemoryModel());
auto addressingModelAttrName = spirv::attributeName<spirv::AddressingModel>();
auto memoryModelAttrName = spirv::attributeName<spirv::MemoryModel>();
elidedAttrs.assign({addressingModelAttrName, memoryModelAttrName,
                    mlir::SymbolTable::getSymbolAttrName()});

if (std::optional<spirv::VerCapExtAttr> triple = getVceTriple()) {
  printer << " requires " << *triple;
  elidedAttrs.push_back(spirv::ModuleOp::getVCETripleAttrName());
}

printer.printOptionalAttrDictWithKeyword((*this)->getAttrs(), elidedAttrs);
printer << ' ';
printer.printRegion(getRegion());
3439}

3441LogicalResult spirv::ModuleOp::verifyRegions() {
Dialect *dialect = (*this)->getDialect();
DenseMap<std::pair<spirv::FuncOp, spirv::ExecutionModel>, spirv::EntryPointOp>
    entryPoints;
mlir::SymbolTable table(*this);

for (auto &op : *getBody()) {
  if (op.getDialect() != dialect)
    return op.emitError("'spirv.module' can only contain spirv.* ops");

  // For EntryPoint op, check that the function and execution model is not
  // duplicated in EntryPointOps. Also verify that the interface specified
  // comes from globalVariables here to make this check cheaper.
  if (auto entryPointOp = dyn_cast<spirv::EntryPointOp>(op)) {
    auto funcOp = table.lookup<spirv::FuncOp>(entryPointOp.getFn());
    if (!funcOp) {
      return entryPointOp.emitError("function '")
             << entryPointOp.getFn() << "' not found in 'spirv.module'";
    }
    if (auto interface = entryPointOp.getInterface()) {
      for (Attribute varRef : interface) {
        auto varSymRef = varRef.dyn_cast<FlatSymbolRefAttr>();
        if (!varSymRef) {
          return entryPointOp.emitError(
                     "expected symbol reference for interface "
                     "specification instead of '")
                 << varRef;
        }
        auto variableOp =
            table.lookup<spirv::GlobalVariableOp>(varSymRef.getValue());
        if (!variableOp) {
          return entryPointOp.emitError("expected spirv.GlobalVariable "
                                        "symbol reference instead of'")
                 << varSymRef << "'";
        }
      }
    }

    auto key = std::pair<spirv::FuncOp, spirv::ExecutionModel>(
        funcOp, entryPointOp.getExecutionModel());
    auto entryPtIt = entryPoints.find(key);
    if (entryPtIt != entryPoints.end()) {
      return entryPointOp.emitError("duplicate of a previous EntryPointOp");
    }
    entryPoints[key] = entryPointOp;
  } else if (auto funcOp = dyn_cast<spirv::FuncOp>(op)) {
    if (funcOp.isExternal())
      return op.emitError("'spirv.module' cannot contain external functions");

    // TODO: move this check to spirv.func.
    for (auto &block : funcOp)
      for (auto &op : block) {
        if (op.getDialect() != dialect)
          return op.emitError(
              "functions in 'spirv.module' can only contain spirv.* ops");
      }
  }
}

return success();
3501}

3503//===----------------------------------------------------------------------===//
3504// spirv.mlir.referenceof
3505//===----------------------------------------------------------------------===//

3507LogicalResult spirv::ReferenceOfOp::verify() {
auto *specConstSym = SymbolTable::lookupNearestSymbolFrom(
    (*this)->getParentOp(), getSpecConstAttr());
Type constType;

auto specConstOp = dyn_cast_or_null<spirv::SpecConstantOp>(specConstSym);
if (specConstOp)
  constType = specConstOp.getDefaultValue().getType();

auto specConstCompositeOp =
    dyn_cast_or_null<spirv::SpecConstantCompositeOp>(specConstSym);
if (specConstCompositeOp)
  constType = specConstCompositeOp.getType();

if (!specConstOp && !specConstCompositeOp)
  return emitOpError(
      "expected spirv.SpecConstant or spirv.SpecConstantComposite symbol");

if (getReference().getType() != constType)
  return emitOpError("result type mismatch with the referenced "
                     "specialization constant's type");

return success();
3530}

3532//===----------------------------------------------------------------------===//
3533// spirv.Return
3534//===----------------------------------------------------------------------===//

3536LogicalResult spirv::ReturnOp::verify() {
// Verification is performed in spirv.func op.
return success();
3539}

3541//===----------------------------------------------------------------------===//
3542// spirv.ReturnValue
3543//===----------------------------------------------------------------------===//

3545LogicalResult spirv::ReturnValueOp::verify() {
// Verification is performed in spirv.func op.
return success();
3548}

3550//===----------------------------------------------------------------------===//
3551// spirv.Select
3552//===----------------------------------------------------------------------===//

3554LogicalResult spirv::SelectOp::verify() {
if (auto conditionTy = getCondition().getType().dyn_cast<VectorType>()) {
  auto resultVectorTy = getResult().getType().dyn_cast<VectorType>();
  if (!resultVectorTy) {
    return emitOpError("result expected to be of vector type when "
                       "condition is of vector type");
  }
  if (resultVectorTy.getNumElements() != conditionTy.getNumElements()) {
    return emitOpError("result should have the same number of elements as "
                       "the condition when condition is of vector type");
  }
}
return success();
3567}

3569//===----------------------------------------------------------------------===//
3570// spirv.mlir.selection
3571//===----------------------------------------------------------------------===//

3573ParseResult spirv::SelectionOp::parse(OpAsmParser &parser,
                                    OperationState &result) {
if (parseControlAttribute<spirv::SelectionControlAttr,
                          spirv::SelectionControl>(parser, result))
  return failure();
return parser.parseRegion(*result.addRegion(), /*arguments=*/{});
3579}

3581void spirv::SelectionOp::print(OpAsmPrinter &printer) {
auto control = getSelectionControl();
if (control != spirv::SelectionControl::None)
  printer << " control(" << spirv::stringifySelectionControl(control) << ")";
printer << ' ';
printer.printRegion(getRegion(), /*printEntryBlockArgs=*/false,
                    /*printBlockTerminators=*/true);
3588}

3590LogicalResult spirv::SelectionOp::verifyRegions() {
auto *op = getOperation();

// We need to verify that the blocks follow the following layout:
//
//                     +--------------+
//                     | header block |
//                     +--------------+
//                          / | \
//                           ...
//
//
//         +---------+   +---------+   +---------+
//         | case #0 |   | case #1 |   | case #2 |  ...
//         +---------+   +---------+   +---------+
//
//
//                           ...
//                          \ | /
//                            v
//                     +-------------+
//                     | merge block |
//                     +-------------+

auto &region = op->getRegion(0);
// Allow empty region as a degenerated case, which can come from
// optimizations.
if (region.empty())
  return success();

// The last block is the merge block.
if (!isMergeBlock(region.back()))
  return emitOpError("last block must be the merge block with only one "
                     "'spirv.mlir.merge' op");

if (std::next(region.begin()) == region.end())
  return emitOpError("must have a selection header block");

return success();
3629}

3631Block *spirv::SelectionOp::getHeaderBlock() {
assert(!getBody().empty() && "op region should not be empty!")(static_cast <bool> (!getBody().empty() && "op region should not be empty!"
) ? void (0) : __assert_fail ("!getBody().empty() && \"op region should not be empty!\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 3632, __extension__
 __PRETTY_FUNCTION__));
// The first block is the loop header block.
return &getBody().front();
3635}

3637Block *spirv::SelectionOp::getMergeBlock() {
assert(!getBody().empty() && "op region should not be empty!")(static_cast <bool> (!getBody().empty() && "op region should not be empty!"
) ? void (0) : __assert_fail ("!getBody().empty() && \"op region should not be empty!\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 3638, __extension__
 __PRETTY_FUNCTION__));
// The last block is the loop merge block.
return &getBody().back();
3641}

3643void spirv::SelectionOp::addMergeBlock() {
assert(getBody().empty() && "entry and merge block already exist")(static_cast <bool> (getBody().empty() && "entry and merge block already exist"
) ? void (0) : __assert_fail ("getBody().empty() && \"entry and merge block already exist\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 3644, __extension__
 __PRETTY_FUNCTION__));
auto *mergeBlock = new Block();
getBody().push_back(mergeBlock);
OpBuilder builder = OpBuilder::atBlockEnd(mergeBlock);

// Add a spirv.mlir.merge op into the merge block.
builder.create<spirv::MergeOp>(getLoc());
3651}

3653spirv::SelectionOp spirv::SelectionOp::createIfThen(
  Location loc, Value condition,
  function_ref<void(OpBuilder &builder)> thenBody, OpBuilder &builder) {
auto selectionOp =
    builder.create<spirv::SelectionOp>(loc, spirv::SelectionControl::None);

selectionOp.addMergeBlock();
Block *mergeBlock = selectionOp.getMergeBlock();
Block *thenBlock = nullptr;

// Build the "then" block.
{
  OpBuilder::InsertionGuard guard(builder);
  thenBlock = builder.createBlock(mergeBlock);
  thenBody(builder);
  builder.create<spirv::BranchOp>(loc, mergeBlock);
}

// Build the header block.
{
  OpBuilder::InsertionGuard guard(builder);
  builder.createBlock(thenBlock);
  builder.create<spirv::BranchConditionalOp>(
      loc, condition, thenBlock,
      /*trueArguments=*/ArrayRef<Value>(), mergeBlock,
      /*falseArguments=*/ArrayRef<Value>());
}

return selectionOp;
3682}

3684//===----------------------------------------------------------------------===//
3685// spirv.SpecConstant
3686//===----------------------------------------------------------------------===//

3688ParseResult spirv::SpecConstantOp::parse(OpAsmParser &parser,
                                       OperationState &result) {
StringAttr nameAttr;
Attribute valueAttr;

if (parser.parseSymbolName(nameAttr, SymbolTable::getSymbolAttrName(),
                           result.attributes))
  return failure();

// Parse optional spec_id.
if (succeeded(parser.parseOptionalKeyword(kSpecIdAttrName))) {
  IntegerAttr specIdAttr;
  if (parser.parseLParen() ||
      parser.parseAttribute(specIdAttr, kSpecIdAttrName, result.attributes) ||
      parser.parseRParen())
    return failure();
}

if (parser.parseEqual() ||
    parser.parseAttribute(valueAttr, kDefaultValueAttrName,
                          result.attributes))
  return failure();

return success();
3712}

3714void spirv::SpecConstantOp::print(OpAsmPrinter &printer) {
printer << ' ';
printer.printSymbolName(getSymName());
if (auto specID = (*this)->getAttrOfType<IntegerAttr>(kSpecIdAttrName))
  printer << ' ' << kSpecIdAttrName << '(' << specID.getInt() << ')';
printer << " = " << getDefaultValue();
3720}

3722LogicalResult spirv::SpecConstantOp::verify() {
if (auto specID = (*this)->getAttrOfType<IntegerAttr>(kSpecIdAttrName))
  if (specID.getValue().isNegative())
    return emitOpError("SpecId cannot be negative");

auto value = getDefaultValue();
if (value.isa<IntegerAttr, FloatAttr>()) {
  // Make sure bitwidth is allowed.
  if (!value.getType().isa<spirv::SPIRVType>())
    return emitOpError("default value bitwidth disallowed");
  return success();
}
return emitOpError(
    "default value can only be a bool, integer, or float scalar");
3736}

3738//===----------------------------------------------------------------------===//
3739// spirv.StoreOp
3740//===----------------------------------------------------------------------===//

3742ParseResult spirv::StoreOp::parse(OpAsmParser &parser, OperationState &result) {
// Parse the storage class specification
spirv::StorageClass storageClass;
SmallVector<OpAsmParser::UnresolvedOperand, 2> operandInfo;
auto loc = parser.getCurrentLocation();
Type elementType;
if (parseEnumStrAttr(storageClass, parser) ||
    parser.parseOperandList(operandInfo, 2) ||
    parseMemoryAccessAttributes(parser, result) || parser.parseColon() ||
    parser.parseType(elementType)) {
  return failure();
}

auto ptrType = spirv::PointerType::get(elementType, storageClass);
if (parser.resolveOperands(operandInfo, {ptrType, elementType}, loc,
                           result.operands)) {
  return failure();
}
return success();
3761}

3763void spirv::StoreOp::print(OpAsmPrinter &printer) {
SmallVector<StringRef, 4> elidedAttrs;
StringRef sc = stringifyStorageClass(
    getPtr().getType().cast<spirv::PointerType>().getStorageClass());
printer << " \"" << sc << "\" " << getPtr() << ", " << getValue();

printMemoryAccessAttribute(*this, printer, elidedAttrs);

printer << " : " << getValue().getType();
printer.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
3773}

3775LogicalResult spirv::StoreOp::verify() {
// SPIR-V spec : "Pointer is the pointer to store through. Its type must be an
// OpTypePointer whose Type operand is the same as the type of Object."
if (failed(verifyLoadStorePtrAndValTypes(*this, getPtr(), getValue())))
  return failure();
return verifyMemoryAccessAttribute(*this);
3781}

3783//===----------------------------------------------------------------------===//
3784// spirv.Unreachable
3785//===----------------------------------------------------------------------===//

3787LogicalResult spirv::UnreachableOp::verify() {
auto *block = (*this)->getBlock();
// Fast track: if this is in entry block, its invalid. Otherwise, if no
// predecessors, it's valid.
if (block->isEntryBlock())
  return emitOpError("cannot be used in reachable block");
if (block->hasNoPredecessors())
  return success();

// TODO: further verification needs to analyze reachability from
// the entry block.

return success();
3800}

3802//===----------------------------------------------------------------------===//
3803// spirv.Variable
3804//===----------------------------------------------------------------------===//

3806ParseResult spirv::VariableOp::parse(OpAsmParser &parser,
                                   OperationState &result) {
// Parse optional initializer
std::optional<OpAsmParser::UnresolvedOperand> initInfo;
if (succeeded(parser.parseOptionalKeyword("init"))) {
  initInfo = OpAsmParser::UnresolvedOperand();
  if (parser.parseLParen() || parser.parseOperand(*initInfo) ||
      parser.parseRParen())
    return failure();
}

if (parseVariableDecorations(parser, result)) {
  return failure();
}

// Parse result pointer type
Type type;
if (parser.parseColon())
  return failure();
auto loc = parser.getCurrentLocation();
if (parser.parseType(type))
  return failure();

auto ptrType = type.dyn_cast<spirv::PointerType>();
if (!ptrType)
  return parser.emitError(loc, "expected spirv.ptr type");
result.addTypes(ptrType);

// Resolve the initializer operand
if (initInfo) {
  if (parser.resolveOperand(*initInfo, ptrType.getPointeeType(),
                            result.operands))
    return failure();
}

auto attr = parser.getBuilder().getAttr<spirv::StorageClassAttr>(
    ptrType.getStorageClass());
result.addAttribute(spirv::attributeName<spirv::StorageClass>(), attr);

return success();
3846}

3848void spirv::VariableOp::print(OpAsmPrinter &printer) {
SmallVector<StringRef, 4> elidedAttrs{
    spirv::attributeName<spirv::StorageClass>()};
// Print optional initializer
if (getNumOperands() != 0)
  printer << " init(" << getInitializer() << ")";

printVariableDecorations(*this, printer, elidedAttrs);
printer << " : " << getType();
3857}

3859LogicalResult spirv::VariableOp::verify() {
// SPIR-V spec: "Storage Class is the Storage Class of the memory holding the
// object. It cannot be Generic. It must be the same as the Storage Class
// operand of the Result Type."
if (getStorageClass() != spirv::StorageClass::Function) {
  return emitOpError(
      "can only be used to model function-level variables. Use "
      "spirv.GlobalVariable for module-level variables.");
}

auto pointerType = getPointer().getType().cast<spirv::PointerType>();
if (getStorageClass() != pointerType.getStorageClass())
  return emitOpError(
      "storage class must match result pointer's storage class");

if (getNumOperands() != 0) {
  // SPIR-V spec: "Initializer must be an <id> from a constant instruction or
  // a global (module scope) OpVariable instruction".
  auto *initOp = getOperand(0).getDefiningOp();
  if (!initOp || !isa<spirv::ConstantOp,    // for normal constant
                      spirv::ReferenceOfOp, // for spec constant
                      spirv::AddressOfOp>(initOp))
    return emitOpError("initializer must be the result of a "
                       "constant or spirv.GlobalVariable op");
}

// TODO: generate these strings using ODS.
auto *op = getOperation();
auto descriptorSetName = llvm::convertToSnakeFromCamelCase(
    stringifyDecoration(spirv::Decoration::DescriptorSet));
auto bindingName = llvm::convertToSnakeFromCamelCase(
    stringifyDecoration(spirv::Decoration::Binding));
auto builtInName = llvm::convertToSnakeFromCamelCase(
    stringifyDecoration(spirv::Decoration::BuiltIn));

for (const auto &attr : {descriptorSetName, bindingName, builtInName}) {
  if (op->getAttr(attr))
    return emitOpError("cannot have '")
           << attr << "' attribute (only allowed in spirv.GlobalVariable)";
}

return success();
3901}

3903//===----------------------------------------------------------------------===//
3904// spirv.VectorShuffle
3905//===----------------------------------------------------------------------===//

3907LogicalResult spirv::VectorShuffleOp::verify() {
VectorType resultType = getType().cast<VectorType>();

size_t numResultElements = resultType.getNumElements();
if (numResultElements != getComponents().size())
  return emitOpError("result type element count (")
         << numResultElements
         << ") mismatch with the number of component selectors ("
         << getComponents().size() << ")";

size_t totalSrcElements =
    getVector1().getType().cast<VectorType>().getNumElements() +
    getVector2().getType().cast<VectorType>().getNumElements();

for (const auto &selector : getComponents().getAsValueRange<IntegerAttr>()) {
  uint32_t index = selector.getZExtValue();
  if (index >= totalSrcElements &&
      index != std::numeric_limits<uint32_t>().max())
    return emitOpError("component selector ")
           << index << " out of range: expected to be in [0, "
           << totalSrcElements << ") or 0xffffffff";
}
return success();
3930}

3932//===----------------------------------------------------------------------===//
3933// spirv.NV.CooperativeMatrixLoad
3934//===----------------------------------------------------------------------===//

3936ParseResult spirv::NVCooperativeMatrixLoadOp::parse(OpAsmParser &parser,
                                                  OperationState &result) {
SmallVector<OpAsmParser::UnresolvedOperand, 3> operandInfo;
Type strideType = parser.getBuilder().getIntegerType(32);
Type columnMajorType = parser.getBuilder().getIntegerType(1);
Type ptrType;
Type elementType;
if (parser.parseOperandList(operandInfo, 3) ||
    parseMemoryAccessAttributes(parser, result) || parser.parseColon() ||
    parser.parseType(ptrType) || parser.parseKeywordType("as", elementType)) {
  return failure();
}
if (parser.resolveOperands(operandInfo,
                           {ptrType, strideType, columnMajorType},
                           parser.getNameLoc(), result.operands)) {
  return failure();
}

result.addTypes(elementType);
return success();
3956}

3958void spirv::NVCooperativeMatrixLoadOp::print(OpAsmPrinter &printer) {
printer << " " << getPointer() << ", " << getStride() << ", "
        << getColumnmajor();
// Print optional memory access attribute.
if (auto memAccess = getMemoryAccess())
  printer << " [\"" << stringifyMemoryAccess(*memAccess) << "\"]";
printer << " : " << getPointer().getType() << " as " << getType();
3965}

3967static LogicalResult verifyPointerAndCoopMatrixType(Operation *op, Type pointer,
                                                  Type coopMatrix) {
Type pointeeType = pointer.cast<spirv::PointerType>().getPointeeType();
if (!pointeeType.isa<spirv::ScalarType>() && !pointeeType.isa<VectorType>())
  return op->emitError(
             "Pointer must point to a scalar or vector type but provided ")
         << pointeeType;
spirv::StorageClass storage =
    pointer.cast<spirv::PointerType>().getStorageClass();
if (storage != spirv::StorageClass::Workgroup &&
    storage != spirv::StorageClass::StorageBuffer &&
    storage != spirv::StorageClass::PhysicalStorageBuffer)
  return op->emitError(
             "Pointer storage class must be Workgroup, StorageBuffer or "
             "PhysicalStorageBufferEXT but provided ")
         << stringifyStorageClass(storage);
return success();
3984}

3986LogicalResult spirv::NVCooperativeMatrixLoadOp::verify() {
return verifyPointerAndCoopMatrixType(*this, getPointer().getType(),
                                      getResult().getType());
3989}

3991//===----------------------------------------------------------------------===//
3992// spirv.NV.CooperativeMatrixStore
3993//===----------------------------------------------------------------------===//

3995ParseResult spirv::NVCooperativeMatrixStoreOp::parse(OpAsmParser &parser,
                                                   OperationState &result) {
SmallVector<OpAsmParser::UnresolvedOperand, 4> operandInfo;
Type strideType = parser.getBuilder().getIntegerType(32);
Type columnMajorType = parser.getBuilder().getIntegerType(1);
Type ptrType;
Type elementType;
if (parser.parseOperandList(operandInfo, 4) ||
    parseMemoryAccessAttributes(parser, result) || parser.parseColon() ||
    parser.parseType(ptrType) || parser.parseComma() ||
    parser.parseType(elementType)) {
  return failure();
}
if (parser.resolveOperands(
        operandInfo, {ptrType, elementType, strideType, columnMajorType},
        parser.getNameLoc(), result.operands)) {
  return failure();
}

return success();
4015}

4017void spirv::NVCooperativeMatrixStoreOp::print(OpAsmPrinter &printer) {
printer << " " << getPointer() << ", " << getObject() << ", " << getStride()
        << ", " << getColumnmajor();
// Print optional memory access attribute.
if (auto memAccess = getMemoryAccess())
  printer << " [\"" << stringifyMemoryAccess(*memAccess) << "\"]";
printer << " : " << getPointer().getType() << ", " << getOperand(1).getType();
4024}

4026LogicalResult spirv::NVCooperativeMatrixStoreOp::verify() {
return verifyPointerAndCoopMatrixType(*this, getPointer().getType(),
                                      getObject().getType());
4029}

4031//===----------------------------------------------------------------------===//
4032// spirv.NV.CooperativeMatrixMulAdd
4033//===----------------------------------------------------------------------===//

4035static LogicalResult
4036verifyCoopMatrixMulAdd(spirv::NVCooperativeMatrixMulAddOp op) {
if (op.getC().getType() != op.getResult().getType())
  return op.emitOpError("result and third operand must have the same type");
auto typeA = op.getA().getType().cast<spirv::CooperativeMatrixNVType>();
auto typeB = op.getB().getType().cast<spirv::CooperativeMatrixNVType>();
auto typeC = op.getC().getType().cast<spirv::CooperativeMatrixNVType>();
auto typeR = op.getResult().getType().cast<spirv::CooperativeMatrixNVType>();
if (typeA.getRows() != typeR.getRows() ||
    typeA.getColumns() != typeB.getRows() ||
    typeB.getColumns() != typeR.getColumns())
  return op.emitOpError("matrix size must match");
if (typeR.getScope() != typeA.getScope() ||
    typeR.getScope() != typeB.getScope() ||
    typeR.getScope() != typeC.getScope())
  return op.emitOpError("matrix scope must match");
if (typeA.getElementType() != typeB.getElementType() ||
    typeR.getElementType() != typeC.getElementType())
  return op.emitOpError("matrix element type must match");
return success();
4055}

4057LogicalResult spirv::NVCooperativeMatrixMulAddOp::verify() {
return verifyCoopMatrixMulAdd(*this);
4059}

4061static LogicalResult
4062verifyPointerAndJointMatrixType(Operation *op, Type pointer, Type jointMatrix) {
Type pointeeType = pointer.cast<spirv::PointerType>().getPointeeType();
if (!pointeeType.isa<spirv::ScalarType>() && !pointeeType.isa<VectorType>())
  return op->emitError(
             "Pointer must point to a scalar or vector type but provided ")
         << pointeeType;
spirv::StorageClass storage =
    pointer.cast<spirv::PointerType>().getStorageClass();
if (storage != spirv::StorageClass::Workgroup &&
    storage != spirv::StorageClass::CrossWorkgroup &&
    storage != spirv::StorageClass::UniformConstant &&
    storage != spirv::StorageClass::Generic)
  return op->emitError("Pointer storage class must be Workgroup or "
                       "CrossWorkgroup but provided ")
         << stringifyStorageClass(storage);
return success();
4078}

4080//===----------------------------------------------------------------------===//
4081// spirv.INTEL.JointMatrixLoad
4082//===----------------------------------------------------------------------===//

4084LogicalResult spirv::INTELJointMatrixLoadOp::verify() {
return verifyPointerAndJointMatrixType(*this, getPointer().getType(),
                                       getResult().getType());
4087}

4089//===----------------------------------------------------------------------===//
4090// spirv.INTEL.JointMatrixStore
4091//===----------------------------------------------------------------------===//

4093LogicalResult spirv::INTELJointMatrixStoreOp::verify() {
return verifyPointerAndJointMatrixType(*this, getPointer().getType(),
                                       getObject().getType());
4096}

4098//===----------------------------------------------------------------------===//
4099// spirv.INTEL.JointMatrixMad
4100//===----------------------------------------------------------------------===//

4102static LogicalResult verifyJointMatrixMad(spirv::INTELJointMatrixMadOp op) {
if (op.getC().getType() != op.getResult().getType())
  return op.emitOpError("result and third operand must have the same type");
auto typeA = op.getA().getType().cast<spirv::JointMatrixINTELType>();
auto typeB = op.getB().getType().cast<spirv::JointMatrixINTELType>();
auto typeC = op.getC().getType().cast<spirv::JointMatrixINTELType>();
auto typeR = op.getResult().getType().cast<spirv::JointMatrixINTELType>();
if (typeA.getRows() != typeR.getRows() ||
    typeA.getColumns() != typeB.getRows() ||
    typeB.getColumns() != typeR.getColumns())
  return op.emitOpError("matrix size must match");
if (typeR.getScope() != typeA.getScope() ||
    typeR.getScope() != typeB.getScope() ||
    typeR.getScope() != typeC.getScope())
  return op.emitOpError("matrix scope must match");
if (typeA.getElementType() != typeB.getElementType() ||
    typeR.getElementType() != typeC.getElementType())
  return op.emitOpError("matrix element type must match");
return success();
4121}

4123LogicalResult spirv::INTELJointMatrixMadOp::verify() {
return verifyJointMatrixMad(*this);
4125}

4127//===----------------------------------------------------------------------===//
4128// spirv.MatrixTimesScalar
4129//===----------------------------------------------------------------------===//

4131LogicalResult spirv::MatrixTimesScalarOp::verify() {
if (auto inputCoopmat =
        getMatrix().getType().dyn_cast<spirv::CooperativeMatrixNVType>()) {
  if (inputCoopmat.getElementType() != getScalar().getType())
    return emitError("input matrix components' type and scaling value must "
                     "have the same type");
  return success();
}

// Check that the scalar type is the same as the matrix element type.
auto inputMatrix = getMatrix().getType().cast<spirv::MatrixType>();
if (getScalar().getType() != inputMatrix.getElementType())
  return emitError("input matrix components' type and scaling value must "
                   "have the same type");

return success();
4147}

4149//===----------------------------------------------------------------------===//
4150// spirv.CopyMemory
4151//===----------------------------------------------------------------------===//

4153void spirv::CopyMemoryOp::print(OpAsmPrinter &printer) {
printer << ' ';

StringRef targetStorageClass = stringifyStorageClass(
    getTarget().getType().cast<spirv::PointerType>().getStorageClass());
printer << " \"" << targetStorageClass << "\" " << getTarget() << ", ";

StringRef sourceStorageClass = stringifyStorageClass(
    getSource().getType().cast<spirv::PointerType>().getStorageClass());
printer << " \"" << sourceStorageClass << "\" " << getSource();

SmallVector<StringRef, 4> elidedAttrs;
printMemoryAccessAttribute(*this, printer, elidedAttrs);
printSourceMemoryAccessAttribute(*this, printer, elidedAttrs,
                                 getSourceMemoryAccess(),
                                 getSourceAlignment());

printer.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);

Type pointeeType =
    getTarget().getType().cast<spirv::PointerType>().getPointeeType();
printer << " : " << pointeeType;
4175}

4177ParseResult spirv::CopyMemoryOp::parse(OpAsmParser &parser,
                                     OperationState &result) {
spirv::StorageClass targetStorageClass;
OpAsmParser::UnresolvedOperand targetPtrInfo;

spirv::StorageClass sourceStorageClass;
1
'sourceStorageClass' declared without an initial value→
OpAsmParser::UnresolvedOperand sourcePtrInfo;

Type elementType;

if (parseEnumStrAttr(targetStorageClass, parser) ||
7
←
Taking false branch→
    parser.parseOperand(targetPtrInfo) || parser.parseComma() ||
    parseEnumStrAttr(sourceStorageClass, parser) ||
2
←
Calling 'parseEnumStrAttr<mlir::spirv::StorageClass>'→
6
←
Returning from 'parseEnumStrAttr<mlir::spirv::StorageClass>'→
    parser.parseOperand(sourcePtrInfo) ||
    parseMemoryAccessAttributes(parser, result)) {
  return failure();
}

if (!parser.parseOptionalComma()) {
  // Parse 2nd memory access attributes.
  if (parseSourceMemoryAccessAttributes(parser, result)) {
    return failure();
  }
}

if (parser.parseColon() || parser.parseType(elementType))
8
←
Taking false branch→
  return failure();

if (parser.parseOptionalAttrDict(result.attributes))
9
←
Taking false branch→
  return failure();

auto targetPtrType = spirv::PointerType::get(elementType, targetStorageClass);
auto sourcePtrType = spirv::PointerType::get(elementType, sourceStorageClass);
10
←
2nd function call argument is an uninitialized value

if (parser.resolveOperand(targetPtrInfo, targetPtrType, result.operands) ||
    parser.resolveOperand(sourcePtrInfo, sourcePtrType, result.operands)) {
  return failure();
}

return success();
4217}

4219LogicalResult spirv::CopyMemoryOp::verify() {
Type targetType =
    getTarget().getType().cast<spirv::PointerType>().getPointeeType();

Type sourceType =
    getSource().getType().cast<spirv::PointerType>().getPointeeType();

if (targetType != sourceType)
  return emitOpError("both operands must be pointers to the same type");

if (failed(verifyMemoryAccessAttribute(*this)))
  return failure();

// TODO - According to the spec:
//
// If two masks are present, the first applies to Target and cannot include
// MakePointerVisible, and the second applies to Source and cannot include
// MakePointerAvailable.
//
// Add such verification here.

return verifySourceMemoryAccessAttribute(*this);
4241}

4243//===----------------------------------------------------------------------===//
4244// spirv.Transpose
4245//===----------------------------------------------------------------------===//

4247LogicalResult spirv::TransposeOp::verify() {
auto inputMatrix = getMatrix().getType().cast<spirv::MatrixType>();
auto resultMatrix = getResult().getType().cast<spirv::MatrixType>();

// Verify that the input and output matrices have correct shapes.
if (inputMatrix.getNumRows() != resultMatrix.getNumColumns())
  return emitError("input matrix rows count must be equal to "
                   "output matrix columns count");

if (inputMatrix.getNumColumns() != resultMatrix.getNumRows())
  return emitError("input matrix columns count must be equal to "
                   "output matrix rows count");

// Verify that the input and output matrices have the same component type
if (inputMatrix.getElementType() != resultMatrix.getElementType())
  return emitError("input and output matrices must have the same "
                   "component type");

return success();
4266}

4268//===----------------------------------------------------------------------===//
4269// spirv.MatrixTimesMatrix
4270//===----------------------------------------------------------------------===//

4272LogicalResult spirv::MatrixTimesMatrixOp::verify() {
auto leftMatrix = getLeftmatrix().getType().cast<spirv::MatrixType>();
auto rightMatrix = getRightmatrix().getType().cast<spirv::MatrixType>();
auto resultMatrix = getResult().getType().cast<spirv::MatrixType>();

// left matrix columns' count and right matrix rows' count must be equal
if (leftMatrix.getNumColumns() != rightMatrix.getNumRows())
  return emitError("left matrix columns' count must be equal to "
                   "the right matrix rows' count");

// right and result matrices columns' count must be the same
if (rightMatrix.getNumColumns() != resultMatrix.getNumColumns())
  return emitError(
      "right and result matrices must have equal columns' count");

// right and result matrices component type must be the same
if (rightMatrix.getElementType() != resultMatrix.getElementType())
  return emitError("right and result matrices' component type must"
                   " be the same");

// left and result matrices component type must be the same
if (leftMatrix.getElementType() != resultMatrix.getElementType())
  return emitError("left and result matrices' component type"
                   " must be the same");

// left and result matrices rows count must be the same
if (leftMatrix.getNumRows() != resultMatrix.getNumRows())
  return emitError("left and result matrices must have equal rows' count");

return success();
4302}

4304//===----------------------------------------------------------------------===//
4305// spirv.SpecConstantComposite
4306//===----------------------------------------------------------------------===//

4308ParseResult spirv::SpecConstantCompositeOp::parse(OpAsmParser &parser,
                                                OperationState &result) {

StringAttr compositeName;
if (parser.parseSymbolName(compositeName, SymbolTable::getSymbolAttrName(),
                           result.attributes))
  return failure();

if (parser.parseLParen())
  return failure();

SmallVector<Attribute, 4> constituents;

do {
  // The name of the constituent attribute isn't important
  const char *attrName = "spec_const";
  FlatSymbolRefAttr specConstRef;
  NamedAttrList attrs;

  if (parser.parseAttribute(specConstRef, Type(), attrName, attrs))
    return failure();

  constituents.push_back(specConstRef);
} while (!parser.parseOptionalComma());

if (parser.parseRParen())
  return failure();

result.addAttribute(kCompositeSpecConstituentsName,
                    parser.getBuilder().getArrayAttr(constituents));

Type type;
if (parser.parseColonType(type))
  return failure();

result.addAttribute(kTypeAttrName, TypeAttr::get(type));

return success();
4346}

4348void spirv::SpecConstantCompositeOp::print(OpAsmPrinter &printer) {
printer << " ";
printer.printSymbolName(getSymName());
printer << " (";
auto constituents = this->getConstituents().getValue();

if (!constituents.empty())
  llvm::interleaveComma(constituents, printer);

printer << ") : " << getType();
4358}

4360LogicalResult spirv::SpecConstantCompositeOp::verify() {
auto cType = getType().dyn_cast<spirv::CompositeType>();
auto constituents = this->getConstituents().getValue();

if (!cType)
  return emitError("result type must be a composite type, but provided ")
         << getType();

if (cType.isa<spirv::CooperativeMatrixNVType>())
  return emitError("unsupported composite type  ") << cType;
if (cType.isa<spirv::JointMatrixINTELType>())
  return emitError("unsupported composite type  ") << cType;
if (constituents.size() != cType.getNumElements())
  return emitError("has incorrect number of operands: expected ")
         << cType.getNumElements() << ", but provided "
         << constituents.size();

for (auto index : llvm::seq<uint32_t>(0, constituents.size())) {
  auto constituent = constituents[index].cast<FlatSymbolRefAttr>();

  auto constituentSpecConstOp =
      dyn_cast<spirv::SpecConstantOp>(SymbolTable::lookupNearestSymbolFrom(
          (*this)->getParentOp(), constituent.getAttr()));

  if (constituentSpecConstOp.getDefaultValue().getType() !=
      cType.getElementType(index))
    return emitError("has incorrect types of operands: expected ")
           << cType.getElementType(index) << ", but provided "
           << constituentSpecConstOp.getDefaultValue().getType();
}

return success();
4392}

4394//===----------------------------------------------------------------------===//
4395// spirv.SpecConstantOperation
4396//===----------------------------------------------------------------------===//

4398ParseResult spirv::SpecConstantOperationOp::parse(OpAsmParser &parser,
                                                OperationState &result) {
Region *body = result.addRegion();

if (parser.parseKeyword("wraps"))
  return failure();

body->push_back(new Block);
Block &block = body->back();
Operation *wrappedOp = parser.parseGenericOperation(&block, block.begin());

if (!wrappedOp)
  return failure();

OpBuilder builder(parser.getContext());
builder.setInsertionPointToEnd(&block);
builder.create<spirv::YieldOp>(wrappedOp->getLoc(), wrappedOp->getResult(0));
result.location = wrappedOp->getLoc();

result.addTypes(wrappedOp->getResult(0).getType());

if (parser.parseOptionalAttrDict(result.attributes))
  return failure();

return success();
4423}

4425void spirv::SpecConstantOperationOp::print(OpAsmPrinter &printer) {
printer << " wraps ";
printer.printGenericOp(&getBody().front().front());
4428}

4430LogicalResult spirv::SpecConstantOperationOp::verifyRegions() {
Block &block = getRegion().getBlocks().front();

if (block.getOperations().size() != 2)
  return emitOpError("expected exactly 2 nested ops");

Operation &enclosedOp = block.getOperations().front();

if (!enclosedOp.hasTrait<OpTrait::spirv::UsableInSpecConstantOp>())
  return emitOpError("invalid enclosed op");

for (auto operand : enclosedOp.getOperands())
  if (!isa<spirv::ConstantOp, spirv::ReferenceOfOp,
           spirv::SpecConstantOperationOp>(operand.getDefiningOp()))
    return emitOpError(
        "invalid operand, must be defined by a constant operation");

return success();
4448}

4450//===----------------------------------------------------------------------===//
4451// spirv.GL.FrexpStruct
4452//===----------------------------------------------------------------------===//

4454LogicalResult spirv::GLFrexpStructOp::verify() {
spirv::StructType structTy =
    getResult().getType().dyn_cast<spirv::StructType>();

if (structTy.getNumElements() != 2)
  return emitError("result type must be a struct type with two memebers");

Type significandTy = structTy.getElementType(0);
Type exponentTy = structTy.getElementType(1);
VectorType exponentVecTy = exponentTy.dyn_cast<VectorType>();
IntegerType exponentIntTy = exponentTy.dyn_cast<IntegerType>();

Type operandTy = getOperand().getType();
VectorType operandVecTy = operandTy.dyn_cast<VectorType>();
FloatType operandFTy = operandTy.dyn_cast<FloatType>();

if (significandTy != operandTy)
  return emitError("member zero of the resulting struct type must be the "
                   "same type as the operand");

if (exponentVecTy) {
  IntegerType componentIntTy =
      exponentVecTy.getElementType().dyn_cast<IntegerType>();
  if (!componentIntTy || componentIntTy.getWidth() != 32)
    return emitError("member one of the resulting struct type must"
                     "be a scalar or vector of 32 bit integer type");
} else if (!exponentIntTy || exponentIntTy.getWidth() != 32) {
  return emitError("member one of the resulting struct type "
                   "must be a scalar or vector of 32 bit integer type");
}

// Check that the two member types have the same number of components
if (operandVecTy && exponentVecTy &&
    (exponentVecTy.getNumElements() == operandVecTy.getNumElements()))
  return success();

if (operandFTy && exponentIntTy)
  return success();

return emitError("member one of the resulting struct type must have the same "
                 "number of components as the operand type");
4495}

4497//===----------------------------------------------------------------------===//
4498// spirv.GL.Ldexp
4499//===----------------------------------------------------------------------===//

4501LogicalResult spirv::GLLdexpOp::verify() {
Type significandType = getX().getType();
Type exponentType = getExp().getType();

if (significandType.isa<FloatType>() != exponentType.isa<IntegerType>())
  return emitOpError("operands must both be scalars or vectors");

auto getNumElements = [](Type type) -> unsigned {
  if (auto vectorType = type.dyn_cast<VectorType>())
    return vectorType.getNumElements();
  return 1;
};

if (getNumElements(significandType) != getNumElements(exponentType))
  return emitOpError("operands must have the same number of elements");

return success();
4518}

4520//===----------------------------------------------------------------------===//
4521// spirv.ImageDrefGather
4522//===----------------------------------------------------------------------===//

4524LogicalResult spirv::ImageDrefGatherOp::verify() {
VectorType resultType = getResult().getType().cast<VectorType>();
auto sampledImageType =
    getSampledimage().getType().cast<spirv::SampledImageType>();
auto imageType = sampledImageType.getImageType().cast<spirv::ImageType>();

if (resultType.getNumElements() != 4)
  return emitOpError("result type must be a vector of four components");

Type elementType = resultType.getElementType();
Type sampledElementType = imageType.getElementType();
if (!sampledElementType.isa<NoneType>() && elementType != sampledElementType)
  return emitOpError(
      "the component type of result must be the same as sampled type of the "
      "underlying image type");

spirv::Dim imageDim = imageType.getDim();
spirv::ImageSamplingInfo imageMS = imageType.getSamplingInfo();

if (imageDim != spirv::Dim::Dim2D && imageDim != spirv::Dim::Cube &&
    imageDim != spirv::Dim::Rect)
  return emitOpError(
      "the Dim operand of the underlying image type must be 2D, Cube, or "
      "Rect");

if (imageMS != spirv::ImageSamplingInfo::SingleSampled)
  return emitOpError("the MS operand of the underlying image type must be 0");

spirv::ImageOperandsAttr attr = getImageoperandsAttr();
auto operandArguments = getOperandArguments();

return verifyImageOperands(*this, attr, operandArguments);
4556}

4558//===----------------------------------------------------------------------===//
4559// spirv.ShiftLeftLogicalOp
4560//===----------------------------------------------------------------------===//

4562LogicalResult spirv::ShiftLeftLogicalOp::verify() {
return verifyShiftOp(*this);
4564}

4566//===----------------------------------------------------------------------===//
4567// spirv.ShiftRightArithmeticOp
4568//===----------------------------------------------------------------------===//

4570LogicalResult spirv::ShiftRightArithmeticOp::verify() {
return verifyShiftOp(*this);
4572}

4574//===----------------------------------------------------------------------===//
4575// spirv.ShiftRightLogicalOp
4576//===----------------------------------------------------------------------===//

4578LogicalResult spirv::ShiftRightLogicalOp::verify() {
return verifyShiftOp(*this);
4580}

4582//===----------------------------------------------------------------------===//
4583// spirv.ImageQuerySize
4584//===----------------------------------------------------------------------===//

4586LogicalResult spirv::ImageQuerySizeOp::verify() {
spirv::ImageType imageType = getImage().getType().cast<spirv::ImageType>();
Type resultType = getResult().getType();

spirv::Dim dim = imageType.getDim();
spirv::ImageSamplingInfo samplingInfo = imageType.getSamplingInfo();
spirv::ImageSamplerUseInfo samplerInfo = imageType.getSamplerUseInfo();
switch (dim) {
case spirv::Dim::Dim1D:
case spirv::Dim::Dim2D:
case spirv::Dim::Dim3D:
case spirv::Dim::Cube:
  if (samplingInfo != spirv::ImageSamplingInfo::MultiSampled &&
      samplerInfo != spirv::ImageSamplerUseInfo::SamplerUnknown &&
      samplerInfo != spirv::ImageSamplerUseInfo::NoSampler)
    return emitError(
        "if Dim is 1D, 2D, 3D, or Cube, "
        "it must also have either an MS of 1 or a Sampled of 0 or 2");
  break;
case spirv::Dim::Buffer:
case spirv::Dim::Rect:
  break;
default:
  return emitError("the Dim operand of the image type must "
                   "be 1D, 2D, 3D, Buffer, Cube, or Rect");
}

unsigned componentNumber = 0;
switch (dim) {
case spirv::Dim::Dim1D:
case spirv::Dim::Buffer:
  componentNumber = 1;
  break;
case spirv::Dim::Dim2D:
case spirv::Dim::Cube:
case spirv::Dim::Rect:
  componentNumber = 2;
  break;
case spirv::Dim::Dim3D:
  componentNumber = 3;
  break;
default:
  break;
}

if (imageType.getArrayedInfo() == spirv::ImageArrayedInfo::Arrayed)
  componentNumber += 1;

unsigned resultComponentNumber = 1;
if (auto resultVectorType = resultType.dyn_cast<VectorType>())
  resultComponentNumber = resultVectorType.getNumElements();

if (componentNumber != resultComponentNumber)
  return emitError("expected the result to have ")
         << componentNumber << " component(s), but found "
         << resultComponentNumber << " component(s)";

return success();
4644}

4646static ParseResult parsePtrAccessChainOpImpl(StringRef opName,
                                           OpAsmParser &parser,
                                           OperationState &state) {
OpAsmParser::UnresolvedOperand ptrInfo;
SmallVector<OpAsmParser::UnresolvedOperand, 4> indicesInfo;
Type type;
auto loc = parser.getCurrentLocation();
SmallVector<Type, 4> indicesTypes;

if (parser.parseOperand(ptrInfo) ||
    parser.parseOperandList(indicesInfo, OpAsmParser::Delimiter::Square) ||
    parser.parseColonType(type) ||
    parser.resolveOperand(ptrInfo, type, state.operands))
  return failure();

// Check that the provided indices list is not empty before parsing their
// type list.
if (indicesInfo.empty())
  return emitError(state.location) << opName << " expected element";

if (parser.parseComma() || parser.parseTypeList(indicesTypes))
  return failure();

// Check that the indices types list is not empty and that it has a one-to-one
// mapping to the provided indices.
if (indicesTypes.size() != indicesInfo.size())
  return emitError(state.location)
         << opName
         << " indices types' count must be equal to indices info count";

if (parser.resolveOperands(indicesInfo, indicesTypes, loc, state.operands))
  return failure();

auto resultType = getElementPtrType(
    type, llvm::makeArrayRef(state.operands).drop_front(2), state.location);
if (!resultType)
  return failure();

state.addTypes(resultType);
return success();
4686}

4688template <typename Op>
4689static auto concatElemAndIndices(Op op) {
SmallVector<Value> ret(op.getIndices().size() + 1);
ret[0] = op.getElement();
llvm::copy(op.getIndices(), ret.begin() + 1);
return ret;
4694}

4696//===----------------------------------------------------------------------===//
4697// spirv.InBoundsPtrAccessChainOp
4698//===----------------------------------------------------------------------===//

4700void spirv::InBoundsPtrAccessChainOp::build(OpBuilder &builder,
                                          OperationState &state,
                                          Value basePtr, Value element,
                                          ValueRange indices) {
auto type = getElementPtrType(basePtr.getType(), indices, state.location);
assert(type && "Unable to deduce return type based on basePtr and indices")(static_cast <bool> (type && "Unable to deduce return type based on basePtr and indices"
) ? void (0) : __assert_fail ("type && \"Unable to deduce return type based on basePtr and indices\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 4705, __extension__
 __PRETTY_FUNCTION__));
build(builder, state, type, basePtr, element, indices);
4707}

4709ParseResult spirv::InBoundsPtrAccessChainOp::parse(OpAsmParser &parser,
                                                 OperationState &result) {
return parsePtrAccessChainOpImpl(
    spirv::InBoundsPtrAccessChainOp::getOperationName(), parser, result);
4713}

4715void spirv::InBoundsPtrAccessChainOp::print(OpAsmPrinter &printer) {
printAccessChain(*this, concatElemAndIndices(*this), printer);
4717}

4719LogicalResult spirv::InBoundsPtrAccessChainOp::verify() {
return verifyAccessChain(*this, getIndices());
4721}

4723//===----------------------------------------------------------------------===//
4724// spirv.PtrAccessChainOp
4725//===----------------------------------------------------------------------===//

4727void spirv::PtrAccessChainOp::build(OpBuilder &builder, OperationState &state,
                                  Value basePtr, Value element,
                                  ValueRange indices) {
auto type = getElementPtrType(basePtr.getType(), indices, state.location);
assert(type && "Unable to deduce return type based on basePtr and indices")(static_cast <bool> (type && "Unable to deduce return type based on basePtr and indices"
) ? void (0) : __assert_fail ("type && \"Unable to deduce return type based on basePtr and indices\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 4731, __extension__
 __PRETTY_FUNCTION__));
build(builder, state, type, basePtr, element, indices);
4733}

4735ParseResult spirv::PtrAccessChainOp::parse(OpAsmParser &parser,
                                         OperationState &result) {
return parsePtrAccessChainOpImpl(spirv::PtrAccessChainOp::getOperationName(),
                                 parser, result);
4739}

4741void spirv::PtrAccessChainOp::print(OpAsmPrinter &printer) {
printAccessChain(*this, concatElemAndIndices(*this), printer);
4743}

4745LogicalResult spirv::PtrAccessChainOp::verify() {
return verifyAccessChain(*this, getIndices());
4747}

4749//===----------------------------------------------------------------------===//
4750// spirv.VectorTimesScalarOp
4751//===----------------------------------------------------------------------===//

4753LogicalResult spirv::VectorTimesScalarOp::verify() {
if (getVector().getType() != getType())
  return emitOpError("vector operand and result type mismatch");
auto scalarType = getType().cast<VectorType>().getElementType();
if (getScalar().getType() != scalarType)
  return emitOpError("scalar operand and result element type match");
return success();
4760}

4762//===----------------------------------------------------------------------===//
4763// Group ops
4764//===----------------------------------------------------------------------===//

4766template <typename Op>
4767static LogicalResult verifyGroupOp(Op op) {
spirv::Scope scope = op.getExecutionScope();
if (scope != spirv::Scope::Workgroup && scope != spirv::Scope::Subgroup)
  return op.emitOpError("execution scope must be 'Workgroup' or 'Subgroup'");

return success();
4773}

4775LogicalResult spirv::GroupIAddOp::verify() { return verifyGroupOp(*this); }

4777LogicalResult spirv::GroupFAddOp::verify() { return verifyGroupOp(*this); }

4779LogicalResult spirv::GroupFMinOp::verify() { return verifyGroupOp(*this); }

4781LogicalResult spirv::GroupUMinOp::verify() { return verifyGroupOp(*this); }

4783LogicalResult spirv::GroupSMinOp::verify() { return verifyGroupOp(*this); }

4785LogicalResult spirv::GroupFMaxOp::verify() { return verifyGroupOp(*this); }

4787LogicalResult spirv::GroupUMaxOp::verify() { return verifyGroupOp(*this); }

4789LogicalResult spirv::GroupSMaxOp::verify() { return verifyGroupOp(*this); }

4791LogicalResult spirv::GroupIMulKHROp::verify() { return verifyGroupOp(*this); }

4793LogicalResult spirv::GroupFMulKHROp::verify() { return verifyGroupOp(*this); }

4795//===----------------------------------------------------------------------===//
4796// Integer Dot Product ops
4797//===----------------------------------------------------------------------===//

4799static LogicalResult verifyIntegerDotProduct(Operation *op) {
assert(llvm::is_contained({2u, 3u}, op->getNumOperands()) &&(static_cast <bool> (llvm::is_contained({2u, 3u}, op->
getNumOperands()) && "Not an integer dot product op?"
) ? void (0) : __assert_fail ("llvm::is_contained({2u, 3u}, op->getNumOperands()) && \"Not an integer dot product op?\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 4801, __extension__
 __PRETTY_FUNCTION__))
       "Not an integer dot product op?")(static_cast <bool> (llvm::is_contained({2u, 3u}, op->
getNumOperands()) && "Not an integer dot product op?"
) ? void (0) : __assert_fail ("llvm::is_contained({2u, 3u}, op->getNumOperands()) && \"Not an integer dot product op?\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 4801, __extension__
 __PRETTY_FUNCTION__));
assert(op->getNumResults() == 1 && "Expected a single result")(static_cast <bool> (op->getNumResults() == 1 &&
 "Expected a single result") ? void (0) : __assert_fail ("op->getNumResults() == 1 && \"Expected a single result\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 4802, __extension__
 __PRETTY_FUNCTION__));

Type factorTy = op->getOperand(0).getType();
if (op->getOperand(1).getType() != factorTy)
  return op->emitOpError("requires the same type for both vector operands");

unsigned expectedNumAttrs = 0;
if (auto intTy = factorTy.dyn_cast<IntegerType>()) {
  ++expectedNumAttrs;
  auto packedVectorFormat =
      op->getAttr(kPackedVectorFormatAttrName)
          .dyn_cast_or_null<spirv::PackedVectorFormatAttr>();
  if (!packedVectorFormat)
    return op->emitOpError("requires Packed Vector Format attribute for "
                           "integer vector operands");

  assert(packedVectorFormat.getValue() ==(static_cast <bool> (packedVectorFormat.getValue() == spirv
::PackedVectorFormat::PackedVectorFormat4x8Bit && "Unknown Packed Vector Format"
) ? void (0) : __assert_fail ("packedVectorFormat.getValue() == spirv::PackedVectorFormat::PackedVectorFormat4x8Bit && \"Unknown Packed Vector Format\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 4820, __extension__
 __PRETTY_FUNCTION__))
             spirv::PackedVectorFormat::PackedVectorFormat4x8Bit &&(static_cast <bool> (packedVectorFormat.getValue() == spirv
::PackedVectorFormat::PackedVectorFormat4x8Bit && "Unknown Packed Vector Format"
) ? void (0) : __assert_fail ("packedVectorFormat.getValue() == spirv::PackedVectorFormat::PackedVectorFormat4x8Bit && \"Unknown Packed Vector Format\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 4820, __extension__
 __PRETTY_FUNCTION__))
         "Unknown Packed Vector Format")(static_cast <bool> (packedVectorFormat.getValue() == spirv
::PackedVectorFormat::PackedVectorFormat4x8Bit && "Unknown Packed Vector Format"
) ? void (0) : __assert_fail ("packedVectorFormat.getValue() == spirv::PackedVectorFormat::PackedVectorFormat4x8Bit && \"Unknown Packed Vector Format\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 4820, __extension__
 __PRETTY_FUNCTION__));
  if (intTy.getWidth() != 32)
    return op->emitOpError(
        llvm::formatv("with specified Packed Vector Format ({0}) requires "
                      "integer vector operands to be 32-bits wide",
                      packedVectorFormat.getValue()));
} else {
  if (op->hasAttr(kPackedVectorFormatAttrName))
    return op->emitOpError(llvm::formatv(
        "with invalid format attribute for vector operands of type '{0}'",
        factorTy));
}

if (op->getAttrs().size() > expectedNumAttrs)
  return op->emitError(
      "op only supports the 'format' #spirv.packed_vector_format attribute");

Type resultTy = op->getResultTypes().front();
bool hasAccumulator = op->getNumOperands() == 3;
if (hasAccumulator && op->getOperand(2).getType() != resultTy)
  return op->emitOpError(
      "requires the same accumulator operand and result types");

unsigned factorBitWidth = getBitWidth(factorTy);
unsigned resultBitWidth = getBitWidth(resultTy);
if (factorBitWidth > resultBitWidth)
  return op->emitOpError(
      llvm::formatv("result type has insufficient bit-width ({0} bits) "
                    "for the specified vector operand type ({1} bits)",
                    resultBitWidth, factorBitWidth));

return success();
4852}

4854static Optional<spirv::Version> getIntegerDotProductMinVersion() {
return spirv::Version::V_1_0; // Available in SPIR-V >= 1.0.
4856}

4858static Optional<spirv::Version> getIntegerDotProductMaxVersion() {
return spirv::Version::V_1_6; // Available in SPIR-V <= 1.6.
4860}

4862static SmallVector<ArrayRef<spirv::Extension>, 1>
4863getIntegerDotProductExtensions() {
// Requires the SPV_KHR_integer_dot_product extension, specified either
// explicitly or implied by target env's SPIR-V version >= 1.6.
static const auto extension = spirv::Extension::SPV_KHR_integer_dot_product;
return {extension};
4868}

4870static SmallVector<ArrayRef<spirv::Capability>, 1>
4871getIntegerDotProductCapabilities(Operation *op) {
// Requires the the DotProduct capability and capabilities that depend on
// exact op types.
static const auto dotProductCap = spirv::Capability::DotProduct;
static const auto dotProductInput4x8BitPackedCap =
    spirv::Capability::DotProductInput4x8BitPacked;
static const auto dotProductInput4x8BitCap =
    spirv::Capability::DotProductInput4x8Bit;
static const auto dotProductInputAllCap =
    spirv::Capability::DotProductInputAll;

SmallVector<ArrayRef<spirv::Capability>, 1> capabilities = {dotProductCap};

Type factorTy = op->getOperand(0).getType();
if (auto intTy = factorTy.dyn_cast<IntegerType>()) {
  auto formatAttr = op->getAttr(kPackedVectorFormatAttrName)
                        .cast<spirv::PackedVectorFormatAttr>();
  if (formatAttr.getValue() ==
      spirv::PackedVectorFormat::PackedVectorFormat4x8Bit)
    capabilities.push_back(dotProductInput4x8BitPackedCap);

  return capabilities;
}

auto vecTy = factorTy.cast<VectorType>();
if (vecTy.getElementTypeBitWidth() == 8) {
  capabilities.push_back(dotProductInput4x8BitCap);
  return capabilities;
}

capabilities.push_back(dotProductInputAllCap);
return capabilities;
4903}

4905#define SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(OpName)                              \
LogicalResult OpName::verify() { return verifyIntegerDotProduct(*this); }    \
SmallVector<ArrayRef<spirv::Extension>, 1> OpName::getExtensions() {         \
  return getIntegerDotProductExtensions();                                   \
}                                                                            \
SmallVector<ArrayRef<spirv::Capability>, 1> OpName::getCapabilities() {      \
  return getIntegerDotProductCapabilities(*this);                            \
}                                                                            \
Optional<spirv::Version> OpName::getMinVersion() {                           \
  return getIntegerDotProductMinVersion();                                   \
}                                                                            \
Optional<spirv::Version> OpName::getMaxVersion() {                           \
  return getIntegerDotProductMaxVersion();                                   \
}

4920SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(spirv::SDotOp)
4921SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(spirv::SUDotOp)
4922SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(spirv::UDotOp)
4923SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(spirv::SDotAccSatOp)
4924SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(spirv::SUDotAccSatOp)
4925SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(spirv::UDotAccSatOp)

4927#undef SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP

4929// TableGen'erated operation interfaces for querying versions, extensions, and
4930// capabilities.
4931#include "mlir/Dialect/SPIRV/IR/SPIRVAvailability.cpp.inc"

4933// TablenGen'erated operation definitions.
4934#define GET_OP_CLASSES
4935#include "mlir/Dialect/SPIRV/IR/SPIRVOps.cpp.inc"

4937namespace mlir {
4938namespace spirv {
4939// TableGen'erated operation availability interface implementations.
4940#include "mlir/Dialect/SPIRV/IR/SPIRVOpAvailabilityImpl.inc"
4941} // namespace spirv
4942} // namespace mlir