/build/source/mlir/include/mlir/IR/Builders.h

Bug Summary

File:	build/source/mlir/include/mlir/IR/Builders.h
Warning:	line 100, column 12 2nd function call argument is an uninitialized value

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name SPIRVOps.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-17/lib/clang/17 -D MLIR_CUDA_CONVERSIONS_ENABLED=1 -D MLIR_ROCM_CONVERSIONS_ENABLED=1 -D _DEBUG -D _GLIBCXX_ASSERTIONS -D _GNU_SOURCE -D _LIBCPP_ENABLE_ASSERTIONS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I tools/mlir/lib/Dialect/SPIRV/IR -I /build/source/mlir/lib/Dialect/SPIRV/IR -I include -I /build/source/llvm/include -I /build/source/mlir/include -I tools/mlir/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-17/lib/clang/17/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/source/= -fcoverage-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/source/= -source-date-epoch 1683717183 -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2023-05-10-133810-16478-1 -x c++ /build/source/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp

/build/source/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp

→

1//===- SPIRVOps.cpp - MLIR SPIR-V operations ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the operations in the SPIR-V dialect.
10//
11//===----------------------------------------------------------------------===//

13#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"

15#include "mlir/Dialect/SPIRV/IR/ParserUtils.h"
16#include "mlir/Dialect/SPIRV/IR/SPIRVAttributes.h"
17#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
18#include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h"
19#include "mlir/Dialect/SPIRV/IR/SPIRVOpTraits.h"
20#include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
21#include "mlir/Dialect/SPIRV/IR/TargetAndABI.h"
22#include "mlir/IR/Builders.h"
23#include "mlir/IR/BuiltinTypes.h"
24#include "mlir/IR/FunctionImplementation.h"
25#include "mlir/IR/OpDefinition.h"
26#include "mlir/IR/OpImplementation.h"
27#include "mlir/IR/Operation.h"
28#include "mlir/IR/TypeUtilities.h"
29#include "mlir/Interfaces/CallInterfaces.h"
30#include "llvm/ADT/APFloat.h"
31#include "llvm/ADT/APInt.h"
32#include "llvm/ADT/ArrayRef.h"
33#include "llvm/ADT/STLExtras.h"
34#include "llvm/ADT/StringExtras.h"
35#include "llvm/Support/FormatVariadic.h"
36#include <cassert>
37#include <numeric>

39using namespace mlir;

41// TODO: generate these strings using ODS.
42constexpr char kAlignmentAttrName[] = "alignment";
43constexpr char kBranchWeightAttrName[] = "branch_weights";
44constexpr char kCallee[] = "callee";
45constexpr char kClusterSize[] = "cluster_size";
46constexpr char kControl[] = "control";
47constexpr char kDefaultValueAttrName[] = "default_value";
48constexpr char kEqualSemanticsAttrName[] = "equal_semantics";
49constexpr char kExecutionScopeAttrName[] = "execution_scope";
50constexpr char kFnNameAttrName[] = "fn";
51constexpr char kGroupOperationAttrName[] = "group_operation";
52constexpr char kIndicesAttrName[] = "indices";
53constexpr char kInitializerAttrName[] = "initializer";
54constexpr char kInterfaceAttrName[] = "interface";
55constexpr char kMemoryAccessAttrName[] = "memory_access";
56constexpr char kMemoryScopeAttrName[] = "memory_scope";
57constexpr char kPackedVectorFormatAttrName[] = "format";
58constexpr char kSemanticsAttrName[] = "semantics";
59constexpr char kSourceAlignmentAttrName[] = "source_alignment";
60constexpr char kSourceMemoryAccessAttrName[] = "source_memory_access";
61constexpr char kSpecIdAttrName[] = "spec_id";
62constexpr char kTypeAttrName[] = "type";
63constexpr char kUnequalSemanticsAttrName[] = "unequal_semantics";
64constexpr char kValueAttrName[] = "value";
65constexpr char kValuesAttrName[] = "values";
66constexpr char kCompositeSpecConstituentsName[] = "constituents";

68//===----------------------------------------------------------------------===//
69// Common utility functions
70//===----------------------------------------------------------------------===//

72static ParseResult parseOneResultSameOperandTypeOp(OpAsmParser &parser,
                                                 OperationState &result) {
SmallVector<OpAsmParser::UnresolvedOperand, 2> ops;
Type type;
// If the operand list is in-between parentheses, then we have a generic form.
// (see the fallback in `printOneResultOp`).
SMLoc loc = parser.getCurrentLocation();
if (!parser.parseOptionalLParen()) {
  if (parser.parseOperandList(ops) || parser.parseRParen() ||
      parser.parseOptionalAttrDict(result.attributes) ||
      parser.parseColon() || parser.parseType(type))
    return failure();
  auto fnType = type.dyn_cast<FunctionType>();
  if (!fnType) {
    parser.emitError(loc, "expected function type");
    return failure();
  }
  if (parser.resolveOperands(ops, fnType.getInputs(), loc, result.operands))
    return failure();
  result.addTypes(fnType.getResults());
  return success();
}
return failure(parser.parseOperandList(ops) ||
               parser.parseOptionalAttrDict(result.attributes) ||
               parser.parseColonType(type) ||
               parser.resolveOperands(ops, type, result.operands) ||
               parser.addTypeToList(type, result.types));
99}

101static void printOneResultOp(Operation *op, OpAsmPrinter &p) {
assert(op->getNumResults() == 1 && "op should have one result")(static_cast <bool> (op->getNumResults() == 1 &&
 "op should have one result") ? void (0) : __assert_fail ("op->getNumResults() == 1 && \"op should have one result\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 102, __extension__
 __PRETTY_FUNCTION__));

// If not all the operand and result types are the same, just use the
// generic assembly form to avoid omitting information in printing.
auto resultType = op->getResult(0).getType();
if (llvm::any_of(op->getOperandTypes(),
                 [&](Type type) { return type != resultType; })) {
  p.printGenericOp(op, /*printOpName=*/false);
  return;
}

p << ' ';
p.printOperands(op->getOperands());
p.printOptionalAttrDict(op->getAttrs());
// Now we can output only one type for all operands and the result.
p << " : " << resultType;
118}

120/// Returns true if the given op is a function-like op or nested in a
121/// function-like op without a module-like op in the middle.
122static bool isNestedInFunctionOpInterface(Operation *op) {
if (!op)
  return false;
if (op->hasTrait<OpTrait::SymbolTable>())
  return false;
if (isa<FunctionOpInterface>(op))
  return true;
return isNestedInFunctionOpInterface(op->getParentOp());
130}

132/// Returns true if the given op is an module-like op that maintains a symbol
133/// table.
134static bool isDirectInModuleLikeOp(Operation *op) {
return op && op->hasTrait<OpTrait::SymbolTable>();
136}

138static LogicalResult extractValueFromConstOp(Operation *op, int32_t &value) {
auto constOp = dyn_cast_or_null<spirv::ConstantOp>(op);
if (!constOp) {
  return failure();
}
auto valueAttr = constOp.getValue();
auto integerValueAttr = valueAttr.dyn_cast<IntegerAttr>();
if (!integerValueAttr) {
  return failure();
}

if (integerValueAttr.getType().isSignlessInteger())
  value = integerValueAttr.getInt();
else
  value = integerValueAttr.getSInt();

return success();
155}

157template <typename Ty>
158static ArrayAttr
159getStrArrayAttrForEnumList(Builder &builder, ArrayRef<Ty> enumValues,
                         function_ref<StringRef(Ty)> stringifyFn) {
if (enumValues.empty()) {
  return nullptr;
}
SmallVector<StringRef, 1> enumValStrs;
enumValStrs.reserve(enumValues.size());
for (auto val : enumValues) {
  enumValStrs.emplace_back(stringifyFn(val));
}
return builder.getStrArrayAttr(enumValStrs);
170}

172/// Parses the next string attribute in `parser` as an enumerant of the given
173/// `EnumClass`.
174template <typename EnumClass>
175static ParseResult
176parseEnumStrAttr(EnumClass &value, OpAsmParser &parser,
               StringRef attrName = spirv::attributeName<EnumClass>()) {
Attribute attrVal;
NamedAttrList attr;
auto loc = parser.getCurrentLocation();
if (parser.parseAttribute(attrVal, parser.getBuilder().getNoneType(),
4
←
Taking false branch→
                          attrName, attr))
  return failure();
if (!attrVal.isa<StringAttr>())
5
←
Taking true branch→
  return parser.emitError(loc, "expected ")
         << attrName << " attribute specified as string";
auto attrOptional =
    spirv::symbolizeEnum<EnumClass>(attrVal.cast<StringAttr>().getValue());
if (!attrOptional)
  return parser.emitError(loc, "invalid ")
         << attrName << " attribute specification: " << attrVal;
value = *attrOptional;
return success();
194}

196/// Parses the next string attribute in `parser` as an enumerant of the given
197/// `EnumClass` and inserts the enumerant into `state` as an 32-bit integer
198/// attribute with the enum class's name as attribute name.
199template <typename EnumAttrClass,
        typename EnumClass = typename EnumAttrClass::ValueType>
201static ParseResult
202parseEnumStrAttr(EnumClass &value, OpAsmParser &parser, OperationState &state,
               StringRef attrName = spirv::attributeName<EnumClass>()) {
if (parseEnumStrAttr(value, parser))
3
←
Calling 'parseEnumStrAttr<mlir::spirv::ExecutionModel>'→
6
←
Returning from 'parseEnumStrAttr<mlir::spirv::ExecutionModel>'→
7
←
Taking false branch→
  return failure();
state.addAttribute(attrName,
                   parser.getBuilder().getAttr<EnumAttrClass>(value));
8
←
Calling 'Builder::getAttr'→
return success();
209}

211/// Parses the next keyword in `parser` as an enumerant of the given `EnumClass`
212/// and inserts the enumerant into `state` as an 32-bit integer attribute with
213/// the enum class's name as attribute name.
214template <typename EnumAttrClass,
        typename EnumClass = typename EnumAttrClass::ValueType>
216static ParseResult
217parseEnumKeywordAttr(EnumClass &value, OpAsmParser &parser,
                   OperationState &state,
                   StringRef attrName = spirv::attributeName<EnumClass>()) {
if (parseEnumKeywordAttr(value, parser))
  return failure();
state.addAttribute(attrName,
                   parser.getBuilder().getAttr<EnumAttrClass>(value));
return success();
225}

227/// Parses Function, Selection and Loop control attributes. If no control is
228/// specified, "None" is used as a default.
229template <typename EnumAttrClass, typename EnumClass>
230static ParseResult
231parseControlAttribute(OpAsmParser &parser, OperationState &state,
                    StringRef attrName = spirv::attributeName<EnumClass>()) {
if (succeeded(parser.parseOptionalKeyword(kControl))) {
  EnumClass control;
  if (parser.parseLParen() ||
      parseEnumKeywordAttr<EnumAttrClass>(control, parser, state) ||
      parser.parseRParen())
    return failure();
  return success();
}
// Set control to "None" otherwise.
Builder builder = parser.getBuilder();
state.addAttribute(attrName,
                   builder.getAttr<EnumAttrClass>(static_cast<EnumClass>(0)));
return success();
246}

248/// Parses optional memory access attributes attached to a memory access
249/// operand/pointer. Specifically, parses the following syntax:
250///     (`[` memory-access `]`)?
251/// where:
252///     memory-access ::= `"None"` | `"Volatile"` | `"Aligned", `
253///         integer-literal | `"NonTemporal"`
254static ParseResult parseMemoryAccessAttributes(OpAsmParser &parser,
                                             OperationState &state) {
// Parse an optional list of attributes staring with '['
if (parser.parseOptionalLSquare()) {
  // Nothing to do
  return success();
}

spirv::MemoryAccess memoryAccessAttr;
if (parseEnumStrAttr<spirv::MemoryAccessAttr>(memoryAccessAttr, parser, state,
                                              kMemoryAccessAttrName))
  return failure();

if (spirv::bitEnumContainsAll(memoryAccessAttr,
                              spirv::MemoryAccess::Aligned)) {
  // Parse integer attribute for alignment.
  Attribute alignmentAttr;
  Type i32Type = parser.getBuilder().getIntegerType(32);
  if (parser.parseComma() ||
      parser.parseAttribute(alignmentAttr, i32Type, kAlignmentAttrName,
                            state.attributes)) {
    return failure();
  }
}
return parser.parseRSquare();
279}

281// TODO Make sure to merge this and the previous function into one template
282// parameterized by memory access attribute name and alignment. Doing so now
283// results in VS2017 in producing an internal error (at the call site) that's
284// not detailed enough to understand what is happening.
285static ParseResult parseSourceMemoryAccessAttributes(OpAsmParser &parser,
                                                   OperationState &state) {
// Parse an optional list of attributes staring with '['
if (parser.parseOptionalLSquare()) {
  // Nothing to do
  return success();
}

spirv::MemoryAccess memoryAccessAttr;
if (parseEnumStrAttr<spirv::MemoryAccessAttr>(memoryAccessAttr, parser, state,
                                              kSourceMemoryAccessAttrName))
  return failure();

if (spirv::bitEnumContainsAll(memoryAccessAttr,
                              spirv::MemoryAccess::Aligned)) {
  // Parse integer attribute for alignment.
  Attribute alignmentAttr;
  Type i32Type = parser.getBuilder().getIntegerType(32);
  if (parser.parseComma() ||
      parser.parseAttribute(alignmentAttr, i32Type, kSourceAlignmentAttrName,
                            state.attributes)) {
    return failure();
  }
}
return parser.parseRSquare();
310}

312template <typename MemoryOpTy>
313static void printMemoryAccessAttribute(
  MemoryOpTy memoryOp, OpAsmPrinter &printer,
  SmallVectorImpl<StringRef> &elidedAttrs,
  std::optional<spirv::MemoryAccess> memoryAccessAtrrValue = std::nullopt,
  std::optional<uint32_t> alignmentAttrValue = std::nullopt) {
// Print optional memory access attribute.
if (auto memAccess = (memoryAccessAtrrValue ? memoryAccessAtrrValue
                                            : memoryOp.getMemoryAccess())) {
  elidedAttrs.push_back(kMemoryAccessAttrName);

  printer << " [\"" << stringifyMemoryAccess(*memAccess) << "\"";

  if (spirv::bitEnumContainsAll(*memAccess, spirv::MemoryAccess::Aligned)) {
    // Print integer alignment attribute.
    if (auto alignment = (alignmentAttrValue ? alignmentAttrValue
                                             : memoryOp.getAlignment())) {
      elidedAttrs.push_back(kAlignmentAttrName);
      printer << ", " << *alignment;
    }
  }
  printer << "]";
}
elidedAttrs.push_back(spirv::attributeName<spirv::StorageClass>());
336}

338// TODO Make sure to merge this and the previous function into one template
339// parameterized by memory access attribute name and alignment. Doing so now
340// results in VS2017 in producing an internal error (at the call site) that's
341// not detailed enough to understand what is happening.
342template <typename MemoryOpTy>
343static void printSourceMemoryAccessAttribute(
  MemoryOpTy memoryOp, OpAsmPrinter &printer,
  SmallVectorImpl<StringRef> &elidedAttrs,
  std::optional<spirv::MemoryAccess> memoryAccessAtrrValue = std::nullopt,
  std::optional<uint32_t> alignmentAttrValue = std::nullopt) {

printer << ", ";

// Print optional memory access attribute.
if (auto memAccess = (memoryAccessAtrrValue ? memoryAccessAtrrValue
                                            : memoryOp.getMemoryAccess())) {
  elidedAttrs.push_back(kSourceMemoryAccessAttrName);

  printer << " [\"" << stringifyMemoryAccess(*memAccess) << "\"";

  if (spirv::bitEnumContainsAll(*memAccess, spirv::MemoryAccess::Aligned)) {
    // Print integer alignment attribute.
    if (auto alignment = (alignmentAttrValue ? alignmentAttrValue
                                             : memoryOp.getAlignment())) {
      elidedAttrs.push_back(kSourceAlignmentAttrName);
      printer << ", " << *alignment;
    }
  }
  printer << "]";
}
elidedAttrs.push_back(spirv::attributeName<spirv::StorageClass>());
369}

371static ParseResult parseImageOperands(OpAsmParser &parser,
                                    spirv::ImageOperandsAttr &attr) {
// Expect image operands
if (parser.parseOptionalLSquare())
  return success();

spirv::ImageOperands imageOperands;
if (parseEnumStrAttr(imageOperands, parser))
  return failure();

attr = spirv::ImageOperandsAttr::get(parser.getContext(), imageOperands);

return parser.parseRSquare();
384}

386static void printImageOperands(OpAsmPrinter &printer, Operation *imageOp,
                             spirv::ImageOperandsAttr attr) {
if (attr) {
  auto strImageOperands = stringifyImageOperands(attr.getValue());
  printer << "[\"" << strImageOperands << "\"]";
}
392}

394template <typename Op>
395static LogicalResult verifyImageOperands(Op imageOp,
                                       spirv::ImageOperandsAttr attr,
                                       Operation::operand_range operands) {
if (!attr) {
  if (operands.empty())
    return success();

  return imageOp.emitError("the Image Operands should encode what operands "
                           "follow, as per Image Operands");
}

// TODO: Add the validation rules for the following Image Operands.
spirv::ImageOperands noSupportOperands =
    spirv::ImageOperands::Bias | spirv::ImageOperands::Lod |
    spirv::ImageOperands::Grad | spirv::ImageOperands::ConstOffset |
    spirv::ImageOperands::Offset | spirv::ImageOperands::ConstOffsets |
    spirv::ImageOperands::Sample | spirv::ImageOperands::MinLod |
    spirv::ImageOperands::MakeTexelAvailable |
    spirv::ImageOperands::MakeTexelVisible |
    spirv::ImageOperands::SignExtend | spirv::ImageOperands::ZeroExtend;

if (spirv::bitEnumContainsAll(attr.getValue(), noSupportOperands))
  llvm_unreachable("unimplemented operands of Image Operands")::llvm::llvm_unreachable_internal("unimplemented operands of Image Operands"
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 417);

return success();
420}

422static LogicalResult verifyCastOp(Operation *op,
                                bool requireSameBitWidth = true,
                                bool skipBitWidthCheck = false) {
// Some CastOps have no limit on bit widths for result and operand type.
if (skipBitWidthCheck)
  return success();

Type operandType = op->getOperand(0).getType();
Type resultType = op->getResult(0).getType();

// ODS checks that result type and operand type have the same shape.
if (auto vectorType = operandType.dyn_cast<VectorType>()) {
  operandType = vectorType.getElementType();
  resultType = resultType.cast<VectorType>().getElementType();
}

if (auto coopMatrixType =
        operandType.dyn_cast<spirv::CooperativeMatrixNVType>()) {
  operandType = coopMatrixType.getElementType();
  resultType =
      resultType.cast<spirv::CooperativeMatrixNVType>().getElementType();
}

if (auto jointMatrixType =
        operandType.dyn_cast<spirv::JointMatrixINTELType>()) {
  operandType = jointMatrixType.getElementType();
  resultType =
      resultType.cast<spirv::JointMatrixINTELType>().getElementType();
}

auto operandTypeBitWidth = operandType.getIntOrFloatBitWidth();
auto resultTypeBitWidth = resultType.getIntOrFloatBitWidth();
auto isSameBitWidth = operandTypeBitWidth == resultTypeBitWidth;

if (requireSameBitWidth) {
  if (!isSameBitWidth) {
    return op->emitOpError(
               "expected the same bit widths for operand type and result "
               "type, but provided ")
           << operandType << " and " << resultType;
  }
  return success();
}

if (isSameBitWidth) {
  return op->emitOpError(
             "expected the different bit widths for operand type and result "
             "type, but provided ")
         << operandType << " and " << resultType;
}
return success();
473}

475template <typename MemoryOpTy>
476static LogicalResult verifyMemoryAccessAttribute(MemoryOpTy memoryOp) {
// ODS checks for attributes values. Just need to verify that if the
// memory-access attribute is Aligned, then the alignment attribute must be
// present.
auto *op = memoryOp.getOperation();
auto memAccessAttr = op->getAttr(kMemoryAccessAttrName);
if (!memAccessAttr) {
  // Alignment attribute shouldn't be present if memory access attribute is
  // not present.
  if (op->getAttr(kAlignmentAttrName)) {
    return memoryOp.emitOpError(
        "invalid alignment specification without aligned memory access "
        "specification");
  }
  return success();
}

auto memAccess = memAccessAttr.template cast<spirv::MemoryAccessAttr>();

if (!memAccess) {
  return memoryOp.emitOpError("invalid memory access specifier: ")
         << memAccessAttr;
}

if (spirv::bitEnumContainsAll(memAccess.getValue(),
                              spirv::MemoryAccess::Aligned)) {
  if (!op->getAttr(kAlignmentAttrName)) {
    return memoryOp.emitOpError("missing alignment value");
  }
} else {
  if (op->getAttr(kAlignmentAttrName)) {
    return memoryOp.emitOpError(
        "invalid alignment specification with non-aligned memory access "
        "specification");
  }
}
return success();
513}

515// TODO Make sure to merge this and the previous function into one template
516// parameterized by memory access attribute name and alignment. Doing so now
517// results in VS2017 in producing an internal error (at the call site) that's
518// not detailed enough to understand what is happening.
519template <typename MemoryOpTy>
520static LogicalResult verifySourceMemoryAccessAttribute(MemoryOpTy memoryOp) {
// ODS checks for attributes values. Just need to verify that if the
// memory-access attribute is Aligned, then the alignment attribute must be
// present.
auto *op = memoryOp.getOperation();
auto memAccessAttr = op->getAttr(kSourceMemoryAccessAttrName);
if (!memAccessAttr) {
  // Alignment attribute shouldn't be present if memory access attribute is
  // not present.
  if (op->getAttr(kSourceAlignmentAttrName)) {
    return memoryOp.emitOpError(
        "invalid alignment specification without aligned memory access "
        "specification");
  }
  return success();
}

auto memAccess = memAccessAttr.template cast<spirv::MemoryAccessAttr>();

if (!memAccess) {
  return memoryOp.emitOpError("invalid memory access specifier: ")
         << memAccess;
}

if (spirv::bitEnumContainsAll(memAccess.getValue(),
                              spirv::MemoryAccess::Aligned)) {
  if (!op->getAttr(kSourceAlignmentAttrName)) {
    return memoryOp.emitOpError("missing alignment value");
  }
} else {
  if (op->getAttr(kSourceAlignmentAttrName)) {
    return memoryOp.emitOpError(
        "invalid alignment specification with non-aligned memory access "
        "specification");
  }
}
return success();
557}

559static LogicalResult
560verifyMemorySemantics(Operation *op, spirv::MemorySemantics memorySemantics) {
// According to the SPIR-V specification:
// "Despite being a mask and allowing multiple bits to be combined, it is
// invalid for more than one of these four bits to be set: Acquire, Release,
// AcquireRelease, or SequentiallyConsistent. Requesting both Acquire and
// Release semantics is done by setting the AcquireRelease bit, not by setting
// two bits."
auto atMostOneInSet = spirv::MemorySemantics::Acquire |
                      spirv::MemorySemantics::Release |
                      spirv::MemorySemantics::AcquireRelease |
                      spirv::MemorySemantics::SequentiallyConsistent;

auto bitCount =
    llvm::popcount(static_cast<uint32_t>(memorySemantics & atMostOneInSet));
if (bitCount > 1) {
  return op->emitError(
      "expected at most one of these four memory constraints "
      "to be set: `Acquire`, `Release`,"
      "`AcquireRelease` or `SequentiallyConsistent`");
}
return success();
581}

583template <typename LoadStoreOpTy>
584static LogicalResult verifyLoadStorePtrAndValTypes(LoadStoreOpTy op, Value ptr,
                                                 Value val) {
// ODS already checks ptr is spirv::PointerType. Just check that the pointee
// type of the pointer and the type of the value are the same
//
// TODO: Check that the value type satisfies restrictions of
// SPIR-V OpLoad/OpStore operations
if (val.getType() !=
    ptr.getType().cast<spirv::PointerType>().getPointeeType()) {
  return op.emitOpError("mismatch in result type and pointer type");
}
return success();
596}

598template <typename BlockReadWriteOpTy>
599static LogicalResult verifyBlockReadWritePtrAndValTypes(BlockReadWriteOpTy op,
                                                      Value ptr, Value val) {
auto valType = val.getType();
if (auto valVecTy = valType.dyn_cast<VectorType>())
  valType = valVecTy.getElementType();

if (valType != ptr.getType().cast<spirv::PointerType>().getPointeeType()) {
  return op.emitOpError("mismatch in result type and pointer type");
}
return success();
609}

611static ParseResult parseVariableDecorations(OpAsmParser &parser,
                                          OperationState &state) {
auto builtInName = llvm::convertToSnakeFromCamelCase(
    stringifyDecoration(spirv::Decoration::BuiltIn));
if (succeeded(parser.parseOptionalKeyword("bind"))) {
  Attribute set, binding;
  // Parse optional descriptor binding
  auto descriptorSetName = llvm::convertToSnakeFromCamelCase(
      stringifyDecoration(spirv::Decoration::DescriptorSet));
  auto bindingName = llvm::convertToSnakeFromCamelCase(
      stringifyDecoration(spirv::Decoration::Binding));
  Type i32Type = parser.getBuilder().getIntegerType(32);
  if (parser.parseLParen() ||
      parser.parseAttribute(set, i32Type, descriptorSetName,
                            state.attributes) ||
      parser.parseComma() ||
      parser.parseAttribute(binding, i32Type, bindingName,
                            state.attributes) ||
      parser.parseRParen()) {
    return failure();
  }
} else if (succeeded(parser.parseOptionalKeyword(builtInName))) {
  StringAttr builtIn;
  if (parser.parseLParen() ||
      parser.parseAttribute(builtIn, builtInName, state.attributes) ||
      parser.parseRParen()) {
    return failure();
  }
}

// Parse other attributes
if (parser.parseOptionalAttrDict(state.attributes))
  return failure();

return success();
646}

648static void printVariableDecorations(Operation *op, OpAsmPrinter &printer,
                                   SmallVectorImpl<StringRef> &elidedAttrs) {
// Print optional descriptor binding
auto descriptorSetName = llvm::convertToSnakeFromCamelCase(
    stringifyDecoration(spirv::Decoration::DescriptorSet));
auto bindingName = llvm::convertToSnakeFromCamelCase(
    stringifyDecoration(spirv::Decoration::Binding));
auto descriptorSet = op->getAttrOfType<IntegerAttr>(descriptorSetName);
auto binding = op->getAttrOfType<IntegerAttr>(bindingName);
if (descriptorSet && binding) {
  elidedAttrs.push_back(descriptorSetName);
  elidedAttrs.push_back(bindingName);
  printer << " bind(" << descriptorSet.getInt() << ", " << binding.getInt()
          << ")";
}

// Print BuiltIn attribute if present
auto builtInName = llvm::convertToSnakeFromCamelCase(
    stringifyDecoration(spirv::Decoration::BuiltIn));
if (auto builtin = op->getAttrOfType<StringAttr>(builtInName)) {
  printer << " " << builtInName << "(\"" << builtin.getValue() << "\")";
  elidedAttrs.push_back(builtInName);
}

printer.printOptionalAttrDict(op->getAttrs(), elidedAttrs);
673}

675// Get bit width of types.
676static unsigned getBitWidth(Type type) {
if (type.isa<spirv::PointerType>()) {
  // Just return 64 bits for pointer types for now.
  // TODO: Make sure not caller relies on the actual pointer width value.
  return 64;
}

if (type.isIntOrFloat())
  return type.getIntOrFloatBitWidth();

if (auto vectorType = type.dyn_cast<VectorType>()) {
  assert(vectorType.getElementType().isIntOrFloat())(static_cast <bool> (vectorType.getElementType().isIntOrFloat
()) ? void (0) : __assert_fail ("vectorType.getElementType().isIntOrFloat()"
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 687, __extension__
 __PRETTY_FUNCTION__));
  return vectorType.getNumElements() *
         vectorType.getElementType().getIntOrFloatBitWidth();
}
llvm_unreachable("unhandled bit width computation for type")::llvm::llvm_unreachable_internal("unhandled bit width computation for type"
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 691);
692}

694/// Walks the given type hierarchy with the given indices, potentially down
695/// to component granularity, to select an element type. Returns null type and
696/// emits errors with the given loc on failure.
697static Type
698getElementType(Type type, ArrayRef<int32_t> indices,
             function_ref<InFlightDiagnostic(StringRef)> emitErrorFn) {
if (indices.empty()) {
  emitErrorFn("expected at least one index for spirv.CompositeExtract");
  return nullptr;
}

for (auto index : indices) {
  if (auto cType = type.dyn_cast<spirv::CompositeType>()) {
    if (cType.hasCompileTimeKnownNumElements() &&
        (index < 0 ||
         static_cast<uint64_t>(index) >= cType.getNumElements())) {
      emitErrorFn("index ") << index << " out of bounds for " << type;
      return nullptr;
    }
    type = cType.getElementType(index);
  } else {
    emitErrorFn("cannot extract from non-composite type ")
        << type << " with index " << index;
    return nullptr;
  }
}
return type;
721}

723static Type
724getElementType(Type type, Attribute indices,
             function_ref<InFlightDiagnostic(StringRef)> emitErrorFn) {
auto indicesArrayAttr = indices.dyn_cast<ArrayAttr>();
if (!indicesArrayAttr) {
  emitErrorFn("expected a 32-bit integer array attribute for 'indices'");
  return nullptr;
}
if (indicesArrayAttr.empty()) {
  emitErrorFn("expected at least one index for spirv.CompositeExtract");
  return nullptr;
}

SmallVector<int32_t, 2> indexVals;
for (auto indexAttr : indicesArrayAttr) {
  auto indexIntAttr = indexAttr.dyn_cast<IntegerAttr>();
  if (!indexIntAttr) {
    emitErrorFn("expected an 32-bit integer for index, but found '")
        << indexAttr << "'";
    return nullptr;
  }
  indexVals.push_back(indexIntAttr.getInt());
}
return getElementType(type, indexVals, emitErrorFn);
747}

749static Type getElementType(Type type, Attribute indices, Location loc) {
auto errorFn = [&](StringRef err) -> InFlightDiagnostic {
  return ::mlir::emitError(loc, err);
};
return getElementType(type, indices, errorFn);
754}

756static Type getElementType(Type type, Attribute indices, OpAsmParser &parser,
                         SMLoc loc) {
auto errorFn = [&](StringRef err) -> InFlightDiagnostic {
  return parser.emitError(loc, err);
};
return getElementType(type, indices, errorFn);
762}

764/// Returns true if the given `block` only contains one `spirv.mlir.merge` op.
765static inline bool isMergeBlock(Block &block) {
return !block.empty() && std::next(block.begin()) == block.end() &&
       isa<spirv::MergeOp>(block.front());
768}

770template <typename ExtendedBinaryOp>
771static LogicalResult verifyArithmeticExtendedBinaryOp(ExtendedBinaryOp op) {
auto resultType = op.getType().template cast<spirv::StructType>();
if (resultType.getNumElements() != 2)
  return op.emitOpError("expected result struct type containing two members");

if (!llvm::all_equal({op.getOperand1().getType(), op.getOperand2().getType(),
                      resultType.getElementType(0),
                      resultType.getElementType(1)}))
  return op.emitOpError(
      "expected all operand types and struct member types are the same");

return success();
783}

785static ParseResult parseArithmeticExtendedBinaryOp(OpAsmParser &parser,
                                                 OperationState &result) {
SmallVector<OpAsmParser::UnresolvedOperand, 2> operands;
if (parser.parseOptionalAttrDict(result.attributes) ||
    parser.parseOperandList(operands) || parser.parseColon())
  return failure();

Type resultType;
SMLoc loc = parser.getCurrentLocation();
if (parser.parseType(resultType))
  return failure();

auto structType = resultType.dyn_cast<spirv::StructType>();
if (!structType || structType.getNumElements() != 2)
  return parser.emitError(loc, "expected spirv.struct type with two members");

SmallVector<Type, 2> operandTypes(2, structType.getElementType(0));
if (parser.resolveOperands(operands, operandTypes, loc, result.operands))
  return failure();

result.addTypes(resultType);
return success();
807}

809static void printArithmeticExtendedBinaryOp(Operation *op,
                                          OpAsmPrinter &printer) {
printer << ' ';
printer.printOptionalAttrDict(op->getAttrs());
printer.printOperands(op->getOperands());
printer << " : " << op->getResultTypes().front();
815}

817//===----------------------------------------------------------------------===//
818// Common parsers and printers
819//===----------------------------------------------------------------------===//

821// Parses an atomic update op. If the update op does not take a value (like
822// AtomicIIncrement) `hasValue` must be false.
823static ParseResult parseAtomicUpdateOp(OpAsmParser &parser,
                                     OperationState &state, bool hasValue) {
spirv::Scope scope;
spirv::MemorySemantics memoryScope;
SmallVector<OpAsmParser::UnresolvedOperand, 2> operandInfo;
OpAsmParser::UnresolvedOperand ptrInfo, valueInfo;
Type type;
SMLoc loc;
if (parseEnumStrAttr<spirv::ScopeAttr>(scope, parser, state,
                                       kMemoryScopeAttrName) ||
    parseEnumStrAttr<spirv::MemorySemanticsAttr>(memoryScope, parser, state,
                                                 kSemanticsAttrName) ||
    parser.parseOperandList(operandInfo, (hasValue ? 2 : 1)) ||
    parser.getCurrentLocation(&loc) || parser.parseColonType(type))
  return failure();

auto ptrType = type.dyn_cast<spirv::PointerType>();
if (!ptrType)
  return parser.emitError(loc, "expected pointer type");

SmallVector<Type, 2> operandTypes;
operandTypes.push_back(ptrType);
if (hasValue)
  operandTypes.push_back(ptrType.getPointeeType());
if (parser.resolveOperands(operandInfo, operandTypes, parser.getNameLoc(),
                           state.operands))
  return failure();
return parser.addTypeToList(ptrType.getPointeeType(), state.types);
851}

853// Prints an atomic update op.
854static void printAtomicUpdateOp(Operation *op, OpAsmPrinter &printer) {
printer << " \"";
auto scopeAttr = op->getAttrOfType<spirv::ScopeAttr>(kMemoryScopeAttrName);
printer << spirv::stringifyScope(scopeAttr.getValue()) << "\" \"";
auto memorySemanticsAttr =
    op->getAttrOfType<spirv::MemorySemanticsAttr>(kSemanticsAttrName);
printer << spirv::stringifyMemorySemantics(memorySemanticsAttr.getValue())
        << "\" " << op->getOperands() << " : " << op->getOperand(0).getType();
862}

864template <typename T>
865static StringRef stringifyTypeName();

867template <>
868StringRef stringifyTypeName<IntegerType>() {
return "integer";
870}

872template <>
873StringRef stringifyTypeName<FloatType>() {
return "float";
875}

877// Verifies an atomic update op.
878template <typename ExpectedElementType>
879static LogicalResult verifyAtomicUpdateOp(Operation *op) {
auto ptrType = op->getOperand(0).getType().cast<spirv::PointerType>();
auto elementType = ptrType.getPointeeType();
if (!elementType.isa<ExpectedElementType>())
  return op->emitOpError() << "pointer operand must point to an "
                           << stringifyTypeName<ExpectedElementType>()
                           << " value, found " << elementType;

if (op->getNumOperands() > 1) {
  auto valueType = op->getOperand(1).getType();
  if (valueType != elementType)
    return op->emitOpError("expected value to have the same type as the "
                           "pointer operand's pointee type ")
           << elementType << ", but found " << valueType;
}
auto memorySemantics =
    op->getAttrOfType<spirv::MemorySemanticsAttr>(kSemanticsAttrName)
        .getValue();
if (failed(verifyMemorySemantics(op, memorySemantics))) {
  return failure();
}
return success();
901}

903static ParseResult parseGroupNonUniformArithmeticOp(OpAsmParser &parser,
                                                  OperationState &state) {
spirv::Scope executionScope;
spirv::GroupOperation groupOperation;
OpAsmParser::UnresolvedOperand valueInfo;
if (parseEnumStrAttr<spirv::ScopeAttr>(executionScope, parser, state,
                                       kExecutionScopeAttrName) ||
    parseEnumStrAttr<spirv::GroupOperationAttr>(groupOperation, parser, state,
                                                kGroupOperationAttrName) ||
    parser.parseOperand(valueInfo))
  return failure();

std::optional<OpAsmParser::UnresolvedOperand> clusterSizeInfo;
if (succeeded(parser.parseOptionalKeyword(kClusterSize))) {
  clusterSizeInfo = OpAsmParser::UnresolvedOperand();
  if (parser.parseLParen() || parser.parseOperand(*clusterSizeInfo) ||
      parser.parseRParen())
    return failure();
}

Type resultType;
if (parser.parseColonType(resultType))
  return failure();

if (parser.resolveOperand(valueInfo, resultType, state.operands))
  return failure();

if (clusterSizeInfo) {
  Type i32Type = parser.getBuilder().getIntegerType(32);
  if (parser.resolveOperand(*clusterSizeInfo, i32Type, state.operands))
    return failure();
}

return parser.addTypeToList(resultType, state.types);
937}

939static void printGroupNonUniformArithmeticOp(Operation *groupOp,
                                           OpAsmPrinter &printer) {
printer
    << " \""
    << stringifyScope(
           groupOp->getAttrOfType<spirv::ScopeAttr>(kExecutionScopeAttrName)
               .getValue())
    << "\" \""
    << stringifyGroupOperation(groupOp
                                   ->getAttrOfType<spirv::GroupOperationAttr>(
                                       kGroupOperationAttrName)
                                   .getValue())
    << "\" " << groupOp->getOperand(0);

if (groupOp->getNumOperands() > 1)
  printer << " " << kClusterSize << '(' << groupOp->getOperand(1) << ')';
printer << " : " << groupOp->getResult(0).getType();
956}

958static LogicalResult verifyGroupNonUniformArithmeticOp(Operation *groupOp) {
spirv::Scope scope =
    groupOp->getAttrOfType<spirv::ScopeAttr>(kExecutionScopeAttrName)
        .getValue();
if (scope != spirv::Scope::Workgroup && scope != spirv::Scope::Subgroup)
  return groupOp->emitOpError(
      "execution scope must be 'Workgroup' or 'Subgroup'");

spirv::GroupOperation operation =
    groupOp->getAttrOfType<spirv::GroupOperationAttr>(kGroupOperationAttrName)
        .getValue();
if (operation == spirv::GroupOperation::ClusteredReduce &&
    groupOp->getNumOperands() == 1)
  return groupOp->emitOpError("cluster size operand must be provided for "
                              "'ClusteredReduce' group operation");
if (groupOp->getNumOperands() > 1) {
  Operation *sizeOp = groupOp->getOperand(1).getDefiningOp();
  int32_t clusterSize = 0;

  // TODO: support specialization constant here.
  if (failed(extractValueFromConstOp(sizeOp, clusterSize)))
    return groupOp->emitOpError(
        "cluster size operand must come from a constant op");

  if (!llvm::isPowerOf2_32(clusterSize))
    return groupOp->emitOpError(
        "cluster size operand must be a power of two");
}
return success();
987}

989/// Result of a logical op must be a scalar or vector of boolean type.
990static Type getUnaryOpResultType(Type operandType) {
Builder builder(operandType.getContext());
Type resultType = builder.getIntegerType(1);
if (auto vecType = operandType.dyn_cast<VectorType>())
  return VectorType::get(vecType.getNumElements(), resultType);
return resultType;
996}

998static LogicalResult verifyShiftOp(Operation *op) {
if (op->getOperand(0).getType() != op->getResult(0).getType()) {
  return op->emitError("expected the same type for the first operand and "
                       "result, but provided ")
         << op->getOperand(0).getType() << " and "
         << op->getResult(0).getType();
}
return success();
1006}

1008//===----------------------------------------------------------------------===//
1009// spirv.AccessChainOp
1010//===----------------------------------------------------------------------===//

1012static Type getElementPtrType(Type type, ValueRange indices, Location baseLoc) {
auto ptrType = type.dyn_cast<spirv::PointerType>();
if (!ptrType) {
  emitError(baseLoc, "'spirv.AccessChain' op expected a pointer "
                     "to composite type, but provided ")
      << type;
  return nullptr;
}

auto resultType = ptrType.getPointeeType();
auto resultStorageClass = ptrType.getStorageClass();
int32_t index = 0;

for (auto indexSSA : indices) {
  auto cType = resultType.dyn_cast<spirv::CompositeType>();
  if (!cType) {
    emitError(
        baseLoc,
        "'spirv.AccessChain' op cannot extract from non-composite type ")
        << resultType << " with index " << index;
    return nullptr;
  }
  index = 0;
  if (resultType.isa<spirv::StructType>()) {
    Operation *op = indexSSA.getDefiningOp();
    if (!op) {
      emitError(baseLoc, "'spirv.AccessChain' op index must be an "
                         "integer spirv.Constant to access "
                         "element of spirv.struct");
      return nullptr;
    }

    // TODO: this should be relaxed to allow
    // integer literals of other bitwidths.
    if (failed(extractValueFromConstOp(op, index))) {
      emitError(
          baseLoc,
          "'spirv.AccessChain' index must be an integer spirv.Constant to "
          "access element of spirv.struct, but provided ")
          << op->getName();
      return nullptr;
    }
    if (index < 0 || static_cast<uint64_t>(index) >= cType.getNumElements()) {
      emitError(baseLoc, "'spirv.AccessChain' op index ")
          << index << " out of bounds for " << resultType;
      return nullptr;
    }
  }
  resultType = cType.getElementType(index);
}
return spirv::PointerType::get(resultType, resultStorageClass);
1063}

1065void spirv::AccessChainOp::build(OpBuilder &builder, OperationState &state,
                               Value basePtr, ValueRange indices) {
auto type = getElementPtrType(basePtr.getType(), indices, state.location);
assert(type && "Unable to deduce return type based on basePtr and indices")(static_cast <bool> (type && "Unable to deduce return type based on basePtr and indices"
) ? void (0) : __assert_fail ("type && \"Unable to deduce return type based on basePtr and indices\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 1068, __extension__
 __PRETTY_FUNCTION__));
build(builder, state, type, basePtr, indices);
1070}

1072ParseResult spirv::AccessChainOp::parse(OpAsmParser &parser,
                                      OperationState &result) {
OpAsmParser::UnresolvedOperand ptrInfo;
SmallVector<OpAsmParser::UnresolvedOperand, 4> indicesInfo;
Type type;
auto loc = parser.getCurrentLocation();
SmallVector<Type, 4> indicesTypes;

if (parser.parseOperand(ptrInfo) ||
    parser.parseOperandList(indicesInfo, OpAsmParser::Delimiter::Square) ||
    parser.parseColonType(type) ||
    parser.resolveOperand(ptrInfo, type, result.operands)) {
  return failure();
}

// Check that the provided indices list is not empty before parsing their
// type list.
if (indicesInfo.empty()) {
  return mlir::emitError(result.location,
                         "'spirv.AccessChain' op expected at "
                         "least one index ");
}

if (parser.parseComma() || parser.parseTypeList(indicesTypes))
  return failure();

// Check that the indices types list is not empty and that it has a one-to-one
// mapping to the provided indices.
if (indicesTypes.size() != indicesInfo.size()) {
  return mlir::emitError(
      result.location, "'spirv.AccessChain' op indices types' count must be "
                       "equal to indices info count");
}

if (parser.resolveOperands(indicesInfo, indicesTypes, loc, result.operands))
  return failure();

auto resultType = getElementPtrType(
    type, llvm::ArrayRef(result.operands).drop_front(), result.location);
if (!resultType) {
  return failure();
}

result.addTypes(resultType);
return success();
1117}

1119template <typename Op>
1120static void printAccessChain(Op op, ValueRange indices, OpAsmPrinter &printer) {
printer << ' ' << op.getBasePtr() << '[' << indices
        << "] : " << op.getBasePtr().getType() << ", " << indices.getTypes();
1123}

1125void spirv::AccessChainOp::print(OpAsmPrinter &printer) {
printAccessChain(*this, getIndices(), printer);
1127}

1129template <typename Op>
1130static LogicalResult verifyAccessChain(Op accessChainOp, ValueRange indices) {
auto resultType = getElementPtrType(accessChainOp.getBasePtr().getType(),
                                    indices, accessChainOp.getLoc());
if (!resultType)
  return failure();

auto providedResultType =
    accessChainOp.getType().template dyn_cast<spirv::PointerType>();
if (!providedResultType)
  return accessChainOp.emitOpError(
             "result type must be a pointer, but provided")
         << providedResultType;

if (resultType != providedResultType)
  return accessChainOp.emitOpError("invalid result type: expected ")
         << resultType << ", but provided " << providedResultType;

return success();
1148}

1150LogicalResult spirv::AccessChainOp::verify() {
return verifyAccessChain(*this, getIndices());
1152}

1154//===----------------------------------------------------------------------===//
1155// spirv.mlir.addressof
1156//===----------------------------------------------------------------------===//

1158void spirv::AddressOfOp::build(OpBuilder &builder, OperationState &state,
                             spirv::GlobalVariableOp var) {
build(builder, state, var.getType(), SymbolRefAttr::get(var));
1161}

1163LogicalResult spirv::AddressOfOp::verify() {
auto varOp = dyn_cast_or_null<spirv::GlobalVariableOp>(
    SymbolTable::lookupNearestSymbolFrom((*this)->getParentOp(),
                                         getVariableAttr()));
if (!varOp) {
  return emitOpError("expected spirv.GlobalVariable symbol");
}
if (getPointer().getType() != varOp.getType()) {
  return emitOpError(
      "result type mismatch with the referenced global variable's type");
}
return success();
1175}

1177template <typename T>
1178static void printAtomicCompareExchangeImpl(T atomOp, OpAsmPrinter &printer) {
printer << " \"" << stringifyScope(atomOp.getMemoryScope()) << "\" \""
        << stringifyMemorySemantics(atomOp.getEqualSemantics()) << "\" \""
        << stringifyMemorySemantics(atomOp.getUnequalSemantics()) << "\" "
        << atomOp.getOperands() << " : " << atomOp.getPointer().getType();
1183}

1185static ParseResult parseAtomicCompareExchangeImpl(OpAsmParser &parser,
                                                OperationState &state) {
spirv::Scope memoryScope;
spirv::MemorySemantics equalSemantics, unequalSemantics;
SmallVector<OpAsmParser::UnresolvedOperand, 3> operandInfo;
Type type;
if (parseEnumStrAttr<spirv::ScopeAttr>(memoryScope, parser, state,
                                       kMemoryScopeAttrName) ||
    parseEnumStrAttr<spirv::MemorySemanticsAttr>(
        equalSemantics, parser, state, kEqualSemanticsAttrName) ||
    parseEnumStrAttr<spirv::MemorySemanticsAttr>(
        unequalSemantics, parser, state, kUnequalSemanticsAttrName) ||
    parser.parseOperandList(operandInfo, 3))
  return failure();

auto loc = parser.getCurrentLocation();
if (parser.parseColonType(type))
  return failure();

auto ptrType = type.dyn_cast<spirv::PointerType>();
if (!ptrType)
  return parser.emitError(loc, "expected pointer type");

if (parser.resolveOperands(
        operandInfo,
        {ptrType, ptrType.getPointeeType(), ptrType.getPointeeType()},
        parser.getNameLoc(), state.operands))
  return failure();

return parser.addTypeToList(ptrType.getPointeeType(), state.types);
1215}

1217template <typename T>
1218static LogicalResult verifyAtomicCompareExchangeImpl(T atomOp) {
// According to the spec:
// "The type of Value must be the same as Result Type. The type of the value
// pointed to by Pointer must be the same as Result Type. This type must also
// match the type of Comparator."
if (atomOp.getType() != atomOp.getValue().getType())
  return atomOp.emitOpError("value operand must have the same type as the op "
                            "result, but found ")
         << atomOp.getValue().getType() << " vs " << atomOp.getType();

if (atomOp.getType() != atomOp.getComparator().getType())
  return atomOp.emitOpError(
             "comparator operand must have the same type as the op "
             "result, but found ")
         << atomOp.getComparator().getType() << " vs " << atomOp.getType();

Type pointeeType = atomOp.getPointer()
                       .getType()
                       .template cast<spirv::PointerType>()
                       .getPointeeType();
if (atomOp.getType() != pointeeType)
  return atomOp.emitOpError(
             "pointer operand's pointee type must have the same "
             "as the op result type, but found ")
         << pointeeType << " vs " << atomOp.getType();

// TODO: Unequal cannot be set to Release or Acquire and Release.
// In addition, Unequal cannot be set to a stronger memory-order then Equal.

return success();
1248}

1250//===----------------------------------------------------------------------===//
1251// spirv.AtomicAndOp
1252//===----------------------------------------------------------------------===//

1254LogicalResult spirv::AtomicAndOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1256}

1258ParseResult spirv::AtomicAndOp::parse(OpAsmParser &parser,
                                    OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1261}
1262void spirv::AtomicAndOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1264}

1266//===----------------------------------------------------------------------===//
1267// spirv.AtomicCompareExchangeOp
1268//===----------------------------------------------------------------------===//

1270LogicalResult spirv::AtomicCompareExchangeOp::verify() {
return ::verifyAtomicCompareExchangeImpl(*this);
1272}

1274ParseResult spirv::AtomicCompareExchangeOp::parse(OpAsmParser &parser,
                                                OperationState &result) {
return ::parseAtomicCompareExchangeImpl(parser, result);
1277}
1278void spirv::AtomicCompareExchangeOp::print(OpAsmPrinter &p) {
::printAtomicCompareExchangeImpl(*this, p);
1280}

1282//===----------------------------------------------------------------------===//
1283// spirv.AtomicCompareExchangeWeakOp
1284//===----------------------------------------------------------------------===//

1286LogicalResult spirv::AtomicCompareExchangeWeakOp::verify() {
return ::verifyAtomicCompareExchangeImpl(*this);
1288}

1290ParseResult spirv::AtomicCompareExchangeWeakOp::parse(OpAsmParser &parser,
                                                    OperationState &result) {
return ::parseAtomicCompareExchangeImpl(parser, result);
1293}
1294void spirv::AtomicCompareExchangeWeakOp::print(OpAsmPrinter &p) {
::printAtomicCompareExchangeImpl(*this, p);
1296}

1298//===----------------------------------------------------------------------===//
1299// spirv.AtomicExchange
1300//===----------------------------------------------------------------------===//

1302void spirv::AtomicExchangeOp::print(OpAsmPrinter &printer) {
printer << " \"" << stringifyScope(getMemoryScope()) << "\" \""
        << stringifyMemorySemantics(getSemantics()) << "\" " << getOperands()
        << " : " << getPointer().getType();
1306}

1308ParseResult spirv::AtomicExchangeOp::parse(OpAsmParser &parser,
                                         OperationState &result) {
spirv::Scope memoryScope;
spirv::MemorySemantics semantics;
SmallVector<OpAsmParser::UnresolvedOperand, 2> operandInfo;
Type type;
if (parseEnumStrAttr<spirv::ScopeAttr>(memoryScope, parser, result,
                                       kMemoryScopeAttrName) ||
    parseEnumStrAttr<spirv::MemorySemanticsAttr>(semantics, parser, result,
                                                 kSemanticsAttrName) ||
    parser.parseOperandList(operandInfo, 2))
  return failure();

auto loc = parser.getCurrentLocation();
if (parser.parseColonType(type))
  return failure();

auto ptrType = type.dyn_cast<spirv::PointerType>();
if (!ptrType)
  return parser.emitError(loc, "expected pointer type");

if (parser.resolveOperands(operandInfo, {ptrType, ptrType.getPointeeType()},
                           parser.getNameLoc(), result.operands))
  return failure();

return parser.addTypeToList(ptrType.getPointeeType(), result.types);
1334}

1336LogicalResult spirv::AtomicExchangeOp::verify() {
if (getType() != getValue().getType())
  return emitOpError("value operand must have the same type as the op "
                     "result, but found ")
         << getValue().getType() << " vs " << getType();

Type pointeeType =
    getPointer().getType().cast<spirv::PointerType>().getPointeeType();
if (getType() != pointeeType)
  return emitOpError("pointer operand's pointee type must have the same "
                     "as the op result type, but found ")
         << pointeeType << " vs " << getType();

return success();
1350}

1352//===----------------------------------------------------------------------===//
1353// spirv.AtomicIAddOp
1354//===----------------------------------------------------------------------===//

1356LogicalResult spirv::AtomicIAddOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1358}

1360ParseResult spirv::AtomicIAddOp::parse(OpAsmParser &parser,
                                     OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1363}
1364void spirv::AtomicIAddOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1366}

1368//===----------------------------------------------------------------------===//
1369// spirv.EXT.AtomicFAddOp
1370//===----------------------------------------------------------------------===//

1372LogicalResult spirv::EXTAtomicFAddOp::verify() {
return ::verifyAtomicUpdateOp<FloatType>(getOperation());
1374}

1376ParseResult spirv::EXTAtomicFAddOp::parse(OpAsmParser &parser,
                                        OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1379}
1380void spirv::EXTAtomicFAddOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1382}

1384//===----------------------------------------------------------------------===//
1385// spirv.AtomicIDecrementOp
1386//===----------------------------------------------------------------------===//

1388LogicalResult spirv::AtomicIDecrementOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1390}

1392ParseResult spirv::AtomicIDecrementOp::parse(OpAsmParser &parser,
                                           OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, false);
1395}
1396void spirv::AtomicIDecrementOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1398}

1400//===----------------------------------------------------------------------===//
1401// spirv.AtomicIIncrementOp
1402//===----------------------------------------------------------------------===//

1404LogicalResult spirv::AtomicIIncrementOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1406}

1408ParseResult spirv::AtomicIIncrementOp::parse(OpAsmParser &parser,
                                           OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, false);
1411}
1412void spirv::AtomicIIncrementOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1414}

1416//===----------------------------------------------------------------------===//
1417// spirv.AtomicISubOp
1418//===----------------------------------------------------------------------===//

1420LogicalResult spirv::AtomicISubOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1422}

1424ParseResult spirv::AtomicISubOp::parse(OpAsmParser &parser,
                                     OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1427}
1428void spirv::AtomicISubOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1430}

1432//===----------------------------------------------------------------------===//
1433// spirv.AtomicOrOp
1434//===----------------------------------------------------------------------===//

1436LogicalResult spirv::AtomicOrOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1438}

1440ParseResult spirv::AtomicOrOp::parse(OpAsmParser &parser,
                                   OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1443}
1444void spirv::AtomicOrOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1446}

1448//===----------------------------------------------------------------------===//
1449// spirv.AtomicSMaxOp
1450//===----------------------------------------------------------------------===//

1452LogicalResult spirv::AtomicSMaxOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1454}

1456ParseResult spirv::AtomicSMaxOp::parse(OpAsmParser &parser,
                                     OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1459}
1460void spirv::AtomicSMaxOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1462}

1464//===----------------------------------------------------------------------===//
1465// spirv.AtomicSMinOp
1466//===----------------------------------------------------------------------===//

1468LogicalResult spirv::AtomicSMinOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1470}

1472ParseResult spirv::AtomicSMinOp::parse(OpAsmParser &parser,
                                     OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1475}
1476void spirv::AtomicSMinOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1478}

1480//===----------------------------------------------------------------------===//
1481// spirv.AtomicUMaxOp
1482//===----------------------------------------------------------------------===//

1484LogicalResult spirv::AtomicUMaxOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1486}

1488ParseResult spirv::AtomicUMaxOp::parse(OpAsmParser &parser,
                                     OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1491}
1492void spirv::AtomicUMaxOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1494}

1496//===----------------------------------------------------------------------===//
1497// spirv.AtomicUMinOp
1498//===----------------------------------------------------------------------===//

1500LogicalResult spirv::AtomicUMinOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1502}

1504ParseResult spirv::AtomicUMinOp::parse(OpAsmParser &parser,
                                     OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1507}
1508void spirv::AtomicUMinOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1510}

1512//===----------------------------------------------------------------------===//
1513// spirv.AtomicXorOp
1514//===----------------------------------------------------------------------===//

1516LogicalResult spirv::AtomicXorOp::verify() {
return ::verifyAtomicUpdateOp<IntegerType>(getOperation());
1518}

1520ParseResult spirv::AtomicXorOp::parse(OpAsmParser &parser,
                                    OperationState &result) {
return ::parseAtomicUpdateOp(parser, result, true);
1523}
1524void spirv::AtomicXorOp::print(OpAsmPrinter &p) {
::printAtomicUpdateOp(*this, p);
1526}

1528//===----------------------------------------------------------------------===//
1529// spirv.BitcastOp
1530//===----------------------------------------------------------------------===//

1532LogicalResult spirv::BitcastOp::verify() {
// TODO: The SPIR-V spec validation rules are different for different
// versions.
auto operandType = getOperand().getType();
auto resultType = getResult().getType();
if (operandType == resultType) {
  return emitError("result type must be different from operand type");
}
if (operandType.isa<spirv::PointerType>() &&
    !resultType.isa<spirv::PointerType>()) {
  return emitError(
      "unhandled bit cast conversion from pointer type to non-pointer type");
}
if (!operandType.isa<spirv::PointerType>() &&
    resultType.isa<spirv::PointerType>()) {
  return emitError(
      "unhandled bit cast conversion from non-pointer type to pointer type");
}
auto operandBitWidth = getBitWidth(operandType);
auto resultBitWidth = getBitWidth(resultType);
if (operandBitWidth != resultBitWidth) {
  return emitOpError("mismatch in result type bitwidth ")
         << resultBitWidth << " and operand type bitwidth "
         << operandBitWidth;
}
return success();
1558}

1560//===----------------------------------------------------------------------===//
1561// spirv.PtrCastToGenericOp
1562//===----------------------------------------------------------------------===//

1564LogicalResult spirv::PtrCastToGenericOp::verify() {
auto operandType = getPointer().getType().cast<spirv::PointerType>();
auto resultType = getResult().getType().cast<spirv::PointerType>();

spirv::StorageClass operandStorage = operandType.getStorageClass();
if (operandStorage != spirv::StorageClass::Workgroup &&
    operandStorage != spirv::StorageClass::CrossWorkgroup &&
    operandStorage != spirv::StorageClass::Function)
  return emitError("pointer must point to the Workgroup, CrossWorkgroup"
                   ", or Function Storage Class");

spirv::StorageClass resultStorage = resultType.getStorageClass();
if (resultStorage != spirv::StorageClass::Generic)
  return emitError("result type must be of storage class Generic");

Type operandPointeeType = operandType.getPointeeType();
Type resultPointeeType = resultType.getPointeeType();
if (operandPointeeType != resultPointeeType)
  return emitOpError("pointer operand's pointee type must have the same "
                     "as the op result type, but found ")
         << operandPointeeType << " vs " << resultPointeeType;
return success();
1586}

1588//===----------------------------------------------------------------------===//
1589// spirv.GenericCastToPtrOp
1590//===----------------------------------------------------------------------===//

1592LogicalResult spirv::GenericCastToPtrOp::verify() {
auto operandType = getPointer().getType().cast<spirv::PointerType>();
auto resultType = getResult().getType().cast<spirv::PointerType>();

spirv::StorageClass operandStorage = operandType.getStorageClass();
if (operandStorage != spirv::StorageClass::Generic)
  return emitError("pointer type must be of storage class Generic");

spirv::StorageClass resultStorage = resultType.getStorageClass();
if (resultStorage != spirv::StorageClass::Workgroup &&
    resultStorage != spirv::StorageClass::CrossWorkgroup &&
    resultStorage != spirv::StorageClass::Function)
  return emitError("result must point to the Workgroup, CrossWorkgroup, "
                   "or Function Storage Class");

Type operandPointeeType = operandType.getPointeeType();
Type resultPointeeType = resultType.getPointeeType();
if (operandPointeeType != resultPointeeType)
  return emitOpError("pointer operand's pointee type must have the same "
                     "as the op result type, but found ")
         << operandPointeeType << " vs " << resultPointeeType;
return success();
1614}

1616//===----------------------------------------------------------------------===//
1617// spirv.GenericCastToPtrExplicitOp
1618//===----------------------------------------------------------------------===//

1620LogicalResult spirv::GenericCastToPtrExplicitOp::verify() {
auto operandType = getPointer().getType().cast<spirv::PointerType>();
auto resultType = getResult().getType().cast<spirv::PointerType>();

spirv::StorageClass operandStorage = operandType.getStorageClass();
if (operandStorage != spirv::StorageClass::Generic)
  return emitError("pointer type must be of storage class Generic");

spirv::StorageClass resultStorage = resultType.getStorageClass();
if (resultStorage != spirv::StorageClass::Workgroup &&
    resultStorage != spirv::StorageClass::CrossWorkgroup &&
    resultStorage != spirv::StorageClass::Function)
  return emitError("result must point to the Workgroup, CrossWorkgroup, "
                   "or Function Storage Class");

Type operandPointeeType = operandType.getPointeeType();
Type resultPointeeType = resultType.getPointeeType();
if (operandPointeeType != resultPointeeType)
  return emitOpError("pointer operand's pointee type must have the same "
                     "as the op result type, but found ")
         << operandPointeeType << " vs " << resultPointeeType;
return success();
1642}

1644//===----------------------------------------------------------------------===//
1645// spirv.BranchOp
1646//===----------------------------------------------------------------------===//

1648SuccessorOperands spirv::BranchOp::getSuccessorOperands(unsigned index) {
assert(index == 0 && "invalid successor index")(static_cast <bool> (index == 0 && "invalid successor index"
) ? void (0) : __assert_fail ("index == 0 && \"invalid successor index\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 1649, __extension__
 __PRETTY_FUNCTION__));
return SuccessorOperands(0, getTargetOperandsMutable());
1651}

1653//===----------------------------------------------------------------------===//
1654// spirv.BranchConditionalOp
1655//===----------------------------------------------------------------------===//

1657SuccessorOperands
1658spirv::BranchConditionalOp::getSuccessorOperands(unsigned index) {
assert(index < 2 && "invalid successor index")(static_cast <bool> (index < 2 && "invalid successor index"
) ? void (0) : __assert_fail ("index < 2 && \"invalid successor index\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 1659, __extension__
 __PRETTY_FUNCTION__));
return SuccessorOperands(index == kTrueIndex
                             ? getTrueTargetOperandsMutable()
                             : getFalseTargetOperandsMutable());
1663}

1665ParseResult spirv::BranchConditionalOp::parse(OpAsmParser &parser,
                                            OperationState &result) {
auto &builder = parser.getBuilder();
OpAsmParser::UnresolvedOperand condInfo;
Block *dest;

// Parse the condition.
Type boolTy = builder.getI1Type();
if (parser.parseOperand(condInfo) ||
    parser.resolveOperand(condInfo, boolTy, result.operands))
  return failure();

// Parse the optional branch weights.
if (succeeded(parser.parseOptionalLSquare())) {
  IntegerAttr trueWeight, falseWeight;
  NamedAttrList weights;

  auto i32Type = builder.getIntegerType(32);
  if (parser.parseAttribute(trueWeight, i32Type, "weight", weights) ||
      parser.parseComma() ||
      parser.parseAttribute(falseWeight, i32Type, "weight", weights) ||
      parser.parseRSquare())
    return failure();

  result.addAttribute(kBranchWeightAttrName,
                      builder.getArrayAttr({trueWeight, falseWeight}));
}

// Parse the true branch.
SmallVector<Value, 4> trueOperands;
if (parser.parseComma() ||
    parser.parseSuccessorAndUseList(dest, trueOperands))
  return failure();
result.addSuccessors(dest);
result.addOperands(trueOperands);

// Parse the false branch.
SmallVector<Value, 4> falseOperands;
if (parser.parseComma() ||
    parser.parseSuccessorAndUseList(dest, falseOperands))
  return failure();
result.addSuccessors(dest);
result.addOperands(falseOperands);
result.addAttribute(spirv::BranchConditionalOp::getOperandSegmentSizeAttr(),
                    builder.getDenseI32ArrayAttr(
                        {1, static_cast<int32_t>(trueOperands.size()),
                         static_cast<int32_t>(falseOperands.size())}));

return success();
1714}

1716void spirv::BranchConditionalOp::print(OpAsmPrinter &printer) {
printer << ' ' << getCondition();

if (auto weights = getBranchWeights()) {
  printer << " [";
  llvm::interleaveComma(weights->getValue(), printer, [&](Attribute a) {
    printer << a.cast<IntegerAttr>().getInt();
  });
  printer << "]";
}

printer << ", ";
printer.printSuccessorAndUseList(getTrueBlock(), getTrueBlockArguments());
printer << ", ";
printer.printSuccessorAndUseList(getFalseBlock(), getFalseBlockArguments());
1731}

1733LogicalResult spirv::BranchConditionalOp::verify() {
if (auto weights = getBranchWeights()) {
  if (weights->getValue().size() != 2) {
    return emitOpError("must have exactly two branch weights");
  }
  if (llvm::all_of(*weights, [](Attribute attr) {
        return attr.cast<IntegerAttr>().getValue().isZero();
      }))
    return emitOpError("branch weights cannot both be zero");
}

return success();
1745}

1747//===----------------------------------------------------------------------===//
1748// spirv.CompositeConstruct
1749//===----------------------------------------------------------------------===//

1751LogicalResult spirv::CompositeConstructOp::verify() {
auto cType = getType().cast<spirv::CompositeType>();
operand_range constituents = this->getConstituents();

if (auto coopType = cType.dyn_cast<spirv::CooperativeMatrixNVType>()) {
  if (constituents.size() != 1)
    return emitOpError("has incorrect number of operands: expected ")
           << "1, but provided " << constituents.size();
  if (coopType.getElementType() != constituents.front().getType())
    return emitOpError("operand type mismatch: expected operand type ")
           << coopType.getElementType() << ", but provided "
           << constituents.front().getType();
  return success();
}

if (auto jointType = cType.dyn_cast<spirv::JointMatrixINTELType>()) {
  if (constituents.size() != 1)
    return emitOpError("has incorrect number of operands: expected ")
           << "1, but provided " << constituents.size();
  if (jointType.getElementType() != constituents.front().getType())
    return emitOpError("operand type mismatch: expected operand type ")
           << jointType.getElementType() << ", but provided "
           << constituents.front().getType();
  return success();
}

if (constituents.size() == cType.getNumElements()) {
  for (auto index : llvm::seq<uint32_t>(0, constituents.size())) {
    if (constituents[index].getType() != cType.getElementType(index)) {
      return emitOpError("operand type mismatch: expected operand type ")
             << cType.getElementType(index) << ", but provided "
             << constituents[index].getType();
    }
  }
  return success();
}

// If not constructing a cooperative matrix type, then we must be constructing
// a vector type.
auto resultType = cType.dyn_cast<VectorType>();
if (!resultType)
  return emitOpError(
      "expected to return a vector or cooperative matrix when the number of "
      "constituents is less than what the result needs");

SmallVector<unsigned> sizes;
for (Value component : constituents) {
  if (!component.getType().isa<VectorType>() &&
      !component.getType().isIntOrFloat())
    return emitOpError("operand type mismatch: expected operand to have "
                       "a scalar or vector type, but provided ")
           << component.getType();

  Type elementType = component.getType();
  if (auto vectorType = component.getType().dyn_cast<VectorType>()) {
    sizes.push_back(vectorType.getNumElements());
    elementType = vectorType.getElementType();
  } else {
    sizes.push_back(1);
  }

  if (elementType != resultType.getElementType())
    return emitOpError("operand element type mismatch: expected to be ")
           << resultType.getElementType() << ", but provided " << elementType;
}
unsigned totalCount = std::accumulate(sizes.begin(), sizes.end(), 0);
if (totalCount != cType.getNumElements())
  return emitOpError("has incorrect number of operands: expected ")
         << cType.getNumElements() << ", but provided " << totalCount;
return success();
1821}

1823//===----------------------------------------------------------------------===//
1824// spirv.CompositeExtractOp
1825//===----------------------------------------------------------------------===//

1827void spirv::CompositeExtractOp::build(OpBuilder &builder, OperationState &state,
                                    Value composite,
                                    ArrayRef<int32_t> indices) {
auto indexAttr = builder.getI32ArrayAttr(indices);
auto elementType =
    getElementType(composite.getType(), indexAttr, state.location);
if (!elementType) {
  return;
}
build(builder, state, elementType, composite, indexAttr);
1837}

1839ParseResult spirv::CompositeExtractOp::parse(OpAsmParser &parser,
                                           OperationState &result) {
OpAsmParser::UnresolvedOperand compositeInfo;
Attribute indicesAttr;
Type compositeType;
SMLoc attrLocation;

if (parser.parseOperand(compositeInfo) ||
    parser.getCurrentLocation(&attrLocation) ||
    parser.parseAttribute(indicesAttr, kIndicesAttrName, result.attributes) ||
    parser.parseColonType(compositeType) ||
    parser.resolveOperand(compositeInfo, compositeType, result.operands)) {
  return failure();
}

Type resultType =
    getElementType(compositeType, indicesAttr, parser, attrLocation);
if (!resultType) {
  return failure();
}
result.addTypes(resultType);
return success();
1861}

1863void spirv::CompositeExtractOp::print(OpAsmPrinter &printer) {
printer << ' ' << getComposite() << getIndices() << " : "
        << getComposite().getType();
1866}

1868LogicalResult spirv::CompositeExtractOp::verify() {
auto indicesArrayAttr = getIndices().dyn_cast<ArrayAttr>();
auto resultType =
    getElementType(getComposite().getType(), indicesArrayAttr, getLoc());
if (!resultType)
  return failure();

if (resultType != getType()) {
  return emitOpError("invalid result type: expected ")
         << resultType << " but provided " << getType();
}

return success();
1881}

1883//===----------------------------------------------------------------------===//
1884// spirv.CompositeInsert
1885//===----------------------------------------------------------------------===//

1887void spirv::CompositeInsertOp::build(OpBuilder &builder, OperationState &state,
                                   Value object, Value composite,
                                   ArrayRef<int32_t> indices) {
auto indexAttr = builder.getI32ArrayAttr(indices);
build(builder, state, composite.getType(), object, composite, indexAttr);
1892}

1894ParseResult spirv::CompositeInsertOp::parse(OpAsmParser &parser,
                                          OperationState &result) {
SmallVector<OpAsmParser::UnresolvedOperand, 2> operands;
Type objectType, compositeType;
Attribute indicesAttr;
auto loc = parser.getCurrentLocation();

return failure(
    parser.parseOperandList(operands, 2) ||
    parser.parseAttribute(indicesAttr, kIndicesAttrName, result.attributes) ||
    parser.parseColonType(objectType) ||
    parser.parseKeywordType("into", compositeType) ||
    parser.resolveOperands(operands, {objectType, compositeType}, loc,
                           result.operands) ||
    parser.addTypesToList(compositeType, result.types));
1909}

1911LogicalResult spirv::CompositeInsertOp::verify() {
auto indicesArrayAttr = getIndices().dyn_cast<ArrayAttr>();
auto objectType =
    getElementType(getComposite().getType(), indicesArrayAttr, getLoc());
if (!objectType)
  return failure();

if (objectType != getObject().getType()) {
  return emitOpError("object operand type should be ")
         << objectType << ", but found " << getObject().getType();
}

if (getComposite().getType() != getType()) {
  return emitOpError("result type should be the same as "
                     "the composite type, but found ")
         << getComposite().getType() << " vs " << getType();
}

return success();
1930}

1932void spirv::CompositeInsertOp::print(OpAsmPrinter &printer) {
printer << " " << getObject() << ", " << getComposite() << getIndices()
        << " : " << getObject().getType() << " into "
        << getComposite().getType();
1936}

1938//===----------------------------------------------------------------------===//
1939// spirv.Constant
1940//===----------------------------------------------------------------------===//

1942ParseResult spirv::ConstantOp::parse(OpAsmParser &parser,
                                   OperationState &result) {
Attribute value;
if (parser.parseAttribute(value, kValueAttrName, result.attributes))
  return failure();

Type type = NoneType::get(parser.getContext());
if (auto typedAttr = value.dyn_cast<TypedAttr>())
  type = typedAttr.getType();
if (type.isa<NoneType, TensorType>()) {
  if (parser.parseColonType(type))
    return failure();
}

return parser.addTypeToList(type, result.types);
1957}

1959void spirv::ConstantOp::print(OpAsmPrinter &printer) {
printer << ' ' << getValue();
if (getType().isa<spirv::ArrayType>())
  printer << " : " << getType();
1963}

1965static LogicalResult verifyConstantType(spirv::ConstantOp op, Attribute value,
                                      Type opType) {
if (value.isa<IntegerAttr, FloatAttr>()) {
  auto valueType = value.cast<TypedAttr>().getType();
  if (valueType != opType)
    return op.emitOpError("result type (")
           << opType << ") does not match value type (" << valueType << ")";
  return success();
}
if (value.isa<DenseIntOrFPElementsAttr, SparseElementsAttr>()) {
  auto valueType = value.cast<TypedAttr>().getType();
  if (valueType == opType)
    return success();
  auto arrayType = opType.dyn_cast<spirv::ArrayType>();
  auto shapedType = valueType.dyn_cast<ShapedType>();
  if (!arrayType)
    return op.emitOpError("result or element type (")
           << opType << ") does not match value type (" << valueType
           << "), must be the same or spirv.array";

  int numElements = arrayType.getNumElements();
  auto opElemType = arrayType.getElementType();
  while (auto t = opElemType.dyn_cast<spirv::ArrayType>()) {
    numElements *= t.getNumElements();
    opElemType = t.getElementType();
  }
  if (!opElemType.isIntOrFloat())
    return op.emitOpError("only support nested array result type");

  auto valueElemType = shapedType.getElementType();
  if (valueElemType != opElemType) {
    return op.emitOpError("result element type (")
           << opElemType << ") does not match value element type ("
           << valueElemType << ")";
  }

  if (numElements != shapedType.getNumElements()) {
    return op.emitOpError("result number of elements (")
           << numElements << ") does not match value number of elements ("
           << shapedType.getNumElements() << ")";
  }
  return success();
}
if (auto arrayAttr = value.dyn_cast<ArrayAttr>()) {
  auto arrayType = opType.dyn_cast<spirv::ArrayType>();
  if (!arrayType)
    return op.emitOpError(
        "must have spirv.array result type for array value");
  Type elemType = arrayType.getElementType();
  for (Attribute element : arrayAttr.getValue()) {
    // Verify array elements recursively.
    if (failed(verifyConstantType(op, element, elemType)))
      return failure();
  }
  return success();
}
return op.emitOpError("cannot have attribute: ") << value;
2022}

2024LogicalResult spirv::ConstantOp::verify() {
// ODS already generates checks to make sure the result type is valid. We just
// need to additionally check that the value's attribute type is consistent
// with the result type.
return verifyConstantType(*this, getValueAttr(), getType());
2029}

2031bool spirv::ConstantOp::isBuildableWith(Type type) {
// Must be valid SPIR-V type first.
if (!type.isa<spirv::SPIRVType>())
  return false;

if (isa<SPIRVDialect>(type.getDialect())) {
  // TODO: support constant struct
  return type.isa<spirv::ArrayType>();
}

return true;
2042}

2044spirv::ConstantOp spirv::ConstantOp::getZero(Type type, Location loc,
                                           OpBuilder &builder) {
if (auto intType = type.dyn_cast<IntegerType>()) {
  unsigned width = intType.getWidth();
  if (width == 1)
    return builder.create<spirv::ConstantOp>(loc, type,
                                             builder.getBoolAttr(false));
  return builder.create<spirv::ConstantOp>(
      loc, type, builder.getIntegerAttr(type, APInt(width, 0)));
}
if (auto floatType = type.dyn_cast<FloatType>()) {
  return builder.create<spirv::ConstantOp>(
      loc, type, builder.getFloatAttr(floatType, 0.0));
}
if (auto vectorType = type.dyn_cast<VectorType>()) {
  Type elemType = vectorType.getElementType();
  if (elemType.isa<IntegerType>()) {
    return builder.create<spirv::ConstantOp>(
        loc, type,
        DenseElementsAttr::get(vectorType,
                               IntegerAttr::get(elemType, 0).getValue()));
  }
  if (elemType.isa<FloatType>()) {
    return builder.create<spirv::ConstantOp>(
        loc, type,
        DenseFPElementsAttr::get(vectorType,
                                 FloatAttr::get(elemType, 0.0).getValue()));
  }
}

llvm_unreachable("unimplemented types for ConstantOp::getZero()")::llvm::llvm_unreachable_internal("unimplemented types for ConstantOp::getZero()"
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 2074);
2075}

2077spirv::ConstantOp spirv::ConstantOp::getOne(Type type, Location loc,
                                          OpBuilder &builder) {
if (auto intType = type.dyn_cast<IntegerType>()) {
  unsigned width = intType.getWidth();
  if (width == 1)
    return builder.create<spirv::ConstantOp>(loc, type,
                                             builder.getBoolAttr(true));
  return builder.create<spirv::ConstantOp>(
      loc, type, builder.getIntegerAttr(type, APInt(width, 1)));
}
if (auto floatType = type.dyn_cast<FloatType>()) {
  return builder.create<spirv::ConstantOp>(
      loc, type, builder.getFloatAttr(floatType, 1.0));
}
if (auto vectorType = type.dyn_cast<VectorType>()) {
  Type elemType = vectorType.getElementType();
  if (elemType.isa<IntegerType>()) {
    return builder.create<spirv::ConstantOp>(
        loc, type,
        DenseElementsAttr::get(vectorType,
                               IntegerAttr::get(elemType, 1).getValue()));
  }
  if (elemType.isa<FloatType>()) {
    return builder.create<spirv::ConstantOp>(
        loc, type,
        DenseFPElementsAttr::get(vectorType,
                                 FloatAttr::get(elemType, 1.0).getValue()));
  }
}

llvm_unreachable("unimplemented types for ConstantOp::getOne()")::llvm::llvm_unreachable_internal("unimplemented types for ConstantOp::getOne()"
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 2107);
2108}

2110void mlir::spirv::ConstantOp::getAsmResultNames(
  llvm::function_ref<void(mlir::Value, llvm::StringRef)> setNameFn) {
Type type = getType();

SmallString<32> specialNameBuffer;
llvm::raw_svector_ostream specialName(specialNameBuffer);
specialName << "cst";

IntegerType intTy = type.dyn_cast<IntegerType>();

if (IntegerAttr intCst = getValue().dyn_cast<IntegerAttr>()) {
  if (intTy && intTy.getWidth() == 1) {
    return setNameFn(getResult(), (intCst.getInt() ? "true" : "false"));
  }

  if (intTy.isSignless()) {
    specialName << intCst.getInt();
  } else if (intTy.isUnsigned()) {
    specialName << intCst.getUInt();
  } else {
    specialName << intCst.getSInt();
  }
}

if (intTy || type.isa<FloatType>()) {
  specialName << '_' << type;
}

if (auto vecType = type.dyn_cast<VectorType>()) {
  specialName << "_vec_";
  specialName << vecType.getDimSize(0);

  Type elementType = vecType.getElementType();

  if (elementType.isa<IntegerType>() || elementType.isa<FloatType>()) {
    specialName << "x" << elementType;
  }
}

setNameFn(getResult(), specialName.str());
2150}

2152void mlir::spirv::AddressOfOp::getAsmResultNames(
  llvm::function_ref<void(mlir::Value, llvm::StringRef)> setNameFn) {
SmallString<32> specialNameBuffer;
llvm::raw_svector_ostream specialName(specialNameBuffer);
specialName << getVariable() << "_addr";
setNameFn(getResult(), specialName.str());
2158}

2160//===----------------------------------------------------------------------===//
2161// spirv.ControlBarrierOp
2162//===----------------------------------------------------------------------===//

2164LogicalResult spirv::ControlBarrierOp::verify() {
return verifyMemorySemantics(getOperation(), getMemorySemantics());
2166}

2168//===----------------------------------------------------------------------===//
2169// spirv.ConvertFToSOp
2170//===----------------------------------------------------------------------===//

2172LogicalResult spirv::ConvertFToSOp::verify() {
return verifyCastOp(*this, /*requireSameBitWidth=*/false,
                    /*skipBitWidthCheck=*/true);
2175}

2177//===----------------------------------------------------------------------===//
2178// spirv.ConvertFToUOp
2179//===----------------------------------------------------------------------===//

2181LogicalResult spirv::ConvertFToUOp::verify() {
return verifyCastOp(*this, /*requireSameBitWidth=*/false,
                    /*skipBitWidthCheck=*/true);
2184}

2186//===----------------------------------------------------------------------===//
2187// spirv.ConvertSToFOp
2188//===----------------------------------------------------------------------===//

2190LogicalResult spirv::ConvertSToFOp::verify() {
return verifyCastOp(*this, /*requireSameBitWidth=*/false,
                    /*skipBitWidthCheck=*/true);
2193}

2195//===----------------------------------------------------------------------===//
2196// spirv.ConvertUToFOp
2197//===----------------------------------------------------------------------===//

2199LogicalResult spirv::ConvertUToFOp::verify() {
return verifyCastOp(*this, /*requireSameBitWidth=*/false,
                    /*skipBitWidthCheck=*/true);
2202}

2204//===----------------------------------------------------------------------===//
2205// spirv.INTELConvertBF16ToFOp
2206//===----------------------------------------------------------------------===//

2208LogicalResult spirv::INTELConvertBF16ToFOp::verify() {
auto operandType = getOperand().getType();
auto resultType = getResult().getType();
// ODS checks that vector result type and vector operand type have the same
// shape.
if (auto vectorType = operandType.dyn_cast<VectorType>()) {
  unsigned operandNumElements = vectorType.getNumElements();
  unsigned resultNumElements = resultType.cast<VectorType>().getNumElements();
  if (operandNumElements != resultNumElements) {
    return emitOpError(
        "operand and result must have same number of elements");
  }
}
return success();
2222}

2224//===----------------------------------------------------------------------===//
2225// spirv.INTELConvertFToBF16Op
2226//===----------------------------------------------------------------------===//

2228LogicalResult spirv::INTELConvertFToBF16Op::verify() {
auto operandType = getOperand().getType();
auto resultType = getResult().getType();
// ODS checks that vector result type and vector operand type have the same
// shape.
if (auto vectorType = operandType.dyn_cast<VectorType>()) {
  unsigned operandNumElements = vectorType.getNumElements();
  unsigned resultNumElements = resultType.cast<VectorType>().getNumElements();
  if (operandNumElements != resultNumElements) {
    return emitOpError(
        "operand and result must have same number of elements");
  }
}
return success();
2242}

2244//===----------------------------------------------------------------------===//
2245// spirv.EntryPoint
2246//===----------------------------------------------------------------------===//

2248void spirv::EntryPointOp::build(OpBuilder &builder, OperationState &state,
                              spirv::ExecutionModel executionModel,
                              spirv::FuncOp function,
                              ArrayRef<Attribute> interfaceVars) {
build(builder, state,
      spirv::ExecutionModelAttr::get(builder.getContext(), executionModel),
      SymbolRefAttr::get(function), builder.getArrayAttr(interfaceVars));
2255}

2257ParseResult spirv::EntryPointOp::parse(OpAsmParser &parser,
                                     OperationState &result) {
spirv::ExecutionModel execModel;
1
'execModel' declared without an initial value→
SmallVector<OpAsmParser::UnresolvedOperand, 0> identifiers;
SmallVector<Type, 0> idTypes;
SmallVector<Attribute, 4> interfaceVars;

FlatSymbolRefAttr fn;
if (parseEnumStrAttr<spirv::ExecutionModelAttr>(execModel, parser, result) ||
2
←
Calling 'parseEnumStrAttr<mlir::spirv::ExecutionModelAttr, mlir::spirv::ExecutionModel>'→
    parser.parseAttribute(fn, Type(), kFnNameAttrName, result.attributes)) {
  return failure();
}

if (!parser.parseOptionalComma()) {
  // Parse the interface variables
  if (parser.parseCommaSeparatedList([&]() -> ParseResult {
        // The name of the interface variable attribute isnt important
        FlatSymbolRefAttr var;
        NamedAttrList attrs;
        if (parser.parseAttribute(var, Type(), "var_symbol", attrs))
          return failure();
        interfaceVars.push_back(var);
        return success();
      }))
    return failure();
}
result.addAttribute(kInterfaceAttrName,
                    parser.getBuilder().getArrayAttr(interfaceVars));
return success();
2286}

2288void spirv::EntryPointOp::print(OpAsmPrinter &printer) {
printer << " \"" << stringifyExecutionModel(getExecutionModel()) << "\" ";
printer.printSymbolName(getFn());
auto interfaceVars = getInterface().getValue();
if (!interfaceVars.empty()) {
  printer << ", ";
  llvm::interleaveComma(interfaceVars, printer);
}
2296}

2298LogicalResult spirv::EntryPointOp::verify() {
// Checks for fn and interface symbol reference are done in spirv::ModuleOp
// verification.
return success();
2302}

2304//===----------------------------------------------------------------------===//
2305// spirv.ExecutionMode
2306//===----------------------------------------------------------------------===//

2308void spirv::ExecutionModeOp::build(OpBuilder &builder, OperationState &state,
                                 spirv::FuncOp function,
                                 spirv::ExecutionMode executionMode,
                                 ArrayRef<int32_t> params) {
build(builder, state, SymbolRefAttr::get(function),
      spirv::ExecutionModeAttr::get(builder.getContext(), executionMode),
      builder.getI32ArrayAttr(params));
2315}

2317ParseResult spirv::ExecutionModeOp::parse(OpAsmParser &parser,
                                        OperationState &result) {
spirv::ExecutionMode execMode;
Attribute fn;
if (parser.parseAttribute(fn, kFnNameAttrName, result.attributes) ||
    parseEnumStrAttr<spirv::ExecutionModeAttr>(execMode, parser, result)) {
  return failure();
}

SmallVector<int32_t, 4> values;
Type i32Type = parser.getBuilder().getIntegerType(32);
while (!parser.parseOptionalComma()) {
  NamedAttrList attr;
  Attribute value;
  if (parser.parseAttribute(value, i32Type, "value", attr)) {
    return failure();
  }
  values.push_back(value.cast<IntegerAttr>().getInt());
}
result.addAttribute(kValuesAttrName,
                    parser.getBuilder().getI32ArrayAttr(values));
return success();
2339}

2341void spirv::ExecutionModeOp::print(OpAsmPrinter &printer) {
printer << " ";
printer.printSymbolName(getFn());
printer << " \"" << stringifyExecutionMode(getExecutionMode()) << "\"";
auto values = this->getValues();
if (values.empty())
  return;
printer << ", ";
llvm::interleaveComma(values, printer, [&](Attribute a) {
  printer << a.cast<IntegerAttr>().getInt();
});
2352}

2354//===----------------------------------------------------------------------===//
2355// spirv.FConvertOp
2356//===----------------------------------------------------------------------===//

2358LogicalResult spirv::FConvertOp::verify() {
return verifyCastOp(*this, /*requireSameBitWidth=*/false);
2360}

2362//===----------------------------------------------------------------------===//
2363// spirv.SConvertOp
2364//===----------------------------------------------------------------------===//

2366LogicalResult spirv::SConvertOp::verify() {
return verifyCastOp(*this, /*requireSameBitWidth=*/false);
2368}

2370//===----------------------------------------------------------------------===//
2371// spirv.UConvertOp
2372//===----------------------------------------------------------------------===//

2374LogicalResult spirv::UConvertOp::verify() {
return verifyCastOp(*this, /*requireSameBitWidth=*/false);
2376}

2378//===----------------------------------------------------------------------===//
2379// spirv.func
2380//===----------------------------------------------------------------------===//

2382ParseResult spirv::FuncOp::parse(OpAsmParser &parser, OperationState &result) {
SmallVector<OpAsmParser::Argument> entryArgs;
SmallVector<DictionaryAttr> resultAttrs;
SmallVector<Type> resultTypes;
auto &builder = parser.getBuilder();

// Parse the name as a symbol.
StringAttr nameAttr;
if (parser.parseSymbolName(nameAttr, SymbolTable::getSymbolAttrName(),
                           result.attributes))
  return failure();

// Parse the function signature.
bool isVariadic = false;
if (function_interface_impl::parseFunctionSignature(
        parser, /*allowVariadic=*/false, entryArgs, isVariadic, resultTypes,
        resultAttrs))
  return failure();

SmallVector<Type> argTypes;
for (auto &arg : entryArgs)
  argTypes.push_back(arg.type);
auto fnType = builder.getFunctionType(argTypes, resultTypes);
result.addAttribute(getFunctionTypeAttrName(result.name),
                    TypeAttr::get(fnType));

// Parse the optional function control keyword.
spirv::FunctionControl fnControl;
if (parseEnumStrAttr<spirv::FunctionControlAttr>(fnControl, parser, result))
  return failure();

// If additional attributes are present, parse them.
if (parser.parseOptionalAttrDictWithKeyword(result.attributes))
  return failure();

// Add the attributes to the function arguments.
assert(resultAttrs.size() == resultTypes.size())(static_cast <bool> (resultAttrs.size() == resultTypes.
size()) ? void (0) : __assert_fail ("resultAttrs.size() == resultTypes.size()"
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 2418, __extension__
 __PRETTY_FUNCTION__));
function_interface_impl::addArgAndResultAttrs(
    builder, result, entryArgs, resultAttrs, getArgAttrsAttrName(result.name),
    getResAttrsAttrName(result.name));

// Parse the optional function body.
auto *body = result.addRegion();
OptionalParseResult parseResult =
    parser.parseOptionalRegion(*body, entryArgs);
return failure(parseResult.has_value() && failed(*parseResult));
2428}

2430void spirv::FuncOp::print(OpAsmPrinter &printer) {
// Print function name, signature, and control.
printer << " ";
printer.printSymbolName(getSymName());
auto fnType = getFunctionType();
function_interface_impl::printFunctionSignature(
    printer, *this, fnType.getInputs(),
    /*isVariadic=*/false, fnType.getResults());
printer << " \"" << spirv::stringifyFunctionControl(getFunctionControl())
        << "\"";
function_interface_impl::printFunctionAttributes(
    printer, *this,
    {spirv::attributeName<spirv::FunctionControl>(),
     getFunctionTypeAttrName(), getArgAttrsAttrName(), getResAttrsAttrName(),
     getFunctionControlAttrName()});

// Print the body if this is not an external function.
Region &body = this->getBody();
if (!body.empty()) {
  printer << ' ';
  printer.printRegion(body, /*printEntryBlockArgs=*/false,
                      /*printBlockTerminators=*/true);
}
2453}

2455LogicalResult spirv::FuncOp::verifyType() {
if (getFunctionType().getNumResults() > 1)
  return emitOpError("cannot have more than one result");
return success();
2459}

2461LogicalResult spirv::FuncOp::verifyBody() {
FunctionType fnType = getFunctionType();

auto walkResult = walk([fnType](Operation *op) -> WalkResult {
  if (auto retOp = dyn_cast<spirv::ReturnOp>(op)) {
    if (fnType.getNumResults() != 0)
      return retOp.emitOpError("cannot be used in functions returning value");
  } else if (auto retOp = dyn_cast<spirv::ReturnValueOp>(op)) {
    if (fnType.getNumResults() != 1)
      return retOp.emitOpError(
                 "returns 1 value but enclosing function requires ")
             << fnType.getNumResults() << " results";

    auto retOperandType = retOp.getValue().getType();
    auto fnResultType = fnType.getResult(0);
    if (retOperandType != fnResultType)
      return retOp.emitOpError(" return value's type (")
             << retOperandType << ") mismatch with function's result type ("
             << fnResultType << ")";
  }
  return WalkResult::advance();
});

// TODO: verify other bits like linkage type.

return failure(walkResult.wasInterrupted());
2487}

2489void spirv::FuncOp::build(OpBuilder &builder, OperationState &state,
                        StringRef name, FunctionType type,
                        spirv::FunctionControl control,
                        ArrayRef<NamedAttribute> attrs) {
state.addAttribute(SymbolTable::getSymbolAttrName(),
                   builder.getStringAttr(name));
state.addAttribute(getFunctionTypeAttrName(state.name), TypeAttr::get(type));
state.addAttribute(spirv::attributeName<spirv::FunctionControl>(),
                   builder.getAttr<spirv::FunctionControlAttr>(control));
state.attributes.append(attrs.begin(), attrs.end());
state.addRegion();
2500}

2502// CallableOpInterface
2503Region *spirv::FuncOp::getCallableRegion() {
return isExternal() ? nullptr : &getBody();
2505}

2507// CallableOpInterface
2508ArrayRef<Type> spirv::FuncOp::getCallableResults() {
return getFunctionType().getResults();
2510}

2512// CallableOpInterface
:mlir::ArrayAttr spirv::FuncOp::getCallableArgAttrs() {
return getArgAttrs().value_or(nullptr);
2515}

2517// CallableOpInterface
:mlir::ArrayAttr spirv::FuncOp::getCallableResAttrs() {
return getResAttrs().value_or(nullptr);
2520}

2522//===----------------------------------------------------------------------===//
2523// spirv.FunctionCall
2524//===----------------------------------------------------------------------===//

2526LogicalResult spirv::FunctionCallOp::verify() {
auto fnName = getCalleeAttr();

auto funcOp = dyn_cast_or_null<spirv::FuncOp>(
    SymbolTable::lookupNearestSymbolFrom((*this)->getParentOp(), fnName));
if (!funcOp) {
  return emitOpError("callee function '")
         << fnName.getValue() << "' not found in nearest symbol table";
}

auto functionType = funcOp.getFunctionType();

if (getNumResults() > 1) {
  return emitOpError(
             "expected callee function to have 0 or 1 result, but provided ")
         << getNumResults();
}

if (functionType.getNumInputs() != getNumOperands()) {
  return emitOpError("has incorrect number of operands for callee: expected ")
         << functionType.getNumInputs() << ", but provided "
         << getNumOperands();
}

for (uint32_t i = 0, e = functionType.getNumInputs(); i != e; ++i) {
  if (getOperand(i).getType() != functionType.getInput(i)) {
    return emitOpError("operand type mismatch: expected operand type ")
           << functionType.getInput(i) << ", but provided "
           << getOperand(i).getType() << " for operand number " << i;
  }
}

if (functionType.getNumResults() != getNumResults()) {
  return emitOpError(
             "has incorrect number of results has for callee: expected ")
         << functionType.getNumResults() << ", but provided "
         << getNumResults();
}

if (getNumResults() &&
    (getResult(0).getType() != functionType.getResult(0))) {
  return emitOpError("result type mismatch: expected ")
         << functionType.getResult(0) << ", but provided "
         << getResult(0).getType();
}

return success();
2573}

2575CallInterfaceCallable spirv::FunctionCallOp::getCallableForCallee() {
return (*this)->getAttrOfType<SymbolRefAttr>(kCallee);
2577}

2579void spirv::FunctionCallOp::setCalleeFromCallable(
  CallInterfaceCallable callee) {
(*this)->setAttr(kCallee, callee.get<SymbolRefAttr>());
2582}

2584Operation::operand_range spirv::FunctionCallOp::getArgOperands() {
return getArguments();
2586}

2588//===----------------------------------------------------------------------===//
2589// spirv.GLFClampOp
2590//===----------------------------------------------------------------------===//

2592ParseResult spirv::GLFClampOp::parse(OpAsmParser &parser,
                                   OperationState &result) {
return parseOneResultSameOperandTypeOp(parser, result);
2595}
2596void spirv::GLFClampOp::print(OpAsmPrinter &p) { printOneResultOp(*this, p); }

2598//===----------------------------------------------------------------------===//
2599// spirv.GLUClampOp
2600//===----------------------------------------------------------------------===//

2602ParseResult spirv::GLUClampOp::parse(OpAsmParser &parser,
                                   OperationState &result) {
return parseOneResultSameOperandTypeOp(parser, result);
2605}
2606void spirv::GLUClampOp::print(OpAsmPrinter &p) { printOneResultOp(*this, p); }

2608//===----------------------------------------------------------------------===//
2609// spirv.GLSClampOp
2610//===----------------------------------------------------------------------===//

2612ParseResult spirv::GLSClampOp::parse(OpAsmParser &parser,
                                   OperationState &result) {
return parseOneResultSameOperandTypeOp(parser, result);
2615}
2616void spirv::GLSClampOp::print(OpAsmPrinter &p) { printOneResultOp(*this, p); }

2618//===----------------------------------------------------------------------===//
2619// spirv.GLFmaOp
2620//===----------------------------------------------------------------------===//

2622ParseResult spirv::GLFmaOp::parse(OpAsmParser &parser, OperationState &result) {
return parseOneResultSameOperandTypeOp(parser, result);
2624}
2625void spirv::GLFmaOp::print(OpAsmPrinter &p) { printOneResultOp(*this, p); }

2627//===----------------------------------------------------------------------===//
2628// spirv.GlobalVariable
2629//===----------------------------------------------------------------------===//

2631void spirv::GlobalVariableOp::build(OpBuilder &builder, OperationState &state,
                                  Type type, StringRef name,
                                  unsigned descriptorSet, unsigned binding) {
build(builder, state, TypeAttr::get(type), builder.getStringAttr(name));
state.addAttribute(
    spirv::SPIRVDialect::getAttributeName(spirv::Decoration::DescriptorSet),
    builder.getI32IntegerAttr(descriptorSet));
state.addAttribute(
    spirv::SPIRVDialect::getAttributeName(spirv::Decoration::Binding),
    builder.getI32IntegerAttr(binding));
2641}

2643void spirv::GlobalVariableOp::build(OpBuilder &builder, OperationState &state,
                                  Type type, StringRef name,
                                  spirv::BuiltIn builtin) {
build(builder, state, TypeAttr::get(type), builder.getStringAttr(name));
state.addAttribute(
    spirv::SPIRVDialect::getAttributeName(spirv::Decoration::BuiltIn),
    builder.getStringAttr(spirv::stringifyBuiltIn(builtin)));
2650}

2652ParseResult spirv::GlobalVariableOp::parse(OpAsmParser &parser,
                                         OperationState &result) {
// Parse variable name.
StringAttr nameAttr;
if (parser.parseSymbolName(nameAttr, SymbolTable::getSymbolAttrName(),
                           result.attributes)) {
  return failure();
}

// Parse optional initializer
if (succeeded(parser.parseOptionalKeyword(kInitializerAttrName))) {
  FlatSymbolRefAttr initSymbol;
  if (parser.parseLParen() ||
      parser.parseAttribute(initSymbol, Type(), kInitializerAttrName,
                            result.attributes) ||
      parser.parseRParen())
    return failure();
}

if (parseVariableDecorations(parser, result)) {
  return failure();
}

Type type;
auto loc = parser.getCurrentLocation();
if (parser.parseColonType(type)) {
  return failure();
}
if (!type.isa<spirv::PointerType>()) {
  return parser.emitError(loc, "expected spirv.ptr type");
}
result.addAttribute(kTypeAttrName, TypeAttr::get(type));

return success();
2686}

2688void spirv::GlobalVariableOp::print(OpAsmPrinter &printer) {
SmallVector<StringRef, 4> elidedAttrs{
    spirv::attributeName<spirv::StorageClass>()};

// Print variable name.
printer << ' ';
printer.printSymbolName(getSymName());
elidedAttrs.push_back(SymbolTable::getSymbolAttrName());

// Print optional initializer
if (auto initializer = this->getInitializer()) {
  printer << " " << kInitializerAttrName << '(';
  printer.printSymbolName(*initializer);
  printer << ')';
  elidedAttrs.push_back(kInitializerAttrName);
}

elidedAttrs.push_back(kTypeAttrName);
printVariableDecorations(*this, printer, elidedAttrs);
printer << " : " << getType();
2708}

2710LogicalResult spirv::GlobalVariableOp::verify() {
if (!getType().isa<spirv::PointerType>())
  return emitOpError("result must be of a !spv.ptr type");

// SPIR-V spec: "Storage Class is the Storage Class of the memory holding the
// object. It cannot be Generic. It must be the same as the Storage Class
// operand of the Result Type."
// Also, Function storage class is reserved by spirv.Variable.
auto storageClass = this->storageClass();
if (storageClass == spirv::StorageClass::Generic ||
    storageClass == spirv::StorageClass::Function) {
  return emitOpError("storage class cannot be '")
         << stringifyStorageClass(storageClass) << "'";
}

if (auto init =
        (*this)->getAttrOfType<FlatSymbolRefAttr>(kInitializerAttrName)) {
  Operation *initOp = SymbolTable::lookupNearestSymbolFrom(
      (*this)->getParentOp(), init.getAttr());
  // TODO: Currently only variable initialization with specialization
  // constants and other variables is supported. They could be normal
  // constants in the module scope as well.
  if (!initOp ||
      !isa<spirv::GlobalVariableOp, spirv::SpecConstantOp>(initOp)) {
    return emitOpError("initializer must be result of a "
                       "spirv.SpecConstant or spirv.GlobalVariable op");
  }
}

return success();
2740}

2742//===----------------------------------------------------------------------===//
2743// spirv.GroupBroadcast
2744//===----------------------------------------------------------------------===//

2746LogicalResult spirv::GroupBroadcastOp::verify() {
spirv::Scope scope = getExecutionScope();
if (scope != spirv::Scope::Workgroup && scope != spirv::Scope::Subgroup)
  return emitOpError("execution scope must be 'Workgroup' or 'Subgroup'");

if (auto localIdTy = getLocalid().getType().dyn_cast<VectorType>())
  if (localIdTy.getNumElements() != 2 && localIdTy.getNumElements() != 3)
    return emitOpError("localid is a vector and can be with only "
                       " 2 or 3 components, actual number is ")
           << localIdTy.getNumElements();

return success();
2758}

2760//===----------------------------------------------------------------------===//
2761// spirv.GroupNonUniformBallotOp
2762//===----------------------------------------------------------------------===//

2764LogicalResult spirv::GroupNonUniformBallotOp::verify() {
spirv::Scope scope = getExecutionScope();
if (scope != spirv::Scope::Workgroup && scope != spirv::Scope::Subgroup)
  return emitOpError("execution scope must be 'Workgroup' or 'Subgroup'");

return success();
2770}

2772//===----------------------------------------------------------------------===//
2773// spirv.GroupNonUniformBroadcast
2774//===----------------------------------------------------------------------===//

2776LogicalResult spirv::GroupNonUniformBroadcastOp::verify() {
spirv::Scope scope = getExecutionScope();
if (scope != spirv::Scope::Workgroup && scope != spirv::Scope::Subgroup)
  return emitOpError("execution scope must be 'Workgroup' or 'Subgroup'");

// SPIR-V spec: "Before version 1.5, Id must come from a
// constant instruction.
auto targetEnv = spirv::getDefaultTargetEnv(getContext());
if (auto spirvModule = (*this)->getParentOfType<spirv::ModuleOp>())
  targetEnv = spirv::lookupTargetEnvOrDefault(spirvModule);

if (targetEnv.getVersion() < spirv::Version::V_1_5) {
  auto *idOp = getId().getDefiningOp();
  if (!idOp || !isa<spirv::ConstantOp,           // for normal constant
                    spirv::ReferenceOfOp>(idOp)) // for spec constant
    return emitOpError("id must be the result of a constant op");
}

return success();
2795}

2797//===----------------------------------------------------------------------===//
2798// spirv.GroupNonUniformShuffle*
2799//===----------------------------------------------------------------------===//

2801template <typename OpTy>
2802static LogicalResult verifyGroupNonUniformShuffleOp(OpTy op) {
spirv::Scope scope = op.getExecutionScope();
if (scope != spirv::Scope::Workgroup && scope != spirv::Scope::Subgroup)
  return op.emitOpError("execution scope must be 'Workgroup' or 'Subgroup'");

if (op.getOperands().back().getType().isSignedInteger())
  return op.emitOpError("second operand must be a singless/unsigned integer");

return success();
2811}

2813LogicalResult spirv::GroupNonUniformShuffleOp::verify() {
return verifyGroupNonUniformShuffleOp(*this);
2815}
2816LogicalResult spirv::GroupNonUniformShuffleDownOp::verify() {
return verifyGroupNonUniformShuffleOp(*this);
2818}
2819LogicalResult spirv::GroupNonUniformShuffleUpOp::verify() {
return verifyGroupNonUniformShuffleOp(*this);
2821}
2822LogicalResult spirv::GroupNonUniformShuffleXorOp::verify() {
return verifyGroupNonUniformShuffleOp(*this);
2824}

2826//===----------------------------------------------------------------------===//
2827// spirv.INTEL.SubgroupBlockRead
2828//===----------------------------------------------------------------------===//

2830ParseResult spirv::INTELSubgroupBlockReadOp::parse(OpAsmParser &parser,
                                                 OperationState &result) {
// Parse the storage class specification
spirv::StorageClass storageClass;
OpAsmParser::UnresolvedOperand ptrInfo;
Type elementType;
if (parseEnumStrAttr(storageClass, parser) || parser.parseOperand(ptrInfo) ||
    parser.parseColon() || parser.parseType(elementType)) {
  return failure();
}

auto ptrType = spirv::PointerType::get(elementType, storageClass);
if (auto valVecTy = elementType.dyn_cast<VectorType>())
  ptrType = spirv::PointerType::get(valVecTy.getElementType(), storageClass);

if (parser.resolveOperand(ptrInfo, ptrType, result.operands)) {
  return failure();
}

result.addTypes(elementType);
return success();
2851}

2853void spirv::INTELSubgroupBlockReadOp::print(OpAsmPrinter &printer) {
printer << " " << getPtr() << " : " << getType();
2855}

2857LogicalResult spirv::INTELSubgroupBlockReadOp::verify() {
if (failed(verifyBlockReadWritePtrAndValTypes(*this, getPtr(), getValue())))
  return failure();

return success();
2862}

2864//===----------------------------------------------------------------------===//
2865// spirv.INTEL.SubgroupBlockWrite
2866//===----------------------------------------------------------------------===//

2868ParseResult spirv::INTELSubgroupBlockWriteOp::parse(OpAsmParser &parser,
                                                  OperationState &result) {
// Parse the storage class specification
spirv::StorageClass storageClass;
SmallVector<OpAsmParser::UnresolvedOperand, 2> operandInfo;
auto loc = parser.getCurrentLocation();
Type elementType;
if (parseEnumStrAttr(storageClass, parser) ||
    parser.parseOperandList(operandInfo, 2) || parser.parseColon() ||
    parser.parseType(elementType)) {
  return failure();
}

auto ptrType = spirv::PointerType::get(elementType, storageClass);
if (auto valVecTy = elementType.dyn_cast<VectorType>())
  ptrType = spirv::PointerType::get(valVecTy.getElementType(), storageClass);

if (parser.resolveOperands(operandInfo, {ptrType, elementType}, loc,
                           result.operands)) {
  return failure();
}
return success();
2890}

2892void spirv::INTELSubgroupBlockWriteOp::print(OpAsmPrinter &printer) {
printer << " " << getPtr() << ", " << getValue() << " : "
        << getValue().getType();
2895}

2897LogicalResult spirv::INTELSubgroupBlockWriteOp::verify() {
if (failed(verifyBlockReadWritePtrAndValTypes(*this, getPtr(), getValue())))
  return failure();

return success();
2902}

2904//===----------------------------------------------------------------------===//
2905// spirv.GroupNonUniformElectOp
2906//===----------------------------------------------------------------------===//

2908LogicalResult spirv::GroupNonUniformElectOp::verify() {
spirv::Scope scope = getExecutionScope();
if (scope != spirv::Scope::Workgroup && scope != spirv::Scope::Subgroup)
  return emitOpError("execution scope must be 'Workgroup' or 'Subgroup'");

return success();
2914}

2916//===----------------------------------------------------------------------===//
2917// spirv.GroupNonUniformFAddOp
2918//===----------------------------------------------------------------------===//

2920LogicalResult spirv::GroupNonUniformFAddOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
2922}

2924ParseResult spirv::GroupNonUniformFAddOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
2927}
2928void spirv::GroupNonUniformFAddOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
2930}

2932//===----------------------------------------------------------------------===//
2933// spirv.GroupNonUniformFMaxOp
2934//===----------------------------------------------------------------------===//

2936LogicalResult spirv::GroupNonUniformFMaxOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
2938}

2940ParseResult spirv::GroupNonUniformFMaxOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
2943}
2944void spirv::GroupNonUniformFMaxOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
2946}

2948//===----------------------------------------------------------------------===//
2949// spirv.GroupNonUniformFMinOp
2950//===----------------------------------------------------------------------===//

2952LogicalResult spirv::GroupNonUniformFMinOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
2954}

2956ParseResult spirv::GroupNonUniformFMinOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
2959}
2960void spirv::GroupNonUniformFMinOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
2962}

2964//===----------------------------------------------------------------------===//
2965// spirv.GroupNonUniformFMulOp
2966//===----------------------------------------------------------------------===//

2968LogicalResult spirv::GroupNonUniformFMulOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
2970}

2972ParseResult spirv::GroupNonUniformFMulOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
2975}
2976void spirv::GroupNonUniformFMulOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
2978}

2980//===----------------------------------------------------------------------===//
2981// spirv.GroupNonUniformIAddOp
2982//===----------------------------------------------------------------------===//

2984LogicalResult spirv::GroupNonUniformIAddOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
2986}

2988ParseResult spirv::GroupNonUniformIAddOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
2991}
2992void spirv::GroupNonUniformIAddOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
2994}

2996//===----------------------------------------------------------------------===//
2997// spirv.GroupNonUniformIMulOp
2998//===----------------------------------------------------------------------===//

3000LogicalResult spirv::GroupNonUniformIMulOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
3002}

3004ParseResult spirv::GroupNonUniformIMulOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
3007}
3008void spirv::GroupNonUniformIMulOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
3010}

3012//===----------------------------------------------------------------------===//
3013// spirv.GroupNonUniformSMaxOp
3014//===----------------------------------------------------------------------===//

3016LogicalResult spirv::GroupNonUniformSMaxOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
3018}

3020ParseResult spirv::GroupNonUniformSMaxOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
3023}
3024void spirv::GroupNonUniformSMaxOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
3026}

3028//===----------------------------------------------------------------------===//
3029// spirv.GroupNonUniformSMinOp
3030//===----------------------------------------------------------------------===//

3032LogicalResult spirv::GroupNonUniformSMinOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
3034}

3036ParseResult spirv::GroupNonUniformSMinOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
3039}
3040void spirv::GroupNonUniformSMinOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
3042}

3044//===----------------------------------------------------------------------===//
3045// spirv.GroupNonUniformUMaxOp
3046//===----------------------------------------------------------------------===//

3048LogicalResult spirv::GroupNonUniformUMaxOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
3050}

3052ParseResult spirv::GroupNonUniformUMaxOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
3055}
3056void spirv::GroupNonUniformUMaxOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
3058}

3060//===----------------------------------------------------------------------===//
3061// spirv.GroupNonUniformUMinOp
3062//===----------------------------------------------------------------------===//

3064LogicalResult spirv::GroupNonUniformUMinOp::verify() {
return verifyGroupNonUniformArithmeticOp(*this);
3066}

3068ParseResult spirv::GroupNonUniformUMinOp::parse(OpAsmParser &parser,
                                              OperationState &result) {
return parseGroupNonUniformArithmeticOp(parser, result);
3071}
3072void spirv::GroupNonUniformUMinOp::print(OpAsmPrinter &p) {
printGroupNonUniformArithmeticOp(*this, p);
3074}

3076//===----------------------------------------------------------------------===//
3077// spirv.IAddCarryOp
3078//===----------------------------------------------------------------------===//

3080LogicalResult spirv::IAddCarryOp::verify() {
return ::verifyArithmeticExtendedBinaryOp(*this);
3082}

3084ParseResult spirv::IAddCarryOp::parse(OpAsmParser &parser,
                                    OperationState &result) {
return ::parseArithmeticExtendedBinaryOp(parser, result);
3087}

3089void spirv::IAddCarryOp::print(OpAsmPrinter &printer) {
::printArithmeticExtendedBinaryOp(*this, printer);
3091}

3093//===----------------------------------------------------------------------===//
3094// spirv.ISubBorrowOp
3095//===----------------------------------------------------------------------===//

3097LogicalResult spirv::ISubBorrowOp::verify() {
return ::verifyArithmeticExtendedBinaryOp(*this);
3099}

3101ParseResult spirv::ISubBorrowOp::parse(OpAsmParser &parser,
                                     OperationState &result) {
return ::parseArithmeticExtendedBinaryOp(parser, result);
3104}

3106void spirv::ISubBorrowOp::print(OpAsmPrinter &printer) {
::printArithmeticExtendedBinaryOp(*this, printer);
3108}

3110//===----------------------------------------------------------------------===//
3111// spirv.SMulExtended
3112//===----------------------------------------------------------------------===//

3114LogicalResult spirv::SMulExtendedOp::verify() {
return ::verifyArithmeticExtendedBinaryOp(*this);
3116}

3118ParseResult spirv::SMulExtendedOp::parse(OpAsmParser &parser,
                                       OperationState &result) {
return ::parseArithmeticExtendedBinaryOp(parser, result);
3121}

3123void spirv::SMulExtendedOp::print(OpAsmPrinter &printer) {
::printArithmeticExtendedBinaryOp(*this, printer);
3125}

3127//===----------------------------------------------------------------------===//
3128// spirv.UMulExtended
3129//===----------------------------------------------------------------------===//

3131LogicalResult spirv::UMulExtendedOp::verify() {
return ::verifyArithmeticExtendedBinaryOp(*this);
3133}

3135ParseResult spirv::UMulExtendedOp::parse(OpAsmParser &parser,
                                       OperationState &result) {
return ::parseArithmeticExtendedBinaryOp(parser, result);
3138}

3140void spirv::UMulExtendedOp::print(OpAsmPrinter &printer) {
::printArithmeticExtendedBinaryOp(*this, printer);
3142}

3144//===----------------------------------------------------------------------===//
3145// spirv.LoadOp
3146//===----------------------------------------------------------------------===//

3148void spirv::LoadOp::build(OpBuilder &builder, OperationState &state,
                        Value basePtr, MemoryAccessAttr memoryAccess,
                        IntegerAttr alignment) {
auto ptrType = basePtr.getType().cast<spirv::PointerType>();
build(builder, state, ptrType.getPointeeType(), basePtr, memoryAccess,
      alignment);
3154}

3156ParseResult spirv::LoadOp::parse(OpAsmParser &parser, OperationState &result) {
// Parse the storage class specification
spirv::StorageClass storageClass;
OpAsmParser::UnresolvedOperand ptrInfo;
Type elementType;
if (parseEnumStrAttr(storageClass, parser) || parser.parseOperand(ptrInfo) ||
    parseMemoryAccessAttributes(parser, result) ||
    parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() ||
    parser.parseType(elementType)) {
  return failure();
}

auto ptrType = spirv::PointerType::get(elementType, storageClass);
if (parser.resolveOperand(ptrInfo, ptrType, result.operands)) {
  return failure();
}

result.addTypes(elementType);
return success();
3175}

3177void spirv::LoadOp::print(OpAsmPrinter &printer) {
SmallVector<StringRef, 4> elidedAttrs;
StringRef sc = stringifyStorageClass(
    getPtr().getType().cast<spirv::PointerType>().getStorageClass());
printer << " \"" << sc << "\" " << getPtr();

printMemoryAccessAttribute(*this, printer, elidedAttrs);

printer.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
printer << " : " << getType();
3187}

3189LogicalResult spirv::LoadOp::verify() {
// SPIR-V spec : "Result Type is the type of the loaded object. It must be a
// type with fixed size; i.e., it cannot be, nor include, any
// OpTypeRuntimeArray types."
if (failed(verifyLoadStorePtrAndValTypes(*this, getPtr(), getValue()))) {
  return failure();
}
return verifyMemoryAccessAttribute(*this);
3197}

3199//===----------------------------------------------------------------------===//
3200// spirv.mlir.loop
3201//===----------------------------------------------------------------------===//

3203void spirv::LoopOp::build(OpBuilder &builder, OperationState &state) {
state.addAttribute("loop_control", builder.getAttr<spirv::LoopControlAttr>(
                                       spirv::LoopControl::None));
state.addRegion();
3207}

3209ParseResult spirv::LoopOp::parse(OpAsmParser &parser, OperationState &result) {
if (parseControlAttribute<spirv::LoopControlAttr, spirv::LoopControl>(parser,
                                                                      result))
  return failure();
return parser.parseRegion(*result.addRegion(), /*arguments=*/{});
3214}

3216void spirv::LoopOp::print(OpAsmPrinter &printer) {
auto control = getLoopControl();
if (control != spirv::LoopControl::None)
  printer << " control(" << spirv::stringifyLoopControl(control) << ")";
printer << ' ';
printer.printRegion(getRegion(), /*printEntryBlockArgs=*/false,
                    /*printBlockTerminators=*/true);
3223}

3225/// Returns true if the given `srcBlock` contains only one `spirv.Branch` to the
3226/// given `dstBlock`.
3227static inline bool hasOneBranchOpTo(Block &srcBlock, Block &dstBlock) {
// Check that there is only one op in the `srcBlock`.
if (!llvm::hasSingleElement(srcBlock))
  return false;

auto branchOp = dyn_cast<spirv::BranchOp>(srcBlock.back());
return branchOp && branchOp.getSuccessor() == &dstBlock;
3234}

3236LogicalResult spirv::LoopOp::verifyRegions() {
auto *op = getOperation();

// We need to verify that the blocks follow the following layout:
//
//                     +-------------+
//                     | entry block |
//                     +-------------+
//                            |
//                            v
//                     +-------------+
//                     | loop header | <-----+
//                     +-------------+       |
//                                           |
//                           ...             |
//                          \ | /            |
//                            v              |
//                    +---------------+      |
//                    | loop continue | -----+
//                    +---------------+
//
//                           ...
//                          \ | /
//                            v
//                     +-------------+
//                     | merge block |
//                     +-------------+

auto &region = op->getRegion(0);
// Allow empty region as a degenerated case, which can come from
// optimizations.
if (region.empty())
  return success();

// The last block is the merge block.
Block &merge = region.back();
if (!isMergeBlock(merge))
  return emitOpError("last block must be the merge block with only one "
                     "'spirv.mlir.merge' op");

if (std::next(region.begin()) == region.end())
  return emitOpError(
      "must have an entry block branching to the loop header block");
// The first block is the entry block.
Block &entry = region.front();

if (std::next(region.begin(), 2) == region.end())
  return emitOpError(
      "must have a loop header block branched from the entry block");
// The second block is the loop header block.
Block &header = *std::next(region.begin(), 1);

if (!hasOneBranchOpTo(entry, header))
  return emitOpError(
      "entry block must only have one 'spirv.Branch' op to the second block");

if (std::next(region.begin(), 3) == region.end())
  return emitOpError(
      "requires a loop continue block branching to the loop header block");
// The second to last block is the loop continue block.
Block &cont = *std::prev(region.end(), 2);

// Make sure that we have a branch from the loop continue block to the loop
// header block.
if (llvm::none_of(
        llvm::seq<unsigned>(0, cont.getNumSuccessors()),
        [&](unsigned index) { return cont.getSuccessor(index) == &header; }))
  return emitOpError("second to last block must be the loop continue "
                     "block that branches to the loop header block");

// Make sure that no other blocks (except the entry and loop continue block)
// branches to the loop header block.
for (auto &block : llvm::make_range(std::next(region.begin(), 2),
                                    std::prev(region.end(), 2))) {
  for (auto i : llvm::seq<unsigned>(0, block.getNumSuccessors())) {
    if (block.getSuccessor(i) == &header) {
      return emitOpError("can only have the entry and loop continue "
                         "block branching to the loop header block");
    }
  }
}

return success();
3319}

3321Block *spirv::LoopOp::getEntryBlock() {
assert(!getBody().empty() && "op region should not be empty!")(static_cast <bool> (!getBody().empty() && "op region should not be empty!"
) ? void (0) : __assert_fail ("!getBody().empty() && \"op region should not be empty!\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 3322, __extension__
 __PRETTY_FUNCTION__));
return &getBody().front();
3324}

3326Block *spirv::LoopOp::getHeaderBlock() {
assert(!getBody().empty() && "op region should not be empty!")(static_cast <bool> (!getBody().empty() && "op region should not be empty!"
) ? void (0) : __assert_fail ("!getBody().empty() && \"op region should not be empty!\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 3327, __extension__
 __PRETTY_FUNCTION__));
// The second block is the loop header block.
return &*std::next(getBody().begin());
3330}

3332Block *spirv::LoopOp::getContinueBlock() {
assert(!getBody().empty() && "op region should not be empty!")(static_cast <bool> (!getBody().empty() && "op region should not be empty!"
) ? void (0) : __assert_fail ("!getBody().empty() && \"op region should not be empty!\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 3333, __extension__
 __PRETTY_FUNCTION__));
// The second to last block is the loop continue block.
return &*std::prev(getBody().end(), 2);
3336}

3338Block *spirv::LoopOp::getMergeBlock() {
assert(!getBody().empty() && "op region should not be empty!")(static_cast <bool> (!getBody().empty() && "op region should not be empty!"
) ? void (0) : __assert_fail ("!getBody().empty() && \"op region should not be empty!\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 3339, __extension__
 __PRETTY_FUNCTION__));
// The last block is the loop merge block.
return &getBody().back();
3342}

3344void spirv::LoopOp::addEntryAndMergeBlock() {
assert(getBody().empty() && "entry and merge block already exist")(static_cast <bool> (getBody().empty() && "entry and merge block already exist"
) ? void (0) : __assert_fail ("getBody().empty() && \"entry and merge block already exist\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 3345, __extension__
 __PRETTY_FUNCTION__));
getBody().push_back(new Block());
auto *mergeBlock = new Block();
getBody().push_back(mergeBlock);
OpBuilder builder = OpBuilder::atBlockEnd(mergeBlock);

// Add a spirv.mlir.merge op into the merge block.
builder.create<spirv::MergeOp>(getLoc());
3353}

3355//===----------------------------------------------------------------------===//
3356// spirv.MemoryBarrierOp
3357//===----------------------------------------------------------------------===//

3359LogicalResult spirv::MemoryBarrierOp::verify() {
return verifyMemorySemantics(getOperation(), getMemorySemantics());
3361}

3363//===----------------------------------------------------------------------===//
3364// spirv.mlir.merge
3365//===----------------------------------------------------------------------===//

3367LogicalResult spirv::MergeOp::verify() {
auto *parentOp = (*this)->getParentOp();
if (!parentOp || !isa<spirv::SelectionOp, spirv::LoopOp>(parentOp))
  return emitOpError(
      "expected parent op to be 'spirv.mlir.selection' or 'spirv.mlir.loop'");

// TODO: This check should be done in `verifyRegions` of parent op.
Block &parentLastBlock = (*this)->getParentRegion()->back();
if (getOperation() != parentLastBlock.getTerminator())
  return emitOpError("can only be used in the last block of "
                     "'spirv.mlir.selection' or 'spirv.mlir.loop'");
return success();
3379}

3381//===----------------------------------------------------------------------===//
3382// spirv.module
3383//===----------------------------------------------------------------------===//

3385void spirv::ModuleOp::build(OpBuilder &builder, OperationState &state,
                          std::optional<StringRef> name) {
OpBuilder::InsertionGuard guard(builder);
builder.createBlock(state.addRegion());
if (name) {
  state.attributes.append(mlir::SymbolTable::getSymbolAttrName(),
                          builder.getStringAttr(*name));
}
3393}

3395void spirv::ModuleOp::build(OpBuilder &builder, OperationState &state,
                          spirv::AddressingModel addressingModel,
                          spirv::MemoryModel memoryModel,
                          std::optional<VerCapExtAttr> vceTriple,
                          std::optional<StringRef> name) {
state.addAttribute(
    "addressing_model",
    builder.getAttr<spirv::AddressingModelAttr>(addressingModel));
state.addAttribute("memory_model",
                   builder.getAttr<spirv::MemoryModelAttr>(memoryModel));
OpBuilder::InsertionGuard guard(builder);
builder.createBlock(state.addRegion());
if (vceTriple)
  state.addAttribute(getVCETripleAttrName(), *vceTriple);
if (name)
  state.addAttribute(mlir::SymbolTable::getSymbolAttrName(),
                     builder.getStringAttr(*name));
3412}

3414ParseResult spirv::ModuleOp::parse(OpAsmParser &parser,
                                 OperationState &result) {
Region *body = result.addRegion();

// If the name is present, parse it.
StringAttr nameAttr;
(void)parser.parseOptionalSymbolName(
    nameAttr, mlir::SymbolTable::getSymbolAttrName(), result.attributes);

// Parse attributes
spirv::AddressingModel addrModel;
spirv::MemoryModel memoryModel;
if (::parseEnumKeywordAttr<spirv::AddressingModelAttr>(addrModel, parser,
                                                       result) ||
    ::parseEnumKeywordAttr<spirv::MemoryModelAttr>(memoryModel, parser,
                                                   result))
  return failure();

if (succeeded(parser.parseOptionalKeyword("requires"))) {
  spirv::VerCapExtAttr vceTriple;
  if (parser.parseAttribute(vceTriple,
                            spirv::ModuleOp::getVCETripleAttrName(),
                            result.attributes))
    return failure();
}

if (parser.parseOptionalAttrDictWithKeyword(result.attributes) ||
    parser.parseRegion(*body, /*arguments=*/{}))
  return failure();

// Make sure we have at least one block.
if (body->empty())
  body->push_back(new Block());

return success();
3449}

3451void spirv::ModuleOp::print(OpAsmPrinter &printer) {
if (std::optional<StringRef> name = getName()) {
  printer << ' ';
  printer.printSymbolName(*name);
}

SmallVector<StringRef, 2> elidedAttrs;

printer << " " << spirv::stringifyAddressingModel(getAddressingModel()) << " "
        << spirv::stringifyMemoryModel(getMemoryModel());
auto addressingModelAttrName = spirv::attributeName<spirv::AddressingModel>();
auto memoryModelAttrName = spirv::attributeName<spirv::MemoryModel>();
elidedAttrs.assign({addressingModelAttrName, memoryModelAttrName,
                    mlir::SymbolTable::getSymbolAttrName()});

if (std::optional<spirv::VerCapExtAttr> triple = getVceTriple()) {
  printer << " requires " << *triple;
  elidedAttrs.push_back(spirv::ModuleOp::getVCETripleAttrName());
}

printer.printOptionalAttrDictWithKeyword((*this)->getAttrs(), elidedAttrs);
printer << ' ';
printer.printRegion(getRegion());
3474}

3476LogicalResult spirv::ModuleOp::verifyRegions() {
Dialect *dialect = (*this)->getDialect();
DenseMap<std::pair<spirv::FuncOp, spirv::ExecutionModel>, spirv::EntryPointOp>
    entryPoints;
mlir::SymbolTable table(*this);

for (auto &op : *getBody()) {
  if (op.getDialect() != dialect)
    return op.emitError("'spirv.module' can only contain spirv.* ops");

  // For EntryPoint op, check that the function and execution model is not
  // duplicated in EntryPointOps. Also verify that the interface specified
  // comes from globalVariables here to make this check cheaper.
  if (auto entryPointOp = dyn_cast<spirv::EntryPointOp>(op)) {
    auto funcOp = table.lookup<spirv::FuncOp>(entryPointOp.getFn());
    if (!funcOp) {
      return entryPointOp.emitError("function '")
             << entryPointOp.getFn() << "' not found in 'spirv.module'";
    }
    if (auto interface = entryPointOp.getInterface()) {
      for (Attribute varRef : interface) {
        auto varSymRef = varRef.dyn_cast<FlatSymbolRefAttr>();
        if (!varSymRef) {
          return entryPointOp.emitError(
                     "expected symbol reference for interface "
                     "specification instead of '")
                 << varRef;
        }
        auto variableOp =
            table.lookup<spirv::GlobalVariableOp>(varSymRef.getValue());
        if (!variableOp) {
          return entryPointOp.emitError("expected spirv.GlobalVariable "
                                        "symbol reference instead of'")
                 << varSymRef << "'";
        }
      }
    }

    auto key = std::pair<spirv::FuncOp, spirv::ExecutionModel>(
        funcOp, entryPointOp.getExecutionModel());
    auto entryPtIt = entryPoints.find(key);
    if (entryPtIt != entryPoints.end()) {
      return entryPointOp.emitError("duplicate of a previous EntryPointOp");
    }
    entryPoints[key] = entryPointOp;
  } else if (auto funcOp = dyn_cast<spirv::FuncOp>(op)) {
    if (funcOp.isExternal())
      return op.emitError("'spirv.module' cannot contain external functions");

    // TODO: move this check to spirv.func.
    for (auto &block : funcOp)
      for (auto &op : block) {
        if (op.getDialect() != dialect)
          return op.emitError(
              "functions in 'spirv.module' can only contain spirv.* ops");
      }
  }
}

return success();
3536}

3538//===----------------------------------------------------------------------===//
3539// spirv.mlir.referenceof
3540//===----------------------------------------------------------------------===//

3542LogicalResult spirv::ReferenceOfOp::verify() {
auto *specConstSym = SymbolTable::lookupNearestSymbolFrom(
    (*this)->getParentOp(), getSpecConstAttr());
Type constType;

auto specConstOp = dyn_cast_or_null<spirv::SpecConstantOp>(specConstSym);
if (specConstOp)
  constType = specConstOp.getDefaultValue().getType();

auto specConstCompositeOp =
    dyn_cast_or_null<spirv::SpecConstantCompositeOp>(specConstSym);
if (specConstCompositeOp)
  constType = specConstCompositeOp.getType();

if (!specConstOp && !specConstCompositeOp)
  return emitOpError(
      "expected spirv.SpecConstant or spirv.SpecConstantComposite symbol");

if (getReference().getType() != constType)
  return emitOpError("result type mismatch with the referenced "
                     "specialization constant's type");

return success();
3565}

3567//===----------------------------------------------------------------------===//
3568// spirv.Return
3569//===----------------------------------------------------------------------===//

3571LogicalResult spirv::ReturnOp::verify() {
// Verification is performed in spirv.func op.
return success();
3574}

3576//===----------------------------------------------------------------------===//
3577// spirv.ReturnValue
3578//===----------------------------------------------------------------------===//

3580LogicalResult spirv::ReturnValueOp::verify() {
// Verification is performed in spirv.func op.
return success();
3583}

3585//===----------------------------------------------------------------------===//
3586// spirv.Select
3587//===----------------------------------------------------------------------===//

3589LogicalResult spirv::SelectOp::verify() {
if (auto conditionTy = getCondition().getType().dyn_cast<VectorType>()) {
  auto resultVectorTy = getResult().getType().dyn_cast<VectorType>();
  if (!resultVectorTy) {
    return emitOpError("result expected to be of vector type when "
                       "condition is of vector type");
  }
  if (resultVectorTy.getNumElements() != conditionTy.getNumElements()) {
    return emitOpError("result should have the same number of elements as "
                       "the condition when condition is of vector type");
  }
}
return success();
3602}

3604//===----------------------------------------------------------------------===//
3605// spirv.mlir.selection
3606//===----------------------------------------------------------------------===//

3608ParseResult spirv::SelectionOp::parse(OpAsmParser &parser,
                                    OperationState &result) {
if (parseControlAttribute<spirv::SelectionControlAttr,
                          spirv::SelectionControl>(parser, result))
  return failure();
return parser.parseRegion(*result.addRegion(), /*arguments=*/{});
3614}

3616void spirv::SelectionOp::print(OpAsmPrinter &printer) {
auto control = getSelectionControl();
if (control != spirv::SelectionControl::None)
  printer << " control(" << spirv::stringifySelectionControl(control) << ")";
printer << ' ';
printer.printRegion(getRegion(), /*printEntryBlockArgs=*/false,
                    /*printBlockTerminators=*/true);
3623}

3625LogicalResult spirv::SelectionOp::verifyRegions() {
auto *op = getOperation();

// We need to verify that the blocks follow the following layout:
//
//                     +--------------+
//                     | header block |
//                     +--------------+
//                          / | \
//                           ...
//
//
//         +---------+   +---------+   +---------+
//         | case #0 |   | case #1 |   | case #2 |  ...
//         +---------+   +---------+   +---------+
//
//
//                           ...
//                          \ | /
//                            v
//                     +-------------+
//                     | merge block |
//                     +-------------+

auto &region = op->getRegion(0);
// Allow empty region as a degenerated case, which can come from
// optimizations.
if (region.empty())
  return success();

// The last block is the merge block.
if (!isMergeBlock(region.back()))
  return emitOpError("last block must be the merge block with only one "
                     "'spirv.mlir.merge' op");

if (std::next(region.begin()) == region.end())
  return emitOpError("must have a selection header block");

return success();
3664}

3666Block *spirv::SelectionOp::getHeaderBlock() {
assert(!getBody().empty() && "op region should not be empty!")(static_cast <bool> (!getBody().empty() && "op region should not be empty!"
) ? void (0) : __assert_fail ("!getBody().empty() && \"op region should not be empty!\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 3667, __extension__
 __PRETTY_FUNCTION__));
// The first block is the loop header block.
return &getBody().front();
3670}

3672Block *spirv::SelectionOp::getMergeBlock() {
assert(!getBody().empty() && "op region should not be empty!")(static_cast <bool> (!getBody().empty() && "op region should not be empty!"
) ? void (0) : __assert_fail ("!getBody().empty() && \"op region should not be empty!\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 3673, __extension__
 __PRETTY_FUNCTION__));
// The last block is the loop merge block.
return &getBody().back();
3676}

3678void spirv::SelectionOp::addMergeBlock() {
assert(getBody().empty() && "entry and merge block already exist")(static_cast <bool> (getBody().empty() && "entry and merge block already exist"
) ? void (0) : __assert_fail ("getBody().empty() && \"entry and merge block already exist\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 3679, __extension__
 __PRETTY_FUNCTION__));
auto *mergeBlock = new Block();
getBody().push_back(mergeBlock);
OpBuilder builder = OpBuilder::atBlockEnd(mergeBlock);

// Add a spirv.mlir.merge op into the merge block.
builder.create<spirv::MergeOp>(getLoc());
3686}

3688spirv::SelectionOp spirv::SelectionOp::createIfThen(
  Location loc, Value condition,
  function_ref<void(OpBuilder &builder)> thenBody, OpBuilder &builder) {
auto selectionOp =
    builder.create<spirv::SelectionOp>(loc, spirv::SelectionControl::None);

selectionOp.addMergeBlock();
Block *mergeBlock = selectionOp.getMergeBlock();
Block *thenBlock = nullptr;

// Build the "then" block.
{
  OpBuilder::InsertionGuard guard(builder);
  thenBlock = builder.createBlock(mergeBlock);
  thenBody(builder);
  builder.create<spirv::BranchOp>(loc, mergeBlock);
}

// Build the header block.
{
  OpBuilder::InsertionGuard guard(builder);
  builder.createBlock(thenBlock);
  builder.create<spirv::BranchConditionalOp>(
      loc, condition, thenBlock,
      /*trueArguments=*/ArrayRef<Value>(), mergeBlock,
      /*falseArguments=*/ArrayRef<Value>());
}

return selectionOp;
3717}

3719//===----------------------------------------------------------------------===//
3720// spirv.SpecConstant
3721//===----------------------------------------------------------------------===//

3723ParseResult spirv::SpecConstantOp::parse(OpAsmParser &parser,
                                       OperationState &result) {
StringAttr nameAttr;
Attribute valueAttr;

if (parser.parseSymbolName(nameAttr, SymbolTable::getSymbolAttrName(),
                           result.attributes))
  return failure();

// Parse optional spec_id.
if (succeeded(parser.parseOptionalKeyword(kSpecIdAttrName))) {
  IntegerAttr specIdAttr;
  if (parser.parseLParen() ||
      parser.parseAttribute(specIdAttr, kSpecIdAttrName, result.attributes) ||
      parser.parseRParen())
    return failure();
}

if (parser.parseEqual() ||
    parser.parseAttribute(valueAttr, kDefaultValueAttrName,
                          result.attributes))
  return failure();

return success();
3747}

3749void spirv::SpecConstantOp::print(OpAsmPrinter &printer) {
printer << ' ';
printer.printSymbolName(getSymName());
if (auto specID = (*this)->getAttrOfType<IntegerAttr>(kSpecIdAttrName))
  printer << ' ' << kSpecIdAttrName << '(' << specID.getInt() << ')';
printer << " = " << getDefaultValue();
3755}

3757LogicalResult spirv::SpecConstantOp::verify() {
if (auto specID = (*this)->getAttrOfType<IntegerAttr>(kSpecIdAttrName))
  if (specID.getValue().isNegative())
    return emitOpError("SpecId cannot be negative");

auto value = getDefaultValue();
if (value.isa<IntegerAttr, FloatAttr>()) {
  // Make sure bitwidth is allowed.
  if (!value.getType().isa<spirv::SPIRVType>())
    return emitOpError("default value bitwidth disallowed");
  return success();
}
return emitOpError(
    "default value can only be a bool, integer, or float scalar");
3771}

3773//===----------------------------------------------------------------------===//
3774// spirv.StoreOp
3775//===----------------------------------------------------------------------===//

3777ParseResult spirv::StoreOp::parse(OpAsmParser &parser, OperationState &result) {
// Parse the storage class specification
spirv::StorageClass storageClass;
SmallVector<OpAsmParser::UnresolvedOperand, 2> operandInfo;
auto loc = parser.getCurrentLocation();
Type elementType;
if (parseEnumStrAttr(storageClass, parser) ||
    parser.parseOperandList(operandInfo, 2) ||
    parseMemoryAccessAttributes(parser, result) || parser.parseColon() ||
    parser.parseType(elementType)) {
  return failure();
}

auto ptrType = spirv::PointerType::get(elementType, storageClass);
if (parser.resolveOperands(operandInfo, {ptrType, elementType}, loc,
                           result.operands)) {
  return failure();
}
return success();
3796}

3798void spirv::StoreOp::print(OpAsmPrinter &printer) {
SmallVector<StringRef, 4> elidedAttrs;
StringRef sc = stringifyStorageClass(
    getPtr().getType().cast<spirv::PointerType>().getStorageClass());
printer << " \"" << sc << "\" " << getPtr() << ", " << getValue();

printMemoryAccessAttribute(*this, printer, elidedAttrs);

printer << " : " << getValue().getType();
printer.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);
3808}

3810LogicalResult spirv::StoreOp::verify() {
// SPIR-V spec : "Pointer is the pointer to store through. Its type must be an
// OpTypePointer whose Type operand is the same as the type of Object."
if (failed(verifyLoadStorePtrAndValTypes(*this, getPtr(), getValue())))
  return failure();
return verifyMemoryAccessAttribute(*this);
3816}

3818//===----------------------------------------------------------------------===//
3819// spirv.Unreachable
3820//===----------------------------------------------------------------------===//

3822LogicalResult spirv::UnreachableOp::verify() {
auto *block = (*this)->getBlock();
// Fast track: if this is in entry block, its invalid. Otherwise, if no
// predecessors, it's valid.
if (block->isEntryBlock())
  return emitOpError("cannot be used in reachable block");
if (block->hasNoPredecessors())
  return success();

// TODO: further verification needs to analyze reachability from
// the entry block.

return success();
3835}

3837//===----------------------------------------------------------------------===//
3838// spirv.Variable
3839//===----------------------------------------------------------------------===//

3841ParseResult spirv::VariableOp::parse(OpAsmParser &parser,
                                   OperationState &result) {
// Parse optional initializer
std::optional<OpAsmParser::UnresolvedOperand> initInfo;
if (succeeded(parser.parseOptionalKeyword("init"))) {
  initInfo = OpAsmParser::UnresolvedOperand();
  if (parser.parseLParen() || parser.parseOperand(*initInfo) ||
      parser.parseRParen())
    return failure();
}

if (parseVariableDecorations(parser, result)) {
  return failure();
}

// Parse result pointer type
Type type;
if (parser.parseColon())
  return failure();
auto loc = parser.getCurrentLocation();
if (parser.parseType(type))
  return failure();

auto ptrType = type.dyn_cast<spirv::PointerType>();
if (!ptrType)
  return parser.emitError(loc, "expected spirv.ptr type");
result.addTypes(ptrType);

// Resolve the initializer operand
if (initInfo) {
  if (parser.resolveOperand(*initInfo, ptrType.getPointeeType(),
                            result.operands))
    return failure();
}

auto attr = parser.getBuilder().getAttr<spirv::StorageClassAttr>(
    ptrType.getStorageClass());
result.addAttribute(spirv::attributeName<spirv::StorageClass>(), attr);

return success();
3881}

3883void spirv::VariableOp::print(OpAsmPrinter &printer) {
SmallVector<StringRef, 4> elidedAttrs{
    spirv::attributeName<spirv::StorageClass>()};
// Print optional initializer
if (getNumOperands() != 0)
  printer << " init(" << getInitializer() << ")";

printVariableDecorations(*this, printer, elidedAttrs);
printer << " : " << getType();
3892}

3894LogicalResult spirv::VariableOp::verify() {
// SPIR-V spec: "Storage Class is the Storage Class of the memory holding the
// object. It cannot be Generic. It must be the same as the Storage Class
// operand of the Result Type."
if (getStorageClass() != spirv::StorageClass::Function) {
  return emitOpError(
      "can only be used to model function-level variables. Use "
      "spirv.GlobalVariable for module-level variables.");
}

auto pointerType = getPointer().getType().cast<spirv::PointerType>();
if (getStorageClass() != pointerType.getStorageClass())
  return emitOpError(
      "storage class must match result pointer's storage class");

if (getNumOperands() != 0) {
  // SPIR-V spec: "Initializer must be an <id> from a constant instruction or
  // a global (module scope) OpVariable instruction".
  auto *initOp = getOperand(0).getDefiningOp();
  if (!initOp || !isa<spirv::ConstantOp,    // for normal constant
                      spirv::ReferenceOfOp, // for spec constant
                      spirv::AddressOfOp>(initOp))
    return emitOpError("initializer must be the result of a "
                       "constant or spirv.GlobalVariable op");
}

// TODO: generate these strings using ODS.
auto *op = getOperation();
auto descriptorSetName = llvm::convertToSnakeFromCamelCase(
    stringifyDecoration(spirv::Decoration::DescriptorSet));
auto bindingName = llvm::convertToSnakeFromCamelCase(
    stringifyDecoration(spirv::Decoration::Binding));
auto builtInName = llvm::convertToSnakeFromCamelCase(
    stringifyDecoration(spirv::Decoration::BuiltIn));

for (const auto &attr : {descriptorSetName, bindingName, builtInName}) {
  if (op->getAttr(attr))
    return emitOpError("cannot have '")
           << attr << "' attribute (only allowed in spirv.GlobalVariable)";
}

return success();
3936}

3938//===----------------------------------------------------------------------===//
3939// spirv.VectorShuffle
3940//===----------------------------------------------------------------------===//

3942LogicalResult spirv::VectorShuffleOp::verify() {
VectorType resultType = getType().cast<VectorType>();

size_t numResultElements = resultType.getNumElements();
if (numResultElements != getComponents().size())
  return emitOpError("result type element count (")
         << numResultElements
         << ") mismatch with the number of component selectors ("
         << getComponents().size() << ")";

size_t totalSrcElements =
    getVector1().getType().cast<VectorType>().getNumElements() +
    getVector2().getType().cast<VectorType>().getNumElements();

for (const auto &selector : getComponents().getAsValueRange<IntegerAttr>()) {
  uint32_t index = selector.getZExtValue();
  if (index >= totalSrcElements &&
      index != std::numeric_limits<uint32_t>().max())
    return emitOpError("component selector ")
           << index << " out of range: expected to be in [0, "
           << totalSrcElements << ") or 0xffffffff";
}
return success();
3965}

3967//===----------------------------------------------------------------------===//
3968// spirv.NV.CooperativeMatrixLoad
3969//===----------------------------------------------------------------------===//

3971ParseResult spirv::NVCooperativeMatrixLoadOp::parse(OpAsmParser &parser,
                                                  OperationState &result) {
SmallVector<OpAsmParser::UnresolvedOperand, 3> operandInfo;
Type strideType = parser.getBuilder().getIntegerType(32);
Type columnMajorType = parser.getBuilder().getIntegerType(1);
Type ptrType;
Type elementType;
if (parser.parseOperandList(operandInfo, 3) ||
    parseMemoryAccessAttributes(parser, result) || parser.parseColon() ||
    parser.parseType(ptrType) || parser.parseKeywordType("as", elementType)) {
  return failure();
}
if (parser.resolveOperands(operandInfo,
                           {ptrType, strideType, columnMajorType},
                           parser.getNameLoc(), result.operands)) {
  return failure();
}

result.addTypes(elementType);
return success();
3991}

3993void spirv::NVCooperativeMatrixLoadOp::print(OpAsmPrinter &printer) {
printer << " " << getPointer() << ", " << getStride() << ", "
        << getColumnmajor();
// Print optional memory access attribute.
if (auto memAccess = getMemoryAccess())
  printer << " [\"" << stringifyMemoryAccess(*memAccess) << "\"]";
printer << " : " << getPointer().getType() << " as " << getType();
4000}

4002static LogicalResult verifyPointerAndCoopMatrixType(Operation *op, Type pointer,
                                                  Type coopMatrix) {
Type pointeeType = pointer.cast<spirv::PointerType>().getPointeeType();
if (!pointeeType.isa<spirv::ScalarType>() && !pointeeType.isa<VectorType>())
  return op->emitError(
             "Pointer must point to a scalar or vector type but provided ")
         << pointeeType;
spirv::StorageClass storage =
    pointer.cast<spirv::PointerType>().getStorageClass();
if (storage != spirv::StorageClass::Workgroup &&
    storage != spirv::StorageClass::StorageBuffer &&
    storage != spirv::StorageClass::PhysicalStorageBuffer)
  return op->emitError(
             "Pointer storage class must be Workgroup, StorageBuffer or "
             "PhysicalStorageBufferEXT but provided ")
         << stringifyStorageClass(storage);
return success();
4019}

4021LogicalResult spirv::NVCooperativeMatrixLoadOp::verify() {
return verifyPointerAndCoopMatrixType(*this, getPointer().getType(),
                                      getResult().getType());
4024}

4026//===----------------------------------------------------------------------===//
4027// spirv.NV.CooperativeMatrixStore
4028//===----------------------------------------------------------------------===//

4030ParseResult spirv::NVCooperativeMatrixStoreOp::parse(OpAsmParser &parser,
                                                   OperationState &result) {
SmallVector<OpAsmParser::UnresolvedOperand, 4> operandInfo;
Type strideType = parser.getBuilder().getIntegerType(32);
Type columnMajorType = parser.getBuilder().getIntegerType(1);
Type ptrType;
Type elementType;
if (parser.parseOperandList(operandInfo, 4) ||
    parseMemoryAccessAttributes(parser, result) || parser.parseColon() ||
    parser.parseType(ptrType) || parser.parseComma() ||
    parser.parseType(elementType)) {
  return failure();
}
if (parser.resolveOperands(
        operandInfo, {ptrType, elementType, strideType, columnMajorType},
        parser.getNameLoc(), result.operands)) {
  return failure();
}

return success();
4050}

4052void spirv::NVCooperativeMatrixStoreOp::print(OpAsmPrinter &printer) {
printer << " " << getPointer() << ", " << getObject() << ", " << getStride()
        << ", " << getColumnmajor();
// Print optional memory access attribute.
if (auto memAccess = getMemoryAccess())
  printer << " [\"" << stringifyMemoryAccess(*memAccess) << "\"]";
printer << " : " << getPointer().getType() << ", " << getOperand(1).getType();
4059}

4061LogicalResult spirv::NVCooperativeMatrixStoreOp::verify() {
return verifyPointerAndCoopMatrixType(*this, getPointer().getType(),
                                      getObject().getType());
4064}

4066//===----------------------------------------------------------------------===//
4067// spirv.NV.CooperativeMatrixMulAdd
4068//===----------------------------------------------------------------------===//

4070static LogicalResult
4071verifyCoopMatrixMulAdd(spirv::NVCooperativeMatrixMulAddOp op) {
if (op.getC().getType() != op.getResult().getType())
  return op.emitOpError("result and third operand must have the same type");
auto typeA = op.getA().getType().cast<spirv::CooperativeMatrixNVType>();
auto typeB = op.getB().getType().cast<spirv::CooperativeMatrixNVType>();
auto typeC = op.getC().getType().cast<spirv::CooperativeMatrixNVType>();
auto typeR = op.getResult().getType().cast<spirv::CooperativeMatrixNVType>();
if (typeA.getRows() != typeR.getRows() ||
    typeA.getColumns() != typeB.getRows() ||
    typeB.getColumns() != typeR.getColumns())
  return op.emitOpError("matrix size must match");
if (typeR.getScope() != typeA.getScope() ||
    typeR.getScope() != typeB.getScope() ||
    typeR.getScope() != typeC.getScope())
  return op.emitOpError("matrix scope must match");
auto elementTypeA = typeA.getElementType();
auto elementTypeB = typeB.getElementType();
if (isa<IntegerType>(elementTypeA) && isa<IntegerType>(elementTypeB)) {
  if (elementTypeA.cast<IntegerType>().getWidth() !=
      elementTypeB.cast<IntegerType>().getWidth())
    return op.emitOpError(
        "matrix A and B integer element types must be the same bit width");
} else if (elementTypeA != elementTypeB) {
  return op.emitOpError(
      "matrix A and B non-integer element types must match");
}
if (typeR.getElementType() != typeC.getElementType())
  return op.emitOpError("matrix accumulator element type must match");
return success();
4100}

4102LogicalResult spirv::NVCooperativeMatrixMulAddOp::verify() {
return verifyCoopMatrixMulAdd(*this);
4104}

4106static LogicalResult
4107verifyPointerAndJointMatrixType(Operation *op, Type pointer, Type jointMatrix) {
Type pointeeType = pointer.cast<spirv::PointerType>().getPointeeType();
if (!pointeeType.isa<spirv::ScalarType>() && !pointeeType.isa<VectorType>())
  return op->emitError(
             "Pointer must point to a scalar or vector type but provided ")
         << pointeeType;
spirv::StorageClass storage =
    pointer.cast<spirv::PointerType>().getStorageClass();
if (storage != spirv::StorageClass::Workgroup &&
    storage != spirv::StorageClass::CrossWorkgroup &&
    storage != spirv::StorageClass::UniformConstant &&
    storage != spirv::StorageClass::Generic)
  return op->emitError("Pointer storage class must be Workgroup or "
                       "CrossWorkgroup but provided ")
         << stringifyStorageClass(storage);
return success();
4123}

4125//===----------------------------------------------------------------------===//
4126// spirv.INTEL.JointMatrixLoad
4127//===----------------------------------------------------------------------===//

4129LogicalResult spirv::INTELJointMatrixLoadOp::verify() {
return verifyPointerAndJointMatrixType(*this, getPointer().getType(),
                                       getResult().getType());
4132}

4134//===----------------------------------------------------------------------===//
4135// spirv.INTEL.JointMatrixStore
4136//===----------------------------------------------------------------------===//

4138LogicalResult spirv::INTELJointMatrixStoreOp::verify() {
return verifyPointerAndJointMatrixType(*this, getPointer().getType(),
                                       getObject().getType());
4141}

4143//===----------------------------------------------------------------------===//
4144// spirv.INTEL.JointMatrixMad
4145//===----------------------------------------------------------------------===//

4147static LogicalResult verifyJointMatrixMad(spirv::INTELJointMatrixMadOp op) {
if (op.getC().getType() != op.getResult().getType())
  return op.emitOpError("result and third operand must have the same type");
auto typeA = op.getA().getType().cast<spirv::JointMatrixINTELType>();
auto typeB = op.getB().getType().cast<spirv::JointMatrixINTELType>();
auto typeC = op.getC().getType().cast<spirv::JointMatrixINTELType>();
auto typeR = op.getResult().getType().cast<spirv::JointMatrixINTELType>();
if (typeA.getRows() != typeR.getRows() ||
    typeA.getColumns() != typeB.getRows() ||
    typeB.getColumns() != typeR.getColumns())
  return op.emitOpError("matrix size must match");
if (typeR.getScope() != typeA.getScope() ||
    typeR.getScope() != typeB.getScope() ||
    typeR.getScope() != typeC.getScope())
  return op.emitOpError("matrix scope must match");
if (typeA.getElementType() != typeB.getElementType() ||
    typeR.getElementType() != typeC.getElementType())
  return op.emitOpError("matrix element type must match");
return success();
4166}

4168LogicalResult spirv::INTELJointMatrixMadOp::verify() {
return verifyJointMatrixMad(*this);
4170}

4172//===----------------------------------------------------------------------===//
4173// spirv.MatrixTimesScalar
4174//===----------------------------------------------------------------------===//

4176LogicalResult spirv::MatrixTimesScalarOp::verify() {
if (auto inputCoopmat =
        getMatrix().getType().dyn_cast<spirv::CooperativeMatrixNVType>()) {
  if (inputCoopmat.getElementType() != getScalar().getType())
    return emitError("input matrix components' type and scaling value must "
                     "have the same type");
  return success();
}

// Check that the scalar type is the same as the matrix element type.
auto inputMatrix = getMatrix().getType().cast<spirv::MatrixType>();
if (getScalar().getType() != inputMatrix.getElementType())
  return emitError("input matrix components' type and scaling value must "
                   "have the same type");

return success();
4192}

4194//===----------------------------------------------------------------------===//
4195// spirv.CopyMemory
4196//===----------------------------------------------------------------------===//

4198void spirv::CopyMemoryOp::print(OpAsmPrinter &printer) {
printer << ' ';

StringRef targetStorageClass = stringifyStorageClass(
    getTarget().getType().cast<spirv::PointerType>().getStorageClass());
printer << " \"" << targetStorageClass << "\" " << getTarget() << ", ";

StringRef sourceStorageClass = stringifyStorageClass(
    getSource().getType().cast<spirv::PointerType>().getStorageClass());
printer << " \"" << sourceStorageClass << "\" " << getSource();

SmallVector<StringRef, 4> elidedAttrs;
printMemoryAccessAttribute(*this, printer, elidedAttrs);
printSourceMemoryAccessAttribute(*this, printer, elidedAttrs,
                                 getSourceMemoryAccess(),
                                 getSourceAlignment());

printer.printOptionalAttrDict((*this)->getAttrs(), elidedAttrs);

Type pointeeType =
    getTarget().getType().cast<spirv::PointerType>().getPointeeType();
printer << " : " << pointeeType;
4220}

4222ParseResult spirv::CopyMemoryOp::parse(OpAsmParser &parser,
                                     OperationState &result) {
spirv::StorageClass targetStorageClass;
OpAsmParser::UnresolvedOperand targetPtrInfo;

spirv::StorageClass sourceStorageClass;
OpAsmParser::UnresolvedOperand sourcePtrInfo;

Type elementType;

if (parseEnumStrAttr(targetStorageClass, parser) ||
    parser.parseOperand(targetPtrInfo) || parser.parseComma() ||
    parseEnumStrAttr(sourceStorageClass, parser) ||
    parser.parseOperand(sourcePtrInfo) ||
    parseMemoryAccessAttributes(parser, result)) {
  return failure();
}

if (!parser.parseOptionalComma()) {
  // Parse 2nd memory access attributes.
  if (parseSourceMemoryAccessAttributes(parser, result)) {
    return failure();
  }
}

if (parser.parseColon() || parser.parseType(elementType))
  return failure();

if (parser.parseOptionalAttrDict(result.attributes))
  return failure();

auto targetPtrType = spirv::PointerType::get(elementType, targetStorageClass);
auto sourcePtrType = spirv::PointerType::get(elementType, sourceStorageClass);

if (parser.resolveOperand(targetPtrInfo, targetPtrType, result.operands) ||
    parser.resolveOperand(sourcePtrInfo, sourcePtrType, result.operands)) {
  return failure();
}

return success();
4262}

4264LogicalResult spirv::CopyMemoryOp::verify() {
Type targetType =
    getTarget().getType().cast<spirv::PointerType>().getPointeeType();

Type sourceType =
    getSource().getType().cast<spirv::PointerType>().getPointeeType();

if (targetType != sourceType)
  return emitOpError("both operands must be pointers to the same type");

if (failed(verifyMemoryAccessAttribute(*this)))
  return failure();

// TODO - According to the spec:
//
// If two masks are present, the first applies to Target and cannot include
// MakePointerVisible, and the second applies to Source and cannot include
// MakePointerAvailable.
//
// Add such verification here.

return verifySourceMemoryAccessAttribute(*this);
4286}

4288//===----------------------------------------------------------------------===//
4289// spirv.Transpose
4290//===----------------------------------------------------------------------===//

4292LogicalResult spirv::TransposeOp::verify() {
auto inputMatrix = getMatrix().getType().cast<spirv::MatrixType>();
auto resultMatrix = getResult().getType().cast<spirv::MatrixType>();

// Verify that the input and output matrices have correct shapes.
if (inputMatrix.getNumRows() != resultMatrix.getNumColumns())
  return emitError("input matrix rows count must be equal to "
                   "output matrix columns count");

if (inputMatrix.getNumColumns() != resultMatrix.getNumRows())
  return emitError("input matrix columns count must be equal to "
                   "output matrix rows count");

// Verify that the input and output matrices have the same component type
if (inputMatrix.getElementType() != resultMatrix.getElementType())
  return emitError("input and output matrices must have the same "
                   "component type");

return success();
4311}

4313//===----------------------------------------------------------------------===//
4314// spirv.MatrixTimesMatrix
4315//===----------------------------------------------------------------------===//

4317LogicalResult spirv::MatrixTimesMatrixOp::verify() {
auto leftMatrix = getLeftmatrix().getType().cast<spirv::MatrixType>();
auto rightMatrix = getRightmatrix().getType().cast<spirv::MatrixType>();
auto resultMatrix = getResult().getType().cast<spirv::MatrixType>();

// left matrix columns' count and right matrix rows' count must be equal
if (leftMatrix.getNumColumns() != rightMatrix.getNumRows())
  return emitError("left matrix columns' count must be equal to "
                   "the right matrix rows' count");

// right and result matrices columns' count must be the same
if (rightMatrix.getNumColumns() != resultMatrix.getNumColumns())
  return emitError(
      "right and result matrices must have equal columns' count");

// right and result matrices component type must be the same
if (rightMatrix.getElementType() != resultMatrix.getElementType())
  return emitError("right and result matrices' component type must"
                   " be the same");

// left and result matrices component type must be the same
if (leftMatrix.getElementType() != resultMatrix.getElementType())
  return emitError("left and result matrices' component type"
                   " must be the same");

// left and result matrices rows count must be the same
if (leftMatrix.getNumRows() != resultMatrix.getNumRows())
  return emitError("left and result matrices must have equal rows' count");

return success();
4347}

4349//===----------------------------------------------------------------------===//
4350// spirv.SpecConstantComposite
4351//===----------------------------------------------------------------------===//

4353ParseResult spirv::SpecConstantCompositeOp::parse(OpAsmParser &parser,
                                                OperationState &result) {

StringAttr compositeName;
if (parser.parseSymbolName(compositeName, SymbolTable::getSymbolAttrName(),
                           result.attributes))
  return failure();

if (parser.parseLParen())
  return failure();

SmallVector<Attribute, 4> constituents;

do {
  // The name of the constituent attribute isn't important
  const char *attrName = "spec_const";
  FlatSymbolRefAttr specConstRef;
  NamedAttrList attrs;

  if (parser.parseAttribute(specConstRef, Type(), attrName, attrs))
    return failure();

  constituents.push_back(specConstRef);
} while (!parser.parseOptionalComma());

if (parser.parseRParen())
  return failure();

result.addAttribute(kCompositeSpecConstituentsName,
                    parser.getBuilder().getArrayAttr(constituents));

Type type;
if (parser.parseColonType(type))
  return failure();

result.addAttribute(kTypeAttrName, TypeAttr::get(type));

return success();
4391}

4393void spirv::SpecConstantCompositeOp::print(OpAsmPrinter &printer) {
printer << " ";
printer.printSymbolName(getSymName());
printer << " (";
auto constituents = this->getConstituents().getValue();

if (!constituents.empty())
  llvm::interleaveComma(constituents, printer);

printer << ") : " << getType();
4403}

4405LogicalResult spirv::SpecConstantCompositeOp::verify() {
auto cType = getType().dyn_cast<spirv::CompositeType>();
auto constituents = this->getConstituents().getValue();

if (!cType)
  return emitError("result type must be a composite type, but provided ")
         << getType();

if (cType.isa<spirv::CooperativeMatrixNVType>())
  return emitError("unsupported composite type  ") << cType;
if (cType.isa<spirv::JointMatrixINTELType>())
  return emitError("unsupported composite type  ") << cType;
if (constituents.size() != cType.getNumElements())
  return emitError("has incorrect number of operands: expected ")
         << cType.getNumElements() << ", but provided "
         << constituents.size();

for (auto index : llvm::seq<uint32_t>(0, constituents.size())) {
  auto constituent = constituents[index].cast<FlatSymbolRefAttr>();

  auto constituentSpecConstOp =
      dyn_cast<spirv::SpecConstantOp>(SymbolTable::lookupNearestSymbolFrom(
          (*this)->getParentOp(), constituent.getAttr()));

  if (constituentSpecConstOp.getDefaultValue().getType() !=
      cType.getElementType(index))
    return emitError("has incorrect types of operands: expected ")
           << cType.getElementType(index) << ", but provided "
           << constituentSpecConstOp.getDefaultValue().getType();
}

return success();
4437}

4439//===----------------------------------------------------------------------===//
4440// spirv.SpecConstantOperation
4441//===----------------------------------------------------------------------===//

4443ParseResult spirv::SpecConstantOperationOp::parse(OpAsmParser &parser,
                                                OperationState &result) {
Region *body = result.addRegion();

if (parser.parseKeyword("wraps"))
  return failure();

body->push_back(new Block);
Block &block = body->back();
Operation *wrappedOp = parser.parseGenericOperation(&block, block.begin());

if (!wrappedOp)
  return failure();

OpBuilder builder(parser.getContext());
builder.setInsertionPointToEnd(&block);
builder.create<spirv::YieldOp>(wrappedOp->getLoc(), wrappedOp->getResult(0));
result.location = wrappedOp->getLoc();

result.addTypes(wrappedOp->getResult(0).getType());

if (parser.parseOptionalAttrDict(result.attributes))
  return failure();

return success();
4468}

4470void spirv::SpecConstantOperationOp::print(OpAsmPrinter &printer) {
printer << " wraps ";
printer.printGenericOp(&getBody().front().front());
4473}

4475LogicalResult spirv::SpecConstantOperationOp::verifyRegions() {
Block &block = getRegion().getBlocks().front();

if (block.getOperations().size() != 2)
  return emitOpError("expected exactly 2 nested ops");

Operation &enclosedOp = block.getOperations().front();

if (!enclosedOp.hasTrait<OpTrait::spirv::UsableInSpecConstantOp>())
  return emitOpError("invalid enclosed op");

for (auto operand : enclosedOp.getOperands())
  if (!isa<spirv::ConstantOp, spirv::ReferenceOfOp,
           spirv::SpecConstantOperationOp>(operand.getDefiningOp()))
    return emitOpError(
        "invalid operand, must be defined by a constant operation");

return success();
4493}

4495//===----------------------------------------------------------------------===//
4496// spirv.GL.FrexpStruct
4497//===----------------------------------------------------------------------===//

4499LogicalResult spirv::GLFrexpStructOp::verify() {
spirv::StructType structTy =
    getResult().getType().dyn_cast<spirv::StructType>();

if (structTy.getNumElements() != 2)
  return emitError("result type must be a struct type with two memebers");

Type significandTy = structTy.getElementType(0);
Type exponentTy = structTy.getElementType(1);
VectorType exponentVecTy = exponentTy.dyn_cast<VectorType>();
IntegerType exponentIntTy = exponentTy.dyn_cast<IntegerType>();

Type operandTy = getOperand().getType();
VectorType operandVecTy = operandTy.dyn_cast<VectorType>();
FloatType operandFTy = operandTy.dyn_cast<FloatType>();

if (significandTy != operandTy)
  return emitError("member zero of the resulting struct type must be the "
                   "same type as the operand");

if (exponentVecTy) {
  IntegerType componentIntTy =
      exponentVecTy.getElementType().dyn_cast<IntegerType>();
  if (!componentIntTy || componentIntTy.getWidth() != 32)
    return emitError("member one of the resulting struct type must"
                     "be a scalar or vector of 32 bit integer type");
} else if (!exponentIntTy || exponentIntTy.getWidth() != 32) {
  return emitError("member one of the resulting struct type "
                   "must be a scalar or vector of 32 bit integer type");
}

// Check that the two member types have the same number of components
if (operandVecTy && exponentVecTy &&
    (exponentVecTy.getNumElements() == operandVecTy.getNumElements()))
  return success();

if (operandFTy && exponentIntTy)
  return success();

return emitError("member one of the resulting struct type must have the same "
                 "number of components as the operand type");
4540}

4542//===----------------------------------------------------------------------===//
4543// spirv.GL.Ldexp
4544//===----------------------------------------------------------------------===//

4546LogicalResult spirv::GLLdexpOp::verify() {
Type significandType = getX().getType();
Type exponentType = getExp().getType();

if (significandType.isa<FloatType>() != exponentType.isa<IntegerType>())
  return emitOpError("operands must both be scalars or vectors");

auto getNumElements = [](Type type) -> unsigned {
  if (auto vectorType = type.dyn_cast<VectorType>())
    return vectorType.getNumElements();
  return 1;
};

if (getNumElements(significandType) != getNumElements(exponentType))
  return emitOpError("operands must have the same number of elements");

return success();
4563}

4565//===----------------------------------------------------------------------===//
4566// spirv.ImageDrefGather
4567//===----------------------------------------------------------------------===//

4569LogicalResult spirv::ImageDrefGatherOp::verify() {
VectorType resultType = getResult().getType().cast<VectorType>();
auto sampledImageType =
    getSampledimage().getType().cast<spirv::SampledImageType>();
auto imageType = sampledImageType.getImageType().cast<spirv::ImageType>();

if (resultType.getNumElements() != 4)
  return emitOpError("result type must be a vector of four components");

Type elementType = resultType.getElementType();
Type sampledElementType = imageType.getElementType();
if (!sampledElementType.isa<NoneType>() && elementType != sampledElementType)
  return emitOpError(
      "the component type of result must be the same as sampled type of the "
      "underlying image type");

spirv::Dim imageDim = imageType.getDim();
spirv::ImageSamplingInfo imageMS = imageType.getSamplingInfo();

if (imageDim != spirv::Dim::Dim2D && imageDim != spirv::Dim::Cube &&
    imageDim != spirv::Dim::Rect)
  return emitOpError(
      "the Dim operand of the underlying image type must be 2D, Cube, or "
      "Rect");

if (imageMS != spirv::ImageSamplingInfo::SingleSampled)
  return emitOpError("the MS operand of the underlying image type must be 0");

spirv::ImageOperandsAttr attr = getImageoperandsAttr();
auto operandArguments = getOperandArguments();

return verifyImageOperands(*this, attr, operandArguments);
4601}

4603//===----------------------------------------------------------------------===//
4604// spirv.ShiftLeftLogicalOp
4605//===----------------------------------------------------------------------===//

4607LogicalResult spirv::ShiftLeftLogicalOp::verify() {
return verifyShiftOp(*this);
4609}

4611//===----------------------------------------------------------------------===//
4612// spirv.ShiftRightArithmeticOp
4613//===----------------------------------------------------------------------===//

4615LogicalResult spirv::ShiftRightArithmeticOp::verify() {
return verifyShiftOp(*this);
4617}

4619//===----------------------------------------------------------------------===//
4620// spirv.ShiftRightLogicalOp
4621//===----------------------------------------------------------------------===//

4623LogicalResult spirv::ShiftRightLogicalOp::verify() {
return verifyShiftOp(*this);
4625}

4627//===----------------------------------------------------------------------===//
4628// spirv.ImageQuerySize
4629//===----------------------------------------------------------------------===//

4631LogicalResult spirv::ImageQuerySizeOp::verify() {
spirv::ImageType imageType = getImage().getType().cast<spirv::ImageType>();
Type resultType = getResult().getType();

spirv::Dim dim = imageType.getDim();
spirv::ImageSamplingInfo samplingInfo = imageType.getSamplingInfo();
spirv::ImageSamplerUseInfo samplerInfo = imageType.getSamplerUseInfo();
switch (dim) {
case spirv::Dim::Dim1D:
case spirv::Dim::Dim2D:
case spirv::Dim::Dim3D:
case spirv::Dim::Cube:
  if (samplingInfo != spirv::ImageSamplingInfo::MultiSampled &&
      samplerInfo != spirv::ImageSamplerUseInfo::SamplerUnknown &&
      samplerInfo != spirv::ImageSamplerUseInfo::NoSampler)
    return emitError(
        "if Dim is 1D, 2D, 3D, or Cube, "
        "it must also have either an MS of 1 or a Sampled of 0 or 2");
  break;
case spirv::Dim::Buffer:
case spirv::Dim::Rect:
  break;
default:
  return emitError("the Dim operand of the image type must "
                   "be 1D, 2D, 3D, Buffer, Cube, or Rect");
}

unsigned componentNumber = 0;
switch (dim) {
case spirv::Dim::Dim1D:
case spirv::Dim::Buffer:
  componentNumber = 1;
  break;
case spirv::Dim::Dim2D:
case spirv::Dim::Cube:
case spirv::Dim::Rect:
  componentNumber = 2;
  break;
case spirv::Dim::Dim3D:
  componentNumber = 3;
  break;
default:
  break;
}

if (imageType.getArrayedInfo() == spirv::ImageArrayedInfo::Arrayed)
  componentNumber += 1;

unsigned resultComponentNumber = 1;
if (auto resultVectorType = resultType.dyn_cast<VectorType>())
  resultComponentNumber = resultVectorType.getNumElements();

if (componentNumber != resultComponentNumber)
  return emitError("expected the result to have ")
         << componentNumber << " component(s), but found "
         << resultComponentNumber << " component(s)";

return success();
4689}

4691static ParseResult parsePtrAccessChainOpImpl(StringRef opName,
                                           OpAsmParser &parser,
                                           OperationState &state) {
OpAsmParser::UnresolvedOperand ptrInfo;
SmallVector<OpAsmParser::UnresolvedOperand, 4> indicesInfo;
Type type;
auto loc = parser.getCurrentLocation();
SmallVector<Type, 4> indicesTypes;

if (parser.parseOperand(ptrInfo) ||
    parser.parseOperandList(indicesInfo, OpAsmParser::Delimiter::Square) ||
    parser.parseColonType(type) ||
    parser.resolveOperand(ptrInfo, type, state.operands))
  return failure();

// Check that the provided indices list is not empty before parsing their
// type list.
if (indicesInfo.empty())
  return emitError(state.location) << opName << " expected element";

if (parser.parseComma() || parser.parseTypeList(indicesTypes))
  return failure();

// Check that the indices types list is not empty and that it has a one-to-one
// mapping to the provided indices.
if (indicesTypes.size() != indicesInfo.size())
  return emitError(state.location)
         << opName
         << " indices types' count must be equal to indices info count";

if (parser.resolveOperands(indicesInfo, indicesTypes, loc, state.operands))
  return failure();

auto resultType = getElementPtrType(
    type, llvm::ArrayRef(state.operands).drop_front(2), state.location);
if (!resultType)
  return failure();

state.addTypes(resultType);
return success();
4731}

4733template <typename Op>
4734static auto concatElemAndIndices(Op op) {
SmallVector<Value> ret(op.getIndices().size() + 1);
ret[0] = op.getElement();
llvm::copy(op.getIndices(), ret.begin() + 1);
return ret;
4739}

4741//===----------------------------------------------------------------------===//
4742// spirv.InBoundsPtrAccessChainOp
4743//===----------------------------------------------------------------------===//

4745void spirv::InBoundsPtrAccessChainOp::build(OpBuilder &builder,
                                          OperationState &state,
                                          Value basePtr, Value element,
                                          ValueRange indices) {
auto type = getElementPtrType(basePtr.getType(), indices, state.location);
assert(type && "Unable to deduce return type based on basePtr and indices")(static_cast <bool> (type && "Unable to deduce return type based on basePtr and indices"
) ? void (0) : __assert_fail ("type && \"Unable to deduce return type based on basePtr and indices\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 4750, __extension__
 __PRETTY_FUNCTION__));
build(builder, state, type, basePtr, element, indices);
4752}

4754ParseResult spirv::InBoundsPtrAccessChainOp::parse(OpAsmParser &parser,
                                                 OperationState &result) {
return parsePtrAccessChainOpImpl(
    spirv::InBoundsPtrAccessChainOp::getOperationName(), parser, result);
4758}

4760void spirv::InBoundsPtrAccessChainOp::print(OpAsmPrinter &printer) {
printAccessChain(*this, concatElemAndIndices(*this), printer);
4762}

4764LogicalResult spirv::InBoundsPtrAccessChainOp::verify() {
return verifyAccessChain(*this, getIndices());
4766}

4768//===----------------------------------------------------------------------===//
4769// spirv.PtrAccessChainOp
4770//===----------------------------------------------------------------------===//

4772void spirv::PtrAccessChainOp::build(OpBuilder &builder, OperationState &state,
                                  Value basePtr, Value element,
                                  ValueRange indices) {
auto type = getElementPtrType(basePtr.getType(), indices, state.location);
assert(type && "Unable to deduce return type based on basePtr and indices")(static_cast <bool> (type && "Unable to deduce return type based on basePtr and indices"
) ? void (0) : __assert_fail ("type && \"Unable to deduce return type based on basePtr and indices\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 4776, __extension__
 __PRETTY_FUNCTION__));
build(builder, state, type, basePtr, element, indices);
4778}

4780ParseResult spirv::PtrAccessChainOp::parse(OpAsmParser &parser,
                                         OperationState &result) {
return parsePtrAccessChainOpImpl(spirv::PtrAccessChainOp::getOperationName(),
                                 parser, result);
4784}

4786void spirv::PtrAccessChainOp::print(OpAsmPrinter &printer) {
printAccessChain(*this, concatElemAndIndices(*this), printer);
4788}

4790LogicalResult spirv::PtrAccessChainOp::verify() {
return verifyAccessChain(*this, getIndices());
4792}

4794//===----------------------------------------------------------------------===//
4795// spirv.VectorTimesScalarOp
4796//===----------------------------------------------------------------------===//

4798LogicalResult spirv::VectorTimesScalarOp::verify() {
if (getVector().getType() != getType())
  return emitOpError("vector operand and result type mismatch");
auto scalarType = getType().cast<VectorType>().getElementType();
if (getScalar().getType() != scalarType)
  return emitOpError("scalar operand and result element type match");
return success();
4805}

4807//===----------------------------------------------------------------------===//
4808// Group ops
4809//===----------------------------------------------------------------------===//

4811template <typename Op>
4812static LogicalResult verifyGroupOp(Op op) {
spirv::Scope scope = op.getExecutionScope();
if (scope != spirv::Scope::Workgroup && scope != spirv::Scope::Subgroup)
  return op.emitOpError("execution scope must be 'Workgroup' or 'Subgroup'");

return success();
4818}

4820LogicalResult spirv::GroupIAddOp::verify() { return verifyGroupOp(*this); }

4822LogicalResult spirv::GroupFAddOp::verify() { return verifyGroupOp(*this); }

4824LogicalResult spirv::GroupFMinOp::verify() { return verifyGroupOp(*this); }

4826LogicalResult spirv::GroupUMinOp::verify() { return verifyGroupOp(*this); }

4828LogicalResult spirv::GroupSMinOp::verify() { return verifyGroupOp(*this); }

4830LogicalResult spirv::GroupFMaxOp::verify() { return verifyGroupOp(*this); }

4832LogicalResult spirv::GroupUMaxOp::verify() { return verifyGroupOp(*this); }

4834LogicalResult spirv::GroupSMaxOp::verify() { return verifyGroupOp(*this); }

4836LogicalResult spirv::GroupIMulKHROp::verify() { return verifyGroupOp(*this); }

4838LogicalResult spirv::GroupFMulKHROp::verify() { return verifyGroupOp(*this); }

4840//===----------------------------------------------------------------------===//
4841// Integer Dot Product ops
4842//===----------------------------------------------------------------------===//

4844static LogicalResult verifyIntegerDotProduct(Operation *op) {
assert(llvm::is_contained({2u, 3u}, op->getNumOperands()) &&(static_cast <bool> (llvm::is_contained({2u, 3u}, op->
getNumOperands()) && "Not an integer dot product op?"
) ? void (0) : __assert_fail ("llvm::is_contained({2u, 3u}, op->getNumOperands()) && \"Not an integer dot product op?\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 4846, __extension__
 __PRETTY_FUNCTION__))
       "Not an integer dot product op?")(static_cast <bool> (llvm::is_contained({2u, 3u}, op->
getNumOperands()) && "Not an integer dot product op?"
) ? void (0) : __assert_fail ("llvm::is_contained({2u, 3u}, op->getNumOperands()) && \"Not an integer dot product op?\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 4846, __extension__
 __PRETTY_FUNCTION__));
assert(op->getNumResults() == 1 && "Expected a single result")(static_cast <bool> (op->getNumResults() == 1 &&
 "Expected a single result") ? void (0) : __assert_fail ("op->getNumResults() == 1 && \"Expected a single result\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 4847, __extension__
 __PRETTY_FUNCTION__));

Type factorTy = op->getOperand(0).getType();
if (op->getOperand(1).getType() != factorTy)
  return op->emitOpError("requires the same type for both vector operands");

unsigned expectedNumAttrs = 0;
if (auto intTy = factorTy.dyn_cast<IntegerType>()) {
  ++expectedNumAttrs;
  auto packedVectorFormat =
      op->getAttr(kPackedVectorFormatAttrName)
          .dyn_cast_or_null<spirv::PackedVectorFormatAttr>();
  if (!packedVectorFormat)
    return op->emitOpError("requires Packed Vector Format attribute for "
                           "integer vector operands");

  assert(packedVectorFormat.getValue() ==(static_cast <bool> (packedVectorFormat.getValue() == spirv
::PackedVectorFormat::PackedVectorFormat4x8Bit && "Unknown Packed Vector Format"
) ? void (0) : __assert_fail ("packedVectorFormat.getValue() == spirv::PackedVectorFormat::PackedVectorFormat4x8Bit && \"Unknown Packed Vector Format\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 4865, __extension__
 __PRETTY_FUNCTION__))
             spirv::PackedVectorFormat::PackedVectorFormat4x8Bit &&(static_cast <bool> (packedVectorFormat.getValue() == spirv
::PackedVectorFormat::PackedVectorFormat4x8Bit && "Unknown Packed Vector Format"
) ? void (0) : __assert_fail ("packedVectorFormat.getValue() == spirv::PackedVectorFormat::PackedVectorFormat4x8Bit && \"Unknown Packed Vector Format\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 4865, __extension__
 __PRETTY_FUNCTION__))
         "Unknown Packed Vector Format")(static_cast <bool> (packedVectorFormat.getValue() == spirv
::PackedVectorFormat::PackedVectorFormat4x8Bit && "Unknown Packed Vector Format"
) ? void (0) : __assert_fail ("packedVectorFormat.getValue() == spirv::PackedVectorFormat::PackedVectorFormat4x8Bit && \"Unknown Packed Vector Format\""
, "mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp", 4865, __extension__
 __PRETTY_FUNCTION__));
  if (intTy.getWidth() != 32)
    return op->emitOpError(
        llvm::formatv("with specified Packed Vector Format ({0}) requires "
                      "integer vector operands to be 32-bits wide",
                      packedVectorFormat.getValue()));
} else {
  if (op->hasAttr(kPackedVectorFormatAttrName))
    return op->emitOpError(llvm::formatv(
        "with invalid format attribute for vector operands of type '{0}'",
        factorTy));
}

if (op->getAttrs().size() > expectedNumAttrs)
  return op->emitError(
      "op only supports the 'format' #spirv.packed_vector_format attribute");

Type resultTy = op->getResultTypes().front();
bool hasAccumulator = op->getNumOperands() == 3;
if (hasAccumulator && op->getOperand(2).getType() != resultTy)
  return op->emitOpError(
      "requires the same accumulator operand and result types");

unsigned factorBitWidth = getBitWidth(factorTy);
unsigned resultBitWidth = getBitWidth(resultTy);
if (factorBitWidth > resultBitWidth)
  return op->emitOpError(
      llvm::formatv("result type has insufficient bit-width ({0} bits) "
                    "for the specified vector operand type ({1} bits)",
                    resultBitWidth, factorBitWidth));

return success();
4897}

4899static std::optional<spirv::Version> getIntegerDotProductMinVersion() {
return spirv::Version::V_1_0; // Available in SPIR-V >= 1.0.
4901}

4903static std::optional<spirv::Version> getIntegerDotProductMaxVersion() {
return spirv::Version::V_1_6; // Available in SPIR-V <= 1.6.
4905}

4907static SmallVector<ArrayRef<spirv::Extension>, 1>
4908getIntegerDotProductExtensions() {
// Requires the SPV_KHR_integer_dot_product extension, specified either
// explicitly or implied by target env's SPIR-V version >= 1.6.
static const auto extension = spirv::Extension::SPV_KHR_integer_dot_product;
return {extension};
4913}

4915static SmallVector<ArrayRef<spirv::Capability>, 1>
4916getIntegerDotProductCapabilities(Operation *op) {
// Requires the the DotProduct capability and capabilities that depend on
// exact op types.
static const auto dotProductCap = spirv::Capability::DotProduct;
static const auto dotProductInput4x8BitPackedCap =
    spirv::Capability::DotProductInput4x8BitPacked;
static const auto dotProductInput4x8BitCap =
    spirv::Capability::DotProductInput4x8Bit;
static const auto dotProductInputAllCap =
    spirv::Capability::DotProductInputAll;

SmallVector<ArrayRef<spirv::Capability>, 1> capabilities = {dotProductCap};

Type factorTy = op->getOperand(0).getType();
if (auto intTy = factorTy.dyn_cast<IntegerType>()) {
  auto formatAttr = op->getAttr(kPackedVectorFormatAttrName)
                        .cast<spirv::PackedVectorFormatAttr>();
  if (formatAttr.getValue() ==
      spirv::PackedVectorFormat::PackedVectorFormat4x8Bit)
    capabilities.push_back(dotProductInput4x8BitPackedCap);

  return capabilities;
}

auto vecTy = factorTy.cast<VectorType>();
if (vecTy.getElementTypeBitWidth() == 8) {
  capabilities.push_back(dotProductInput4x8BitCap);
  return capabilities;
}

capabilities.push_back(dotProductInputAllCap);
return capabilities;
4948}

4950#define SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(OpName)                              \
LogicalResult OpName::verify() { return verifyIntegerDotProduct(*this); }    \
SmallVector<ArrayRef<spirv::Extension>, 1> OpName::getExtensions() {         \
  return getIntegerDotProductExtensions();                                   \
}                                                                            \
SmallVector<ArrayRef<spirv::Capability>, 1> OpName::getCapabilities() {      \
  return getIntegerDotProductCapabilities(*this);                            \
}                                                                            \
std::optional<spirv::Version> OpName::getMinVersion() {                      \
  return getIntegerDotProductMinVersion();                                   \
}                                                                            \
std::optional<spirv::Version> OpName::getMaxVersion() {                      \
  return getIntegerDotProductMaxVersion();                                   \
}

4965SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(spirv::SDotOp)
4966SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(spirv::SUDotOp)
4967SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(spirv::UDotOp)
4968SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(spirv::SDotAccSatOp)
4969SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(spirv::SUDotAccSatOp)
4970SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(spirv::UDotAccSatOp)

4972#undef SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP

4974// TableGen'erated operation interfaces for querying versions, extensions, and
4975// capabilities.
4976#include "mlir/Dialect/SPIRV/IR/SPIRVAvailability.cpp.inc"

4978// TablenGen'erated operation definitions.
4979#define GET_OP_CLASSES
4980#include "mlir/Dialect/SPIRV/IR/SPIRVOps.cpp.inc"

4982namespace mlir {
4983namespace spirv {
4984// TableGen'erated operation availability interface implementations.
4985#include "mlir/Dialect/SPIRV/IR/SPIRVOpAvailabilityImpl.inc"
4986} // namespace spirv
4987} // namespace mlir

←

/build/source/mlir/include/mlir/IR/Builders.h

1//===- Builders.h - Helpers for constructing MLIR Classes -------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//

9#ifndef MLIR_IR_BUILDERS_H
10#define MLIR_IR_BUILDERS_H

12#include "mlir/IR/OpDefinition.h"
13#include "llvm/Support/Compiler.h"
14#include <optional>

16namespace mlir {

18class AffineExpr;
19class IRMapping;
20class UnknownLoc;
21class FileLineColLoc;
22class Type;
23class PrimitiveType;
24class IntegerType;
25class FloatType;
26class FunctionType;
27class IndexType;
28class MemRefType;
29class VectorType;
30class RankedTensorType;
31class UnrankedTensorType;
32class TupleType;
33class NoneType;
34class BoolAttr;
35class IntegerAttr;
36class FloatAttr;
37class StringAttr;
38class TypeAttr;
39class ArrayAttr;
40class SymbolRefAttr;
41class ElementsAttr;
42class DenseElementsAttr;
43class DenseIntElementsAttr;
44class AffineMapAttr;
45class AffineMap;
46class UnitAttr;

48/// This class is a general helper class for creating context-global objects
49/// like types, attributes, and affine expressions.
50class Builder {
51public:
explicit Builder(MLIRContext *context) : context(context) {}
explicit Builder(Operation *op) : Builder(op->getContext()) {}

MLIRContext *getContext() const { return context; }

// Locations.
Location getUnknownLoc();
Location getFusedLoc(ArrayRef<Location> locs,
                     Attribute metadata = Attribute());

// Types.
FloatType getFloat8E5M2Type();
FloatType getFloat8E4M3FNType();
FloatType getFloat8E5M2FNUZType();
FloatType getFloat8E4M3FNUZType();
FloatType getFloat8E4M3B11FNUZType();
FloatType getBF16Type();
FloatType getF16Type();
FloatType getF32Type();
FloatType getF64Type();
FloatType getF80Type();
FloatType getF128Type();

IndexType getIndexType();

IntegerType getI1Type();
IntegerType getI2Type();
IntegerType getI4Type();
IntegerType getI8Type();
IntegerType getI16Type();
IntegerType getI32Type();
IntegerType getI64Type();
IntegerType getIntegerType(unsigned width);
IntegerType getIntegerType(unsigned width, bool isSigned);
FunctionType getFunctionType(TypeRange inputs, TypeRange results);
TupleType getTupleType(TypeRange elementTypes);
NoneType getNoneType();

/// Get or construct an instance of the type `Ty` with provided arguments.
template <typename Ty, typename... Args>
Ty getType(Args &&...args) {
  return Ty::get(context, std::forward<Args>(args)...);
}

/// Get or construct an instance of the attribute `Attr` with provided
/// arguments.
template <typename Attr, typename... Args>
Attr getAttr(Args &&...args) {
  return Attr::get(context, std::forward<Args>(args)...);
9
←
Calling 'forward<mlir::spirv::ExecutionModel &>'→
10
←
Returning from 'forward<mlir::spirv::ExecutionModel &>'→
11
←
2nd function call argument is an uninitialized value
}

// Attributes.
NamedAttribute getNamedAttr(StringRef name, Attribute val);

UnitAttr getUnitAttr();
BoolAttr getBoolAttr(bool value);
DictionaryAttr getDictionaryAttr(ArrayRef<NamedAttribute> value);
IntegerAttr getIntegerAttr(Type type, int64_t value);
IntegerAttr getIntegerAttr(Type type, const APInt &value);
FloatAttr getFloatAttr(Type type, double value);
FloatAttr getFloatAttr(Type type, const APFloat &value);
StringAttr getStringAttr(const Twine &bytes);
ArrayAttr getArrayAttr(ArrayRef<Attribute> value);

// Returns a 0-valued attribute of the given `type`. This function only
// supports boolean, integer, and 16-/32-/64-bit float types, and vector or
// ranked tensor of them. Returns null attribute otherwise.
TypedAttr getZeroAttr(Type type);

// Convenience methods for fixed types.
FloatAttr getF16FloatAttr(float value);
FloatAttr getF32FloatAttr(float value);
FloatAttr getF64FloatAttr(double value);

IntegerAttr getI8IntegerAttr(int8_t value);
IntegerAttr getI16IntegerAttr(int16_t value);
IntegerAttr getI32IntegerAttr(int32_t value);
IntegerAttr getI64IntegerAttr(int64_t value);
IntegerAttr getIndexAttr(int64_t value);

/// Signed and unsigned integer attribute getters.
IntegerAttr getSI32IntegerAttr(int32_t value);
IntegerAttr getUI32IntegerAttr(uint32_t value);

/// Vector-typed DenseIntElementsAttr getters. `values` must not be empty.
DenseIntElementsAttr getBoolVectorAttr(ArrayRef<bool> values);
DenseIntElementsAttr getI32VectorAttr(ArrayRef<int32_t> values);
DenseIntElementsAttr getI64VectorAttr(ArrayRef<int64_t> values);
DenseIntElementsAttr getIndexVectorAttr(ArrayRef<int64_t> values);

/// Tensor-typed DenseIntElementsAttr getters. `values` can be empty.
/// These are generally preferable for representing general lists of integers
/// as attributes.
DenseIntElementsAttr getI32TensorAttr(ArrayRef<int32_t> values);
DenseIntElementsAttr getI64TensorAttr(ArrayRef<int64_t> values);
DenseIntElementsAttr getIndexTensorAttr(ArrayRef<int64_t> values);

/// Tensor-typed DenseArrayAttr getters.
DenseBoolArrayAttr getDenseBoolArrayAttr(ArrayRef<bool> values);
DenseI8ArrayAttr getDenseI8ArrayAttr(ArrayRef<int8_t> values);
DenseI16ArrayAttr getDenseI16ArrayAttr(ArrayRef<int16_t> values);
DenseI32ArrayAttr getDenseI32ArrayAttr(ArrayRef<int32_t> values);
DenseI64ArrayAttr getDenseI64ArrayAttr(ArrayRef<int64_t> values);
DenseF32ArrayAttr getDenseF32ArrayAttr(ArrayRef<float> values);
DenseF64ArrayAttr getDenseF64ArrayAttr(ArrayRef<double> values);

ArrayAttr getAffineMapArrayAttr(ArrayRef<AffineMap> values);
ArrayAttr getBoolArrayAttr(ArrayRef<bool> values);
ArrayAttr getI32ArrayAttr(ArrayRef<int32_t> values);
ArrayAttr getI64ArrayAttr(ArrayRef<int64_t> values);
ArrayAttr getIndexArrayAttr(ArrayRef<int64_t> values);
ArrayAttr getF32ArrayAttr(ArrayRef<float> values);
ArrayAttr getF64ArrayAttr(ArrayRef<double> values);
ArrayAttr getStrArrayAttr(ArrayRef<StringRef> values);
ArrayAttr getTypeArrayAttr(TypeRange values);

// Affine expressions and affine maps.
AffineExpr getAffineDimExpr(unsigned position);
AffineExpr getAffineSymbolExpr(unsigned position);
AffineExpr getAffineConstantExpr(int64_t constant);

// Special cases of affine maps and integer sets
/// Returns a zero result affine map with no dimensions or symbols: () -> ().
AffineMap getEmptyAffineMap();
/// Returns a single constant result affine map with 0 dimensions and 0
/// symbols.  One constant result: () -> (val).
AffineMap getConstantAffineMap(int64_t val);
// One dimension id identity map: (i) -> (i).
AffineMap getDimIdentityMap();
// Multi-dimensional identity map: (d0, d1, d2) -> (d0, d1, d2).
AffineMap getMultiDimIdentityMap(unsigned rank);
// One symbol identity map: ()[s] -> (s).
AffineMap getSymbolIdentityMap();

/// Returns a map that shifts its (single) input dimension by 'shift'.
/// (d0) -> (d0 + shift)
AffineMap getSingleDimShiftAffineMap(int64_t shift);

/// Returns an affine map that is a translation (shift) of all result
/// expressions in 'map' by 'shift'.
/// Eg: input: (d0, d1)[s0] -> (d0, d1 + s0), shift = 2
///   returns:    (d0, d1)[s0] -> (d0 + 2, d1 + s0 + 2)
AffineMap getShiftedAffineMap(AffineMap map, int64_t shift);

196protected:
MLIRContext *context;
198};

200/// This class helps build Operations. Operations that are created are
201/// automatically inserted at an insertion point. The builder is copyable.
202class OpBuilder : public Builder {
203public:
struct Listener;

/// Create a builder with the given context.
explicit OpBuilder(MLIRContext *ctx, Listener *listener = nullptr)
    : Builder(ctx), listener(listener) {}

/// Create a builder and set the insertion point to the start of the region.
explicit OpBuilder(Region *region, Listener *listener = nullptr)
    : OpBuilder(region->getContext(), listener) {
  if (!region->empty())
    setInsertionPoint(&region->front(), region->front().begin());
}
explicit OpBuilder(Region &region, Listener *listener = nullptr)
    : OpBuilder(&region, listener) {}

/// Create a builder and set insertion point to the given operation, which
/// will cause subsequent insertions to go right before it.
explicit OpBuilder(Operation *op, Listener *listener = nullptr)
    : OpBuilder(op->getContext(), listener) {
  setInsertionPoint(op);
}

OpBuilder(Block *block, Block::iterator insertPoint,
          Listener *listener = nullptr)
    : OpBuilder(block->getParent()->getContext(), listener) {
  setInsertionPoint(block, insertPoint);
}

/// Create a builder and set the insertion point to before the first operation
/// in the block but still inside the block.
static OpBuilder atBlockBegin(Block *block, Listener *listener = nullptr) {
  return OpBuilder(block, block->begin(), listener);
}

/// Create a builder and set the insertion point to after the last operation
/// in the block but still inside the block.
static OpBuilder atBlockEnd(Block *block, Listener *listener = nullptr) {
  return OpBuilder(block, block->end(), listener);
}

/// Create a builder and set the insertion point to before the block
/// terminator.
static OpBuilder atBlockTerminator(Block *block,
                                   Listener *listener = nullptr) {
  auto *terminator = block->getTerminator();
  assert(terminator != nullptr && "the block has no terminator")(static_cast <bool> (terminator != nullptr && "the block has no terminator"
) ? void (0) : __assert_fail ("terminator != nullptr && \"the block has no terminator\""
, "mlir/include/mlir/IR/Builders.h", 249, __extension__ __PRETTY_FUNCTION__
));
  return OpBuilder(block, Block::iterator(terminator), listener);
}

//===--------------------------------------------------------------------===//
// Listeners
//===--------------------------------------------------------------------===//

/// Base class for listeners.
struct ListenerBase {
  /// The kind of listener.
  enum class Kind {
    /// OpBuilder::Listener or user-derived class.
    OpBuilderListener = 0,

    /// RewriterBase::Listener or user-derived class.
    RewriterBaseListener = 1
  };

  Kind getKind() const { return kind; }

protected:
  ListenerBase(Kind kind) : kind(kind) {}

private:
  const Kind kind;
};

/// This class represents a listener that may be used to hook into various
/// actions within an OpBuilder.
struct Listener : public ListenerBase {
  Listener() : ListenerBase(ListenerBase::Kind::OpBuilderListener) {}

  virtual ~Listener() = default;

  /// Notification handler for when an operation is inserted into the builder.
  /// `op` is the operation that was inserted.
  virtual void notifyOperationInserted(Operation *op) {}

  /// Notification handler for when a block is created using the builder.
  /// `block` is the block that was created.
  virtual void notifyBlockCreated(Block *block) {}

protected:
  Listener(Kind kind) : ListenerBase(kind) {}
};

/// Sets the listener of this builder to the one provided.
void setListener(Listener *newListener) { listener = newListener; }

/// Returns the current listener of this builder, or nullptr if this builder
/// doesn't have a listener.
Listener *getListener() const { return listener; }

//===--------------------------------------------------------------------===//
// Insertion Point Management
//===--------------------------------------------------------------------===//

/// This class represents a saved insertion point.
class InsertPoint {
public:
  /// Creates a new insertion point which doesn't point to anything.
  InsertPoint() = default;

  /// Creates a new insertion point at the given location.
  InsertPoint(Block *insertBlock, Block::iterator insertPt)
      : block(insertBlock), point(insertPt) {}

  /// Returns true if this insert point is set.
  bool isSet() const { return (block != nullptr); }

  Block *getBlock() const { return block; }
  Block::iterator getPoint() const { return point; }

private:
  Block *block = nullptr;
  Block::iterator point;
};

/// RAII guard to reset the insertion point of the builder when destroyed.
class InsertionGuard {
public:
  InsertionGuard(OpBuilder &builder)
      : builder(&builder), ip(builder.saveInsertionPoint()) {}

  ~InsertionGuard() {
    if (builder)
      builder->restoreInsertionPoint(ip);
  }

  InsertionGuard(const InsertionGuard &) = delete;
  InsertionGuard &operator=(const InsertionGuard &) = delete;

  /// Implement the move constructor to clear the builder field of `other`.
  /// That way it does not restore the insertion point upon destruction as
  /// that should be done exclusively by the just constructed InsertionGuard.
  InsertionGuard(InsertionGuard &&other) noexcept
      : builder(other.builder), ip(other.ip) {
    other.builder = nullptr;
  }

  InsertionGuard &operator=(InsertionGuard &&other) = delete;

private:
  OpBuilder *builder;
  OpBuilder::InsertPoint ip;
};

/// Reset the insertion point to no location.  Creating an operation without a
/// set insertion point is an error, but this can still be useful when the
/// current insertion point a builder refers to is being removed.
void clearInsertionPoint() {
  this->block = nullptr;
  insertPoint = Block::iterator();
}

/// Return a saved insertion point.
InsertPoint saveInsertionPoint() const {
  return InsertPoint(getInsertionBlock(), getInsertionPoint());
}

/// Restore the insert point to a previously saved point.
void restoreInsertionPoint(InsertPoint ip) {
  if (ip.isSet())
    setInsertionPoint(ip.getBlock(), ip.getPoint());
  else
    clearInsertionPoint();
}

/// Set the insertion point to the specified location.
void setInsertionPoint(Block *block, Block::iterator insertPoint) {
  // TODO: check that insertPoint is in this rather than some other block.
  this->block = block;
  this->insertPoint = insertPoint;
}

/// Sets the insertion point to the specified operation, which will cause
/// subsequent insertions to go right before it.
void setInsertionPoint(Operation *op) {
  setInsertionPoint(op->getBlock(), Block::iterator(op));
}

/// Sets the insertion point to the node after the specified operation, which
/// will cause subsequent insertions to go right after it.
void setInsertionPointAfter(Operation *op) {
  setInsertionPoint(op->getBlock(), ++Block::iterator(op));
}

/// Sets the insertion point to the node after the specified value. If value
/// has a defining operation, sets the insertion point to the node after such
/// defining operation. This will cause subsequent insertions to go right
/// after it. Otherwise, value is a BlockArgument. Sets the insertion point to
/// the start of its block.
void setInsertionPointAfterValue(Value val) {
  if (Operation *op = val.getDefiningOp()) {
    setInsertionPointAfter(op);
  } else {
    auto blockArg = val.cast<BlockArgument>();
    setInsertionPointToStart(blockArg.getOwner());
  }
}

/// Sets the insertion point to the start of the specified block.
void setInsertionPointToStart(Block *block) {
  setInsertionPoint(block, block->begin());
}

/// Sets the insertion point to the end of the specified block.
void setInsertionPointToEnd(Block *block) {
  setInsertionPoint(block, block->end());
}

/// Return the block the current insertion point belongs to.  Note that the
/// insertion point is not necessarily the end of the block.
Block *getInsertionBlock() const { return block; }

/// Returns the current insertion point of the builder.
Block::iterator getInsertionPoint() const { return insertPoint; }

/// Returns the current block of the builder.
Block *getBlock() const { return block; }

//===--------------------------------------------------------------------===//
// Block Creation
//===--------------------------------------------------------------------===//

/// Add new block with 'argTypes' arguments and set the insertion point to the
/// end of it. The block is inserted at the provided insertion point of
/// 'parent'. `locs` contains the locations of the inserted arguments, and
/// should match the size of `argTypes`.
Block *createBlock(Region *parent, Region::iterator insertPt = {},
                   TypeRange argTypes = std::nullopt,
                   ArrayRef<Location> locs = std::nullopt);

/// Add new block with 'argTypes' arguments and set the insertion point to the
/// end of it. The block is placed before 'insertBefore'. `locs` contains the
/// locations of the inserted arguments, and should match the size of
/// `argTypes`.
Block *createBlock(Block *insertBefore, TypeRange argTypes = std::nullopt,
                   ArrayRef<Location> locs = std::nullopt);

//===--------------------------------------------------------------------===//
// Operation Creation
//===--------------------------------------------------------------------===//

/// Insert the given operation at the current insertion point and return it.
Operation *insert(Operation *op);

/// Creates an operation given the fields represented as an OperationState.
Operation *create(const OperationState &state);

/// Creates an operation with the given fields.
Operation *create(Location loc, StringAttr opName, ValueRange operands,
                  TypeRange types = {},
                  ArrayRef<NamedAttribute> attributes = {},
                  BlockRange successors = {},
                  MutableArrayRef<std::unique_ptr<Region>> regions = {});

467private:
/// Helper for sanity checking preconditions for create* methods below.
template <typename OpT>
RegisteredOperationName getCheckRegisteredInfo(MLIRContext *ctx) {
  std::optional<RegisteredOperationName> opName =
      RegisteredOperationName::lookup(OpT::getOperationName(), ctx);
  if (LLVM_UNLIKELY(!opName)__builtin_expect((bool)(!opName), false)) {
    llvm::report_fatal_error(
        "Building op `" + OpT::getOperationName() +
        "` but it isn't registered in this MLIRContext: the dialect may not "
        "be loaded or this operation isn't registered by the dialect. See "
        "also https://mlir.llvm.org/getting_started/Faq/"
        "#registered-loaded-dependent-whats-up-with-dialects-management");
  }
  return *opName;
}

484public:
/// Create an operation of specific op type at the current insertion point.
template <typename OpTy, typename... Args>
OpTy create(Location location, Args &&...args) {
  OperationState state(location,
                       getCheckRegisteredInfo<OpTy>(location.getContext()));
  OpTy::build(*this, state, std::forward<Args>(args)...);
  auto *op = create(state);
  auto result = dyn_cast<OpTy>(op);
  assert(result && "builder didn't return the right type")(static_cast <bool> (result && "builder didn't return the right type"
) ? void (0) : __assert_fail ("result && \"builder didn't return the right type\""
, "mlir/include/mlir/IR/Builders.h", 493, __extension__ __PRETTY_FUNCTION__
));
  return result;
}

/// Create an operation of specific op type at the current insertion point,
/// and immediately try to fold it. This functions populates 'results' with
/// the results after folding the operation.
template <typename OpTy, typename... Args>
void createOrFold(SmallVectorImpl<Value> &results, Location location,
                  Args &&...args) {
  // Create the operation without using 'create' as we don't want to
  // insert it yet.
  OperationState state(location,
                       getCheckRegisteredInfo<OpTy>(location.getContext()));
  OpTy::build(*this, state, std::forward<Args>(args)...);
  Operation *op = Operation::create(state);

  // Fold the operation. If successful destroy it, otherwise insert it.
  if (succeeded(tryFold(op, results)))
    op->destroy();
  else
    insert(op);
}

/// Overload to create or fold a single result operation.
template <typename OpTy, typename... Args>
std::enable_if_t<OpTy::template hasTrait<OpTrait::OneResult>(), Value>
createOrFold(Location location, Args &&...args) {
  SmallVector<Value, 1> results;
  createOrFold<OpTy>(results, location, std::forward<Args>(args)...);
  return results.front();
}

/// Overload to create or fold a zero result operation.
template <typename OpTy, typename... Args>
std::enable_if_t<OpTy::template hasTrait<OpTrait::ZeroResults>(), OpTy>
createOrFold(Location location, Args &&...args) {
  auto op = create<OpTy>(location, std::forward<Args>(args)...);
  SmallVector<Value, 0> unused;
  (void)tryFold(op.getOperation(), unused);

  // Folding cannot remove a zero-result operation, so for convenience we
  // continue to return it.
  return op;
}

/// Attempts to fold the given operation and places new results within
/// 'results'. Returns success if the operation was folded, failure otherwise.
/// Note: This function does not erase the operation on a successful fold.
LogicalResult tryFold(Operation *op, SmallVectorImpl<Value> &results);

/// Creates a deep copy of the specified operation, remapping any operands
/// that use values outside of the operation using the map that is provided
/// ( leaving them alone if no entry is present).  Replaces references to
/// cloned sub-operations to the corresponding operation that is copied,
/// and adds those mappings to the map.
Operation *clone(Operation &op, IRMapping &mapper);
Operation *clone(Operation &op);

/// Creates a deep copy of this operation but keep the operation regions
/// empty. Operands are remapped using `mapper` (if present), and `mapper` is
/// updated to contain the results.
Operation *cloneWithoutRegions(Operation &op, IRMapping &mapper) {
  return insert(op.cloneWithoutRegions(mapper));
}
Operation *cloneWithoutRegions(Operation &op) {
  return insert(op.cloneWithoutRegions());
}
template <typename OpT>
OpT cloneWithoutRegions(OpT op) {
  return cast<OpT>(cloneWithoutRegions(*op.getOperation()));
}

566protected:
/// The optional listener for events of this builder.
Listener *listener;

570private:
/// The current block this builder is inserting into.
Block *block = nullptr;
/// The insertion point within the block that this builder is inserting
/// before.
Block::iterator insertPoint;
576};

578} // namespace mlir

580#endif