/build/source/mlir/lib/Dialect/Vector/IR/VectorOps.cpp

Bug Summary

File:	build/source/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
Warning:	line 5547, column 23 1st function call argument is an uninitialized value

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name VectorOps.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-16/lib/clang/16 -I tools/mlir/lib/Dialect/Vector/IR -I /build/source/mlir/lib/Dialect/Vector/IR -I include -I /build/source/llvm/include -I /build/source/mlir/include -I tools/mlir/include -D MLIR_CUDA_CONVERSIONS_ENABLED=1 -D MLIR_ROCM_CONVERSIONS_ENABLED=1 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-16/lib/clang/16/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/source/= -fcoverage-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/source/= -source-date-epoch 1671487667 -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-12-20-010714-16201-1 -x c++ /build/source/mlir/lib/Dialect/Vector/IR/VectorOps.cpp

/build/source/mlir/lib/Dialect/Vector/IR/VectorOps.cpp

→

1//===- VectorOps.cpp - MLIR Vector Dialect Operations ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements convenience types for working with super-vectorization
10// operations, in particular super-vector loads and stores.
11//
12//===----------------------------------------------------------------------===//

14#include "mlir/Dialect/Vector/IR/VectorOps.h"

16#include "mlir/Dialect/Arith/IR/Arith.h"
17#include "mlir/Dialect/Arith/Utils/Utils.h"
18#include "mlir/Dialect/MemRef/IR/MemRef.h"
19#include "mlir/Dialect/Tensor/IR/Tensor.h"
20#include "mlir/Dialect/Utils/IndexingUtils.h"
21#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
22#include "mlir/IR/AffineExpr.h"
23#include "mlir/IR/AffineMap.h"
24#include "mlir/IR/BlockAndValueMapping.h"
25#include "mlir/IR/Builders.h"
26#include "mlir/IR/BuiltinAttributes.h"
27#include "mlir/IR/BuiltinOps.h"
28#include "mlir/IR/BuiltinTypes.h"
29#include "mlir/IR/DialectImplementation.h"
30#include "mlir/IR/OpImplementation.h"
31#include "mlir/IR/PatternMatch.h"
32#include "mlir/IR/TypeUtilities.h"
33#include "mlir/Support/LLVM.h"
34#include "llvm/ADT/ArrayRef.h"
35#include "llvm/ADT/STLExtras.h"
36#include "llvm/ADT/SmallVector.h"
37#include "llvm/ADT/StringSet.h"
38#include "llvm/ADT/TypeSwitch.h"
39#include "llvm/ADT/bit.h"

41#include <cassert>
42#include <cstdint>
43#include <numeric>

45#include "mlir/Dialect/Vector/IR/VectorOpsDialect.cpp.inc"
46// Pull in all enum type and utility function definitions.
47#include "mlir/Dialect/Vector/IR/VectorOpsEnums.cpp.inc"

49using namespace mlir;
50using namespace mlir::vector;

52/// Helper enum to classify mask value.
53enum class MaskFormat {
AllTrue = 0,
AllFalse = 1,
Unknown = 2,
57};

59/// Helper method to classify a mask value. Currently, the method
60/// looks "under the hood" of a constant value with dense attributes
61/// and a constant mask operation (since the client may be called at
62/// various stages during progressive lowering).
63static MaskFormat getMaskFormat(Value mask) {
if (auto c = mask.getDefiningOp<arith::ConstantOp>()) {
  // Inspect constant dense values. We count up for bits that
  // are set, count down for bits that are cleared, and bail
  // when a mix is detected.
  if (auto denseElts = c.getValue().dyn_cast<DenseIntElementsAttr>()) {
    int64_t val = 0;
    for (bool b : denseElts.getValues<bool>())
      if (b && val >= 0)
        val++;
      else if (!b && val <= 0)
        val--;
      else
        return MaskFormat::Unknown;
    if (val > 0)
      return MaskFormat::AllTrue;
    if (val < 0)
      return MaskFormat::AllFalse;
  }
} else if (auto m = mask.getDefiningOp<ConstantMaskOp>()) {
  // Inspect constant mask index. If the index exceeds the
  // dimension size, all bits are set. If the index is zero
  // or less, no bits are set.
  ArrayAttr masks = m.getMaskDimSizes();
  auto shape = m.getType().getShape();
  bool allTrue = true;
  bool allFalse = true;
  for (auto [maskIdx, dimSize] : llvm::zip_equal(masks, shape)) {
    int64_t i = maskIdx.cast<IntegerAttr>().getInt();
    if (i < dimSize)
      allTrue = false;
    if (i > 0)
      allFalse = false;
  }
  if (allTrue)
    return MaskFormat::AllTrue;
  if (allFalse)
    return MaskFormat::AllFalse;
}
return MaskFormat::Unknown;
103}

105/// Default callback to build a region with a 'vector.yield' terminator with no
106/// arguments.
107void mlir::vector::buildTerminatedBody(OpBuilder &builder, Location loc) {
builder.create<vector::YieldOp>(loc);
109}

111// Helper for verifying combining kinds in contractions and reductions.
112static bool isSupportedCombiningKind(CombiningKind combiningKind,
                                   Type elementType) {
switch (combiningKind) {
case CombiningKind::ADD:
case CombiningKind::MUL:
  return elementType.isIntOrIndexOrFloat();
case CombiningKind::MINUI:
case CombiningKind::MINSI:
case CombiningKind::MAXUI:
case CombiningKind::MAXSI:
case CombiningKind::AND:
case CombiningKind::OR:
case CombiningKind::XOR:
  return elementType.isIntOrIndex();
case CombiningKind::MINF:
case CombiningKind::MAXF:
  return elementType.isa<FloatType>();
}
return false;
131}

133/// Return true if the last dimension of the MemRefType has unit stride. Also
134/// return true for memrefs with no strides.
135bool mlir::vector::isLastMemrefDimUnitStride(MemRefType type) {
int64_t offset;
SmallVector<int64_t> strides;
auto successStrides = getStridesAndOffset(type, strides, offset);
return succeeded(successStrides) && (strides.empty() || strides.back() == 1);
140}

142AffineMap mlir::vector::getTransferMinorIdentityMap(ShapedType shapedType,
                                                  VectorType vectorType) {
int64_t elementVectorRank = 0;
VectorType elementVectorType =
    shapedType.getElementType().dyn_cast<VectorType>();
if (elementVectorType)
  elementVectorRank += elementVectorType.getRank();
// 0-d transfers are to/from tensor<t>/memref<t> and vector<1xt>.
// TODO: replace once we have 0-d vectors.
if (shapedType.getRank() == 0 &&
    vectorType.getShape() == ArrayRef<int64_t>{1})
  return AffineMap::get(
      /*numDims=*/0, /*numSymbols=*/0,
      getAffineConstantExpr(0, shapedType.getContext()));
return AffineMap::getMinorIdentityMap(
    shapedType.getRank(), vectorType.getRank() - elementVectorRank,
    shapedType.getContext());
159}

161bool mlir::vector::checkSameValueRAW(vector::TransferWriteOp defWrite,
                                   vector::TransferReadOp read) {
return !defWrite.hasOutOfBoundsDim() && !defWrite.getMask() &&
       !read.getMask() && defWrite.getIndices() == read.getIndices() &&
       defWrite.getVectorType() == read.getVectorType() &&
       defWrite.getPermutationMap() == read.getPermutationMap();
167}

169bool mlir::vector::checkSameValueWAW(vector::TransferWriteOp write,
                                   vector::TransferWriteOp priorWrite) {
return priorWrite.getIndices() == write.getIndices() &&
       priorWrite.getMask() == write.getMask() &&
       priorWrite.getVectorType() == write.getVectorType() &&
       priorWrite.getPermutationMap() == write.getPermutationMap();
175}

177bool mlir::vector::isDisjointTransferIndices(
  VectorTransferOpInterface transferA, VectorTransferOpInterface transferB) {
// For simplicity only look at transfer of same type.
if (transferA.getVectorType() != transferB.getVectorType())
  return false;
unsigned rankOffset = transferA.getLeadingShapedRank();
for (unsigned i = 0, e = transferA.indices().size(); i < e; i++) {
  auto indexA = transferA.indices()[i].getDefiningOp<arith::ConstantOp>();
  auto indexB = transferB.indices()[i].getDefiningOp<arith::ConstantOp>();
  // If any of the indices are dynamic we cannot prove anything.
  if (!indexA || !indexB)
    continue;

  if (i < rankOffset) {
    // For leading dimensions, if we can prove that index are different we
    // know we are accessing disjoint slices.
    if (indexA.getValue().cast<IntegerAttr>().getInt() !=
        indexB.getValue().cast<IntegerAttr>().getInt())
      return true;
  } else {
    // For this dimension, we slice a part of the memref we need to make sure
    // the intervals accessed don't overlap.
    int64_t distance =
        std::abs(indexA.getValue().cast<IntegerAttr>().getInt() -
                 indexB.getValue().cast<IntegerAttr>().getInt());
    if (distance >= transferA.getVectorType().getDimSize(i - rankOffset))
      return true;
  }
}
return false;
207}

209bool mlir::vector::isDisjointTransferSet(VectorTransferOpInterface transferA,
                                       VectorTransferOpInterface transferB) {
if (transferA.source() != transferB.source())
  return false;
return isDisjointTransferIndices(transferA, transferB);
214}

216// Helper to iterate over n-D vector slice elements. Calculate the next
217// `position` in the n-D vector of size `shape`, applying an offset `offsets`.
218// Modifies the `position` in place. Returns a failure when `position` becomes
219// the end position.
220static LogicalResult incSlicePosition(MutableArrayRef<int64_t> position,
                                    ArrayRef<int64_t> shape,
                                    ArrayRef<int64_t> offsets) {
for (auto [posInDim, dimSize, offsetInDim] :
     llvm::reverse(llvm::zip_equal(position, shape, offsets))) {
  ++posInDim;
  if (posInDim < dimSize + offsetInDim)
    return success();

  // Carry the overflow to the next loop iteration.
  posInDim = offsetInDim;
}

return failure();
234}

236//===----------------------------------------------------------------------===//
237// CombiningKindAttr
238//===----------------------------------------------------------------------===//

240namespace mlir {
241namespace vector {
242namespace detail {
243struct BitmaskEnumStorage : public AttributeStorage {
using KeyTy = uint64_t;

BitmaskEnumStorage(KeyTy val) : value(val) {}

bool operator==(const KeyTy &key) const { return value == key; }

static BitmaskEnumStorage *construct(AttributeStorageAllocator &allocator,
                                     const KeyTy &key) {
  return new (allocator.allocate<BitmaskEnumStorage>())
      BitmaskEnumStorage(key);
}

KeyTy value = 0;
257};
258} // namespace detail
259} // namespace vector
260} // namespace mlir

262//===----------------------------------------------------------------------===//
263// VectorDialect
264//===----------------------------------------------------------------------===//

266void VectorDialect::initialize() {
addAttributes<
268#define GET_ATTRDEF_LIST
269#include "mlir/Dialect/Vector/IR/VectorOpsAttrDefs.cpp.inc"
    >();

addOperations<
273#define GET_OP_LIST
274#include "mlir/Dialect/Vector/IR/VectorOps.cpp.inc"
    >();
276}

278/// Materialize a single constant operation from a given attribute value with
279/// the desired resultant type.
280Operation *VectorDialect::materializeConstant(OpBuilder &builder,
                                            Attribute value, Type type,
                                            Location loc) {
return builder.create<arith::ConstantOp>(loc, type, value);
284}

286IntegerType vector::getVectorSubscriptType(Builder &builder) {
return builder.getIntegerType(64);
288}

290ArrayAttr vector::getVectorSubscriptAttr(Builder &builder,
                                       ArrayRef<int64_t> values) {
return builder.getI64ArrayAttr(values);
293}

295//===----------------------------------------------------------------------===//
296// MultiDimReductionOp
297//===----------------------------------------------------------------------===//

299void vector::MultiDimReductionOp::build(OpBuilder &builder,
                                      OperationState &result, Value source,
                                      Value acc, ArrayRef<bool> reductionMask,
                                      CombiningKind kind) {
SmallVector<int64_t> reductionDims;
for (const auto &en : llvm::enumerate(reductionMask))
  if (en.value())
    reductionDims.push_back(en.index());
build(builder, result, kind, source, acc,
      builder.getI64ArrayAttr(reductionDims));
309}

311OpFoldResult MultiDimReductionOp::fold(ArrayRef<Attribute> operands) {
// Single parallel dim, this is a noop.
if (getSourceVectorType().getRank() == 1 && !isReducedDim(0))
  return getSource();
return {};
316}

318std::optional<SmallVector<int64_t, 4>>
319MultiDimReductionOp::getShapeForUnroll() {
return llvm::to_vector<4>(getSourceVectorType().getShape());
321}

323LogicalResult MultiDimReductionOp::verify() {
SmallVector<int64_t> targetShape;
Type inferredReturnType;
for (auto it : llvm::enumerate(getSourceVectorType().getShape()))
  if (!llvm::any_of(getReductionDims().getValue(), [&](Attribute attr) {
        return attr.cast<IntegerAttr>().getValue() == it.index();
      }))
    targetShape.push_back(it.value());
// TODO: update to also allow 0-d vectors when available.
if (targetShape.empty())
  inferredReturnType = getSourceVectorType().getElementType();
else
  inferredReturnType =
      VectorType::get(targetShape, getSourceVectorType().getElementType());
if (getType() != inferredReturnType)
  return emitOpError() << "destination type " << getType()
                       << " is incompatible with source type "
                       << getSourceVectorType();

return success();
343}

345namespace {
346// Only unit dimensions that are being reduced are folded. If the dimension is
347// unit, but not reduced, it is not folded, thereby keeping the output type the
348// same. If not all dimensions which are reduced are of unit dimension, this
349// transformation does nothing. This is just a generalization of
350// ElideSingleElementReduction for ReduceOp.
351struct ElideUnitDimsInMultiDimReduction
  : public OpRewritePattern<MultiDimReductionOp> {
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(MultiDimReductionOp reductionOp,
                              PatternRewriter &rewriter) const override {
  ArrayRef<int64_t> shape = reductionOp.getSourceVectorType().getShape();
  for (const auto &dim : enumerate(shape)) {
    if (reductionOp.isReducedDim(dim.index()) && dim.value() != 1)
      return failure();
  }
  Location loc = reductionOp.getLoc();
  Value acc = reductionOp.getAcc();
  Value cast;
  if (reductionOp.getDestType().isa<VectorType>()) {
    cast = rewriter.create<vector::ShapeCastOp>(
        loc, reductionOp.getDestType(), reductionOp.getSource());
  } else {
    // This means we are reducing all the dimensions, and all reduction
    // dimensions are of size 1. So a simple extraction would do.
    cast = rewriter.create<vector::ExtractOp>(
        loc, reductionOp.getDestType(), reductionOp.getSource(),
        rewriter.getI64ArrayAttr(SmallVector<int64_t>(shape.size(), 0)));
  }

  Value result = vector::makeArithReduction(rewriter, loc,
                                            reductionOp.getKind(), acc, cast);
  rewriter.replaceOp(reductionOp, result);
  return success();
}
381};
382} // namespace

384void MultiDimReductionOp::getCanonicalizationPatterns(
  RewritePatternSet &results, MLIRContext *context) {
results.add<ElideUnitDimsInMultiDimReduction>(context);
387}

389//===----------------------------------------------------------------------===//
390// ReductionOp
391//===----------------------------------------------------------------------===//

393void vector::ReductionOp::build(OpBuilder &builder, OperationState &result,
                              CombiningKind kind, Value vector) {
build(builder, result, kind, vector, /*acc=*/Value());
396}

398void vector::ReductionOp::build(OpBuilder &builder, OperationState &result,
                              CombiningKind kind, Value vector, Value acc) {
build(builder, result, vector.getType().cast<VectorType>().getElementType(),
      kind, vector, acc);
402}

404LogicalResult ReductionOp::verify() {
// Verify for 0-D and 1-D vector.
int64_t rank = getVectorType().getRank();
if (rank > 1)
  return emitOpError("unsupported reduction rank: ") << rank;

// Verify supported reduction kind.
Type eltType = getDest().getType();
if (!isSupportedCombiningKind(getKind(), eltType))
  return emitOpError("unsupported reduction type '")
         << eltType << "' for kind '" << stringifyCombiningKind(getKind())
         << "'";

return success();
418}

420ParseResult ReductionOp::parse(OpAsmParser &parser, OperationState &result) {
SmallVector<OpAsmParser::UnresolvedOperand, 2> operandsInfo;
Type redType;
Type resType;
CombiningKindAttr kindAttr;
if (parser.parseCustomAttributeWithFallback(kindAttr, Type{}, "kind",
                                            result.attributes) ||
    parser.parseComma() || parser.parseOperandList(operandsInfo) ||
    parser.parseColonType(redType) ||
    parser.parseKeywordType("into", resType) ||
    (!operandsInfo.empty() &&
     parser.resolveOperand(operandsInfo[0], redType, result.operands)) ||
    (operandsInfo.size() > 1 &&
     parser.resolveOperand(operandsInfo[1], resType, result.operands)) ||
    parser.addTypeToList(resType, result.types))
  return failure();
if (operandsInfo.empty() || operandsInfo.size() > 2)
  return parser.emitError(parser.getNameLoc(),
                          "unsupported number of operands");
return success();
440}

442void ReductionOp::print(OpAsmPrinter &p) {
p << " ";
getKindAttr().print(p);
p << ", " << getVector();
if (getAcc())
  p << ", " << getAcc();
p << " : " << getVector().getType() << " into " << getDest().getType();
449}

451// MaskableOpInterface methods.

453/// Returns the mask type expected by this operation.
454Type ReductionOp::getExpectedMaskType() {
auto vecType = getVectorType();
return vecType.cloneWith(std::nullopt,
                         IntegerType::get(vecType.getContext(), /*width=*/1));
458}

460Value mlir::vector::getVectorReductionOp(arith::AtomicRMWKind op,
                                       OpBuilder &builder, Location loc,
                                       Value vector) {
switch (op) {
case arith::AtomicRMWKind::addf:
case arith::AtomicRMWKind::addi:
  return builder.create<vector::ReductionOp>(vector.getLoc(),
                                             CombiningKind::ADD, vector);
case arith::AtomicRMWKind::mulf:
case arith::AtomicRMWKind::muli:
  return builder.create<vector::ReductionOp>(vector.getLoc(),
                                             CombiningKind::MUL, vector);
case arith::AtomicRMWKind::minf:
  return builder.create<vector::ReductionOp>(vector.getLoc(),
                                             CombiningKind::MINF, vector);
case arith::AtomicRMWKind::mins:
  return builder.create<vector::ReductionOp>(vector.getLoc(),
                                             CombiningKind::MINSI, vector);
case arith::AtomicRMWKind::minu:
  return builder.create<vector::ReductionOp>(vector.getLoc(),
                                             CombiningKind::MINUI, vector);
case arith::AtomicRMWKind::maxf:
  return builder.create<vector::ReductionOp>(vector.getLoc(),
                                             CombiningKind::MAXF, vector);
case arith::AtomicRMWKind::maxs:
  return builder.create<vector::ReductionOp>(vector.getLoc(),
                                             CombiningKind::MAXSI, vector);
case arith::AtomicRMWKind::maxu:
  return builder.create<vector::ReductionOp>(vector.getLoc(),
                                             CombiningKind::MAXUI, vector);
case arith::AtomicRMWKind::andi:
  return builder.create<vector::ReductionOp>(vector.getLoc(),
                                             CombiningKind::AND, vector);
case arith::AtomicRMWKind::ori:
  return builder.create<vector::ReductionOp>(vector.getLoc(),
                                             CombiningKind::OR, vector);
// TODO: Add remaining reduction operations.
default:
  (void)emitOptionalError(loc, "Reduction operation type not supported");
  break;
}
return nullptr;
502}

504std::optional<SmallVector<int64_t, 4>> ReductionOp::getShapeForUnroll() {
return llvm::to_vector<4>(getVectorType().getShape());
506}

508namespace {
509struct ElideSingleElementReduction : public OpRewritePattern<ReductionOp> {
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(ReductionOp reductionOp,
                              PatternRewriter &rewriter) const override {
  if (reductionOp.getVectorType().getDimSize(0) != 1)
    return failure();

  Location loc = reductionOp.getLoc();
  Value result = rewriter.create<ExtractOp>(loc, reductionOp.getType(),
                                            reductionOp.getVector(),
                                            rewriter.getI64ArrayAttr(0));

  if (Value acc = reductionOp.getAcc())
    result = vector::makeArithReduction(rewriter, loc, reductionOp.getKind(),
                                        result, acc);

  rewriter.replaceOp(reductionOp, result);
  return success();
}
529};
530} // namespace

532void ReductionOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                            MLIRContext *context) {
results.add<ElideSingleElementReduction>(context);
535}

537//===----------------------------------------------------------------------===//
538// ContractionOp
539//===----------------------------------------------------------------------===//

541void vector::ContractionOp::build(OpBuilder &builder, OperationState &result,
                                Value lhs, Value rhs, Value acc,
                                ArrayRef<ArrayRef<AffineExpr>> indexingExprs,
                                ArrayRef<IteratorType> iteratorTypes) {
result.addOperands({lhs, rhs, acc});
result.addTypes(acc.getType());
result.addAttribute(getIndexingMapsAttrName(result.name),
                    builder.getAffineMapArrayAttr(
                        AffineMap::inferFromExprList(indexingExprs)));
result.addAttribute(
    getIteratorTypesAttrName(result.name),
    builder.getArrayAttr(llvm::to_vector(llvm::map_range(
        iteratorTypes, [&](IteratorType t) -> mlir::Attribute {
          return IteratorTypeAttr::get(builder.getContext(), t);
        }))));
556}

558void vector::ContractionOp::build(OpBuilder &builder, OperationState &result,
                                Value lhs, Value rhs, Value acc,
                                ArrayAttr indexingMaps,
                                ArrayAttr iteratorTypes) {
build(builder, result, lhs, rhs, acc, indexingMaps, iteratorTypes,
      ContractionOp::getDefaultKind());
564}

566void vector::ContractionOp::build(OpBuilder &builder, OperationState &result,
                                Value lhs, Value rhs, Value acc,
                                ArrayAttr indexingMaps,
                                ArrayAttr iteratorTypes, CombiningKind kind) {
result.addOperands({lhs, rhs, acc});
result.addTypes(acc.getType());
result.addAttribute(getIndexingMapsAttrName(result.name), indexingMaps);
result.addAttribute(getIteratorTypesAttrName(result.name), iteratorTypes);
result.addAttribute(getKindAttrName(result.name),
                    CombiningKindAttr::get(builder.getContext(), kind));
576}

578ParseResult ContractionOp::parse(OpAsmParser &parser, OperationState &result) {
OpAsmParser::UnresolvedOperand lhsInfo;
OpAsmParser::UnresolvedOperand rhsInfo;
OpAsmParser::UnresolvedOperand accInfo;
SmallVector<OpAsmParser::UnresolvedOperand, 2> masksInfo;
SmallVector<Type, 2> types;
Type resultType;
auto loc = parser.getCurrentLocation();
DictionaryAttr dictAttr;
// TODO: Unify linalg op attribute parsing.
if (parser.parseAttribute(dictAttr, "_", result.attributes) ||
    parser.parseOperand(lhsInfo) || parser.parseComma() ||
    parser.parseOperand(rhsInfo) || parser.parseComma() ||
    parser.parseOperand(accInfo) ||
    parser.parseTrailingOperandList(masksInfo) ||
    parser.parseOptionalAttrDict(result.attributes) ||
    parser.parseColonTypeList(types) ||
    parser.parseKeywordType("into", resultType) ||
    parser.resolveOperand(lhsInfo, types[0], result.operands) ||
    parser.resolveOperand(rhsInfo, types[1], result.operands) ||
    parser.resolveOperand(accInfo, resultType, result.operands) ||
    parser.addTypeToList(resultType, result.types))
  return failure();
result.attributes.assign(dictAttr.getValue().begin(),
                         dictAttr.getValue().end());

// Convert array of string into an array of IteratyType enums. This is needed,
// because tests still use the old format when 'iterator_types' attribute is
// represented as an array of strings.
// TODO: Remove this conversion once tests are fixed.
ArrayAttr iteratorTypes =
    result.attributes.get(getIteratorTypesAttrName(result.name))
        .cast<ArrayAttr>();

SmallVector<Attribute> iteratorTypeAttrs;

for (StringRef s : iteratorTypes.getAsValueRange<StringAttr>()) {
  auto maybeIteratorType = symbolizeIteratorType(s);
  if (!maybeIteratorType.has_value())
    return parser.emitError(loc) << "unexpected iterator_type (" << s << ")";

  iteratorTypeAttrs.push_back(
      IteratorTypeAttr::get(parser.getContext(), maybeIteratorType.value()));
}
result.attributes.set(getIteratorTypesAttrName(result.name),
                      parser.getBuilder().getArrayAttr(iteratorTypeAttrs));

if (!result.attributes.get(getKindAttrName(result.name))) {
  result.addAttribute(
      getKindAttrName(result.name),
      CombiningKindAttr::get(result.getContext(),
                             ContractionOp::getDefaultKind()));
}
if (masksInfo.empty())
  return success();
if (masksInfo.size() != 2)
  return parser.emitError(parser.getNameLoc(),
                          "expected zero or exactly 2 vector mask operands");
auto lhsType = types[0].cast<VectorType>();
auto rhsType = types[1].cast<VectorType>();
auto maskElementType = parser.getBuilder().getI1Type();
std::array<Type, 2> maskTypes = {
    VectorType::Builder(lhsType).setElementType(maskElementType),
    VectorType::Builder(rhsType).setElementType(maskElementType)};
if (parser.resolveOperands(masksInfo, maskTypes, loc, result.operands))
  return failure();
return success();
645}

647void ContractionOp::print(OpAsmPrinter &p) {
// TODO: Unify printing code with linalg ops.
auto attrNames = getTraitAttrNames();
llvm::StringSet<> traitAttrsSet;
traitAttrsSet.insert(attrNames.begin(), attrNames.end());
SmallVector<NamedAttribute, 8> attrs;
for (auto attr : (*this)->getAttrs()) {
  if (attr.getName() == getIteratorTypesAttrName()) {
    auto iteratorTypes =
        attr.getValue()
            .cast<ArrayAttr>()
            .getAsValueRange<IteratorTypeAttr, IteratorType>();
    // Convert IteratorType enums into the string representation. This is
    // needed, because tests still use the old format when 'iterator_types'
    // attribute is represented as an array of strings.
    // TODO: Remove this conversion once tests are fixed.
    SmallVector<Attribute> iteratorTypeNames = llvm::to_vector(
        llvm::map_range(iteratorTypes, [&](IteratorType t) -> Attribute {
          return StringAttr::get(getContext(), stringifyIteratorType(t));
        }));

    attrs.emplace_back(getIteratorTypesAttrName(),
                       ArrayAttr::get(getContext(), iteratorTypeNames));
  } else if (traitAttrsSet.count(attr.getName().strref()) > 0)
    attrs.push_back(attr);
}

auto dictAttr = DictionaryAttr::get(getContext(), attrs);
p << " " << dictAttr << " " << getLhs() << ", ";
p << getRhs() << ", " << getAcc();
if (getMasks().size() == 2)
  p << ", " << getMasks();

p.printOptionalAttrDict((*this)->getAttrs(), attrNames);
p << " : " << getLhs().getType() << ", " << getRhs().getType() << " into "
  << getResultType();
683}

685static bool verifyDimMap(VectorType lhsType, VectorType rhsType,
                       const std::vector<std::pair<int64_t, int64_t>> &map) {
for (auto &dimPair : map) {
  if (dimPair.first < 0 || dimPair.first >= lhsType.getRank() ||
      dimPair.second < 0 || dimPair.second >= rhsType.getRank() ||
      lhsType.getDimSize(dimPair.first) != rhsType.getDimSize(dimPair.second))
    return false;
}
return true;
694}

696static LogicalResult verifyOutputShape(
  ContractionOp op, VectorType lhsType, VectorType rhsType, Type accType,
  Type resType,
  const std::vector<std::pair<int64_t, int64_t>> &contractingDimMap,
  const std::vector<std::pair<int64_t, int64_t>> &batchDimMap) {
DenseSet<int64_t> lhsContractingDimSet;
DenseSet<int64_t> rhsContractingDimSet;
for (auto &dimPair : contractingDimMap) {
  lhsContractingDimSet.insert(dimPair.first);
  rhsContractingDimSet.insert(dimPair.second);
}
DenseSet<int64_t> rhsBatchDimSet;
for (auto &dimPair : batchDimMap)
  rhsBatchDimSet.insert(dimPair.second);

// Add free and batch dimensions from 'lhsType' to 'expectedResultDims'.
SmallVector<int64_t, 4> expectedResultDims;
for (int64_t i = 0, e = lhsType.getRank(); i < e; ++i) {
  if (lhsContractingDimSet.count(i) > 0)
    continue;
  expectedResultDims.push_back(lhsType.getDimSize(i));
}

// Add free dimensions from 'rhsType' to 'expectedResultDims'.
for (int64_t i = 0, e = rhsType.getRank(); i < e; ++i) {
  if (rhsContractingDimSet.count(i) > 0 || rhsBatchDimSet.count(i) > 0)
    continue;
  expectedResultDims.push_back(rhsType.getDimSize(i));
}

// Verify 'expectedResultDims'.
if (expectedResultDims.empty()) {
  // No batch or free dimension implies a scalar result.
  if (resType.isa<VectorType>() || accType.isa<VectorType>())
    return op.emitOpError("invalid accumulator/result vector shape");
} else {
  // At least one batch or free dimension implies a vector result.
  auto resVectorType = resType.dyn_cast<VectorType>();
  auto accVectorType = accType.dyn_cast<VectorType>();
  if (!resVectorType || !accVectorType)
    return op.emitOpError("invalid accumulator/result vector shape");

  // Infer expected result vector type. Lhs + rhs map and lhs + rhs vector
  // types fully define the result vector type. This assumes the affine maps
  // are well-formed, which must have been verified already.
  MLIRContext *ctx = op.getContext();
  AffineMap lhsMap = op.getIndexingMapsArray()[0];
  AffineMap rhsMap = op.getIndexingMapsArray()[1];
  if (getUnusedDimsBitVector({lhsMap, rhsMap}).any())
    return op.emitOpError(
        "expected all dimensions to be either a LHS or a RHS dimension");
  SmallVector<AffineExpr, 4> extents(lhsMap.getNumInputs());
  for (auto pair :
       {std::make_pair(lhsType, lhsMap), std::make_pair(rhsType, rhsMap)}) {
    VectorType v = pair.first;
    auto map = pair.second;
    for (unsigned idx = 0, e = v.getRank(); idx < e; ++idx) {
      unsigned pos = map.getDimPosition(idx);
      if (!extents[pos])
        extents[pos] = getAffineConstantExpr(v.getShape()[idx], ctx);
    }
  }
  if (!llvm::all_of(extents, [](AffineExpr e) { return e; }))
    return op.emitOpError("expected all dimensions to get an extent as "
                          "either a LHS or a RHS dimension");

  AffineMap resMap = op.getIndexingMapsArray()[2];
  auto extentsMap = AffineMap::get(/*dimCount=*/extents.size(),
                                   /*symCount=*/0, extents, ctx);
  // Compose the resMap with the extentsMap, which is a constant map.
  AffineMap expectedMap = simplifyAffineMap(resMap.compose(extentsMap));
  assert(llvm::all_of((static_cast <bool> (llvm::all_of( expectedMap.getResults
(), [](AffineExpr e) { return e.isa<AffineConstantExpr>
(); }) && "expected constant extent along all dimensions."
) ? void (0) : __assert_fail ("llvm::all_of( expectedMap.getResults(), [](AffineExpr e) { return e.isa<AffineConstantExpr>(); }) && \"expected constant extent along all dimensions.\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 770, __extension__
 __PRETTY_FUNCTION__))
             expectedMap.getResults(),(static_cast <bool> (llvm::all_of( expectedMap.getResults
(), [](AffineExpr e) { return e.isa<AffineConstantExpr>
(); }) && "expected constant extent along all dimensions."
) ? void (0) : __assert_fail ("llvm::all_of( expectedMap.getResults(), [](AffineExpr e) { return e.isa<AffineConstantExpr>(); }) && \"expected constant extent along all dimensions.\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 770, __extension__
 __PRETTY_FUNCTION__))
             [](AffineExpr e) { return e.isa<AffineConstantExpr>(); }) &&(static_cast <bool> (llvm::all_of( expectedMap.getResults
(), [](AffineExpr e) { return e.isa<AffineConstantExpr>
(); }) && "expected constant extent along all dimensions."
) ? void (0) : __assert_fail ("llvm::all_of( expectedMap.getResults(), [](AffineExpr e) { return e.isa<AffineConstantExpr>(); }) && \"expected constant extent along all dimensions.\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 770, __extension__
 __PRETTY_FUNCTION__))
         "expected constant extent along all dimensions.")(static_cast <bool> (llvm::all_of( expectedMap.getResults
(), [](AffineExpr e) { return e.isa<AffineConstantExpr>
(); }) && "expected constant extent along all dimensions."
) ? void (0) : __assert_fail ("llvm::all_of( expectedMap.getResults(), [](AffineExpr e) { return e.isa<AffineConstantExpr>(); }) && \"expected constant extent along all dimensions.\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 770, __extension__
 __PRETTY_FUNCTION__));
  // Extract the expected shape and build the type.
  auto expectedShape = llvm::to_vector<4>(
      llvm::map_range(expectedMap.getResults(), [](AffineExpr e) {
        return e.cast<AffineConstantExpr>().getValue();
      }));
  auto expected =
      VectorType::get(expectedShape, resVectorType.getElementType());
  if (resVectorType != expected || accVectorType != expected)
    return op.emitOpError(
               "invalid accumulator/result vector shape, expected: ")
           << expected;
}
return success();
784}

786LogicalResult ContractionOp::verify() {
auto lhsType = getLhsType();
auto rhsType = getRhsType();
auto accType = getAccType();
auto resType = getResultType();

// Verify that an indexing map was specified for each vector operand.
if (getIndexingMapsArray().size() != 3)
  return emitOpError("expected an indexing map for each vector operand");

// Verify that each index map has 'numIterators' inputs, no symbols, and
// that the number of map outputs equals the rank of its associated
// vector operand.
unsigned numIterators = getIteratorTypes().getValue().size();
for (const auto &it : llvm::enumerate(getIndexingMapsArray())) {
  auto index = it.index();
  auto map = it.value();
  if (map.getNumSymbols() != 0)
    return emitOpError("expected indexing map ")
           << index << " to have no symbols";
  auto vectorType = getOperand(index).getType().dyn_cast<VectorType>();
  unsigned rank = vectorType ? vectorType.getShape().size() : 0;
  // Verify that the map has the right number of inputs, outputs, and indices.
  // This also correctly accounts for (..) -> () for rank-0 results.
  if (map.getNumDims() != numIterators)
    return emitOpError("expected indexing map ")
           << index << " to have " << numIterators << " number of inputs";
  if (map.getNumResults() != rank)
    return emitOpError("expected indexing map ")
           << index << " to have " << rank << " number of outputs";
  if (!map.isProjectedPermutation())
    return emitOpError("expected indexing map ")
           << index << " to be a projected permutation of its inputs";
}

auto contractingDimMap = getContractingDimMap();
auto batchDimMap = getBatchDimMap();

// Verify at least one contracting dimension pair was specified.
if (contractingDimMap.empty())
  return emitOpError("expected at least one contracting dimension pair");

// Verify contracting dimension map was properly constructed.
if (!verifyDimMap(lhsType, rhsType, contractingDimMap))
  return emitOpError("invalid contracting dimension map");

// Verify batch dimension map was properly constructed.
if (!verifyDimMap(lhsType, rhsType, batchDimMap))
  return emitOpError("invalid batch dimension map");

// Verify 'accType' and 'resType' shape.
if (failed(verifyOutputShape(*this, lhsType, rhsType, accType, resType,
                             contractingDimMap, batchDimMap)))
  return failure();

// Verify that either two vector masks are set or none are set.
auto lhsMaskType = getLHSVectorMaskType();
auto rhsMaskType = getRHSVectorMaskType();
if ((lhsMaskType && !rhsMaskType) || (!lhsMaskType && rhsMaskType))
  return emitOpError("invalid number of vector masks specified");
if (lhsMaskType && rhsMaskType) {
  // Verify mask rank == argument rank.
  if (lhsMaskType.getShape().size() != lhsType.getShape().size() ||
      rhsMaskType.getShape().size() != rhsType.getShape().size())
    return emitOpError("invalid vector mask rank");
}

// Verify supported combining kind.
auto vectorType = resType.dyn_cast<VectorType>();
auto elementType = vectorType ? vectorType.getElementType() : resType;
if (!isSupportedCombiningKind(getKind(), elementType))
  return emitOpError("unsupported contraction type");

return success();
860}

862SmallVector<StringRef> ContractionOp::getTraitAttrNames() {
return SmallVector<StringRef>{getIndexingMapsAttrName(),
                              getIteratorTypesAttrName(), getKindAttrName()};
865}

867static int64_t getResultIndex(AffineMap map, AffineExpr targetExpr) {
for (int64_t i = 0, e = map.getNumResults(); i < e; ++i)
  if (targetExpr == map.getResult(i))
    return i;
return -1;
872}

874static std::vector<std::pair<int64_t, int64_t>>
875getDimMap(ArrayRef<AffineMap> indexingMaps, ArrayAttr iteratorTypes,
        IteratorType targetIteratorType, MLIRContext *context) {
std::vector<std::pair<int64_t, int64_t>> dimMap;
for (const auto &it : llvm::enumerate(iteratorTypes)) {
  auto iteratorType = it.value().cast<IteratorTypeAttr>().getValue();
  if (iteratorType != targetIteratorType)
    continue;
  // Search lhs/rhs map results for 'targetExpr'.
  auto targetExpr = getAffineDimExpr(it.index(), context);
  int64_t lhsDim = getResultIndex(indexingMaps[0], targetExpr);
  int64_t rhsDim = getResultIndex(indexingMaps[1], targetExpr);
  if (lhsDim >= 0 && rhsDim >= 0)
    dimMap.emplace_back(lhsDim, rhsDim);
}
return dimMap;
890}

892void ContractionOp::getIterationBounds(
  SmallVectorImpl<int64_t> &iterationBounds) {
auto lhsShape = getLhsType().getShape();
auto resVectorType = getResultType().dyn_cast<VectorType>();
SmallVector<AffineMap, 4> indexingMaps(getIndexingMapsArray());
SmallVector<int64_t, 2> iterationShape;
for (const auto &it : llvm::enumerate(getIteratorTypes())) {
  // Search lhs/rhs map results for 'targetExpr'.
  auto targetExpr = getAffineDimExpr(it.index(), getContext());
  auto iteratorType = it.value().cast<IteratorTypeAttr>().getValue();
  if (iteratorType == IteratorType::reduction) {
    // Get reduction dim size from lhs shape (same size in rhsShape).
    int64_t lhsDimIndex = getResultIndex(indexingMaps[0], targetExpr);
    assert(lhsDimIndex >= 0)(static_cast <bool> (lhsDimIndex >= 0) ? void (0) : __assert_fail
 ("lhsDimIndex >= 0", "mlir/lib/Dialect/Vector/IR/VectorOps.cpp"
, 905, __extension__ __PRETTY_FUNCTION__));
    iterationBounds.push_back(lhsShape[lhsDimIndex]);
    continue;
  }
  // Get parallel dimension size from result shape.
  int64_t resDimIndex = getResultIndex(indexingMaps[2], targetExpr);
  assert(resDimIndex >= 0)(static_cast <bool> (resDimIndex >= 0) ? void (0) : __assert_fail
 ("resDimIndex >= 0", "mlir/lib/Dialect/Vector/IR/VectorOps.cpp"
, 911, __extension__ __PRETTY_FUNCTION__));
  assert(resVectorType != nullptr)(static_cast <bool> (resVectorType != nullptr) ? void (
0) : __assert_fail ("resVectorType != nullptr", "mlir/lib/Dialect/Vector/IR/VectorOps.cpp"
, 912, __extension__ __PRETTY_FUNCTION__));
  iterationBounds.push_back(resVectorType.getShape()[resDimIndex]);
}
915}

917void ContractionOp::getIterationIndexMap(
  std::vector<DenseMap<int64_t, int64_t>> &iterationIndexMap) {
unsigned numMaps = getIndexingMapsArray().size();
iterationIndexMap.resize(numMaps);
for (const auto &it : llvm::enumerate(getIndexingMapsArray())) {
  auto index = it.index();
  auto map = it.value();
  for (unsigned i = 0, e = map.getNumResults(); i < e; ++i) {
    auto dim = map.getResult(i).cast<AffineDimExpr>();
    iterationIndexMap[index][dim.getPosition()] = i;
  }
}
929}

931std::vector<std::pair<int64_t, int64_t>> ContractionOp::getContractingDimMap() {
SmallVector<AffineMap, 4> indexingMaps(getIndexingMapsArray());
return getDimMap(indexingMaps, getIteratorTypes(), IteratorType::reduction,
                 getContext());
935}

937std::vector<std::pair<int64_t, int64_t>> ContractionOp::getBatchDimMap() {
SmallVector<AffineMap, 4> indexingMaps(getIndexingMapsArray());
return getDimMap(indexingMaps, getIteratorTypes(), IteratorType::parallel,
                 getContext());
941}

943std::optional<SmallVector<int64_t, 4>> ContractionOp::getShapeForUnroll() {
SmallVector<int64_t, 4> shape;
getIterationBounds(shape);
return shape;
947}

949/// Return a fused vector::ContractionOp which represents a patterns such as:
950///
951/// ```mlir
952///    %c0 = vector.constant 0: ...
953///    %c = vector.contract %a, %b, %c0: ...
954///    %e = add %c, %d: ...
955/// ```
956///
957/// by:
958///
959/// ```mlir
960///    %e = vector.contract %a, %b, %d: ...
961/// ```
962///
963/// Return null if the canonicalization does not apply.
964// TODO: This should be a folding of Add into Contract in core but while they
965// live in different dialects, it is not possible without unnatural
966// dependencies.
967template <typename AddOpType>
968struct CanonicalizeContractAdd : public OpRewritePattern<AddOpType> {
using OpRewritePattern<AddOpType>::OpRewritePattern;

LogicalResult matchAndRewrite(AddOpType addOp,
                              PatternRewriter &rewriter) const override {
  auto canonicalize = [&](Value maybeContraction,
                          Value otherOperand) -> vector::ContractionOp {
    vector::ContractionOp contractionOp =
        dyn_cast_or_null<vector::ContractionOp>(
            maybeContraction.getDefiningOp());
    if (!contractionOp)
      return vector::ContractionOp();
    if (auto maybeZero = dyn_cast_or_null<arith::ConstantOp>(
            contractionOp.getAcc().getDefiningOp())) {
      if (maybeZero.getValue() ==
          rewriter.getZeroAttr(contractionOp.getAcc().getType())) {
        BlockAndValueMapping bvm;
        bvm.map(contractionOp.getAcc(), otherOperand);
        auto newContraction =
            cast<vector::ContractionOp>(rewriter.clone(*contractionOp, bvm));
        rewriter.replaceOp(addOp, newContraction.getResult());
        return newContraction;
      }
    }
    return vector::ContractionOp();
  };

  Value a = addOp->getOperand(0), b = addOp->getOperand(1);
  vector::ContractionOp contract = canonicalize(a, b);
  contract = contract ? contract : canonicalize(b, a);
  return contract ? success() : failure();
}
1000};

1002void ContractionOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                              MLIRContext *context) {
results.add<CanonicalizeContractAdd<arith::AddIOp>,
            CanonicalizeContractAdd<arith::AddFOp>>(context);
1006}

1008//===----------------------------------------------------------------------===//
1009// ExtractElementOp
1010//===----------------------------------------------------------------------===//

1012void vector::ExtractElementOp::build(OpBuilder &builder, OperationState &result,
                                   Value source) {
result.addOperands({source});
result.addTypes(source.getType().cast<VectorType>().getElementType());
1016}

1018void vector::ExtractElementOp::build(OpBuilder &builder, OperationState &result,
                                   Value source, Value position) {
result.addOperands({source, position});
result.addTypes(source.getType().cast<VectorType>().getElementType());
1022}

1024LogicalResult vector::ExtractElementOp::verify() {
VectorType vectorType = getVectorType();
if (vectorType.getRank() == 0) {
  if (getPosition())
    return emitOpError("expected position to be empty with 0-D vector");
  return success();
}
if (vectorType.getRank() != 1)
  return emitOpError("unexpected >1 vector rank");
if (!getPosition())
  return emitOpError("expected position for 1-D vector");
return success();
1036}

1038OpFoldResult vector::ExtractElementOp::fold(ArrayRef<Attribute> operands) {
// Skip the 0-D vector here now.
if (operands.size() < 2)
  return {};

Attribute src = operands[0];
Attribute pos = operands[1];

// Fold extractelement (splat X) -> X.
if (auto splat = getVector().getDefiningOp<vector::SplatOp>())
  return splat.getInput();

if (!pos || !src)
  return {};

auto srcElements = src.cast<DenseElementsAttr>().getValues<Attribute>();

auto attr = pos.dyn_cast<IntegerAttr>();
uint64_t posIdx = attr.getInt();

return srcElements[posIdx];
1059}

1061//===----------------------------------------------------------------------===//
1062// ExtractOp
1063//===----------------------------------------------------------------------===//

1065void vector::ExtractOp::build(OpBuilder &builder, OperationState &result,
                            Value source, ArrayRef<int64_t> position) {
build(builder, result, source, getVectorSubscriptAttr(builder, position));
1068}

1070// Convenience builder which assumes the values are constant indices.
1071void vector::ExtractOp::build(OpBuilder &builder, OperationState &result,
                            Value source, ValueRange position) {
SmallVector<int64_t, 4> positionConstants =
    llvm::to_vector<4>(llvm::map_range(position, [](Value pos) {
      return pos.getDefiningOp<arith::ConstantIndexOp>().value();
    }));
build(builder, result, source, positionConstants);
1078}

1080LogicalResult
1081ExtractOp::inferReturnTypes(MLIRContext *, std::optional<Location>,
                          ValueRange operands, DictionaryAttr attributes,
                          RegionRange,
                          SmallVectorImpl<Type> &inferredReturnTypes) {
ExtractOp::Adaptor op(operands, attributes);
auto vectorType = op.getVector().getType().cast<VectorType>();
if (static_cast<int64_t>(op.getPosition().size()) == vectorType.getRank()) {
  inferredReturnTypes.push_back(vectorType.getElementType());
} else {
  auto n =
      std::min<size_t>(op.getPosition().size(), vectorType.getRank() - 1);
  inferredReturnTypes.push_back(VectorType::get(
      vectorType.getShape().drop_front(n), vectorType.getElementType()));
}
return success();
1096}

1098bool ExtractOp::isCompatibleReturnTypes(TypeRange l, TypeRange r) {
// Allow extracting 1-element vectors instead of scalars.
auto isCompatible = [](TypeRange l, TypeRange r) {
  auto vectorType = l.front().dyn_cast<VectorType>();
  return vectorType && vectorType.getShape().equals({1}) &&
         vectorType.getElementType() == r.front();
};
if (l.size() == 1 && r.size() == 1 &&
    (isCompatible(l, r) || isCompatible(r, l)))
  return true;
return l == r;
1109}

1111LogicalResult vector::ExtractOp::verify() {
auto positionAttr = getPosition().getValue();
if (positionAttr.size() > static_cast<unsigned>(getVectorType().getRank()))
  return emitOpError(
      "expected position attribute of rank smaller than vector rank");
for (const auto &en : llvm::enumerate(positionAttr)) {
  auto attr = en.value().dyn_cast<IntegerAttr>();
  if (!attr || attr.getInt() < 0 ||
      attr.getInt() >= getVectorType().getDimSize(en.index()))
    return emitOpError("expected position attribute #")
           << (en.index() + 1)
           << " to be a non-negative integer smaller than the corresponding "
              "vector dimension";
}
return success();
1126}

1128template <typename IntType>
1129static SmallVector<IntType> extractVector(ArrayAttr arrayAttr) {
return llvm::to_vector<4>(llvm::map_range(
    arrayAttr.getAsRange<IntegerAttr>(),
    [](IntegerAttr attr) { return static_cast<IntType>(attr.getInt()); }));
1133}

1135/// Fold the result of chains of ExtractOp in place by simply concatenating the
1136/// positions.
1137static LogicalResult foldExtractOpFromExtractChain(ExtractOp extractOp) {
if (!extractOp.getVector().getDefiningOp<ExtractOp>())
  return failure();

SmallVector<int64_t, 4> globalPosition;
ExtractOp currentOp = extractOp;
auto extrPos = extractVector<int64_t>(currentOp.getPosition());
globalPosition.append(extrPos.rbegin(), extrPos.rend());
while (ExtractOp nextOp = currentOp.getVector().getDefiningOp<ExtractOp>()) {
  currentOp = nextOp;
  auto extrPos = extractVector<int64_t>(currentOp.getPosition());
  globalPosition.append(extrPos.rbegin(), extrPos.rend());
}
extractOp.setOperand(currentOp.getVector());
// OpBuilder is only used as a helper to build an I64ArrayAttr.
OpBuilder b(extractOp.getContext());
std::reverse(globalPosition.begin(), globalPosition.end());
extractOp->setAttr(ExtractOp::getPositionAttrStrName(),
                   b.getI64ArrayAttr(globalPosition));
return success();
1157}

1159namespace {
1160/// Fold an ExtractOp that is fed by a chain of InsertOps and TransposeOps.
1161/// Walk back a chain of InsertOp/TransposeOp until we hit a match.
1162/// Compose TransposeOp permutations as we walk back.
1163/// This helper class keeps an updated extraction position `extractPosition`
1164/// with extra trailing sentinels.
1165/// The sentinels encode the internal transposition status of the result vector.
1166/// As we iterate, extractPosition is permuted and updated.
1167class ExtractFromInsertTransposeChainState {
1168public:
ExtractFromInsertTransposeChainState(ExtractOp e);

/// Iterate over producing insert and transpose ops until we find a fold.
Value fold();

1174private:
/// Return true if the vector at position `a` is contained within the vector
/// at position `b`. Under insert/extract semantics, this is the same as `a`
/// is a prefix of `b`.
template <typename ContainerA, typename ContainerB>
bool isContainedWithin(const ContainerA &a, const ContainerB &b) {
  return a.size() <= b.size() &&
         std::equal(a.begin(), a.begin() + a.size(), b.begin());
}

/// Return true if the vector at position `a` intersects the vector at
/// position `b`. Under insert/extract semantics, this is the same as equality
/// of all entries of `a` that are >=0 with the corresponding entries of b.
/// Comparison is on the common prefix (i.e. zip).
template <typename ContainerA, typename ContainerB>
bool intersectsWhereNonNegative(const ContainerA &a, const ContainerB &b) {
  for (auto [elemA, elemB] : llvm::zip(a, b)) {
    if (elemA < 0 || elemB < 0)
      continue;
    if (elemA != elemB)
      return false;
  }
  return true;
}

/// Folding is only possible in the absence of an internal permutation in the
/// result vector.
bool canFold() {
  return (sentinels ==
          makeArrayRef(extractPosition).drop_front(extractedRank));
}

// Helper to get the next defining op of interest.
void updateStateForNextIteration(Value v) {
  nextInsertOp = v.getDefiningOp<vector::InsertOp>();
  nextTransposeOp = v.getDefiningOp<vector::TransposeOp>();
};

// Case 1. If we hit a transpose, just compose the map and iterate.
// Invariant: insert + transpose do not change rank, we can always compose.
LogicalResult handleTransposeOp();

// Case 2: the insert position matches extractPosition exactly, early return.
LogicalResult handleInsertOpWithMatchingPos(Value &res);

/// Case 3: if the insert position is a prefix of extractPosition, extract a
/// portion of the source of the insert.
/// Example:
/// ```
/// %ins = vector.insert %source, %vest[1]: vector<3x4> into vector<2x3x4x5>
/// // extractPosition == [1, 2, 3]
/// %ext = vector.extract %ins[1, 0]: vector<3x4x5>
/// // can fold to vector.extract %source[0, 3]
/// %ext = vector.extract %source[3]: vector<5x6>
/// ```
/// To traverse through %source, we need to set the leading dims to 0 and
/// drop the extra leading dims.
/// This method updates the internal state.
LogicalResult handleInsertOpWithPrefixPos(Value &res);

/// Try to fold in place to extract(source, extractPosition) and return the
/// folded result. Return null if folding is not possible (e.g. due to an
/// internal tranposition in the result).
Value tryToFoldExtractOpInPlace(Value source);

ExtractOp extractOp;
int64_t vectorRank;
int64_t extractedRank;

InsertOp nextInsertOp;
TransposeOp nextTransposeOp;

/// Sentinel values that encode the internal permutation status of the result.
/// They are set to (-1, ... , -k) at the beginning and appended to
/// `extractPosition`.
/// In the end, the tail of `extractPosition` must be exactly `sentinels` to
/// ensure that there is no internal transposition.
/// Internal transposition cannot be accounted for with a folding pattern.
// TODO: We could relax the internal transposition with an extra transposition
// operation in a future canonicalizer.
SmallVector<int64_t> sentinels;
SmallVector<int64_t> extractPosition;
1256};
1257} // namespace

1259ExtractFromInsertTransposeChainState::ExtractFromInsertTransposeChainState(
  ExtractOp e)
  : extractOp(e), vectorRank(extractOp.getVectorType().getRank()),
    extractedRank(extractOp.getPosition().size()) {
assert(vectorRank >= extractedRank && "extracted pos overflow")(static_cast <bool> (vectorRank >= extractedRank &&
 "extracted pos overflow") ? void (0) : __assert_fail ("vectorRank >= extractedRank && \"extracted pos overflow\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 1263, __extension__
 __PRETTY_FUNCTION__));
sentinels.reserve(vectorRank - extractedRank);
for (int64_t i = 0, e = vectorRank - extractedRank; i < e; ++i)
  sentinels.push_back(-(i + 1));
extractPosition = extractVector<int64_t>(extractOp.getPosition());
llvm::append_range(extractPosition, sentinels);
1269}

1271// Case 1. If we hit a transpose, just compose the map and iterate.
1272// Invariant: insert + transpose do not change rank, we can always compose.
1273LogicalResult ExtractFromInsertTransposeChainState::handleTransposeOp() {
if (!nextTransposeOp)
  return failure();
auto permutation = extractVector<unsigned>(nextTransposeOp.getTransp());
AffineMap m = inversePermutation(
    AffineMap::getPermutationMap(permutation, extractOp.getContext()));
extractPosition = applyPermutationMap(m, makeArrayRef(extractPosition));
return success();
1281}

1283// Case 2: the insert position matches extractPosition exactly, early return.
1284LogicalResult
1285ExtractFromInsertTransposeChainState::handleInsertOpWithMatchingPos(
  Value &res) {
auto insertedPos = extractVector<int64_t>(nextInsertOp.getPosition());
if (makeArrayRef(insertedPos) !=
    llvm::makeArrayRef(extractPosition).take_front(extractedRank))
  return failure();
// Case 2.a. early-exit fold.
res = nextInsertOp.getSource();
// Case 2.b. if internal transposition is present, canFold will be false.
return success();
1295}

1297/// Case 3: if inserted position is a prefix of extractPosition,
1298/// extract a portion of the source of the insertion.
1299/// This method updates the internal state.
1300LogicalResult
1301ExtractFromInsertTransposeChainState::handleInsertOpWithPrefixPos(Value &res) {
auto insertedPos = extractVector<int64_t>(nextInsertOp.getPosition());
if (!isContainedWithin(insertedPos, extractPosition))
  return failure();
// Set leading dims to zero.
std::fill_n(extractPosition.begin(), insertedPos.size(), 0);
// Drop extra leading dims.
extractPosition.erase(extractPosition.begin(),
                      extractPosition.begin() + insertedPos.size());
extractedRank = extractPosition.size() - sentinels.size();
// Case 3.a. early-exit fold (break and delegate to post-while path).
res = nextInsertOp.getSource();
// Case 3.b. if internal transposition is present, canFold will be false.
return success();
1315}

1317/// Try to fold in place to extract(source, extractPosition) and return the
1318/// folded result. Return null if folding is not possible (e.g. due to an
1319/// internal tranposition in the result).
1320Value ExtractFromInsertTransposeChainState::tryToFoldExtractOpInPlace(
  Value source) {
// If we can't fold (either internal transposition, or nothing to fold), bail.
bool nothingToFold = (source == extractOp.getVector());
if (nothingToFold || !canFold())
  return Value();
// Otherwise, fold by updating the op inplace and return its result.
OpBuilder b(extractOp.getContext());
extractOp->setAttr(
    extractOp.getPositionAttrName(),
    b.getI64ArrayAttr(
        makeArrayRef(extractPosition).take_front(extractedRank)));
extractOp.getVectorMutable().assign(source);
return extractOp.getResult();
1334}

1336/// Iterate over producing insert and transpose ops until we find a fold.
1337Value ExtractFromInsertTransposeChainState::fold() {
Value valueToExtractFrom = extractOp.getVector();
updateStateForNextIteration(valueToExtractFrom);
while (nextInsertOp || nextTransposeOp) {
  // Case 1. If we hit a transpose, just compose the map and iterate.
  // Invariant: insert + transpose do not change rank, we can always compose.
  if (succeeded(handleTransposeOp())) {
    valueToExtractFrom = nextTransposeOp.getVector();
    updateStateForNextIteration(valueToExtractFrom);
    continue;
  }

  Value result;
  // Case 2: the position match exactly.
  if (succeeded(handleInsertOpWithMatchingPos(result)))
    return result;

  // Case 3: if the inserted position is a prefix of extractPosition, we can
  // just extract a portion of the source of the insert.
  if (succeeded(handleInsertOpWithPrefixPos(result)))
    return tryToFoldExtractOpInPlace(result);

  // Case 4: extractPositionRef intersects insertedPosRef on non-sentinel
  // values. This is a more difficult case and we bail.
  auto insertedPos = extractVector<int64_t>(nextInsertOp.getPosition());
  if (isContainedWithin(extractPosition, insertedPos) ||
      intersectsWhereNonNegative(extractPosition, insertedPos))
    return Value();

  // Case 5: No intersection, we forward the extract to insertOp.dest().
  valueToExtractFrom = nextInsertOp.getDest();
  updateStateForNextIteration(valueToExtractFrom);
}
// If after all this we can fold, go for it.
return tryToFoldExtractOpInPlace(valueToExtractFrom);
1372}

1374/// Fold extractOp with scalar result coming from BroadcastOp or SplatOp.
1375static Value foldExtractFromBroadcast(ExtractOp extractOp) {
Operation *defOp = extractOp.getVector().getDefiningOp();
if (!defOp || !isa<vector::BroadcastOp, SplatOp>(defOp))
  return Value();
Value source = defOp->getOperand(0);
if (extractOp.getType() == source.getType())
  return source;
auto getRank = [](Type type) {
  return type.isa<VectorType>() ? type.cast<VectorType>().getRank() : 0;
};
// If splat or broadcast from a scalar, just return the source scalar.
unsigned broadcastSrcRank = getRank(source.getType());
if (broadcastSrcRank == 0)
  return source;

unsigned extractResultRank = getRank(extractOp.getType());
if (extractResultRank >= broadcastSrcRank)
  return Value();
// Check that the dimension of the result haven't been broadcasted.
auto extractVecType = extractOp.getType().dyn_cast<VectorType>();
auto broadcastVecType = source.getType().dyn_cast<VectorType>();
if (extractVecType && broadcastVecType &&
    extractVecType.getShape() !=
        broadcastVecType.getShape().take_back(extractResultRank))
  return Value();

auto broadcastOp = cast<vector::BroadcastOp>(defOp);
int64_t rankDiff = broadcastSrcRank - extractResultRank;
// Detect all the positions that come from "dim-1" broadcasting.
// These dimensions correspond to "dim-1" broadcasted dims; set the mathching
// extract position to `0` when extracting from the source operand.
llvm::SetVector<int64_t> broadcastedUnitDims =
    broadcastOp.computeBroadcastedUnitDims();
auto extractPos = extractVector<int64_t>(extractOp.getPosition());
for (int64_t i = rankDiff, e = extractPos.size(); i < e; ++i)
  if (broadcastedUnitDims.contains(i))
    extractPos[i] = 0;
// `rankDiff` leading dimensions correspond to new broadcasted dims, drop the
// matching extract position when extracting from the source operand.
extractPos.erase(extractPos.begin(),
                 std::next(extractPos.begin(), extractPos.size() - rankDiff));
// OpBuilder is only used as a helper to build an I64ArrayAttr.
OpBuilder b(extractOp.getContext());
extractOp.setOperand(source);
extractOp->setAttr(ExtractOp::getPositionAttrStrName(),
                   b.getI64ArrayAttr(extractPos));
return extractOp.getResult();
1422}

1424// Fold extractOp with source coming from ShapeCast op.
1425static Value foldExtractFromShapeCast(ExtractOp extractOp) {
auto shapeCastOp = extractOp.getVector().getDefiningOp<vector::ShapeCastOp>();
if (!shapeCastOp)
  return Value();
// Get the nth dimension size starting from lowest dimension.
auto getDimReverse = [](VectorType type, int64_t n) {
  return type.getShape().take_back(n + 1).front();
};
int64_t destinationRank =
    extractOp.getType().isa<VectorType>()
        ? extractOp.getType().cast<VectorType>().getRank()
        : 0;
if (destinationRank > shapeCastOp.getSourceVectorType().getRank())
  return Value();
if (destinationRank > 0) {
  auto destinationType = extractOp.getResult().getType().cast<VectorType>();
  for (int64_t i = 0; i < destinationRank; i++) {
    // The lowest dimension of of the destination must match the lowest
    // dimension of the shapecast op source.
    // TODO: This case could be support in a canonicalization pattern.
    if (getDimReverse(shapeCastOp.getSourceVectorType(), i) !=
        getDimReverse(destinationType, i))
      return Value();
  }
}
// Extract the strides associated with the extract op vector source. Then use
// this to calculate a linearized position for the extract.
auto extractedPos = extractVector<int64_t>(extractOp.getPosition());
std::reverse(extractedPos.begin(), extractedPos.end());
SmallVector<int64_t, 4> strides;
int64_t stride = 1;
for (int64_t i = 0, e = extractedPos.size(); i < e; i++) {
  strides.push_back(stride);
  stride *= getDimReverse(extractOp.getVectorType(), i + destinationRank);
}

int64_t position = linearize(extractedPos, strides);
// Then extract the strides associated to the shapeCast op vector source and
// delinearize the position using those strides.
SmallVector<int64_t, 4> newStrides;
int64_t numDimension =
    shapeCastOp.getSourceVectorType().getRank() - destinationRank;
stride = 1;
for (int64_t i = 0; i < numDimension; i++) {
  newStrides.push_back(stride);
  stride *=
      getDimReverse(shapeCastOp.getSourceVectorType(), i + destinationRank);
}
std::reverse(newStrides.begin(), newStrides.end());
SmallVector<int64_t, 4> newPosition = delinearize(newStrides, position);
// OpBuilder is only used as a helper to build an I64ArrayAttr.
OpBuilder b(extractOp.getContext());
extractOp->setAttr(ExtractOp::getPositionAttrStrName(),
                   b.getI64ArrayAttr(newPosition));
extractOp.setOperand(shapeCastOp.getSource());
return extractOp.getResult();
1481}

1483/// Fold an ExtractOp from ExtractStridedSliceOp.
1484static Value foldExtractFromExtractStrided(ExtractOp extractOp) {
auto extractStridedSliceOp =
    extractOp.getVector().getDefiningOp<vector::ExtractStridedSliceOp>();
if (!extractStridedSliceOp)
  return Value();
// Return if 'extractStridedSliceOp' has non-unit strides.
if (extractStridedSliceOp.hasNonUnitStrides())
  return Value();

// Trim offsets for dimensions fully extracted.
auto sliceOffsets =
    extractVector<int64_t>(extractStridedSliceOp.getOffsets());
while (!sliceOffsets.empty()) {
  size_t lastOffset = sliceOffsets.size() - 1;
  if (sliceOffsets.back() != 0 ||
      extractStridedSliceOp.getType().getDimSize(lastOffset) !=
          extractStridedSliceOp.getVectorType().getDimSize(lastOffset))
    break;
  sliceOffsets.pop_back();
}
unsigned destinationRank = 0;
if (auto vecType = extractOp.getType().dyn_cast<VectorType>())
  destinationRank = vecType.getRank();
// The dimensions of the result need to be untouched by the
// extractStridedSlice op.
if (destinationRank >
    extractStridedSliceOp.getVectorType().getRank() - sliceOffsets.size())
  return Value();
auto extractedPos = extractVector<int64_t>(extractOp.getPosition());
assert(extractedPos.size() >= sliceOffsets.size())(static_cast <bool> (extractedPos.size() >= sliceOffsets
.size()) ? void (0) : __assert_fail ("extractedPos.size() >= sliceOffsets.size()"
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 1513, __extension__
 __PRETTY_FUNCTION__));
for (size_t i = 0, e = sliceOffsets.size(); i < e; i++)
  extractedPos[i] = extractedPos[i] + sliceOffsets[i];
extractOp.getVectorMutable().assign(extractStridedSliceOp.getVector());
// OpBuilder is only used as a helper to build an I64ArrayAttr.
OpBuilder b(extractOp.getContext());
extractOp->setAttr(ExtractOp::getPositionAttrStrName(),
                   b.getI64ArrayAttr(extractedPos));
return extractOp.getResult();
1522}

1524/// Fold extract_op fed from a chain of insertStridedSlice ops.
1525static Value foldExtractStridedOpFromInsertChain(ExtractOp op) {
int64_t destinationRank = op.getType().isa<VectorType>()
                              ? op.getType().cast<VectorType>().getRank()
                              : 0;
auto insertOp = op.getVector().getDefiningOp<InsertStridedSliceOp>();
while (insertOp) {
  int64_t insertRankDiff = insertOp.getDestVectorType().getRank() -
                           insertOp.getSourceVectorType().getRank();
  if (destinationRank > insertOp.getSourceVectorType().getRank())
    return Value();
  auto insertOffsets = extractVector<int64_t>(insertOp.getOffsets());
  auto extractOffsets = extractVector<int64_t>(op.getPosition());

  if (llvm::any_of(insertOp.getStrides(), [](Attribute attr) {
        return attr.cast<IntegerAttr>().getInt() != 1;
      }))
    return Value();
  bool disjoint = false;
  SmallVector<int64_t, 4> offsetDiffs;
  for (unsigned dim = 0, e = extractOffsets.size(); dim < e; ++dim) {
    int64_t start = insertOffsets[dim];
    int64_t size =
        (dim < insertRankDiff)
            ? 1
            : insertOp.getSourceVectorType().getDimSize(dim - insertRankDiff);
    int64_t end = start + size;
    int64_t offset = extractOffsets[dim];
    // Check if the start of the extract offset is in the interval inserted.
    if (start <= offset && offset < end) {
      if (dim >= insertRankDiff)
        offsetDiffs.push_back(offset - start);
      continue;
    }
    disjoint = true;
    break;
  }
  // The extract element chunk overlap with the vector inserted.
  if (!disjoint) {
    // If any of the inner dimensions are only partially inserted we have a
    // partial overlap.
    int64_t srcRankDiff =
        insertOp.getSourceVectorType().getRank() - destinationRank;
    for (int64_t i = 0; i < destinationRank; i++) {
      if (insertOp.getSourceVectorType().getDimSize(i + srcRankDiff) !=
          insertOp.getDestVectorType().getDimSize(i + srcRankDiff +
                                                  insertRankDiff))
        return Value();
    }
    op.getVectorMutable().assign(insertOp.getSource());
    // OpBuilder is only used as a helper to build an I64ArrayAttr.
    OpBuilder b(op.getContext());
    op->setAttr(ExtractOp::getPositionAttrStrName(),
                b.getI64ArrayAttr(offsetDiffs));
    return op.getResult();
  }
  // If the chunk extracted is disjoint from the chunk inserted, keep
  // looking in the insert chain.
  insertOp = insertOp.getDest().getDefiningOp<InsertStridedSliceOp>();
}
return Value();
1585}

1587OpFoldResult ExtractOp::fold(ArrayRef<Attribute>) {
if (getPosition().empty())
  return getVector();
if (succeeded(foldExtractOpFromExtractChain(*this)))
  return getResult();
if (auto res = ExtractFromInsertTransposeChainState(*this).fold())
  return res;
if (auto res = foldExtractFromBroadcast(*this))
  return res;
if (auto res = foldExtractFromShapeCast(*this))
  return res;
if (auto val = foldExtractFromExtractStrided(*this))
  return val;
if (auto val = foldExtractStridedOpFromInsertChain(*this))
  return val;
return OpFoldResult();
1603}

1605namespace {

1607// Pattern to rewrite a ExtractOp(Broadcast) -> Broadcast.
1608class ExtractOpFromBroadcast final : public OpRewritePattern<ExtractOp> {
1609public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(ExtractOp extractOp,
                              PatternRewriter &rewriter) const override {
  Operation *defOp = extractOp.getVector().getDefiningOp();
  if (!defOp || !isa<vector::BroadcastOp, SplatOp>(defOp))
    return failure();

  Value source = defOp->getOperand(0);
  if (extractOp.getType() == source.getType())
    return failure();
  auto getRank = [](Type type) {
    return type.isa<VectorType>() ? type.cast<VectorType>().getRank() : 0;
  };
  unsigned broadcastSrcRank = getRank(source.getType());
  unsigned extractResultRank = getRank(extractOp.getType());
  // We only consider the case where the rank of the source is less than or
  // equal to the rank of the extract dst. The other cases are handled in the
  // folding patterns.
  if (extractResultRank < broadcastSrcRank)
    return failure();
  rewriter.replaceOpWithNewOp<vector::BroadcastOp>(
      extractOp, extractOp.getType(), source);
  return success();
}
1635};

1637// Pattern to rewrite a ExtractOp(splat ConstantOp) -> ConstantOp.
1638class ExtractOpSplatConstantFolder final : public OpRewritePattern<ExtractOp> {
1639public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(ExtractOp extractOp,
                              PatternRewriter &rewriter) const override {
  // Return if 'ExtractOp' operand is not defined by a splat vector
  // ConstantOp.
  Value sourceVector = extractOp.getVector();
  Attribute vectorCst;
  if (!matchPattern(sourceVector, m_Constant(&vectorCst)))
    return failure();
  auto splat = vectorCst.dyn_cast<SplatElementsAttr>();
  if (!splat)
    return failure();
  Attribute newAttr = splat.getSplatValue<Attribute>();
  if (auto vecDstType = extractOp.getType().dyn_cast<VectorType>())
    newAttr = DenseElementsAttr::get(vecDstType, newAttr);
  rewriter.replaceOpWithNewOp<arith::ConstantOp>(extractOp, newAttr);
  return success();
}
1659};

1661// Pattern to rewrite a ExtractOp(non-splat ConstantOp)[...] -> ConstantOp.
1662class ExtractOpNonSplatConstantFolder final
  : public OpRewritePattern<ExtractOp> {
1664public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(ExtractOp extractOp,
                              PatternRewriter &rewriter) const override {
  // Return if 'ExtractOp' operand is not defined by a compatible vector
  // ConstantOp.
  Value sourceVector = extractOp.getVector();
  Attribute vectorCst;
  if (!matchPattern(sourceVector, m_Constant(&vectorCst)))
    return failure();

  auto vecTy = sourceVector.getType().cast<VectorType>();
  if (vecTy.isScalable())
    return failure();

  // The splat case is handled by `ExtractOpSplatConstantFolder`.
  auto dense = vectorCst.dyn_cast<DenseElementsAttr>();
  if (!dense || dense.isSplat())
    return failure();

  // Calculate the linearized position of the continuous chunk of elements to
  // extract.
  llvm::SmallVector<int64_t> completePositions(vecTy.getRank(), 0);
  copy(getI64SubArray(extractOp.getPosition()), completePositions.begin());
  int64_t elemBeginPosition =
      linearize(completePositions, computeStrides(vecTy.getShape()));
  auto denseValuesBegin = dense.value_begin<Attribute>() + elemBeginPosition;

  Attribute newAttr;
  if (auto resVecTy = extractOp.getType().dyn_cast<VectorType>()) {
    SmallVector<Attribute> elementValues(
        denseValuesBegin, denseValuesBegin + resVecTy.getNumElements());
    newAttr = DenseElementsAttr::get(resVecTy, elementValues);
  } else {
    newAttr = *denseValuesBegin;
  }

  rewriter.replaceOpWithNewOp<arith::ConstantOp>(extractOp, newAttr);
  return success();
}
1705};

1707} // namespace

1709void ExtractOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                          MLIRContext *context) {
results.add<ExtractOpSplatConstantFolder, ExtractOpNonSplatConstantFolder,
            ExtractOpFromBroadcast>(context);
1713}

1715static void populateFromInt64AttrArray(ArrayAttr arrayAttr,
                                     SmallVectorImpl<int64_t> &results) {
for (auto attr : arrayAttr)
  results.push_back(attr.cast<IntegerAttr>().getInt());
1719}

1721//===----------------------------------------------------------------------===//
1722// FmaOp
1723//===----------------------------------------------------------------------===//

1725std::optional<SmallVector<int64_t, 4>> FMAOp::getShapeForUnroll() {
return llvm::to_vector<4>(getVectorType().getShape());
1727}

1729//===----------------------------------------------------------------------===//
1730// BroadcastOp
1731//===----------------------------------------------------------------------===//

1733/// Return the dimensions of the result vector that were formerly ones in the
1734/// source tensor and thus correspond to "dim-1" broadcasting.
1735static llvm::SetVector<int64_t>
1736computeBroadcastedUnitDims(ArrayRef<int64_t> srcShape,
                         ArrayRef<int64_t> dstShape) {
int64_t rankDiff = dstShape.size() - srcShape.size();
int64_t dstDim = rankDiff;
llvm::SetVector<int64_t> res;
for (auto [s1, s2] :
     llvm::zip_equal(srcShape, dstShape.drop_front(rankDiff))) {
  if (s1 != s2) {
    assert(s1 == 1 && "expected dim-1 broadcasting")(static_cast <bool> (s1 == 1 && "expected dim-1 broadcasting"
) ? void (0) : __assert_fail ("s1 == 1 && \"expected dim-1 broadcasting\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 1744, __extension__
 __PRETTY_FUNCTION__));
    res.insert(dstDim);
  }
  ++dstDim;
}
return res;
1750}

1752llvm::SetVector<int64_t> BroadcastOp::computeBroadcastedUnitDims() {
// Scalar broadcast is without any unit dim broadcast.
auto srcVectorType = getSourceType().dyn_cast<VectorType>();
if (!srcVectorType)
  return {};
return ::computeBroadcastedUnitDims(srcVectorType.getShape(),
                                    getVectorType().getShape());
1759}

1761/// Broadcast `value` to a vector of `dstShape`, knowing that exactly the
1762/// `broadcastedDims` dimensions in the dstShape are broadcasted.
1763/// This requires (and asserts) that the broadcast is free of dim-1
1764/// broadcasting.
1765/// Since vector.broadcast only allows expanding leading dimensions, an extra
1766/// vector.transpose may be inserted to make the broadcast possible.
1767/// `value`, `dstShape` and `broadcastedDims` must be properly specified or
1768/// the helper will assert. This means:
1769///   1. `dstShape` must not be empty.
1770///   2. `broadcastedDims` must be confined to [0 .. rank(value.getVectorType)]
1771///   2. `dstShape` trimmed of the dimensions specified in `broadcastedDims`
1772//       must match the `value` shape.
1773Value BroadcastOp::createOrFoldBroadcastOp(
  OpBuilder &b, Value value, ArrayRef<int64_t> dstShape,
  const llvm::SetVector<int64_t> &broadcastedDims) {
assert(!dstShape.empty() && "unexpected empty dst shape")(static_cast <bool> (!dstShape.empty() && "unexpected empty dst shape"
) ? void (0) : __assert_fail ("!dstShape.empty() && \"unexpected empty dst shape\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 1776, __extension__
 __PRETTY_FUNCTION__));

// Well-formedness check.
SmallVector<int64_t> checkShape;
for (int i = 0, e = dstShape.size(); i < e; ++i) {
  if (broadcastedDims.contains(i))
    continue;
  checkShape.push_back(dstShape[i]);
}
assert(broadcastedDims.size() == dstShape.size() - checkShape.size() &&(static_cast <bool> (broadcastedDims.size() == dstShape
.size() - checkShape.size() && "ill-formed broadcastedDims contains values not confined to "
 "destVectorShape") ? void (0) : __assert_fail ("broadcastedDims.size() == dstShape.size() - checkShape.size() && \"ill-formed broadcastedDims contains values not confined to \" \"destVectorShape\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 1787, __extension__
 __PRETTY_FUNCTION__))
       "ill-formed broadcastedDims contains values not confined to "(static_cast <bool> (broadcastedDims.size() == dstShape
.size() - checkShape.size() && "ill-formed broadcastedDims contains values not confined to "
 "destVectorShape") ? void (0) : __assert_fail ("broadcastedDims.size() == dstShape.size() - checkShape.size() && \"ill-formed broadcastedDims contains values not confined to \" \"destVectorShape\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 1787, __extension__
 __PRETTY_FUNCTION__))
       "destVectorShape")(static_cast <bool> (broadcastedDims.size() == dstShape
.size() - checkShape.size() && "ill-formed broadcastedDims contains values not confined to "
 "destVectorShape") ? void (0) : __assert_fail ("broadcastedDims.size() == dstShape.size() - checkShape.size() && \"ill-formed broadcastedDims contains values not confined to \" \"destVectorShape\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 1787, __extension__
 __PRETTY_FUNCTION__));

Location loc = value.getLoc();
Type elementType = getElementTypeOrSelf(value.getType());
VectorType srcVectorType = value.getType().dyn_cast<VectorType>();
VectorType dstVectorType = VectorType::get(dstShape, elementType);

// Step 2. If scalar -> dstShape broadcast, just do it.
if (!srcVectorType) {
  assert(checkShape.empty() &&(static_cast <bool> (checkShape.empty() && "ill-formed createOrFoldBroadcastOp arguments"
) ? void (0) : __assert_fail ("checkShape.empty() && \"ill-formed createOrFoldBroadcastOp arguments\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 1797, __extension__
 __PRETTY_FUNCTION__))
         "ill-formed createOrFoldBroadcastOp arguments")(static_cast <bool> (checkShape.empty() && "ill-formed createOrFoldBroadcastOp arguments"
) ? void (0) : __assert_fail ("checkShape.empty() && \"ill-formed createOrFoldBroadcastOp arguments\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 1797, __extension__
 __PRETTY_FUNCTION__));
  return b.createOrFold<vector::BroadcastOp>(loc, dstVectorType, value);
}

assert(srcVectorType.getShape().equals(checkShape) &&(static_cast <bool> (srcVectorType.getShape().equals(checkShape
) && "ill-formed createOrFoldBroadcastOp arguments") ?
 void (0) : __assert_fail ("srcVectorType.getShape().equals(checkShape) && \"ill-formed createOrFoldBroadcastOp arguments\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 1802, __extension__
 __PRETTY_FUNCTION__))
       "ill-formed createOrFoldBroadcastOp arguments")(static_cast <bool> (srcVectorType.getShape().equals(checkShape
) && "ill-formed createOrFoldBroadcastOp arguments") ?
 void (0) : __assert_fail ("srcVectorType.getShape().equals(checkShape) && \"ill-formed createOrFoldBroadcastOp arguments\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 1802, __extension__
 __PRETTY_FUNCTION__));

// Step 3. Since vector.broadcast only allows creating leading dims,
//   vector -> dstShape broadcast may require a transpose.
// Traverse the dims in order and construct:
//   1. The leading entries of the broadcastShape that is guaranteed to be
//      achievable by a simple broadcast.
//   2. The induced permutation for the subsequent vector.transpose that will
//      bring us from `broadcastShape` back to he desired `dstShape`.
// If the induced permutation is not the identity, create a vector.transpose.
SmallVector<int64_t> broadcastShape, permutation(dstShape.size(), -1);
broadcastShape.reserve(dstShape.size());
// Consider the example:
//   srcShape     = 2x4
//   dstShape     = 1x2x3x4x5
//   broadcastedDims = [0, 2, 4]
//
// We want to build:
//   broadcastShape  = 1x3x5x2x4
//   permutation     = [0, 2, 4,                 1, 3]
//                      ---V---           -----V-----
//            leading broadcast part      src shape part
//
// Note that the trailing dims of broadcastShape are exactly the srcShape
// by construction.
// nextSrcShapeDim is used to keep track of where in the permutation the
// "src shape part" occurs.
int64_t nextSrcShapeDim = broadcastedDims.size();
for (int64_t i = 0, e = dstShape.size(); i < e; ++i) {
  if (broadcastedDims.contains(i)) {
    // 3.a. For each dim in the dst shape, if it is a broadcasted dim,
    // bring it to the head of the broadcastShape.
    // It will need to be permuted back from `broadcastShape.size() - 1` into
    // position `i`.
    broadcastShape.push_back(dstShape[i]);
    permutation[i] = broadcastShape.size() - 1;
  } else {
    // 3.b. Otherwise, the dim is not broadcasted, it comes from the src
    // shape and needs to be permuted into position `i`.
    // Don't touch `broadcastShape` here, the whole srcShape will be
    // appended after.
    permutation[i] = nextSrcShapeDim++;
  }
}
// 3.c. Append the srcShape.
llvm::append_range(broadcastShape, srcVectorType.getShape());

// Ensure there are no dim-1 broadcasts.
assert(::computeBroadcastedUnitDims(srcVectorType.getShape(), broadcastShape)(static_cast <bool> (::computeBroadcastedUnitDims(srcVectorType
.getShape(), broadcastShape) .empty() && "unexpected dim-1 broadcast"
) ? void (0) : __assert_fail ("::computeBroadcastedUnitDims(srcVectorType.getShape(), broadcastShape) .empty() && \"unexpected dim-1 broadcast\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 1852, __extension__
 __PRETTY_FUNCTION__))
           .empty() &&(static_cast <bool> (::computeBroadcastedUnitDims(srcVectorType
.getShape(), broadcastShape) .empty() && "unexpected dim-1 broadcast"
) ? void (0) : __assert_fail ("::computeBroadcastedUnitDims(srcVectorType.getShape(), broadcastShape) .empty() && \"unexpected dim-1 broadcast\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 1852, __extension__
 __PRETTY_FUNCTION__))
       "unexpected dim-1 broadcast")(static_cast <bool> (::computeBroadcastedUnitDims(srcVectorType
.getShape(), broadcastShape) .empty() && "unexpected dim-1 broadcast"
) ? void (0) : __assert_fail ("::computeBroadcastedUnitDims(srcVectorType.getShape(), broadcastShape) .empty() && \"unexpected dim-1 broadcast\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 1852, __extension__
 __PRETTY_FUNCTION__));

VectorType broadcastType = VectorType::get(broadcastShape, elementType);
assert(vector::isBroadcastableTo(value.getType(), broadcastType) ==(static_cast <bool> (vector::isBroadcastableTo(value.getType
(), broadcastType) == vector::BroadcastableToResult::Success &&
 "must be broadcastable") ? void (0) : __assert_fail ("vector::isBroadcastableTo(value.getType(), broadcastType) == vector::BroadcastableToResult::Success && \"must be broadcastable\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 1857, __extension__
 __PRETTY_FUNCTION__))
           vector::BroadcastableToResult::Success &&(static_cast <bool> (vector::isBroadcastableTo(value.getType
(), broadcastType) == vector::BroadcastableToResult::Success &&
 "must be broadcastable") ? void (0) : __assert_fail ("vector::isBroadcastableTo(value.getType(), broadcastType) == vector::BroadcastableToResult::Success && \"must be broadcastable\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 1857, __extension__
 __PRETTY_FUNCTION__))
       "must be broadcastable")(static_cast <bool> (vector::isBroadcastableTo(value.getType
(), broadcastType) == vector::BroadcastableToResult::Success &&
 "must be broadcastable") ? void (0) : __assert_fail ("vector::isBroadcastableTo(value.getType(), broadcastType) == vector::BroadcastableToResult::Success && \"must be broadcastable\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 1857, __extension__
 __PRETTY_FUNCTION__));
Value res = b.createOrFold<vector::BroadcastOp>(loc, broadcastType, value);
// Step 4. If we find any dimension that indeed needs to be permuted,
// immediately return a new vector.transpose.
for (int64_t i = 0, e = permutation.size(); i < e; ++i)
  if (permutation[i] != i)
    return b.createOrFold<vector::TransposeOp>(loc, res, permutation);
// Otherwise return res.
return res;
1866}

1868BroadcastableToResult
1869mlir::vector::isBroadcastableTo(Type srcType, VectorType dstVectorType,
                              std::pair<int, int> *mismatchingDims) {
// Broadcast scalar to vector of the same element type.
if (srcType.isIntOrIndexOrFloat() && dstVectorType &&
    getElementTypeOrSelf(srcType) == getElementTypeOrSelf(dstVectorType))
  return BroadcastableToResult::Success;
// From now on, only vectors broadcast.
VectorType srcVectorType = srcType.dyn_cast<VectorType>();
if (!srcVectorType)
  return BroadcastableToResult::SourceTypeNotAVector;

int64_t srcRank = srcVectorType.getRank();
int64_t dstRank = dstVectorType.getRank();
if (srcRank > dstRank)
  return BroadcastableToResult::SourceRankHigher;
// Source has an exact match or singleton value for all trailing dimensions
// (all leading dimensions are simply duplicated).
int64_t lead = dstRank - srcRank;
for (int64_t r = 0; r < srcRank; ++r) {
  int64_t srcDim = srcVectorType.getDimSize(r);
  int64_t dstDim = dstVectorType.getDimSize(lead + r);
  if (srcDim != 1 && srcDim != dstDim) {
    if (mismatchingDims) {
      mismatchingDims->first = srcDim;
      mismatchingDims->second = dstDim;
    }
    return BroadcastableToResult::DimensionMismatch;
  }
}

return BroadcastableToResult::Success;
1900}

1902LogicalResult BroadcastOp::verify() {
std::pair<int, int> mismatchingDims;
BroadcastableToResult res =
    isBroadcastableTo(getSourceType(), getVectorType(), &mismatchingDims);
if (res == BroadcastableToResult::Success)
  return success();
if (res == BroadcastableToResult::SourceRankHigher)
  return emitOpError("source rank higher than destination rank");
if (res == BroadcastableToResult::DimensionMismatch)
  return emitOpError("dimension mismatch (")
         << mismatchingDims.first << " vs. " << mismatchingDims.second << ")";
if (res == BroadcastableToResult::SourceTypeNotAVector)
  return emitOpError("source type is not a vector");
llvm_unreachable("unexpected vector.broadcast op error")::llvm::llvm_unreachable_internal("unexpected vector.broadcast op error"
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 1915);
1916}

1918OpFoldResult BroadcastOp::fold(ArrayRef<Attribute> operands) {
if (getSourceType() == getVectorType())
  return getSource();
if (!operands[0])
  return {};
auto vectorType = getVectorType();
if (operands[0].isa<IntegerAttr, FloatAttr>())
  return DenseElementsAttr::get(vectorType, operands[0]);
if (auto attr = operands[0].dyn_cast<SplatElementsAttr>())
  return DenseElementsAttr::get(vectorType, attr.getSplatValue<Attribute>());
return {};
1929}

1931namespace {

1933// Fold broadcast1(broadcast2(x)) into broadcast1(x).
1934struct BroadcastFolder : public OpRewritePattern<BroadcastOp> {
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(BroadcastOp broadcastOp,
                              PatternRewriter &rewriter) const override {
  auto srcBroadcast = broadcastOp.getSource().getDefiningOp<BroadcastOp>();
  if (!srcBroadcast)
    return failure();
  rewriter.replaceOpWithNewOp<BroadcastOp>(
      broadcastOp, broadcastOp.getVectorType(), srcBroadcast.getSource());
  return success();
}
1946};
1947} // namespace

1949void BroadcastOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                            MLIRContext *context) {
// BroadcastToShapeCast is not a default canonicalization, it is opt-in by
// calling `populateCastAwayVectorLeadingOneDimPatterns`
results.add<BroadcastFolder>(context);
1954}

1956//===----------------------------------------------------------------------===//
1957// ShuffleOp
1958//===----------------------------------------------------------------------===//

1960void ShuffleOp::build(OpBuilder &builder, OperationState &result, Value v1,
                    Value v2, ArrayRef<int64_t> mask) {
build(builder, result, v1, v2, getVectorSubscriptAttr(builder, mask));
1963}

1965LogicalResult ShuffleOp::verify() {
VectorType resultType = getVectorType();
VectorType v1Type = getV1VectorType();
VectorType v2Type = getV2VectorType();
// Verify ranks.
int64_t resRank = resultType.getRank();
int64_t v1Rank = v1Type.getRank();
int64_t v2Rank = v2Type.getRank();
bool wellFormed0DCase = v1Rank == 0 && v2Rank == 0 && resRank == 1;
bool wellFormedNDCase = v1Rank == resRank && v2Rank == resRank;
if (!wellFormed0DCase && !wellFormedNDCase)
  return emitOpError("rank mismatch");

// Verify all but leading dimension sizes.
for (int64_t r = 1; r < v1Rank; ++r) {
  int64_t resDim = resultType.getDimSize(r);
  int64_t v1Dim = v1Type.getDimSize(r);
  int64_t v2Dim = v2Type.getDimSize(r);
  if (resDim != v1Dim || v1Dim != v2Dim)
    return emitOpError("dimension mismatch");
}
// Verify mask length.
auto maskAttr = getMask().getValue();
int64_t maskLength = maskAttr.size();
if (maskLength <= 0)
  return emitOpError("invalid mask length");
if (maskLength != resultType.getDimSize(0))
  return emitOpError("mask length mismatch");
// Verify all indices.
int64_t indexSize = (v1Type.getRank() == 0 ? 1 : v1Type.getDimSize(0)) +
                    (v2Type.getRank() == 0 ? 1 : v2Type.getDimSize(0));
for (const auto &en : llvm::enumerate(maskAttr)) {
  auto attr = en.value().dyn_cast<IntegerAttr>();
  if (!attr || attr.getInt() < 0 || attr.getInt() >= indexSize)
    return emitOpError("mask index #") << (en.index() + 1) << " out of range";
}
return success();
2002}

2004LogicalResult
2005ShuffleOp::inferReturnTypes(MLIRContext *, std::optional<Location>,
                          ValueRange operands, DictionaryAttr attributes,
                          RegionRange,
                          SmallVectorImpl<Type> &inferredReturnTypes) {
ShuffleOp::Adaptor op(operands, attributes);
auto v1Type = op.getV1().getType().cast<VectorType>();
auto v1Rank = v1Type.getRank();
// Construct resulting type: leading dimension matches mask
// length, all trailing dimensions match the operands.
SmallVector<int64_t, 4> shape;
shape.reserve(v1Rank);
shape.push_back(std::max<size_t>(1, op.getMask().size()));
// In the 0-D case there is no trailing shape to append.
if (v1Rank > 0)
  llvm::append_range(shape, v1Type.getShape().drop_front());
inferredReturnTypes.push_back(
    VectorType::get(shape, v1Type.getElementType()));
return success();
2023}

2025static bool isStepIndexArray(ArrayAttr idxArr, uint64_t begin, size_t width) {
uint64_t expected = begin;
return idxArr.size() == width &&
       llvm::all_of(idxArr.getAsValueRange<IntegerAttr>(),
                    [&expected](auto attr) {
                      return attr.getZExtValue() == expected++;
                    });
2032}

2034OpFoldResult vector::ShuffleOp::fold(ArrayRef<Attribute> operands) {
VectorType v1Type = getV1VectorType();
// For consistency: 0-D shuffle return type is 1-D, this cannot be a folding
// but must be a canonicalization into a vector.broadcast.
if (v1Type.getRank() == 0)
  return {};

// fold shuffle V1, V2, [0, 1, 2, 3] : <4xi32>, <2xi32> -> V1
if (!v1Type.isScalable() &&
    isStepIndexArray(getMask(), 0, v1Type.getDimSize(0)))
  return getV1();
// fold shuffle V1, V2, [4, 5] : <4xi32>, <2xi32> -> V2
if (!getV1VectorType().isScalable() && !getV2VectorType().isScalable() &&
    isStepIndexArray(getMask(), getV1VectorType().getDimSize(0),
                     getV2VectorType().getDimSize(0)))
  return getV2();

Attribute lhs = operands.front(), rhs = operands.back();
if (!lhs || !rhs)
  return {};

auto lhsType = lhs.cast<DenseElementsAttr>().getType().cast<VectorType>();
// Only support 1-D for now to avoid complicated n-D DenseElementsAttr
// manipulation.
if (lhsType.getRank() != 1)
  return {};
int64_t lhsSize = lhsType.getDimSize(0);

SmallVector<Attribute> results;
auto lhsElements = lhs.cast<DenseElementsAttr>().getValues<Attribute>();
auto rhsElements = rhs.cast<DenseElementsAttr>().getValues<Attribute>();
for (const auto &index : this->getMask().getAsValueRange<IntegerAttr>()) {
  int64_t i = index.getZExtValue();
  if (i >= lhsSize) {
    results.push_back(rhsElements[i - lhsSize]);
  } else {
    results.push_back(lhsElements[i]);
  }
}

return DenseElementsAttr::get(getVectorType(), results);
2075}

2077namespace {

2079// Pattern to rewrite a 0-D shuffle with [0] or [1] mask returning a 1-D vector
2080// to a broadcast.
2081struct Canonicalize0DShuffleOp : public OpRewritePattern<ShuffleOp> {
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(ShuffleOp shuffleOp,
                              PatternRewriter &rewriter) const override {
  VectorType v1VectorType = shuffleOp.getV1VectorType();
  ArrayAttr mask = shuffleOp.getMask();
  if (v1VectorType.getRank() > 0)
    return failure();
  if (mask.size() != 1)
    return failure();
  Type resType = VectorType::Builder(v1VectorType).setShape({1});
  if (mask[0].cast<IntegerAttr>().getInt() == 0)
    rewriter.replaceOpWithNewOp<vector::BroadcastOp>(shuffleOp, resType,
                                                     shuffleOp.getV1());
  else
    rewriter.replaceOpWithNewOp<vector::BroadcastOp>(shuffleOp, resType,
                                                     shuffleOp.getV2());
  return success();
}
2101};

2103/// Pattern to rewrite a ShuffleOp(SplatOp, SplatOp) to SplatOp.
2104class ShuffleSplat final : public OpRewritePattern<ShuffleOp> {
2105public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(ShuffleOp op,
                              PatternRewriter &rewriter) const override {
  auto v1Splat = op.getV1().getDefiningOp<SplatOp>();
  auto v2Splat = op.getV2().getDefiningOp<SplatOp>();

  if (!v1Splat || !v2Splat)
    return failure();

  if (v1Splat.getInput() != v2Splat.getInput())
    return failure();

  rewriter.replaceOpWithNewOp<SplatOp>(op, op.getType(), v1Splat.getInput());
  return success();
}
2122};

2124} // namespace

2126void ShuffleOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                          MLIRContext *context) {
results.add<ShuffleSplat, Canonicalize0DShuffleOp>(context);
2129}

2131//===----------------------------------------------------------------------===//
2132// InsertElementOp
2133//===----------------------------------------------------------------------===//

2135void InsertElementOp::build(OpBuilder &builder, OperationState &result,
                          Value source, Value dest) {
build(builder, result, source, dest, {});
2138}

2140LogicalResult InsertElementOp::verify() {
auto dstVectorType = getDestVectorType();
if (dstVectorType.getRank() == 0) {
  if (getPosition())
    return emitOpError("expected position to be empty with 0-D vector");
  return success();
}
if (dstVectorType.getRank() != 1)
  return emitOpError("unexpected >1 vector rank");
if (!getPosition())
  return emitOpError("expected position for 1-D vector");
return success();
2152}

2154OpFoldResult vector::InsertElementOp::fold(ArrayRef<Attribute> operands) {
// Skip the 0-D vector here.
if (operands.size() < 3)
  return {};

Attribute src = operands[0];
Attribute dst = operands[1];
Attribute pos = operands[2];
if (!src || !dst || !pos)
  return {};

auto dstElements = dst.cast<DenseElementsAttr>().getValues<Attribute>();

SmallVector<Attribute> results(dstElements);

auto attr = pos.dyn_cast<IntegerAttr>();
uint64_t posIdx = attr.getInt();

results[posIdx] = src;

return DenseElementsAttr::get(getDestVectorType(), results);
2175}

2177//===----------------------------------------------------------------------===//
2178// InsertOp
2179//===----------------------------------------------------------------------===//

2181void InsertOp::build(OpBuilder &builder, OperationState &result, Value source,
                   Value dest, ArrayRef<int64_t> position) {
result.addOperands({source, dest});
auto positionAttr = getVectorSubscriptAttr(builder, position);
result.addTypes(dest.getType());
result.addAttribute(getPositionAttrStrName(), positionAttr);
2187}

2189// Convenience builder which assumes the values are constant indices.
2190void InsertOp::build(OpBuilder &builder, OperationState &result, Value source,
                   Value dest, ValueRange position) {
SmallVector<int64_t, 4> positionConstants =
    llvm::to_vector<4>(llvm::map_range(position, [](Value pos) {
      return pos.getDefiningOp<arith::ConstantIndexOp>().value();
    }));
build(builder, result, source, dest, positionConstants);
2197}

2199LogicalResult InsertOp::verify() {
auto positionAttr = getPosition().getValue();
auto destVectorType = getDestVectorType();
if (positionAttr.size() > static_cast<unsigned>(destVectorType.getRank()))
  return emitOpError(
      "expected position attribute of rank smaller than dest vector rank");
auto srcVectorType = getSourceType().dyn_cast<VectorType>();
if (srcVectorType &&
    (static_cast<unsigned>(srcVectorType.getRank()) + positionAttr.size() !=
     static_cast<unsigned>(destVectorType.getRank())))
  return emitOpError("expected position attribute rank + source rank to "
                     "match dest vector rank");
if (!srcVectorType &&
    (positionAttr.size() != static_cast<unsigned>(destVectorType.getRank())))
  return emitOpError(
      "expected position attribute rank to match the dest vector rank");
for (const auto &en : llvm::enumerate(positionAttr)) {
  auto attr = en.value().dyn_cast<IntegerAttr>();
  if (!attr || attr.getInt() < 0 ||
      attr.getInt() >= destVectorType.getDimSize(en.index()))
    return emitOpError("expected position attribute #")
           << (en.index() + 1)
           << " to be a non-negative integer smaller than the corresponding "
              "dest vector dimension";
}
return success();
2225}

2227namespace {

2229// If insertOp is only inserting unit dimensions it can be transformed to a
2230// broadcast.
2231class InsertToBroadcast final : public OpRewritePattern<InsertOp> {
2232public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(InsertOp insertOp,
                              PatternRewriter &rewriter) const override {
  auto srcVecType = insertOp.getSourceType().dyn_cast<VectorType>();
  if (!srcVecType || insertOp.getDestVectorType().getNumElements() !=
                         srcVecType.getNumElements())
    return failure();
  rewriter.replaceOpWithNewOp<BroadcastOp>(
      insertOp, insertOp.getDestVectorType(), insertOp.getSource());
  return success();
}
2245};

2247/// Pattern to rewrite a InsertOp(SplatOp, SplatOp) to SplatOp.
2248class InsertSplatToSplat final : public OpRewritePattern<InsertOp> {
2249public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(InsertOp op,
                              PatternRewriter &rewriter) const override {
  auto srcSplat = op.getSource().getDefiningOp<SplatOp>();
  auto dstSplat = op.getDest().getDefiningOp<SplatOp>();

  if (!srcSplat || !dstSplat)
    return failure();

  if (srcSplat.getInput() != dstSplat.getInput())
    return failure();

  rewriter.replaceOpWithNewOp<SplatOp>(op, op.getType(), srcSplat.getInput());
  return success();
}
2266};

2268// Pattern to rewrite a InsertOp(ConstantOp into ConstantOp) -> ConstantOp.
2269class InsertOpConstantFolder final : public OpRewritePattern<InsertOp> {
2270public:
using OpRewritePattern::OpRewritePattern;

// Do not create constants with more than `vectorSizeFoldThreashold` elements,
// unless the source vector constant has a single use.
static constexpr int64_t vectorSizeFoldThreshold = 256;

LogicalResult matchAndRewrite(InsertOp op,
                              PatternRewriter &rewriter) const override {
  // Return if 'InsertOp' operand is not defined by a compatible vector
  // ConstantOp.
  TypedValue<VectorType> destVector = op.getDest();
  Attribute vectorDestCst;
  if (!matchPattern(destVector, m_Constant(&vectorDestCst)))
    return failure();

  VectorType destTy = destVector.getType();
  if (destTy.isScalable())
    return failure();

  // Make sure we do not create too many large constants.
  if (destTy.getNumElements() > vectorSizeFoldThreshold &&
      !destVector.hasOneUse())
    return failure();

  auto denseDest = vectorDestCst.cast<DenseElementsAttr>();

  Value sourceValue = op.getSource();
  Attribute sourceCst;
  if (!matchPattern(sourceValue, m_Constant(&sourceCst)))
    return failure();

  // Calculate the linearized position of the continuous chunk of elements to
  // insert.
  llvm::SmallVector<int64_t> completePositions(destTy.getRank(), 0);
  copy(getI64SubArray(op.getPosition()), completePositions.begin());
  int64_t insertBeginPosition =
      linearize(completePositions, computeStrides(destTy.getShape()));

  SmallVector<Attribute> insertedValues;
  if (auto denseSource = sourceCst.dyn_cast<DenseElementsAttr>())
    llvm::append_range(insertedValues, denseSource.getValues<Attribute>());
  else
    insertedValues.push_back(sourceCst);

  auto allValues = llvm::to_vector(denseDest.getValues<Attribute>());
  copy(insertedValues, allValues.begin() + insertBeginPosition);
  auto newAttr = DenseElementsAttr::get(destTy, allValues);

  rewriter.replaceOpWithNewOp<arith::ConstantOp>(op, newAttr);
  return success();
}
2322};

2324} // namespace

2326void InsertOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                         MLIRContext *context) {
results.add<InsertToBroadcast, BroadcastFolder, InsertSplatToSplat,
            InsertOpConstantFolder>(context);
2330}

2332// Eliminates insert operations that produce values identical to their source
2333// value. This happens when the source and destination vectors have identical
2334// sizes.
2335OpFoldResult vector::InsertOp::fold(ArrayRef<Attribute> operands) {
if (getPosition().empty())
  return getSource();
return {};
2339}

2341//===----------------------------------------------------------------------===//
2342// InsertStridedSliceOp
2343//===----------------------------------------------------------------------===//

2345void InsertStridedSliceOp::build(OpBuilder &builder, OperationState &result,
                               Value source, Value dest,
                               ArrayRef<int64_t> offsets,
                               ArrayRef<int64_t> strides) {
result.addOperands({source, dest});
auto offsetsAttr = getVectorSubscriptAttr(builder, offsets);
auto stridesAttr = getVectorSubscriptAttr(builder, strides);
result.addTypes(dest.getType());
result.addAttribute(getOffsetsAttrStrName(), offsetsAttr);
result.addAttribute(getStridesAttrStrName(), stridesAttr);
2355}

2357// TODO: Should be moved to Tablegen ConfinedAttr attributes.
2358template <typename OpType>
2359static LogicalResult isIntegerArrayAttrSmallerThanShape(OpType op,
                                                      ArrayAttr arrayAttr,
                                                      ArrayRef<int64_t> shape,
                                                      StringRef attrName) {
if (arrayAttr.size() > shape.size())
  return op.emitOpError("expected ")
         << attrName << " attribute of rank smaller than vector rank";
return success();
2367}

2369// Returns true if all integers in `arrayAttr` are in the half-open [min, max}
2370// interval. If `halfOpen` is true then the admissible interval is [min, max).
2371// Otherwise, the admissible interval is [min, max].
2372template <typename OpType>
2373static LogicalResult
2374isIntegerArrayAttrConfinedToRange(OpType op, ArrayAttr arrayAttr, int64_t min,
                                int64_t max, StringRef attrName,
                                bool halfOpen = true) {
for (auto attr : arrayAttr) {
  auto val = attr.cast<IntegerAttr>().getInt();
  auto upper = max;
  if (!halfOpen)
    upper += 1;
  if (val < min || val >= upper)
    return op.emitOpError("expected ") << attrName << " to be confined to ["
                                       << min << ", " << upper << ")";
}
return success();
2387}

2389// Returns true if all integers in `arrayAttr` are in the half-open [min, max}
2390// interval. If `halfOpen` is true then the admissible interval is [min, max).
2391// Otherwise, the admissible interval is [min, max].
2392template <typename OpType>
2393static LogicalResult
2394isIntegerArrayAttrConfinedToShape(OpType op, ArrayAttr arrayAttr,
                                ArrayRef<int64_t> shape, StringRef attrName,
                                bool halfOpen = true, int64_t min = 0) {
for (auto [index, attrDimPair] :
     llvm::enumerate(llvm::zip_first(arrayAttr, shape))) {
  int64_t val =
      std::get<0>(attrDimPair).template cast<IntegerAttr>().getInt();
  int64_t max = std::get<1>(attrDimPair);
  if (!halfOpen)
    max += 1;
  if (val < min || val >= max)
    return op.emitOpError("expected ")
           << attrName << " dimension " << index << " to be confined to ["
           << min << ", " << max << ")";
}
return success();
2410}

2412// Returns true if all integers in `arrayAttr` are in the interval [min, max}.
2413// interval. If `halfOpen` is true then the admissible interval is [min, max).
2414// Otherwise, the admissible interval is [min, max].
2415template <typename OpType>
2416static LogicalResult isSumOfIntegerArrayAttrConfinedToShape(
  OpType op, ArrayAttr arrayAttr1, ArrayAttr arrayAttr2,
  ArrayRef<int64_t> shape, StringRef attrName1, StringRef attrName2,
  bool halfOpen = true, int64_t min = 1) {
assert(arrayAttr1.size() <= shape.size())(static_cast <bool> (arrayAttr1.size() <= shape.size
()) ? void (0) : __assert_fail ("arrayAttr1.size() <= shape.size()"
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 2420, __extension__
 __PRETTY_FUNCTION__));
assert(arrayAttr2.size() <= shape.size())(static_cast <bool> (arrayAttr2.size() <= shape.size
()) ? void (0) : __assert_fail ("arrayAttr2.size() <= shape.size()"
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 2421, __extension__
 __PRETTY_FUNCTION__));
for (auto [index, it] :
     llvm::enumerate(llvm::zip(arrayAttr1, arrayAttr2, shape))) {
  auto val1 = std::get<0>(it).template cast<IntegerAttr>().getInt();
  auto val2 = std::get<1>(it).template cast<IntegerAttr>().getInt();
  int64_t max = std::get<2>(it);
  if (!halfOpen)
    max += 1;
  if (val1 + val2 < 0 || val1 + val2 >= max)
    return op.emitOpError("expected sum(")
           << attrName1 << ", " << attrName2 << ") dimension " << index
           << " to be confined to [" << min << ", " << max << ")";
}
return success();
2435}

2437static ArrayAttr makeI64ArrayAttr(ArrayRef<int64_t> values,
                                MLIRContext *context) {
auto attrs = llvm::map_range(values, [context](int64_t v) -> Attribute {
  return IntegerAttr::get(IntegerType::get(context, 64), APInt(64, v));
});
return ArrayAttr::get(context, llvm::to_vector<8>(attrs));
2443}

2445LogicalResult InsertStridedSliceOp::verify() {
auto sourceVectorType = getSourceVectorType();
auto destVectorType = getDestVectorType();
auto offsets = getOffsetsAttr();
auto strides = getStridesAttr();
if (offsets.size() != static_cast<unsigned>(destVectorType.getRank()))
  return emitOpError(
      "expected offsets of same size as destination vector rank");
if (strides.size() != static_cast<unsigned>(sourceVectorType.getRank()))
  return emitOpError("expected strides of same size as source vector rank");
if (sourceVectorType.getRank() > destVectorType.getRank())
  return emitOpError(
      "expected source rank to be smaller than destination rank");

auto sourceShape = sourceVectorType.getShape();
auto destShape = destVectorType.getShape();
SmallVector<int64_t, 4> sourceShapeAsDestShape(
    destShape.size() - sourceShape.size(), 0);
sourceShapeAsDestShape.append(sourceShape.begin(), sourceShape.end());
auto offName = InsertStridedSliceOp::getOffsetsAttrName();
auto stridesName = InsertStridedSliceOp::getStridesAttrName();
if (failed(isIntegerArrayAttrConfinedToShape(*this, offsets, destShape,
                                             offName)) ||
    failed(isIntegerArrayAttrConfinedToRange(*this, strides, 1, 1,
                                             stridesName,
                                             /*halfOpen=*/false)) ||
    failed(isSumOfIntegerArrayAttrConfinedToShape(
        *this, offsets,
        makeI64ArrayAttr(sourceShapeAsDestShape, getContext()), destShape,
        offName, "source vector shape",
        /*halfOpen=*/false, /*min=*/1)))
  return failure();

return success();
2479}

2481namespace {
2482/// Pattern to rewrite an InsertStridedSliceOp(SplatOp(X):src_type,
2483/// SplatOp(X):dst_type) to SplatOp(X):dst_type.
2484class FoldInsertStridedSliceSplat final
  : public OpRewritePattern<InsertStridedSliceOp> {
2486public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(InsertStridedSliceOp insertStridedSliceOp,
                              PatternRewriter &rewriter) const override {
  auto srcSplatOp =
      insertStridedSliceOp.getSource().getDefiningOp<vector::SplatOp>();
  auto destSplatOp =
      insertStridedSliceOp.getDest().getDefiningOp<vector::SplatOp>();

  if (!srcSplatOp || !destSplatOp)
    return failure();

  if (srcSplatOp.getInput() != destSplatOp.getInput())
    return failure();

  rewriter.replaceOp(insertStridedSliceOp, insertStridedSliceOp.getDest());
  return success();
}
2505};

2507/// Pattern to rewrite an InsertStridedSliceOp(ExtractStridedSliceOp(dst), dst)
2508/// to dst.
2509class FoldInsertStridedSliceOfExtract final
  : public OpRewritePattern<InsertStridedSliceOp> {
2511public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(InsertStridedSliceOp insertStridedSliceOp,
                              PatternRewriter &rewriter) const override {
  auto extractStridedSliceOp =
      insertStridedSliceOp.getSource()
          .getDefiningOp<vector::ExtractStridedSliceOp>();

  if (!extractStridedSliceOp)
    return failure();

  if (extractStridedSliceOp.getOperand() != insertStridedSliceOp.getDest())
    return failure();

  // Check if have the same strides and offsets.
  if (extractStridedSliceOp.getStrides() !=
          insertStridedSliceOp.getStrides() ||
      extractStridedSliceOp.getOffsets() != insertStridedSliceOp.getOffsets())
    return failure();

  rewriter.replaceOp(insertStridedSliceOp, insertStridedSliceOp.getDest());
  return success();
}
2535};

2537// Pattern to rewrite an InsertStridedSliceOp(ConstantOp into ConstantOp) ->
2538// ConstantOp.
2539class InsertStridedSliceConstantFolder final
  : public OpRewritePattern<InsertStridedSliceOp> {
2541public:
using OpRewritePattern::OpRewritePattern;

// Do not create constants with more than `vectorSizeFoldThreashold` elements,
// unless the source vector constant has a single use.
static constexpr int64_t vectorSizeFoldThreshold = 256;

LogicalResult matchAndRewrite(InsertStridedSliceOp op,
                              PatternRewriter &rewriter) const override {
  // Return if 'InsertOp' operand is not defined by a compatible vector
  // ConstantOp.
  TypedValue<VectorType> destVector = op.getDest();
  Attribute vectorDestCst;
  if (!matchPattern(destVector, m_Constant(&vectorDestCst)))
    return failure();

  VectorType destTy = destVector.getType();
  if (destTy.isScalable())
    return failure();

  // Make sure we do not create too many large constants.
  if (destTy.getNumElements() > vectorSizeFoldThreshold &&
      !destVector.hasOneUse())
    return failure();

  auto denseDest = vectorDestCst.cast<DenseElementsAttr>();

  TypedValue<VectorType> sourceValue = op.getSource();
  Attribute sourceCst;
  if (!matchPattern(sourceValue, m_Constant(&sourceCst)))
    return failure();

  // TODO: Handle non-unit strides when they become available.
  if (op.hasNonUnitStrides())
    return failure();

  VectorType sliceVecTy = sourceValue.getType();
  ArrayRef<int64_t> sliceShape = sliceVecTy.getShape();
  int64_t rankDifference = destTy.getRank() - sliceVecTy.getRank();
  SmallVector<int64_t, 4> offsets = getI64SubArray(op.getOffsets());
  SmallVector<int64_t, 4> destStrides = computeStrides(destTy.getShape());

  // Calcualte the destination element indices by enumerating all slice
  // positions within the destination and linearizing them. The enumeration
  // order is lexicographic which yields a sequence of monotonically
  // increasing linearized position indices.
  // Because the destination may have higher dimensionality then the slice,
  // we keep track of two overlapping sets of positions and offsets.
  auto denseSlice = sourceCst.cast<DenseElementsAttr>();
  auto sliceValuesIt = denseSlice.value_begin<Attribute>();
  auto newValues = llvm::to_vector(denseDest.getValues<Attribute>());
  SmallVector<int64_t> currDestPosition(offsets.begin(), offsets.end());
  MutableArrayRef<int64_t> currSlicePosition(
      currDestPosition.begin() + rankDifference, currDestPosition.end());
  ArrayRef<int64_t> sliceOffsets(offsets.begin() + rankDifference,
                                 offsets.end());
  do {
    int64_t linearizedPosition = linearize(currDestPosition, destStrides);
    assert(linearizedPosition < destTy.getNumElements() && "Invalid index")(static_cast <bool> (linearizedPosition < destTy.getNumElements
() && "Invalid index") ? void (0) : __assert_fail ("linearizedPosition < destTy.getNumElements() && \"Invalid index\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 2599, __extension__
 __PRETTY_FUNCTION__));
    assert(sliceValuesIt != denseSlice.value_end<Attribute>() &&(static_cast <bool> (sliceValuesIt != denseSlice.value_end
<Attribute>() && "Invalid slice element") ? void
 (0) : __assert_fail ("sliceValuesIt != denseSlice.value_end<Attribute>() && \"Invalid slice element\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 2601, __extension__
 __PRETTY_FUNCTION__))
           "Invalid slice element")(static_cast <bool> (sliceValuesIt != denseSlice.value_end
<Attribute>() && "Invalid slice element") ? void
 (0) : __assert_fail ("sliceValuesIt != denseSlice.value_end<Attribute>() && \"Invalid slice element\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 2601, __extension__
 __PRETTY_FUNCTION__));
    newValues[linearizedPosition] = *sliceValuesIt;
    ++sliceValuesIt;
  } while (succeeded(
      incSlicePosition(currSlicePosition, sliceShape, sliceOffsets)));

  auto newAttr = DenseElementsAttr::get(destTy, newValues);
  rewriter.replaceOpWithNewOp<arith::ConstantOp>(op, newAttr);
  return success();
}
2611};

2613} // namespace

2615void vector::InsertStridedSliceOp::getCanonicalizationPatterns(
  RewritePatternSet &results, MLIRContext *context) {
results.add<FoldInsertStridedSliceSplat, FoldInsertStridedSliceOfExtract,
            InsertStridedSliceConstantFolder>(context);
2619}

2621OpFoldResult InsertStridedSliceOp::fold(ArrayRef<Attribute> operands) {
if (getSourceVectorType() == getDestVectorType())
  return getSource();
return {};
2625}

2627//===----------------------------------------------------------------------===//
2628// OuterProductOp
2629//===----------------------------------------------------------------------===//

2631/// Build an op without mask, use the type of `acc` as the return type.
2632void OuterProductOp::build(OpBuilder &builder, OperationState &result,
                         Value lhs, Value rhs, Value acc) {
result.addOperands({lhs, rhs, acc});
result.addTypes(acc.getType());
2636}

2638void OuterProductOp::print(OpAsmPrinter &p) {
p << " " << getLhs() << ", " << getRhs();
if (!getAcc().empty()) {
  p << ", " << getAcc();
  p.printOptionalAttrDict((*this)->getAttrs());
}
p << " : " << getLhs().getType() << ", " << getRhs().getType();
2645}

2647ParseResult OuterProductOp::parse(OpAsmParser &parser, OperationState &result) {
SmallVector<OpAsmParser::UnresolvedOperand, 3> operandsInfo;
Type tLHS, tRHS;
if (parser.parseOperandList(operandsInfo) ||
    parser.parseOptionalAttrDict(result.attributes) ||
    parser.parseColonType(tLHS) || parser.parseComma() ||
    parser.parseType(tRHS))
  return failure();
if (operandsInfo.size() < 2)
  return parser.emitError(parser.getNameLoc(),
                          "expected at least 2 operands");
VectorType vLHS = tLHS.dyn_cast<VectorType>();
VectorType vRHS = tRHS.dyn_cast<VectorType>();
if (!vLHS)
  return parser.emitError(parser.getNameLoc(),
                          "expected vector type for operand #1");
VectorType resType =
    vRHS ? VectorType::get({vLHS.getDimSize(0), vRHS.getDimSize(0)},
                           vLHS.getElementType())
         : VectorType::get({vLHS.getDimSize(0)}, vLHS.getElementType());

if (!result.attributes.get(OuterProductOp::getKindAttrStrName())) {
  result.attributes.append(
      OuterProductOp::getKindAttrStrName(),
      CombiningKindAttr::get(result.getContext(),
                             OuterProductOp::getDefaultKind()));
}

return failure(
    parser.resolveOperand(operandsInfo[0], tLHS, result.operands) ||
    parser.resolveOperand(operandsInfo[1], tRHS, result.operands) ||
    (operandsInfo.size() > 2 &&
     parser.resolveOperand(operandsInfo[2], resType, result.operands)) ||
    parser.addTypeToList(resType, result.types));
2681}

2683LogicalResult OuterProductOp::verify() {
Type tRHS = getOperandTypeRHS();
VectorType vLHS = getOperandVectorTypeLHS(),
           vRHS = tRHS.dyn_cast<VectorType>(),
           vACC = getOperandVectorTypeACC(), vRES = getVectorType();

if (vLHS.getRank() != 1)
  return emitOpError("expected 1-d vector for operand #1");

if (vRHS) {
  // Proper OUTER operation.
  if (vRHS.getRank() != 1)
    return emitOpError("expected 1-d vector for operand #2");
  if (vRES.getRank() != 2)
    return emitOpError("expected 2-d vector result");
  if (vLHS.getDimSize(0) != vRES.getDimSize(0))
    return emitOpError("expected #1 operand dim to match result dim #1");
  if (vRHS.getDimSize(0) != vRES.getDimSize(1))
    return emitOpError("expected #2 operand dim to match result dim #2");
} else {
  // An AXPY operation.
  if (vRES.getRank() != 1)
    return emitOpError("expected 1-d vector result");
  if (vLHS.getDimSize(0) != vRES.getDimSize(0))
    return emitOpError("expected #1 operand dim to match result dim #1");
}

if (vACC && vACC != vRES)
  return emitOpError("expected operand #3 of same type as result type");

// Verify supported combining kind.
if (!isSupportedCombiningKind(getKind(), vRES.getElementType()))
  return emitOpError("unsupported outerproduct type");

return success();
2718}

2720//===----------------------------------------------------------------------===//
2721// ReshapeOp
2722//===----------------------------------------------------------------------===//

2724LogicalResult ReshapeOp::verify() {
// Verify that rank(numInputs/outputs) + numFixedVec dim matches vec rank.
auto inputVectorType = getInputVectorType();
auto outputVectorType = getOutputVectorType();
int64_t inputShapeRank = getNumInputShapeSizes();
int64_t outputShapeRank = getNumOutputShapeSizes();
SmallVector<int64_t, 4> fixedVectorSizes;
getFixedVectorSizes(fixedVectorSizes);
int64_t numFixedVectorSizes = fixedVectorSizes.size();

if (inputVectorType.getRank() != inputShapeRank + numFixedVectorSizes)
  return emitError("invalid input shape for vector type ") << inputVectorType;

if (outputVectorType.getRank() != outputShapeRank + numFixedVectorSizes)
  return emitError("invalid output shape for vector type ")
         << outputVectorType;

// Verify that the 'fixedVectorSizes' match an input/output vector shape
// suffix.
unsigned inputVectorRank = inputVectorType.getRank();
for (unsigned i = 0; i < numFixedVectorSizes; ++i) {
  unsigned index = inputVectorRank - numFixedVectorSizes - i;
  if (fixedVectorSizes[i] != inputVectorType.getShape()[index])
    return emitError("fixed vector size must match input vector for dim ")
           << i;
}

unsigned outputVectorRank = outputVectorType.getRank();
for (unsigned i = 0; i < numFixedVectorSizes; ++i) {
  unsigned index = outputVectorRank - numFixedVectorSizes - i;
  if (fixedVectorSizes[i] != outputVectorType.getShape()[index])
    return emitError("fixed vector size must match output vector for dim ")
           << i;
}

// If all shape operands are produced by constant ops, verify that product
// of dimensions for input/output shape match.
auto isDefByConstant = [](Value operand) {
  return isa_and_nonnull<arith::ConstantIndexOp>(operand.getDefiningOp());
};
if (llvm::all_of(getInputShape(), isDefByConstant) &&
    llvm::all_of(getOutputShape(), isDefByConstant)) {
  int64_t numInputElements = 1;
  for (auto operand : getInputShape())
    numInputElements *=
        cast<arith::ConstantIndexOp>(operand.getDefiningOp()).value();
  int64_t numOutputElements = 1;
  for (auto operand : getOutputShape())
    numOutputElements *=
        cast<arith::ConstantIndexOp>(operand.getDefiningOp()).value();
  if (numInputElements != numOutputElements)
    return emitError("product of input and output shape sizes must match");
}
return success();
2778}

2780void ReshapeOp::getFixedVectorSizes(SmallVectorImpl<int64_t> &results) {
populateFromInt64AttrArray(getFixedVectorSizes(), results);
2782}

2784//===----------------------------------------------------------------------===//
2785// ExtractStridedSliceOp
2786//===----------------------------------------------------------------------===//

2788// Inference works as follows:
2789//   1. Add 'sizes' from prefix of dims in 'offsets'.
2790//   2. Add sizes from 'vectorType' for remaining dims.
2791static Type inferStridedSliceOpResultType(VectorType vectorType,
                                        ArrayAttr offsets, ArrayAttr sizes,
                                        ArrayAttr strides) {
assert(offsets.size() == sizes.size() && offsets.size() == strides.size())(static_cast <bool> (offsets.size() == sizes.size() &&
 offsets.size() == strides.size()) ? void (0) : __assert_fail
 ("offsets.size() == sizes.size() && offsets.size() == strides.size()"
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 2794, __extension__
 __PRETTY_FUNCTION__));
SmallVector<int64_t, 4> shape;
shape.reserve(vectorType.getRank());
unsigned idx = 0;
for (unsigned e = offsets.size(); idx < e; ++idx)
  shape.push_back(sizes[idx].cast<IntegerAttr>().getInt());
for (unsigned e = vectorType.getShape().size(); idx < e; ++idx)
  shape.push_back(vectorType.getShape()[idx]);

return VectorType::get(shape, vectorType.getElementType());
2804}

2806void ExtractStridedSliceOp::build(OpBuilder &builder, OperationState &result,
                                Value source, ArrayRef<int64_t> offsets,
                                ArrayRef<int64_t> sizes,
                                ArrayRef<int64_t> strides) {
result.addOperands(source);
auto offsetsAttr = getVectorSubscriptAttr(builder, offsets);
auto sizesAttr = getVectorSubscriptAttr(builder, sizes);
auto stridesAttr = getVectorSubscriptAttr(builder, strides);
result.addTypes(
    inferStridedSliceOpResultType(source.getType().cast<VectorType>(),
                                  offsetsAttr, sizesAttr, stridesAttr));
result.addAttribute(getOffsetsAttrStrName(), offsetsAttr);
result.addAttribute(getSizesAttrStrName(), sizesAttr);
result.addAttribute(getStridesAttrStrName(), stridesAttr);
2820}

2822LogicalResult ExtractStridedSliceOp::verify() {
auto type = getVectorType();
auto offsets = getOffsetsAttr();
auto sizes = getSizesAttr();
auto strides = getStridesAttr();
if (offsets.size() != sizes.size() || offsets.size() != strides.size())
  return emitOpError(
      "expected offsets, sizes and strides attributes of same size");

auto shape = type.getShape();
auto offName = getOffsetsAttrName();
auto sizesName = getSizesAttrName();
auto stridesName = getStridesAttrName();
if (failed(
        isIntegerArrayAttrSmallerThanShape(*this, offsets, shape, offName)) ||
    failed(
        isIntegerArrayAttrSmallerThanShape(*this, sizes, shape, sizesName)) ||
    failed(isIntegerArrayAttrSmallerThanShape(*this, strides, shape,
                                              stridesName)) ||
    failed(
        isIntegerArrayAttrConfinedToShape(*this, offsets, shape, offName)) ||
    failed(isIntegerArrayAttrConfinedToShape(*this, sizes, shape, sizesName,
                                             /*halfOpen=*/false,
                                             /*min=*/1)) ||
    failed(isIntegerArrayAttrConfinedToRange(*this, strides, 1, 1,
                                             stridesName,
                                             /*halfOpen=*/false)) ||
    failed(isSumOfIntegerArrayAttrConfinedToShape(*this, offsets, sizes,
                                                  shape, offName, sizesName,
                                                  /*halfOpen=*/false)))
  return failure();

auto resultType =
    inferStridedSliceOpResultType(getVectorType(), offsets, sizes, strides);
if (getResult().getType() != resultType)
  return emitOpError("expected result type to be ") << resultType;

return success();
2860}

2862// When the source of ExtractStrided comes from a chain of InsertStrided ops try
2863// to use the source of the InsertStrided ops if we can detect that the
2864// extracted vector is a subset of one of the vector inserted.
2865static LogicalResult
2866foldExtractStridedOpFromInsertChain(ExtractStridedSliceOp op) {
// Helper to extract integer out of ArrayAttr.
auto getElement = [](ArrayAttr array, int idx) {
  return array[idx].cast<IntegerAttr>().getInt();
};
ArrayAttr extractOffsets = op.getOffsets();
ArrayAttr extractStrides = op.getStrides();
ArrayAttr extractSizes = op.getSizes();
auto insertOp = op.getVector().getDefiningOp<InsertStridedSliceOp>();
while (insertOp) {
  if (op.getVectorType().getRank() !=
      insertOp.getSourceVectorType().getRank())
    return failure();
  ArrayAttr insertOffsets = insertOp.getOffsets();
  ArrayAttr insertStrides = insertOp.getStrides();
  // If the rank of extract is greater than the rank of insert, we are likely
  // extracting a partial chunk of the vector inserted.
  if (extractOffsets.size() > insertOffsets.size())
    return failure();
  bool patialoverlap = false;
  bool disjoint = false;
  SmallVector<int64_t, 4> offsetDiffs;
  for (unsigned dim = 0, e = extractOffsets.size(); dim < e; ++dim) {
    if (getElement(extractStrides, dim) != getElement(insertStrides, dim))
      return failure();
    int64_t start = getElement(insertOffsets, dim);
    int64_t end = start + insertOp.getSourceVectorType().getDimSize(dim);
    int64_t offset = getElement(extractOffsets, dim);
    int64_t size = getElement(extractSizes, dim);
    // Check if the start of the extract offset is in the interval inserted.
    if (start <= offset && offset < end) {
      // If the extract interval overlaps but is not fully included we may
      // have a partial overlap that will prevent any folding.
      if (offset + size > end)
        patialoverlap = true;
      offsetDiffs.push_back(offset - start);
      continue;
    }
    disjoint = true;
    break;
  }
  // The extract element chunk is a subset of the insert element.
  if (!disjoint && !patialoverlap) {
    op.setOperand(insertOp.getSource());
    // OpBuilder is only used as a helper to build an I64ArrayAttr.
    OpBuilder b(op.getContext());
    op->setAttr(ExtractStridedSliceOp::getOffsetsAttrStrName(),
                b.getI64ArrayAttr(offsetDiffs));
    return success();
  }
  // If the chunk extracted is disjoint from the chunk inserted, keep looking
  // in the insert chain.
  if (disjoint)
    insertOp = insertOp.getDest().getDefiningOp<InsertStridedSliceOp>();
  else {
    // The extracted vector partially overlap the inserted vector, we cannot
    // fold.
    return failure();
  }
}
return failure();
2927}

2929OpFoldResult ExtractStridedSliceOp::fold(ArrayRef<Attribute> operands) {
if (getVectorType() == getResult().getType())
  return getVector();
if (succeeded(foldExtractStridedOpFromInsertChain(*this)))
  return getResult();
return {};
2935}

2937void ExtractStridedSliceOp::getOffsets(SmallVectorImpl<int64_t> &results) {
populateFromInt64AttrArray(getOffsets(), results);
2939}

2941namespace {

2943// Pattern to rewrite an ExtractStridedSliceOp(ConstantMaskOp) to
2944// ConstantMaskOp.
2945class StridedSliceConstantMaskFolder final
  : public OpRewritePattern<ExtractStridedSliceOp> {
2947public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(ExtractStridedSliceOp extractStridedSliceOp,
                              PatternRewriter &rewriter) const override {
  // Return if 'extractStridedSliceOp' operand is not defined by a
  // ConstantMaskOp.
  auto *defOp = extractStridedSliceOp.getVector().getDefiningOp();
  auto constantMaskOp = dyn_cast_or_null<ConstantMaskOp>(defOp);
  if (!constantMaskOp)
    return failure();
  // Return if 'extractStridedSliceOp' has non-unit strides.
  if (extractStridedSliceOp.hasNonUnitStrides())
    return failure();
  // Gather constant mask dimension sizes.
  SmallVector<int64_t, 4> maskDimSizes;
  populateFromInt64AttrArray(constantMaskOp.getMaskDimSizes(), maskDimSizes);
  // Gather strided slice offsets and sizes.
  SmallVector<int64_t, 4> sliceOffsets;
  populateFromInt64AttrArray(extractStridedSliceOp.getOffsets(),
                             sliceOffsets);
  SmallVector<int64_t, 4> sliceSizes;
  populateFromInt64AttrArray(extractStridedSliceOp.getSizes(), sliceSizes);

  // Compute slice of vector mask region.
  SmallVector<int64_t, 4> sliceMaskDimSizes;
  sliceMaskDimSizes.reserve(maskDimSizes.size());
  for (auto [maskDimSize, sliceOffset, sliceSize] :
       llvm::zip(maskDimSizes, sliceOffsets, sliceSizes)) {
    int64_t sliceMaskDimSize = std::max(
        static_cast<int64_t>(0),
        std::min(sliceOffset + sliceSize, maskDimSize) - sliceOffset);
    sliceMaskDimSizes.push_back(sliceMaskDimSize);
  }
  // Add unchanged dimensions.
  if (sliceMaskDimSizes.size() < maskDimSizes.size())
    for (size_t i = sliceMaskDimSizes.size(); i < maskDimSizes.size(); ++i)
      sliceMaskDimSizes.push_back(maskDimSizes[i]);
  // If any of 'sliceMaskDimSizes' are zero, then set all to zero (masked
  // region is a conjunction of mask dim intervals).
  if (llvm::is_contained(sliceMaskDimSizes, 0))
    sliceMaskDimSizes.assign(maskDimSizes.size(), 0);

  // Replace 'extractStridedSliceOp' with ConstantMaskOp with sliced mask
  // region.
  rewriter.replaceOpWithNewOp<ConstantMaskOp>(
      extractStridedSliceOp, extractStridedSliceOp.getResult().getType(),
      vector::getVectorSubscriptAttr(rewriter, sliceMaskDimSizes));
  return success();
}
2997};

2999// Pattern to rewrite a ExtractStridedSliceOp(splat ConstantOp) -> ConstantOp.
3000class StridedSliceSplatConstantFolder final
  : public OpRewritePattern<ExtractStridedSliceOp> {
3002public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(ExtractStridedSliceOp extractStridedSliceOp,
                              PatternRewriter &rewriter) const override {
  // Return if 'ExtractStridedSliceOp' operand is not defined by a splat
  // ConstantOp.
  Value sourceVector = extractStridedSliceOp.getVector();
  Attribute vectorCst;
  if (!matchPattern(sourceVector, m_Constant(&vectorCst)))
    return failure();

  auto splat = vectorCst.dyn_cast<SplatElementsAttr>();
  if (!splat)
    return failure();

  auto newAttr = SplatElementsAttr::get(extractStridedSliceOp.getType(),
                                        splat.getSplatValue<Attribute>());
  rewriter.replaceOpWithNewOp<arith::ConstantOp>(extractStridedSliceOp,
                                                 newAttr);
  return success();
}
3024};

3026// Pattern to rewrite a ExtractStridedSliceOp(non-splat ConstantOp) ->
3027// ConstantOp.
3028class StridedSliceNonSplatConstantFolder final
  : public OpRewritePattern<ExtractStridedSliceOp> {
3030public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(ExtractStridedSliceOp extractStridedSliceOp,
                              PatternRewriter &rewriter) const override {
  // Return if 'ExtractStridedSliceOp' operand is not defined by a non-splat
  // ConstantOp.
  Value sourceVector = extractStridedSliceOp.getVector();
  Attribute vectorCst;
  if (!matchPattern(sourceVector, m_Constant(&vectorCst)))
    return failure();

  // The splat case is handled by `StridedSliceSplatConstantFolder`.
  auto dense = vectorCst.dyn_cast<DenseElementsAttr>();
  if (!dense || dense.isSplat())
    return failure();

  // TODO: Handle non-unit strides when they become available.
  if (extractStridedSliceOp.hasNonUnitStrides())
    return failure();

  auto sourceVecTy = sourceVector.getType().cast<VectorType>();
  ArrayRef<int64_t> sourceShape = sourceVecTy.getShape();
  SmallVector<int64_t, 4> sourceStrides = computeStrides(sourceShape);

  VectorType sliceVecTy = extractStridedSliceOp.getType();
  ArrayRef<int64_t> sliceShape = sliceVecTy.getShape();
  int64_t sliceRank = sliceVecTy.getRank();

  // Expand offsets and sizes to match the vector rank.
  SmallVector<int64_t, 4> offsets(sliceRank, 0);
  copy(getI64SubArray(extractStridedSliceOp.getOffsets()), offsets.begin());

  SmallVector<int64_t, 4> sizes(sourceShape.begin(), sourceShape.end());
  copy(getI64SubArray(extractStridedSliceOp.getSizes()), sizes.begin());

  // Calculate the slice elements by enumerating all slice positions and
  // linearizing them. The enumeration order is lexicographic which yields a
  // sequence of monotonically increasing linearized position indices.
  auto denseValuesBegin = dense.value_begin<Attribute>();
  SmallVector<Attribute> sliceValues;
  sliceValues.reserve(sliceVecTy.getNumElements());
  SmallVector<int64_t> currSlicePosition(offsets.begin(), offsets.end());
  do {
    int64_t linearizedPosition = linearize(currSlicePosition, sourceStrides);
    assert(linearizedPosition < sourceVecTy.getNumElements() &&(static_cast <bool> (linearizedPosition < sourceVecTy
.getNumElements() && "Invalid index") ? void (0) : __assert_fail
 ("linearizedPosition < sourceVecTy.getNumElements() && \"Invalid index\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 3076, __extension__
 __PRETTY_FUNCTION__))
           "Invalid index")(static_cast <bool> (linearizedPosition < sourceVecTy
.getNumElements() && "Invalid index") ? void (0) : __assert_fail
 ("linearizedPosition < sourceVecTy.getNumElements() && \"Invalid index\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 3076, __extension__
 __PRETTY_FUNCTION__));
    sliceValues.push_back(*(denseValuesBegin + linearizedPosition));
  } while (
      succeeded(incSlicePosition(currSlicePosition, sliceShape, offsets)));

  assert(static_cast<int64_t>(sliceValues.size()) ==(static_cast <bool> (static_cast<int64_t>(sliceValues
.size()) == sliceVecTy.getNumElements() && "Invalid number of slice elements"
) ? void (0) : __assert_fail ("static_cast<int64_t>(sliceValues.size()) == sliceVecTy.getNumElements() && \"Invalid number of slice elements\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 3083, __extension__
 __PRETTY_FUNCTION__))
             sliceVecTy.getNumElements() &&(static_cast <bool> (static_cast<int64_t>(sliceValues
.size()) == sliceVecTy.getNumElements() && "Invalid number of slice elements"
) ? void (0) : __assert_fail ("static_cast<int64_t>(sliceValues.size()) == sliceVecTy.getNumElements() && \"Invalid number of slice elements\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 3083, __extension__
 __PRETTY_FUNCTION__))
         "Invalid number of slice elements")(static_cast <bool> (static_cast<int64_t>(sliceValues
.size()) == sliceVecTy.getNumElements() && "Invalid number of slice elements"
) ? void (0) : __assert_fail ("static_cast<int64_t>(sliceValues.size()) == sliceVecTy.getNumElements() && \"Invalid number of slice elements\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 3083, __extension__
 __PRETTY_FUNCTION__));
  auto newAttr = DenseElementsAttr::get(sliceVecTy, sliceValues);
  rewriter.replaceOpWithNewOp<arith::ConstantOp>(extractStridedSliceOp,
                                                 newAttr);
  return success();
}
3089};

3091// Pattern to rewrite an ExtractStridedSliceOp(BroadcastOp) to
3092// BroadcastOp(ExtractStrideSliceOp).
3093class StridedSliceBroadcast final
  : public OpRewritePattern<ExtractStridedSliceOp> {
3095public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(ExtractStridedSliceOp op,
                              PatternRewriter &rewriter) const override {
  auto broadcast = op.getVector().getDefiningOp<BroadcastOp>();
  if (!broadcast)
    return failure();
  auto srcVecType = broadcast.getSource().getType().dyn_cast<VectorType>();
  unsigned srcRank = srcVecType ? srcVecType.getRank() : 0;
  auto dstVecType = op.getType().cast<VectorType>();
  unsigned dstRank = dstVecType.getRank();
  unsigned rankDiff = dstRank - srcRank;
  // Check if the most inner dimensions of the source of the broadcast are the
  // same as the destination of the extract. If this is the case we can just
  // use a broadcast as the original dimensions are untouched.
  bool lowerDimMatch = true;
  for (unsigned i = 0; i < srcRank; i++) {
    if (srcVecType.getDimSize(i) != dstVecType.getDimSize(i + rankDiff)) {
      lowerDimMatch = false;
      break;
    }
  }
  Value source = broadcast.getSource();
  // If the inner dimensions don't match, it means we need to extract from the
  // source of the orignal broadcast and then broadcast the extracted value.
  // We also need to handle degenerated cases where the source is effectively
  // just a single scalar.
  bool isScalarSrc = (srcRank == 0 || srcVecType.getNumElements() == 1);
  if (!lowerDimMatch && !isScalarSrc) {
    source = rewriter.create<ExtractStridedSliceOp>(
        op->getLoc(), source,
        getI64SubArray(op.getOffsets(), /* dropFront=*/rankDiff),
        getI64SubArray(op.getSizes(), /* dropFront=*/rankDiff),
        getI64SubArray(op.getStrides(), /* dropFront=*/rankDiff));
  }
  rewriter.replaceOpWithNewOp<BroadcastOp>(op, op.getType(), source);
  return success();
}
3134};

3136/// Pattern to rewrite an ExtractStridedSliceOp(SplatOp) to SplatOp.
3137class StridedSliceSplat final : public OpRewritePattern<ExtractStridedSliceOp> {
3138public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(ExtractStridedSliceOp op,
                              PatternRewriter &rewriter) const override {
  auto splat = op.getVector().getDefiningOp<SplatOp>();
  if (!splat)
    return failure();
  rewriter.replaceOpWithNewOp<SplatOp>(op, op.getType(), splat.getInput());
  return success();
}
3149};

3151} // namespace

3153void ExtractStridedSliceOp::getCanonicalizationPatterns(
  RewritePatternSet &results, MLIRContext *context) {
// Pattern to rewrite a ExtractStridedSliceOp(ConstantMaskOp) ->
// ConstantMaskOp and ExtractStridedSliceOp(ConstantOp) -> ConstantOp.
results.add<StridedSliceConstantMaskFolder, StridedSliceSplatConstantFolder,
            StridedSliceNonSplatConstantFolder, StridedSliceBroadcast,
            StridedSliceSplat>(context);
3160}

3162//===----------------------------------------------------------------------===//
3163// TransferReadOp
3164//===----------------------------------------------------------------------===//

3166/// 1. Builder that sets padding to zero and an empty mask (variant with attrs).
3167void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                         VectorType vectorType, Value source,
                         ValueRange indices, AffineMapAttr permutationMapAttr,
                         /*optional*/ ArrayAttr inBoundsAttr) {
Type elemType = source.getType().cast<ShapedType>().getElementType();
Value padding = builder.create<arith::ConstantOp>(
    result.location, elemType, builder.getZeroAttr(elemType));
build(builder, result, vectorType, source, indices, permutationMapAttr,
      padding, /*mask=*/Value(), inBoundsAttr);
3176}

3178/// 2. Builder that sets padding to zero an empty mask (variant without attrs).
3179void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                         VectorType vectorType, Value source,
                         ValueRange indices, AffineMap permutationMap,
                         std::optional<ArrayRef<bool>> inBounds) {
auto permutationMapAttr = AffineMapAttr::get(permutationMap);
auto inBoundsAttr = (inBounds && !inBounds.value().empty())
                        ? builder.getBoolArrayAttr(inBounds.value())
                        : ArrayAttr();
build(builder, result, vectorType, source, indices, permutationMapAttr,
      inBoundsAttr);
3189}

3191/// 3. Builder that sets permutation map to 'getMinorIdentityMap'.
3192void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                         VectorType vectorType, Value source,
                         ValueRange indices, Value padding,
                         std::optional<ArrayRef<bool>> inBounds) {
AffineMap permutationMap = getTransferMinorIdentityMap(
    source.getType().cast<ShapedType>(), vectorType);
auto permutationMapAttr = AffineMapAttr::get(permutationMap);
auto inBoundsAttr = (inBounds && !inBounds.value().empty())
                        ? builder.getBoolArrayAttr(inBounds.value())
                        : ArrayAttr();
build(builder, result, vectorType, source, indices, permutationMapAttr,
      padding,
      /*mask=*/Value(), inBoundsAttr);
3205}

3207/// 4. Builder that sets padding to zero and permutation map to
3208/// 'getMinorIdentityMap'.
3209void TransferReadOp::build(OpBuilder &builder, OperationState &result,
                         VectorType vectorType, Value source,
                         ValueRange indices,
                         std::optional<ArrayRef<bool>> inBounds) {
Type elemType = source.getType().cast<ShapedType>().getElementType();
Value padding = builder.create<arith::ConstantOp>(
    result.location, elemType, builder.getZeroAttr(elemType));
build(builder, result, vectorType, source, indices, padding, inBounds);
3217}

3219template <typename EmitFun>
3220static LogicalResult verifyPermutationMap(AffineMap permutationMap,
                                        EmitFun emitOpError) {
SmallVector<bool, 8> seen(permutationMap.getNumInputs(), false);
for (auto expr : permutationMap.getResults()) {
  auto dim = expr.dyn_cast<AffineDimExpr>();
  auto zero = expr.dyn_cast<AffineConstantExpr>();
  if (zero) {
    if (zero.getValue() != 0) {
      return emitOpError(
          "requires a projected permutation_map (at most one dim or the zero "
          "constant can appear in each result)");
    }
    continue;
  }
  if (!dim) {
    return emitOpError("requires a projected permutation_map (at most one "
                       "dim or the zero constant can appear in each result)");
  }
  if (seen[dim.getPosition()]) {
    return emitOpError(
        "requires a permutation_map that is a permutation (found one dim "
        "used more than once)");
  }
  seen[dim.getPosition()] = true;
}
return success();
3246}

3248static LogicalResult
3249verifyTransferOp(VectorTransferOpInterface op, ShapedType shapedType,
               VectorType vectorType, VectorType maskType,
               VectorType inferredMaskType, AffineMap permutationMap,
               ArrayAttr inBounds) {
if (op->hasAttr("masked")) {
  return op->emitOpError("masked attribute has been removed. "
                         "Use in_bounds instead.");
}

if (!shapedType.isa<MemRefType, RankedTensorType>())
  return op->emitOpError(
      "requires source to be a memref or ranked tensor type");

auto elementType = shapedType.getElementType();
DataLayout dataLayout = DataLayout::closest(op);
if (auto vectorElementType = elementType.dyn_cast<VectorType>()) {
  // Memref or tensor has vector element type.
  unsigned sourceVecSize =
      dataLayout.getTypeSizeInBits(vectorElementType.getElementType()) *
      vectorElementType.getShape().back();
  unsigned resultVecSize =
      dataLayout.getTypeSizeInBits(vectorType.getElementType()) *
      vectorType.getShape().back();
  if (resultVecSize % sourceVecSize != 0)
    return op->emitOpError(
        "requires the bitwidth of the minor 1-D vector to be an integral "
        "multiple of the bitwidth of the minor 1-D vector of the source");

  unsigned sourceVecEltRank = vectorElementType.getRank();
  unsigned resultVecRank = vectorType.getRank();
  if (sourceVecEltRank > resultVecRank)
    return op->emitOpError(
        "requires source vector element and vector result ranks to match.");
  unsigned rankOffset = resultVecRank - sourceVecEltRank;
  // Check that permutation map results match 'rankOffset' of vector type.
  if (permutationMap.getNumResults() != rankOffset)
    return op->emitOpError("requires a permutation_map with result dims of "
                           "the same rank as the vector type");

  if (maskType)
    return op->emitOpError("does not support masks with vector element type");
} else {
  // Memref or tensor has scalar element type.
  unsigned minorSize =
      vectorType.getRank() == 0 ? 1 : vectorType.getShape().back();
  unsigned resultVecSize =
      dataLayout.getTypeSizeInBits(vectorType.getElementType()) * minorSize;
  if (resultVecSize % dataLayout.getTypeSizeInBits(elementType) != 0)
    return op->emitOpError(
        "requires the bitwidth of the minor 1-D vector to be an integral "
        "multiple of the bitwidth of the source element type");

  // Check that permutation map results match rank of vector type.
  if (permutationMap.getNumResults() != vectorType.getRank())
    return op->emitOpError("requires a permutation_map with result dims of "
                           "the same rank as the vector type");
}

if (permutationMap.getNumSymbols() != 0)
  return op->emitOpError("requires permutation_map without symbols");

if (permutationMap.getNumInputs() != shapedType.getRank())
  return op->emitOpError("requires a permutation_map with input dims of the "
                         "same rank as the source type");

if (maskType && maskType != inferredMaskType)
  return op->emitOpError("inferred mask type (")
         << inferredMaskType << ") and mask operand type (" << maskType
         << ") don't match";

if (inBounds) {
  if (permutationMap.getNumResults() != static_cast<int64_t>(inBounds.size()))
    return op->emitOpError("expects the optional in_bounds attr of same rank "
                           "as permutation_map results: ")
           << AffineMapAttr::get(permutationMap)
           << " vs inBounds of size: " << inBounds.size();
  for (unsigned int i = 0; i < permutationMap.getNumResults(); ++i)
    if (permutationMap.getResult(i).isa<AffineConstantExpr>() &&
        !inBounds.getValue()[i].cast<BoolAttr>().getValue())
      return op->emitOpError("requires broadcast dimensions to be in-bounds");
}

return success();
3332}

3334static void printTransferAttrs(OpAsmPrinter &p, VectorTransferOpInterface op) {
SmallVector<StringRef, 3> elidedAttrs;
elidedAttrs.push_back(TransferReadOp::getOperandSegmentSizeAttr());
if (op.permutation_map().isMinorIdentity())
  elidedAttrs.push_back(op.getPermutationMapAttrStrName());
bool elideInBounds = true;
if (auto inBounds = op.in_bounds()) {
  for (auto attr : *inBounds) {
    if (attr.template cast<BoolAttr>().getValue()) {
      elideInBounds = false;
      break;
    }
  }
}
if (elideInBounds)
  elidedAttrs.push_back(op.getInBoundsAttrStrName());
p.printOptionalAttrDict(op->getAttrs(), elidedAttrs);
3351}

3353void TransferReadOp::print(OpAsmPrinter &p) {
p << " " << getSource() << "[" << getIndices() << "], " << getPadding();
if (getMask())
  p << ", " << getMask();
printTransferAttrs(p, *this);
p << " : " << getShapedType() << ", " << getVectorType();
3359}

3361/// Infers the mask type for a transfer read given its vector type and
3362/// permutation map. The mask in a transfer read operation applies to the
3363/// tensor/buffer reading part of it and its type should match the shape read
3364/// *before* any permutation or broadcasting.
3365static VectorType inferTransferReadMaskType(VectorType vecType,
                                          AffineMap permMap) {
auto i1Type = IntegerType::get(permMap.getContext(), 1);
AffineMap invPermMap = inversePermutation(compressUnusedDims(permMap));
assert(invPermMap && "Inversed permutation map couldn't be computed")(static_cast <bool> (invPermMap && "Inversed permutation map couldn't be computed"
) ? void (0) : __assert_fail ("invPermMap && \"Inversed permutation map couldn't be computed\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 3369, __extension__
 __PRETTY_FUNCTION__));
SmallVector<int64_t, 8> maskShape = invPermMap.compose(vecType.getShape());
return VectorType::get(maskShape, i1Type);
3372}

3374ParseResult TransferReadOp::parse(OpAsmParser &parser, OperationState &result) {
auto &builder = parser.getBuilder();
SMLoc typesLoc;
OpAsmParser::UnresolvedOperand sourceInfo;
SmallVector<OpAsmParser::UnresolvedOperand, 8> indexInfo;
OpAsmParser::UnresolvedOperand paddingInfo;
SmallVector<Type, 2> types;
OpAsmParser::UnresolvedOperand maskInfo;
// Parsing with support for paddingValue.
if (parser.parseOperand(sourceInfo) ||
    parser.parseOperandList(indexInfo, OpAsmParser::Delimiter::Square) ||
    parser.parseComma() || parser.parseOperand(paddingInfo))
  return failure();
ParseResult hasMask = parser.parseOptionalComma();
if (hasMask.succeeded()) {
  if (parser.parseOperand(maskInfo))
    return failure();
}
if (parser.parseOptionalAttrDict(result.attributes) ||
    parser.getCurrentLocation(&typesLoc) || parser.parseColonTypeList(types))
  return failure();
if (types.size() != 2)
  return parser.emitError(typesLoc, "requires two types");
auto indexType = builder.getIndexType();
auto shapedType = types[0].dyn_cast<ShapedType>();
if (!shapedType || !shapedType.isa<MemRefType, RankedTensorType>())
  return parser.emitError(typesLoc, "requires memref or ranked tensor type");
VectorType vectorType = types[1].dyn_cast<VectorType>();
if (!vectorType)
  return parser.emitError(typesLoc, "requires vector type");
auto permMapAttrName = TransferReadOp::getPermutationMapAttrStrName();
Attribute permMapAttr = result.attributes.get(permMapAttrName);
AffineMap permMap;
if (!permMapAttr) {
  permMap = getTransferMinorIdentityMap(shapedType, vectorType);
  result.attributes.set(permMapAttrName, AffineMapAttr::get(permMap));
} else {
  permMap = permMapAttr.cast<AffineMapAttr>().getValue();
}
if (parser.resolveOperand(sourceInfo, shapedType, result.operands) ||
    parser.resolveOperands(indexInfo, indexType, result.operands) ||
    parser.resolveOperand(paddingInfo, shapedType.getElementType(),
                          result.operands))
  return failure();
if (hasMask.succeeded()) {
  if (shapedType.getElementType().dyn_cast<VectorType>())
    return parser.emitError(
        maskInfo.location, "does not support masks with vector element type");
  // Instead of adding the mask type as an op type, compute it based on the
  // vector type and the permutation map (to keep the type signature small).
  auto maskType = inferTransferReadMaskType(vectorType, permMap);
  if (parser.resolveOperand(maskInfo, maskType, result.operands))
    return failure();
}
result.addAttribute(TransferReadOp::getOperandSegmentSizeAttr(),
                    builder.getDenseI32ArrayAttr(
                        {1, static_cast<int32_t>(indexInfo.size()), 1,
                         static_cast<int32_t>(hasMask.succeeded())}));
return parser.addTypeToList(vectorType, result.types);
3433}

3435LogicalResult TransferReadOp::verify() {
// Consistency of elemental types in source and vector.
ShapedType shapedType = getShapedType();
VectorType vectorType = getVectorType();
VectorType maskType = getMaskType();
auto paddingType = getPadding().getType();
auto permutationMap = getPermutationMap();
VectorType inferredMaskType =
    maskType ? inferTransferReadMaskType(vectorType, permutationMap)
             : VectorType();
auto sourceElementType = shapedType.getElementType();

if (static_cast<int64_t>(getIndices().size()) != shapedType.getRank())
  return emitOpError("requires ") << shapedType.getRank() << " indices";

if (failed(verifyTransferOp(cast<VectorTransferOpInterface>(getOperation()),
                            shapedType, vectorType, maskType,
                            inferredMaskType, permutationMap,
                            getInBounds() ? *getInBounds() : ArrayAttr())))
  return failure();

if (auto sourceVectorElementType = sourceElementType.dyn_cast<VectorType>()) {
  // Source has vector element type.
  // Check that 'sourceVectorElementType' and 'paddingType' types match.
  if (sourceVectorElementType != paddingType)
    return emitOpError(
        "requires source element type and padding type to match.");

} else {
  // Check that 'paddingType' is valid to store in a vector type.
  if (!VectorType::isValidElementType(paddingType))
    return emitOpError("requires valid padding vector elemental type");

  // Check that padding type and vector element types match.
  if (paddingType != sourceElementType)
    return emitOpError(
        "requires formal padding and source of the same elemental type");
}

return verifyPermutationMap(permutationMap,
                            [&](Twine t) { return emitOpError(t); });
3476}

3478// MaskableOpInterface methods.

3480/// Returns the mask type expected by this operation. Mostly used for
3481/// verification purposes. It requires the operation to be vectorized."
3482Type TransferReadOp::getExpectedMaskType() {
return inferTransferReadMaskType(getVectorType(), getPermutationMap());
3484}

3486template <typename TransferOp>
3487static bool isInBounds(TransferOp op, int64_t resultIdx, int64_t indicesIdx) {
// TODO: support more aggressive createOrFold on:
// `op.indices()[indicesIdx] + vectorType < dim(op.source(), indicesIdx)`
if (op.getShapedType().isDynamicDim(indicesIdx))
  return false;
Value index = op.getIndices()[indicesIdx];
auto cstOp = index.getDefiningOp<arith::ConstantIndexOp>();
if (!cstOp)
  return false;

int64_t sourceSize = op.getShapedType().getDimSize(indicesIdx);
int64_t vectorSize = op.getVectorType().getDimSize(resultIdx);

return cstOp.value() + vectorSize <= sourceSize;
3501}

3503template <typename TransferOp>
3504static LogicalResult foldTransferInBoundsAttribute(TransferOp op) {
// TODO: support 0-d corner case.
// TODO: Be less conservative.
if (op.getTransferRank() == 0)
  return failure();
AffineMap permutationMap = op.getPermutationMap();
bool changed = false;
SmallVector<bool, 4> newInBounds;
newInBounds.reserve(op.getTransferRank());
for (unsigned i = 0; i < op.getTransferRank(); ++i) {
  // Already marked as in-bounds, nothing to see here.
  if (op.isDimInBounds(i)) {
    newInBounds.push_back(true);
    continue;
  }
  // Currently out-of-bounds, check whether we can statically determine it is
  // inBounds.
  auto dimExpr = permutationMap.getResult(i).dyn_cast<AffineDimExpr>();
  assert(dimExpr && "Broadcast dims must be in-bounds")(static_cast <bool> (dimExpr && "Broadcast dims must be in-bounds"
) ? void (0) : __assert_fail ("dimExpr && \"Broadcast dims must be in-bounds\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 3522, __extension__
 __PRETTY_FUNCTION__));
  auto inBounds =
      isInBounds(op, /*resultIdx=*/i, /*indicesIdx=*/dimExpr.getPosition());
  newInBounds.push_back(inBounds);
  // We commit the pattern if it is "more inbounds".
  changed |= inBounds;
}
if (!changed)
  return failure();
// OpBuilder is only used as a helper to build an I64ArrayAttr.
OpBuilder b(op.getContext());
op->setAttr(TransferOp::getInBoundsAttrStrName(),
            b.getBoolArrayAttr(newInBounds));
return success();
3536}

3538///  ```
3539///  %w0 = vector.transfer_write %v0, %arg0[%c1, %c0] {in_bounds = [true, true]}
3540///    : vector<1x4xf32>, tensor<4x4xf32>
3541///  %0 = vector.transfer_read %w0[%c1, %c0], %cf0 {in_bounds = [true, true]}
3542///    : tensor<4x4xf32>, vector<1x4xf32>
3543///  ```
3544///  -> Folds into
3545///  ```
3546///  %v0
3547///  ```
3548static Value foldRAW(TransferReadOp readOp) {
if (!readOp.getShapedType().isa<RankedTensorType>())
  return {};
auto defWrite = readOp.getSource().getDefiningOp<vector::TransferWriteOp>();
while (defWrite) {
  if (checkSameValueRAW(defWrite, readOp))
    return defWrite.getVector();
  if (!isDisjointTransferIndices(
          cast<VectorTransferOpInterface>(defWrite.getOperation()),
          cast<VectorTransferOpInterface>(readOp.getOperation())))
    break;
  defWrite = defWrite.getSource().getDefiningOp<vector::TransferWriteOp>();
}
return {};
3562}

3564OpFoldResult TransferReadOp::fold(ArrayRef<Attribute>) {
if (Value vec = foldRAW(*this))
  return vec;
/// transfer_read(memrefcast) -> transfer_read
if (succeeded(foldTransferInBoundsAttribute(*this)))
  return getResult();
if (succeeded(memref::foldMemRefCast(*this)))
  return getResult();
if (succeeded(tensor::foldTensorCast(*this)))
  return getResult();
return OpFoldResult();
3575}

3577std::optional<SmallVector<int64_t, 4>> TransferReadOp::getShapeForUnroll() {
return llvm::to_vector<4>(getVectorType().getShape());
3579}

3581void TransferReadOp::getEffects(
  SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
      &effects) {
if (getShapedType().isa<MemRefType>())
  effects.emplace_back(MemoryEffects::Read::get(), getSource(),
                       SideEffects::DefaultResource::get());
3587}

3589/// Returns true if all rank reduced in the given `extractOp` happen in leading
3590/// dimensions earlier than last `trailingRank` dimensions.
3591static bool areAllRankReducedLeadingDim(tensor::ExtractSliceOp extractOp,
                                      unsigned trailingRank) {
// If no ranks are reduced at all, it's a degenerated case; always true.
if (extractOp.getSourceType().getRank() == extractOp.getType().getRank())
  return true;

RankedTensorType inferredType = extractOp.inferResultType(
    extractOp.getSourceType(), extractOp.getMixedOffsets(),
    extractOp.getMixedSizes(), extractOp.getMixedStrides());
return extractOp.getType().getShape().take_back(trailingRank) ==
       inferredType.getShape().take_back(trailingRank);
3602}

3604namespace {
3605/// Fold transfer_reads of a tensor.extract_slice op. E.g.:
3606///
3607/// ```
3608/// %0 = tensor.extract_slice %t[%a, %b] [%c, %d] [1, 1]
3609///     : tensor<?x?xf32> to tensor<?x?xf32>
3610/// %1 = vector.transfer_read %0[%e, %f], %cst {in_bounds = [true, true]}
3611///     : tensor<?x?xf32>, vector<4x5xf32>
3612/// ```
3613/// is rewritten to:
3614/// ```
3615/// %p0 = arith.addi %a, %e : index
3616/// %p1 = arith.addi %b, %f : index
3617/// %1 = vector.transfer_read %t[%p0, %p1], %cst {in_bounds = [true, true]}
3618///     : tensor<?x?xf32>, vector<4x5xf32>
3619/// ```
3620struct FoldExtractSliceIntoTransferRead
  : public OpRewritePattern<TransferReadOp> {
3622public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(TransferReadOp xferOp,
                              PatternRewriter &rewriter) const override {
  // TODO: support 0-d corner case.
  if (xferOp.getTransferRank() == 0)
    return failure();
  if (xferOp.hasOutOfBoundsDim())
    return failure();
  if (!xferOp.getPermutationMap().isMinorIdentity())
    return failure();
  if (xferOp.getMask())
    return failure();
  auto extractOp = xferOp.getSource().getDefiningOp<tensor::ExtractSliceOp>();
  if (!extractOp)
    return failure();
  if (!extractOp.hasUnitStride())
    return failure();

  // Bail on illegal rank-reduction: we need to check that the rank-reduced
  // dims are exactly the leading dims. I.e. the following is illegal:
  // ```
  //    %0 = tensor.extract_slice %t[0,0,0][2,1,4][1,1,1] :
  //      tensor<2x1x4xf32> to tensor<2x4xf32>
  //    %1 = vector.transfer_read %0[0,0], %cst :
  //      tensor<2x4xf32>, vector<2x4xf32>
  // ```
  //
  // Cannot fold into:
  // ```
  //    %0 = vector.transfer_read %t[0,0,0], %cst :
  //      tensor<2x1x4xf32>, vector<2x4xf32>
  // ```
  // For this, check the trailing `vectorRank` dims of the extract_slice
  // result tensor match the trailing dims of the inferred result tensor.
  if (!areAllRankReducedLeadingDim(extractOp, extractOp.getType().getRank()))
    return failure();

  int64_t rankReduced =
      extractOp.getSourceType().getRank() - extractOp.getType().getRank();

  SmallVector<Value> newIndices;
  // In case this is a rank-reducing ExtractSliceOp, copy rank-reduced
  // indices first.
  for (int64_t i = 0; i < rankReduced; ++i) {
    OpFoldResult offset = extractOp.getMixedOffsets()[i];
    newIndices.push_back(getValueOrCreateConstantIndexOp(
        rewriter, extractOp.getLoc(), offset));
  }
  for (const auto &it : llvm::enumerate(xferOp.getIndices())) {
    OpFoldResult offset =
        extractOp.getMixedOffsets()[it.index() + rankReduced];
    newIndices.push_back(rewriter.create<arith::AddIOp>(
        xferOp->getLoc(), it.value(),
        getValueOrCreateConstantIndexOp(rewriter, extractOp.getLoc(),
                                        offset)));
  }
  SmallVector<bool> inBounds(xferOp.getTransferRank(), true);
  rewriter.replaceOpWithNewOp<TransferReadOp>(
      xferOp, xferOp.getVectorType(), extractOp.getSource(), newIndices,
      xferOp.getPadding(), ArrayRef<bool>{inBounds});

  return success();
}
3687};

3689/// Store to load forwarding for transfer operations with permuation maps.
3690/// Even if the permutation maps are different we can still propagate the store
3691/// into the load if the size of the dimensions read and written match. Then we
3692/// can replace the transfer_read + transfer_write by vector.broadcast and
3693/// vector.transpose.
3694/// Example:
3695/// ```
3696/// %w0 = vector.transfer_write %v0, %arg0[%c0, %c0, %c0]
3697///  {in_bounds = [true, true],
3698///   permutation_map = affine_map<(d0, d1, d2) -> (d2, d1)>} :
3699///   vector<4x1xf32>, tensor<4x4x4xf32>
3700///  %r = vector.transfer_read %w0[%c0, %c0, %c0], %cf0
3701///   {in_bounds = [true, true, true, true],
3702///   permutation_map = affine_map<(d0, d1, d2) -> (d1, 0, d2, 0)>} :
3703///   tensor<4x4x4xf32>, vector<1x100x4x5xf32>
3704/// ```
3705/// To:
3706/// ```
3707/// %0 = vector.broadcast %arg1 : vector<4x1xf32> to vector<100x5x4x1xf32>
3708/// %r = vector.transpose %0, [3, 0, 2, 1] :
3709///   vector<100x5x4x1xf32> to vector<1x100x4x5xf32>
3710/// ```
3711struct TransferReadAfterWriteToBroadcast
  : public OpRewritePattern<TransferReadOp> {
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(TransferReadOp readOp,
                              PatternRewriter &rewriter) const override {
  if (readOp.hasOutOfBoundsDim() ||
      !readOp.getShapedType().isa<RankedTensorType>())
    return failure();
  auto defWrite = readOp.getSource().getDefiningOp<vector::TransferWriteOp>();
  if (!defWrite)
    return failure();

  SmallVector<int64_t> readDims = readOp.getTransferChunkAccessed();
  Value vec;
  if (readOp.getIndices() == defWrite.getIndices() &&
      readOp.getMask() == defWrite.getMask()) {
    SmallVector<int64_t> writeDims = defWrite.getTransferChunkAccessed();
    // TODO: If the writeDim is a superset of the read dims we could do an
    // extract_strided_slice.
    if (writeDims == readDims)
      vec = defWrite.getVector();
  }
  // TODO: loop through the chain of transfer_write if we can prove that they
  // don't overlap with the transfer_read. This requires improving
  // `isDisjointTransferIndices` helper.
  if (!vec)
    return failure();
  SmallVector<unsigned> permutation;
  AffineMap readMap = compressUnusedDims(readOp.getPermutationMap());
  AffineMap writeMap = compressUnusedDims(defWrite.getPermutationMap());
  AffineMap map = readMap.compose(writeMap);
  if (map.getNumResults() == 0)
    return failure();
  // Calculate the permuation to apply to go from the vector stored to the
  // vector read.
  if (!map.isPermutationOfMinorIdentityWithBroadcasting(permutation))
    return failure();

  Location loc = readOp.getLoc();
  // Calculate the broadcast shape by applying the reverse permuation to the
  // final shape we want.
  ArrayRef<int64_t> destShape = readOp.getVectorType().getShape();
  SmallVector<int64_t> broadcastShape(destShape.size());
  for (const auto &pos : llvm::enumerate(permutation))
    broadcastShape[pos.value()] = destShape[pos.index()];
  VectorType broadcastedType = VectorType::get(
      broadcastShape, defWrite.getVectorType().getElementType());
  vec = rewriter.create<vector::BroadcastOp>(loc, broadcastedType, vec);
  SmallVector<int64_t> transposePerm(permutation.begin(), permutation.end());
  rewriter.replaceOpWithNewOp<vector::TransposeOp>(readOp, vec,
                                                   transposePerm);
  return success();
}
3765};
3766} // namespace

3768void TransferReadOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                               MLIRContext *context) {
results
    .add<FoldExtractSliceIntoTransferRead, TransferReadAfterWriteToBroadcast>(
        context);
3773}

3775//===----------------------------------------------------------------------===//
3776// TransferWriteOp
3777//===----------------------------------------------------------------------===//

3779/// 1. Builder with type inference.
3780void TransferWriteOp::build(OpBuilder &builder, OperationState &result,
                          Value vector, Value dest, ValueRange indices,
                          AffineMapAttr permutationMapAttr,
                          /*optional*/ Value mask,
                          /*optional*/ ArrayAttr inBoundsAttr) {
Type resultType = dest.getType().dyn_cast<RankedTensorType>();
build(builder, result, resultType, vector, dest, indices, permutationMapAttr,
      mask, inBoundsAttr);
3788}

3790/// 2. Builder with type inference that sets an empty mask (variant with attrs).
3791void TransferWriteOp::build(OpBuilder &builder, OperationState &result,
                          Value vector, Value dest, ValueRange indices,
                          AffineMapAttr permutationMapAttr,
                          /*optional*/ ArrayAttr inBoundsAttr) {
build(builder, result, vector, dest, indices, permutationMapAttr,
      /*mask=*/Value(), inBoundsAttr);
3797}

3799/// 3. Builder with type inference that sets an empty mask (variant without
3800/// attrs)
3801void TransferWriteOp::build(OpBuilder &builder, OperationState &result,
                          Value vector, Value dest, ValueRange indices,
                          AffineMap permutationMap,
                          std::optional<ArrayRef<bool>> inBounds) {
auto permutationMapAttr = AffineMapAttr::get(permutationMap);
auto inBoundsAttr = (inBounds && !inBounds.value().empty())
                        ? builder.getBoolArrayAttr(inBounds.value())
                        : ArrayAttr();
build(builder, result, vector, dest, indices, permutationMapAttr,
      /*mask=*/Value(), inBoundsAttr);
3811}

3813/// 4. Builder with type inference that sets an empty mask and sets permutation
3814///    map to 'getMinorIdentityMap'.
3815void TransferWriteOp::build(OpBuilder &builder, OperationState &result,
                          Value vector, Value dest, ValueRange indices,
                          std::optional<ArrayRef<bool>> inBounds) {
auto vectorType = vector.getType().cast<VectorType>();
AffineMap permutationMap = getTransferMinorIdentityMap(
    dest.getType().cast<ShapedType>(), vectorType);
build(builder, result, vector, dest, indices, permutationMap, inBounds);
3822}

3824/// Infers the mask type for a transfer write given its vector type and
3825/// permutation map. The mask in a transfer read operation applies to the
3826/// tensor/buffer writing part of it and its type should match the shape written
3827/// *after* any permutation.
3828static VectorType inferTransferWriteMaskType(VectorType vecType,
                                           AffineMap permMap) {
auto i1Type = IntegerType::get(permMap.getContext(), 1);
SmallVector<int64_t, 8> maskShape =
    compressUnusedDims(permMap).compose(vecType.getShape());
return VectorType::get(maskShape, i1Type);
3834}

3836ParseResult TransferWriteOp::parse(OpAsmParser &parser,
                                 OperationState &result) {
auto &builder = parser.getBuilder();
SMLoc typesLoc;
OpAsmParser::UnresolvedOperand vectorInfo, sourceInfo;
SmallVector<OpAsmParser::UnresolvedOperand, 8> indexInfo;
SmallVector<Type, 2> types;
OpAsmParser::UnresolvedOperand maskInfo;
if (parser.parseOperand(vectorInfo) || parser.parseComma() ||
    parser.parseOperand(sourceInfo) ||
    parser.parseOperandList(indexInfo, OpAsmParser::Delimiter::Square))
  return failure();
ParseResult hasMask = parser.parseOptionalComma();
if (hasMask.succeeded() && parser.parseOperand(maskInfo))
  return failure();
if (parser.parseOptionalAttrDict(result.attributes) ||
    parser.getCurrentLocation(&typesLoc) || parser.parseColonTypeList(types))
  return failure();
if (types.size() != 2)
  return parser.emitError(typesLoc, "requires two types");
auto indexType = builder.getIndexType();
VectorType vectorType = types[0].dyn_cast<VectorType>();
if (!vectorType)
  return parser.emitError(typesLoc, "requires vector type");
ShapedType shapedType = types[1].dyn_cast<ShapedType>();
if (!shapedType || !shapedType.isa<MemRefType, RankedTensorType>())
  return parser.emitError(typesLoc, "requires memref or ranked tensor type");
auto permMapAttrName = TransferWriteOp::getPermutationMapAttrStrName();
auto permMapAttr = result.attributes.get(permMapAttrName);
AffineMap permMap;
if (!permMapAttr) {
  permMap = getTransferMinorIdentityMap(shapedType, vectorType);
  result.attributes.set(permMapAttrName, AffineMapAttr::get(permMap));
} else {
  permMap = permMapAttr.cast<AffineMapAttr>().getValue();
}
if (parser.resolveOperand(vectorInfo, vectorType, result.operands) ||
    parser.resolveOperand(sourceInfo, shapedType, result.operands) ||
    parser.resolveOperands(indexInfo, indexType, result.operands))
  return failure();
if (hasMask.succeeded()) {
  if (shapedType.getElementType().dyn_cast<VectorType>())
    return parser.emitError(
        maskInfo.location, "does not support masks with vector element type");
  auto maskType = inferTransferWriteMaskType(vectorType, permMap);
  if (parser.resolveOperand(maskInfo, maskType, result.operands))
    return failure();
}
result.addAttribute(TransferWriteOp::getOperandSegmentSizeAttr(),
                    builder.getDenseI32ArrayAttr(
                        {1, 1, static_cast<int32_t>(indexInfo.size()),
                         static_cast<int32_t>(hasMask.succeeded())}));
return failure(shapedType.isa<RankedTensorType>() &&
               parser.addTypeToList(shapedType, result.types));
3890}

3892void TransferWriteOp::print(OpAsmPrinter &p) {
p << " " << getVector() << ", " << getSource() << "[" << getIndices() << "]";
if (getMask())
  p << ", " << getMask();
printTransferAttrs(p, *this);
p << " : " << getVectorType() << ", " << getShapedType();
3898}

3900LogicalResult TransferWriteOp::verify() {
// Consistency of elemental types in shape and vector.
ShapedType shapedType = getShapedType();
VectorType vectorType = getVectorType();
VectorType maskType = getMaskType();
auto permutationMap = getPermutationMap();
VectorType inferredMaskType =
    maskType ? inferTransferWriteMaskType(vectorType, permutationMap)
             : VectorType();

if (llvm::size(getIndices()) != shapedType.getRank())
  return emitOpError("requires ") << shapedType.getRank() << " indices";

// We do not allow broadcast dimensions on TransferWriteOps for the moment,
// as the semantics is unclear. This can be revisited later if necessary.
if (hasBroadcastDim())
  return emitOpError("should not have broadcast dimensions");

if (failed(verifyTransferOp(cast<VectorTransferOpInterface>(getOperation()),
                            shapedType, vectorType, maskType,
                            inferredMaskType, permutationMap,
                            getInBounds() ? *getInBounds() : ArrayAttr())))
  return failure();

return verifyPermutationMap(permutationMap,
                            [&](Twine t) { return emitOpError(t); });
3926}

3928// MaskableOpInterface methods.

3930/// Returns the mask type expected by this operation. Mostly used for
3931/// verification purposes.
3932Type TransferWriteOp::getExpectedMaskType() {
return inferTransferWriteMaskType(getVectorType(), getPermutationMap());
3934}

3936/// Fold:
3937/// ```
3938///    %t1 = ...
3939///    %v = vector.transfer_read %t0[%c0...], {in_bounds = [true...]} :
3940///      tensor<static_sizesxf32>, vector<static_sizesxf32>
3941///    %t2 = vector.transfer_write %v, %t1[%c0...] {in_bounds = [true...]} :
3942///      vector<static_sizesxf32>, tensor<static_sizesxf32>
3943/// ```
3944///
3945/// into:
3946///
3947/// ```
3948///    %t0
3949/// ```
3950///
3951/// The producer of t1 may or may not be DCE'd depending on whether it is a
3952/// block argument or has side effects.
3953static LogicalResult foldReadInitWrite(TransferWriteOp write,
                                     ArrayRef<Attribute>,
                                     SmallVectorImpl<OpFoldResult> &results) {
// TODO: support 0-d corner case.
if (write.getTransferRank() == 0)
  return failure();
auto rankedTensorType =
    write.getSource().getType().dyn_cast<RankedTensorType>();
// If not operating on tensors, bail.
if (!rankedTensorType)
  return failure();
// If no read, bail.
auto read = write.getVector().getDefiningOp<vector::TransferReadOp>();
if (!read)
  return failure();
// TODO: support 0-d corner case.
if (read.getTransferRank() == 0)
  return failure();
// For now, only accept minor identity. Future: composition is minor identity.
if (!read.getPermutationMap().isMinorIdentity() ||
    !write.getPermutationMap().isMinorIdentity())
  return failure();
// Bail on mismatching ranks.
if (read.getTransferRank() != write.getTransferRank())
  return failure();
// Bail on potential out-of-bounds accesses.
if (read.hasOutOfBoundsDim() || write.hasOutOfBoundsDim())
  return failure();
// Tensor types must be the same.
if (read.getSource().getType() != rankedTensorType)
  return failure();
// Vector types must be the same.
if (read.getVectorType() != write.getVectorType())
  return failure();
// Vector and Tensor shapes must match.
if (read.getVectorType().getShape() != rankedTensorType.getShape())
  return failure();
// If any index is nonzero.
auto isNotConstantZero = [](Value v) {
  auto cstOp = v.getDefiningOp<arith::ConstantIndexOp>();
  return !cstOp || cstOp.value() != 0;
};
if (llvm::any_of(read.getIndices(), isNotConstantZero) ||
    llvm::any_of(write.getIndices(), isNotConstantZero))
  return failure();
// Success.
results.push_back(read.getSource());
return success();
4001}

4003static bool checkSameValueWAR(vector::TransferReadOp read,
                            vector::TransferWriteOp write) {
return read.getSource() == write.getSource() &&
       read.getIndices() == write.getIndices() &&
       read.getPermutationMap() == write.getPermutationMap() &&
       read.getVectorType() == write.getVectorType() && !read.getMask() &&
       !write.getMask();
4010}
4011/// Fold transfer_write write after read:
4012/// ```
4013///    %t0 = ...
4014///    %v = vector.transfer_read %t0[%c0...] :
4015///      tensor<static_sizesxf32>, vector<static_sizesxf32>
4016///    %t1 = vector.transfer_write %v, %t0[%c0...] :
4017///      vector<static_sizesxf32>, tensor<static_sizesxf32>
4018/// ```
4019///
4020/// into:
4021///
4022/// ```
4023///    %t0
4024/// ```
4025static LogicalResult foldWAR(TransferWriteOp write,
                           SmallVectorImpl<OpFoldResult> &results) {
if (!write.getSource().getType().isa<RankedTensorType>())
  return failure();
auto read = write.getVector().getDefiningOp<vector::TransferReadOp>();
if (!read)
  return failure();

if (!checkSameValueWAR(read, write))
  return failure();
results.push_back(read.getSource());
return success();
4037}

4039LogicalResult TransferWriteOp::fold(ArrayRef<Attribute> operands,
                                  SmallVectorImpl<OpFoldResult> &results) {
if (succeeded(foldReadInitWrite(*this, operands, results)))
  return success();
if (succeeded(foldWAR(*this, results)))
  return success();
if (succeeded(foldTransferInBoundsAttribute(*this)))
  return success();
return memref::foldMemRefCast(*this);
4048}

4050std::optional<SmallVector<int64_t, 4>> TransferWriteOp::getShapeForUnroll() {
return llvm::to_vector<4>(getVectorType().getShape());
4052}

4054void TransferWriteOp::getEffects(
  SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
      &effects) {
if (getShapedType().isa<MemRefType>())
  effects.emplace_back(MemoryEffects::Write::get(), getSource(),
                       SideEffects::DefaultResource::get());
4060}

4062namespace {
4063/// Remove dead transfer write from the SSA chain so that it an be eliminated by
4064/// DCE
4065/// ```
4066///  %w0 = vector.transfer_write %v0, %arg0[%c1, %c0] {in_bounds = [true, true]}
4067///    : vector<1x4xf32>, tensor<4x4xf32>
4068///  %w1 = vector.transfer_write %v0, %w0[%c2, %c0] {in_bounds = [true, true]}
4069///    : vector<1x4xf32>, tensor<4x4xf32>
4070///  %w2 = vector.transfer_write %v1, %w1[%c1, %c0] {in_bounds = [true, true]}
4071///    : vector<1x4xf32>, tensor<4x4xf32>
4072/// ```
4073///
4074/// into:
4075///
4076/// ```
4077///  %w0 = vector.transfer_write %v0, %arg0[%c1, %c0] {in_bounds = [true, true]}
4078///    : vector<1x4xf32>, tensor<4x4xf32>
4079///  %w1 = vector.transfer_write %v0, %arg0[%c2, %c0] {in_bounds = [true, true]}
4080///    : vector<1x4xf32>, tensor<4x4xf32>
4081///  %w2 = vector.transfer_write %v1, %w1[%c1, %c0] {in_bounds = [true, true]}
4082///    : vector<1x4xf32>, tensor<4x4xf32>
4083/// ```
4084///
4085/// `%w0 = vector.transfer_write` op will be removed by DCE if it doesn't have
4086/// any other uses.
4087class FoldWaw final : public OpRewritePattern<TransferWriteOp> {
4088public:
using OpRewritePattern::OpRewritePattern;
LogicalResult matchAndRewrite(TransferWriteOp writeOp,
                              PatternRewriter &rewriter) const override {
  if (!writeOp.getShapedType().isa<RankedTensorType>())
    return failure();
  vector::TransferWriteOp writeToModify = writeOp;

  auto defWrite =
      writeOp.getSource().getDefiningOp<vector::TransferWriteOp>();
  while (defWrite) {
    if (checkSameValueWAW(writeOp, defWrite)) {
      writeToModify.getSourceMutable().assign(defWrite.getSource());
      return success();
    }
    if (!isDisjointTransferIndices(
            cast<VectorTransferOpInterface>(defWrite.getOperation()),
            cast<VectorTransferOpInterface>(writeOp.getOperation())))
      break;
    // If the previous write op doesn't have any other use we an safely look
    // at the previous store to see if it can be removed.
    if (!defWrite->hasOneUse())
      break;
    writeToModify = defWrite;
    defWrite = defWrite.getSource().getDefiningOp<vector::TransferWriteOp>();
  }
  return failure();
}
4116};

4118/// Fold tensor.insert_slice into vector.transfer_write if the transfer_write
4119/// could directly write to the insert_slice's destination. E.g.:
4120///
4121/// ```
4122/// %0 = vector.transfer_write %v, %t1[%c0, %c0] {in_bounds = [true, true]}
4123///     : vector<4x5xf32>, tensor<4x5xf32>
4124/// %1 = tensor.insert_slice %0 into %t2[%a, %b] [4, 5] [1, 1]
4125///     : tensor<4x5xf32> into tensor<?x?xf32>
4126/// ```
4127/// is rewritten to:
4128/// ```
4129/// %1 = vector.transfer_write %v, %t2[%a, %b] {in_bounds = [true, true]}
4130///     : vector<4x5xf32>, tensor<?x?xf32>
4131/// ```
4132struct FoldInsertSliceIntoTransferWrite
  : public OpRewritePattern<tensor::InsertSliceOp> {
4134public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(tensor::InsertSliceOp insertOp,
                              PatternRewriter &rewriter) const override {
  if (!insertOp.hasUnitStride())
    return failure();

  auto xferOp = insertOp.getSource().getDefiningOp<TransferWriteOp>();
  if (!xferOp)
    return failure();
  // TODO: support 0-d corner case.
  if (xferOp.getTransferRank() == 0)
    return failure();

  if (xferOp.hasOutOfBoundsDim())
    return failure();
  if (xferOp.getVectorType().getRank() != xferOp.getShapedType().getRank())
    return failure();
  if (xferOp.getMask())
    return failure();
  // Fold only if the TransferWriteOp completely overwrites the `source` with
  // a vector. I.e., the result of the TransferWriteOp is a new tensor whose
  // content is the data of the vector.
  if (!llvm::equal(xferOp.getVectorType().getShape(),
                   xferOp.getShapedType().getShape()))
    return failure();
  if (!xferOp.getPermutationMap().isIdentity())
    return failure();

  // Bail on illegal rank-reduction: we need to check that the rank-reduced
  // dims are exactly the leading dims. I.e. the following is illegal:
  // ```
  //    %0 = vector.transfer_write %v, %t[0,0], %cst :
  //      vector<2x4xf32>, tensor<2x4xf32>
  //    %1 = tensor.insert_slice %0 into %tt[0,0,0][2,1,4][1,1,1] :
  //      tensor<2x4xf32> into tensor<2x1x4xf32>
  // ```
  //
  // Cannot fold into:
  // ```
  //    %0 = vector.transfer_write %v, %t[0,0,0], %cst :
  //      vector<2x4xf32>, tensor<2x1x4xf32>
  // ```
  // For this, check the trailing `vectorRank` dims of the insert_slice result
  // tensor match the trailing dims of the inferred result tensor.
  int64_t rankReduced =
      insertOp.getType().getRank() - insertOp.getSourceType().getRank();
  int64_t vectorRank = xferOp.getVectorType().getRank();
  RankedTensorType inferredSourceTensorType =
      tensor::ExtractSliceOp::inferResultType(
          insertOp.getType(), insertOp.getMixedOffsets(),
          insertOp.getMixedSizes(), insertOp.getMixedStrides());
  auto actualSourceTensorShape = insertOp.getSourceType().getShape();
  if (rankReduced > 0 &&
      actualSourceTensorShape.take_back(vectorRank) !=
          inferredSourceTensorType.getShape().take_back(vectorRank))
    return failure();

  SmallVector<Value> indices = getValueOrCreateConstantIndexOp(
      rewriter, insertOp.getLoc(), insertOp.getMixedOffsets());
  SmallVector<bool> inBounds(xferOp.getTransferRank(), true);
  rewriter.replaceOpWithNewOp<TransferWriteOp>(insertOp, xferOp.getVector(),
                                               insertOp.getDest(), indices,
                                               ArrayRef<bool>{inBounds});
  return success();
}
4201};

4203/// Rewrite tensor::ExtractSliceOp(vector::TransferWriteOp) to
4204/// vector::TransferWriteOp(tensor::ExtractSliceOp) if the full slice is
4205/// overwritten and inserted into another tensor. After this rewrite, the
4206/// operations bufferize in-place since all of them work on the same slice.
4207///
4208/// For example:
4209/// ```mlir
4210///   %0 = vector.transfer_write %vec, %init_tensor[%c0, %c0]
4211///        : vector<8x16xf32>, tensor<8x16xf32>
4212///   %1 = tensor.extract_slice %0[0, 0] [%sz0, %sz1] [1, 1]
4213///        : tensor<8x16xf32> to tensor<?x?xf32>
4214///   %r = tensor.insert_slice %1 into %iter_arg[%iv0, %iv1] [%sz0, %sz1] [1, 1]
4215///        : tensor<?x?xf32> into tensor<27x37xf32>
4216/// ```
4217/// folds to
4218/// ```mlir
4219///   %0 = tensor.extract_slice %iter_arg[%iv0, %iv1] [%sz0, %sz1] [1, 1]
4220///        : tensor<27x37xf32> to tensor<?x?xf32>
4221///   %1 = vector.transfer_write %vec, %0[%c0, %c0]
4222///        : vector<8x16xf32>, tensor<?x?xf32>
4223///   %r = tensor.insert_slice %1 into %iter_arg[%iv0, %iv1] [%sz0, %sz1] [1, 1]
4224///        : tensor<?x?xf32> into tensor<27x37xf32>
4225/// ```
4226struct SwapExtractSliceOfTransferWrite
  : public OpRewritePattern<tensor::InsertSliceOp> {
4228public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(tensor::InsertSliceOp insertOp,
                              PatternRewriter &rewriter) const override {
  if (!insertOp.hasUnitStride())
    return failure();
  auto extractOp =
      insertOp.getSource().getDefiningOp<tensor::ExtractSliceOp>();
  if (!extractOp || !extractOp.hasUnitStride() || !extractOp->hasOneUse())
    return failure();
  auto transferOp = extractOp.getSource().getDefiningOp<TransferWriteOp>();
  if (!transferOp || !transferOp->hasOneUse())
    return failure();

  // Fail if vector::TransferWriteOp or tensor::ExtractSliceOp is
  // rank-reducing.
  if (insertOp.getSourceType().getRank() != transferOp.getTransferRank()) {
    return rewriter.notifyMatchFailure(insertOp,
                                       "use-def chain is rank-reducing");
  }

  // Fail if tensor::ExtractSliceOp has non-zero offset.
  if (!extractOp.hasZeroOffset()) {
    return rewriter.notifyMatchFailure(insertOp,
                                       "ExtractSliceOp has non-zero offset");
  }

  // Fail if tensor::TransferWriteOp has non-zero offset.
  if (!llvm::all_of(transferOp.getIndices(), [](Value value) {
        return getConstantIntValue(value) == static_cast<int64_t>(0);
      })) {
    return rewriter.notifyMatchFailure(insertOp,
                                       "TranferWriteOp has non-zero offset");
  }

  // Fail if tensor::ExtractSliceOp and tensor::InsertSliceOp sizes differ.
  for (auto [insertSize, extractSize] :
       llvm::zip_equal(insertOp.getMixedSizes(), extractOp.getMixedSizes())) {
    if (!isEqualConstantIntOrValue(insertSize, extractSize)) {
      return rewriter.notifyMatchFailure(
          insertOp, "InsertSliceOp and ExtractSliceOp sizes differ");
    }
  }

  // Fail if the vector::TransferWriteOp may not overwrite the full tensor.
  assert(transferOp.getVectorType().hasStaticShape() &&(static_cast <bool> (transferOp.getVectorType().hasStaticShape
() && "expected vector to have a static shape") ? void
 (0) : __assert_fail ("transferOp.getVectorType().hasStaticShape() && \"expected vector to have a static shape\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 4275, __extension__
 __PRETTY_FUNCTION__))
         "expected vector to have a static shape")(static_cast <bool> (transferOp.getVectorType().hasStaticShape
() && "expected vector to have a static shape") ? void
 (0) : __assert_fail ("transferOp.getVectorType().hasStaticShape() && \"expected vector to have a static shape\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 4275, __extension__
 __PRETTY_FUNCTION__));
  ArrayRef<int64_t> vectorShape = transferOp.getVectorType().getShape();
  SmallVector<int64_t> resultShape = applyPermutationMap(
      transferOp.getPermutationMap(), transferOp.getShapedType().getShape());
  if (transferOp.getMask() || !vectorShape.equals(resultShape)) {
    return rewriter.notifyMatchFailure(
        insertOp, "TransferWriteOp may not write the full tensor.");
  }

  // Swap the tensor::ExtractSliceOp in front of the vector::TransferWriteOp.
  SmallVector<int64_t> newResultShape = applyPermutationMap(
      transferOp.getPermutationMap(), insertOp.getSourceType().getShape());
  SmallVector<bool> newInBounds;
  for (const auto &en : enumerate(newResultShape))
    newInBounds.push_back(en.value() == vectorShape[en.index()]);
  auto newExtractOp = rewriter.create<tensor::ExtractSliceOp>(
      extractOp.getLoc(), insertOp.getSourceType(), insertOp.getDest(),
      insertOp.getMixedOffsets(), insertOp.getMixedSizes(),
      insertOp.getMixedStrides());
  auto newTransferWriteOp = rewriter.create<TransferWriteOp>(
      transferOp.getLoc(), transferOp.getVector(), newExtractOp.getResult(),
      transferOp.getIndices(), transferOp.getPermutationMapAttr(),
      rewriter.getBoolArrayAttr(newInBounds));
  rewriter.updateRootInPlace(insertOp, [&]() {
    insertOp.getSourceMutable().assign(newTransferWriteOp.getResult());
  });
  return success();
}
4303};

4305} // namespace

4307void TransferWriteOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                                MLIRContext *context) {
results.add<FoldWaw, FoldInsertSliceIntoTransferWrite,
            SwapExtractSliceOfTransferWrite>(context);
4311}

4313//===----------------------------------------------------------------------===//
4314// LoadOp
4315//===----------------------------------------------------------------------===//

4317static LogicalResult verifyLoadStoreMemRefLayout(Operation *op,
                                               MemRefType memRefTy) {
if (!isLastMemrefDimUnitStride(memRefTy))
  return op->emitOpError("most minor memref dim must have unit stride");
return success();
4322}

4324LogicalResult vector::LoadOp::verify() {
VectorType resVecTy = getVectorType();
MemRefType memRefTy = getMemRefType();

if (failed(verifyLoadStoreMemRefLayout(*this, memRefTy)))
  return failure();

// Checks for vector memrefs.
Type memElemTy = memRefTy.getElementType();
if (auto memVecTy = memElemTy.dyn_cast<VectorType>()) {
  if (memVecTy != resVecTy)
    return emitOpError("base memref and result vector types should match");
  memElemTy = memVecTy.getElementType();
}

if (resVecTy.getElementType() != memElemTy)
  return emitOpError("base and result element types should match");
if (llvm::size(getIndices()) != memRefTy.getRank())
  return emitOpError("requires ") << memRefTy.getRank() << " indices";
return success();
4344}

4346OpFoldResult LoadOp::fold(ArrayRef<Attribute>) {
if (succeeded(memref::foldMemRefCast(*this)))
  return getResult();
return OpFoldResult();
4350}

4352//===----------------------------------------------------------------------===//
4353// StoreOp
4354//===----------------------------------------------------------------------===//

4356LogicalResult vector::StoreOp::verify() {
VectorType valueVecTy = getVectorType();
MemRefType memRefTy = getMemRefType();

if (failed(verifyLoadStoreMemRefLayout(*this, memRefTy)))
  return failure();

// Checks for vector memrefs.
Type memElemTy = memRefTy.getElementType();
if (auto memVecTy = memElemTy.dyn_cast<VectorType>()) {
  if (memVecTy != valueVecTy)
    return emitOpError(
        "base memref and valueToStore vector types should match");
  memElemTy = memVecTy.getElementType();
}

if (valueVecTy.getElementType() != memElemTy)
  return emitOpError("base and valueToStore element type should match");
if (llvm::size(getIndices()) != memRefTy.getRank())
  return emitOpError("requires ") << memRefTy.getRank() << " indices";
return success();
4377}

4379LogicalResult StoreOp::fold(ArrayRef<Attribute> operands,
                          SmallVectorImpl<OpFoldResult> &results) {
return memref::foldMemRefCast(*this);
4382}

4384//===----------------------------------------------------------------------===//
4385// MaskedLoadOp
4386//===----------------------------------------------------------------------===//

4388LogicalResult MaskedLoadOp::verify() {
VectorType maskVType = getMaskVectorType();
VectorType passVType = getPassThruVectorType();
VectorType resVType = getVectorType();
MemRefType memType = getMemRefType();

if (resVType.getElementType() != memType.getElementType())
  return emitOpError("base and result element type should match");
if (llvm::size(getIndices()) != memType.getRank())
  return emitOpError("requires ") << memType.getRank() << " indices";
if (resVType.getDimSize(0) != maskVType.getDimSize(0))
  return emitOpError("expected result dim to match mask dim");
if (resVType != passVType)
  return emitOpError("expected pass_thru of same type as result type");
return success();
4403}

4405namespace {
4406class MaskedLoadFolder final : public OpRewritePattern<MaskedLoadOp> {
4407public:
using OpRewritePattern::OpRewritePattern;
LogicalResult matchAndRewrite(MaskedLoadOp load,
                              PatternRewriter &rewriter) const override {
  switch (getMaskFormat(load.getMask())) {
  case MaskFormat::AllTrue:
    rewriter.replaceOpWithNewOp<vector::LoadOp>(
        load, load.getType(), load.getBase(), load.getIndices());
    return success();
  case MaskFormat::AllFalse:
    rewriter.replaceOp(load, load.getPassThru());
    return success();
  case MaskFormat::Unknown:
    return failure();
  }
  llvm_unreachable("Unexpected 1DMaskFormat on MaskedLoad")::llvm::llvm_unreachable_internal("Unexpected 1DMaskFormat on MaskedLoad"
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 4422);
}
4424};
4425} // namespace

4427void MaskedLoadOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                             MLIRContext *context) {
results.add<MaskedLoadFolder>(context);
4430}

4432OpFoldResult MaskedLoadOp::fold(ArrayRef<Attribute>) {
if (succeeded(memref::foldMemRefCast(*this)))
  return getResult();
return OpFoldResult();
4436}

4438//===----------------------------------------------------------------------===//
4439// MaskedStoreOp
4440//===----------------------------------------------------------------------===//

4442LogicalResult MaskedStoreOp::verify() {
VectorType maskVType = getMaskVectorType();
VectorType valueVType = getVectorType();
MemRefType memType = getMemRefType();

if (valueVType.getElementType() != memType.getElementType())
  return emitOpError("base and valueToStore element type should match");
if (llvm::size(getIndices()) != memType.getRank())
  return emitOpError("requires ") << memType.getRank() << " indices";
if (valueVType.getDimSize(0) != maskVType.getDimSize(0))
  return emitOpError("expected valueToStore dim to match mask dim");
return success();
4454}

4456namespace {
4457class MaskedStoreFolder final : public OpRewritePattern<MaskedStoreOp> {
4458public:
using OpRewritePattern::OpRewritePattern;
LogicalResult matchAndRewrite(MaskedStoreOp store,
                              PatternRewriter &rewriter) const override {
  switch (getMaskFormat(store.getMask())) {
  case MaskFormat::AllTrue:
    rewriter.replaceOpWithNewOp<vector::StoreOp>(
        store, store.getValueToStore(), store.getBase(), store.getIndices());
    return success();
  case MaskFormat::AllFalse:
    rewriter.eraseOp(store);
    return success();
  case MaskFormat::Unknown:
    return failure();
  }
  llvm_unreachable("Unexpected 1DMaskFormat on MaskedStore")::llvm::llvm_unreachable_internal("Unexpected 1DMaskFormat on MaskedStore"
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 4473);
}
4475};
4476} // namespace

4478void MaskedStoreOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                              MLIRContext *context) {
results.add<MaskedStoreFolder>(context);
4481}

4483LogicalResult MaskedStoreOp::fold(ArrayRef<Attribute> operands,
                                SmallVectorImpl<OpFoldResult> &results) {
return memref::foldMemRefCast(*this);
4486}

4488//===----------------------------------------------------------------------===//
4489// GatherOp
4490//===----------------------------------------------------------------------===//

4492LogicalResult GatherOp::verify() {
VectorType indVType = getIndexVectorType();
VectorType maskVType = getMaskVectorType();
VectorType resVType = getVectorType();
ShapedType baseType = getBaseType();

if (!baseType.isa<MemRefType, RankedTensorType>())
  return emitOpError("requires base to be a memref or ranked tensor type");

if (resVType.getElementType() != baseType.getElementType())
  return emitOpError("base and result element type should match");
if (llvm::size(getIndices()) != baseType.getRank())
  return emitOpError("requires ") << baseType.getRank() << " indices";
if (resVType.getShape() != indVType.getShape())
  return emitOpError("expected result dim to match indices dim");
if (resVType.getShape() != maskVType.getShape())
  return emitOpError("expected result dim to match mask dim");
if (resVType != getPassThruVectorType())
  return emitOpError("expected pass_thru of same type as result type");
return success();
4512}

4514namespace {
4515class GatherFolder final : public OpRewritePattern<GatherOp> {
4516public:
using OpRewritePattern::OpRewritePattern;
LogicalResult matchAndRewrite(GatherOp gather,
                              PatternRewriter &rewriter) const override {
  switch (getMaskFormat(gather.getMask())) {
  case MaskFormat::AllTrue:
    return failure(); // no unmasked equivalent
  case MaskFormat::AllFalse:
    rewriter.replaceOp(gather, gather.getPassThru());
    return success();
  case MaskFormat::Unknown:
    return failure();
  }
  llvm_unreachable("Unexpected 1DMaskFormat on GatherFolder")::llvm::llvm_unreachable_internal("Unexpected 1DMaskFormat on GatherFolder"
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 4529);
}
4531};
4532} // namespace

4534void GatherOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                         MLIRContext *context) {
results.add<GatherFolder>(context);
4537}

4539//===----------------------------------------------------------------------===//
4540// ScatterOp
4541//===----------------------------------------------------------------------===//

4543LogicalResult ScatterOp::verify() {
VectorType indVType = getIndexVectorType();
VectorType maskVType = getMaskVectorType();
VectorType valueVType = getVectorType();
MemRefType memType = getMemRefType();

if (valueVType.getElementType() != memType.getElementType())
  return emitOpError("base and valueToStore element type should match");
if (llvm::size(getIndices()) != memType.getRank())
  return emitOpError("requires ") << memType.getRank() << " indices";
if (valueVType.getDimSize(0) != indVType.getDimSize(0))
  return emitOpError("expected valueToStore dim to match indices dim");
if (valueVType.getDimSize(0) != maskVType.getDimSize(0))
  return emitOpError("expected valueToStore dim to match mask dim");
return success();
4558}

4560namespace {
4561class ScatterFolder final : public OpRewritePattern<ScatterOp> {
4562public:
using OpRewritePattern::OpRewritePattern;
LogicalResult matchAndRewrite(ScatterOp scatter,
                              PatternRewriter &rewriter) const override {
  switch (getMaskFormat(scatter.getMask())) {
  case MaskFormat::AllTrue:
    return failure(); // no unmasked equivalent
  case MaskFormat::AllFalse:
    rewriter.eraseOp(scatter);
    return success();
  case MaskFormat::Unknown:
    return failure();
  }
  llvm_unreachable("Unexpected 1DMaskFormat on ScatterFolder")::llvm::llvm_unreachable_internal("Unexpected 1DMaskFormat on ScatterFolder"
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 4575);
}
4577};
4578} // namespace

4580void ScatterOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                          MLIRContext *context) {
results.add<ScatterFolder>(context);
4583}

4585//===----------------------------------------------------------------------===//
4586// ExpandLoadOp
4587//===----------------------------------------------------------------------===//

4589LogicalResult ExpandLoadOp::verify() {
VectorType maskVType = getMaskVectorType();
VectorType passVType = getPassThruVectorType();
VectorType resVType = getVectorType();
MemRefType memType = getMemRefType();

if (resVType.getElementType() != memType.getElementType())
  return emitOpError("base and result element type should match");
if (llvm::size(getIndices()) != memType.getRank())
  return emitOpError("requires ") << memType.getRank() << " indices";
if (resVType.getDimSize(0) != maskVType.getDimSize(0))
  return emitOpError("expected result dim to match mask dim");
if (resVType != passVType)
  return emitOpError("expected pass_thru of same type as result type");
return success();
4604}

4606namespace {
4607class ExpandLoadFolder final : public OpRewritePattern<ExpandLoadOp> {
4608public:
using OpRewritePattern::OpRewritePattern;
LogicalResult matchAndRewrite(ExpandLoadOp expand,
                              PatternRewriter &rewriter) const override {
  switch (getMaskFormat(expand.getMask())) {
  case MaskFormat::AllTrue:
    rewriter.replaceOpWithNewOp<vector::LoadOp>(
        expand, expand.getType(), expand.getBase(), expand.getIndices());
    return success();
  case MaskFormat::AllFalse:
    rewriter.replaceOp(expand, expand.getPassThru());
    return success();
  case MaskFormat::Unknown:
    return failure();
  }
  llvm_unreachable("Unexpected 1DMaskFormat on ExpandLoadFolder")::llvm::llvm_unreachable_internal("Unexpected 1DMaskFormat on ExpandLoadFolder"
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 4623);
}
4625};
4626} // namespace

4628void ExpandLoadOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                             MLIRContext *context) {
results.add<ExpandLoadFolder>(context);
4631}

4633//===----------------------------------------------------------------------===//
4634// CompressStoreOp
4635//===----------------------------------------------------------------------===//

4637LogicalResult CompressStoreOp::verify() {
VectorType maskVType = getMaskVectorType();
VectorType valueVType = getVectorType();
MemRefType memType = getMemRefType();

if (valueVType.getElementType() != memType.getElementType())
  return emitOpError("base and valueToStore element type should match");
if (llvm::size(getIndices()) != memType.getRank())
  return emitOpError("requires ") << memType.getRank() << " indices";
if (valueVType.getDimSize(0) != maskVType.getDimSize(0))
  return emitOpError("expected valueToStore dim to match mask dim");
return success();
4649}

4651namespace {
4652class CompressStoreFolder final : public OpRewritePattern<CompressStoreOp> {
4653public:
using OpRewritePattern::OpRewritePattern;
LogicalResult matchAndRewrite(CompressStoreOp compress,
                              PatternRewriter &rewriter) const override {
  switch (getMaskFormat(compress.getMask())) {
  case MaskFormat::AllTrue:
    rewriter.replaceOpWithNewOp<vector::StoreOp>(
        compress, compress.getValueToStore(), compress.getBase(),
        compress.getIndices());
    return success();
  case MaskFormat::AllFalse:
    rewriter.eraseOp(compress);
    return success();
  case MaskFormat::Unknown:
    return failure();
  }
  llvm_unreachable("Unexpected 1DMaskFormat on CompressStoreFolder")::llvm::llvm_unreachable_internal("Unexpected 1DMaskFormat on CompressStoreFolder"
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 4669);
}
4671};
4672} // namespace

4674void CompressStoreOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                                MLIRContext *context) {
results.add<CompressStoreFolder>(context);
4677}

4679//===----------------------------------------------------------------------===//
4680// ShapeCastOp
4681//===----------------------------------------------------------------------===//

4683/// Returns true if each element of 'a' is equal to the product of a contiguous
4684/// sequence of the elements of 'b'. Returns false otherwise.
4685static bool isValidShapeCast(ArrayRef<int64_t> a, ArrayRef<int64_t> b) {
unsigned rankA = a.size();
unsigned rankB = b.size();
assert(rankA < rankB)(static_cast <bool> (rankA < rankB) ? void (0) : __assert_fail
 ("rankA < rankB", "mlir/lib/Dialect/Vector/IR/VectorOps.cpp"
, 4688, __extension__ __PRETTY_FUNCTION__));

unsigned i = 0;
unsigned j = 0;
while (i < rankA && j < rankB) {
  int64_t dimA = a[i];
  int64_t dimB = 1;
  while (dimB < dimA && j < rankB)
    dimB *= b[j++];
  if (dimA != dimB)
    break;
  ++i;

  // Handle the case when trailing dimensions are of size 1.
  // Include them into the contiguous sequence.
  auto isOne = [](int64_t v) { return v == 1; };
  if (i < rankA && llvm::all_of(a.slice(i), isOne))
    i = rankA;
  if (j < rankB && llvm::all_of(b.slice(j), isOne))
    j = rankB;
}

return i == rankA && j == rankB;
4711}

4713static LogicalResult verifyVectorShapeCast(Operation *op,
                                         VectorType sourceVectorType,
                                         VectorType resultVectorType) {
// Check that element type is the same.
if (sourceVectorType.getElementType() != resultVectorType.getElementType())
  return op->emitOpError("source/result vectors must have same element type");
auto sourceShape = sourceVectorType.getShape();
auto resultShape = resultVectorType.getShape();

// Check that product of source dim sizes matches product of result dim sizes.
int64_t sourceDimProduct = std::accumulate(
    sourceShape.begin(), sourceShape.end(), 1LL, std::multiplies<int64_t>{});
int64_t resultDimProduct = std::accumulate(
    resultShape.begin(), resultShape.end(), 1LL, std::multiplies<int64_t>{});
if (sourceDimProduct != resultDimProduct)
  return op->emitOpError("source/result number of elements must match");

// Check that expanding/contracting rank cases.
unsigned sourceRank = sourceVectorType.getRank();
unsigned resultRank = resultVectorType.getRank();
if (sourceRank < resultRank) {
  if (!isValidShapeCast(sourceShape, resultShape))
    return op->emitOpError("invalid shape cast");
} else if (sourceRank > resultRank) {
  if (!isValidShapeCast(resultShape, sourceShape))
    return op->emitOpError("invalid shape cast");
}
return success();
4741}

4743LogicalResult ShapeCastOp::verify() {
auto sourceVectorType = getSource().getType().dyn_cast_or_null<VectorType>();
auto resultVectorType = getResult().getType().dyn_cast_or_null<VectorType>();

// Check if source/result are of vector type.
if (sourceVectorType && resultVectorType)
  return verifyVectorShapeCast(*this, sourceVectorType, resultVectorType);

return success();
4752}

4754OpFoldResult ShapeCastOp::fold(ArrayRef<Attribute> operands) {
// No-op shape cast.
if (getSource().getType() == getResult().getType())
  return getSource();

// Canceling shape casts.
if (auto otherOp = getSource().getDefiningOp<ShapeCastOp>()) {
  if (getResult().getType() == otherOp.getSource().getType())
    return otherOp.getSource();

  // Only allows valid transitive folding.
  VectorType srcType = otherOp.getSource().getType().cast<VectorType>();
  VectorType resultType = getResult().getType().cast<VectorType>();
  if (srcType.getRank() < resultType.getRank()) {
    if (!isValidShapeCast(srcType.getShape(), resultType.getShape()))
      return {};
  } else if (srcType.getRank() > resultType.getRank()) {
    if (!isValidShapeCast(resultType.getShape(), srcType.getShape()))
      return {};
  } else {
    return {};
  }

  setOperand(otherOp.getSource());
  return getResult();
}

// Cancelling broadcast and shape cast ops.
if (auto bcastOp = getSource().getDefiningOp<BroadcastOp>()) {
  if (bcastOp.getSourceType() == getType())
    return bcastOp.getSource();
}

return {};
4788}

4790namespace {
4791// Pattern to rewrite a ShapeCast(splat ConstantOp) -> ConstantOp.
4792class ShapeCastConstantFolder final : public OpRewritePattern<ShapeCastOp> {
4793public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(ShapeCastOp shapeCastOp,
                              PatternRewriter &rewriter) const override {
  auto constantOp =
      shapeCastOp.getSource().getDefiningOp<arith::ConstantOp>();
  if (!constantOp)
    return failure();
  // Only handle splat for now.
  auto dense = constantOp.getValue().dyn_cast<SplatElementsAttr>();
  if (!dense)
    return failure();
  auto newAttr =
      DenseElementsAttr::get(shapeCastOp.getType().cast<VectorType>(),
                             dense.getSplatValue<Attribute>());
  rewriter.replaceOpWithNewOp<arith::ConstantOp>(shapeCastOp, newAttr);
  return success();
}
4812};

4814/// Pattern to rewrite a ShapeCast(Broadcast) -> Broadcast.
4815/// This only applies when the shape of the broadcast source is a suffix of the
4816/// shape of the result (i.e. when broadcast without reshape is expressive
4817/// enough to capture the result in a single op).
4818class ShapeCastBroadcastFolder final : public OpRewritePattern<ShapeCastOp> {
4819public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(ShapeCastOp shapeCastOp,
                              PatternRewriter &rewriter) const override {
  auto broadcastOp =
      shapeCastOp.getSource().getDefiningOp<vector::BroadcastOp>();
  if (!broadcastOp)
    return failure();

  auto broadcastSourceVectorType =
      broadcastOp.getSourceType().dyn_cast<VectorType>();
  auto broadcastSourceShape = broadcastSourceVectorType
                                  ? broadcastSourceVectorType.getShape()
                                  : ArrayRef<int64_t>{};
  auto shapeCastTargetShape = shapeCastOp.getResultVectorType().getShape();

  // Bail if `broadcastSourceShape` is not a suffix of the result.
  bool isSuffix = (broadcastSourceShape == shapeCastTargetShape.take_back(
                                               broadcastSourceShape.size()));
  if (!isSuffix)
    return failure();

  rewriter.replaceOpWithNewOp<vector::BroadcastOp>(
      shapeCastOp, shapeCastOp.getResultVectorType(),
      broadcastOp.getSource());
  return success();
}
4847};

4849} // namespace

4851void ShapeCastOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                            MLIRContext *context) {
results.add<ShapeCastConstantFolder, ShapeCastBroadcastFolder>(context);
4854}

4856//===----------------------------------------------------------------------===//
4857// VectorBitCastOp
4858//===----------------------------------------------------------------------===//

4860LogicalResult BitCastOp::verify() {
auto sourceVectorType = getSourceVectorType();
auto resultVectorType = getResultVectorType();

for (int64_t i = 0, e = sourceVectorType.getRank() - 1; i < e; i++) {
  if (sourceVectorType.getDimSize(i) != resultVectorType.getDimSize(i))
    return emitOpError("dimension size mismatch at: ") << i;
}

DataLayout dataLayout = DataLayout::closest(*this);
auto sourceElementBits =
    dataLayout.getTypeSizeInBits(sourceVectorType.getElementType());
auto resultElementBits =
    dataLayout.getTypeSizeInBits(resultVectorType.getElementType());

if (sourceVectorType.getRank() == 0) {
  if (sourceElementBits != resultElementBits)
    return emitOpError("source/result bitwidth of the 0-D vector element "
                       "types must be equal");
} else if (sourceElementBits * sourceVectorType.getShape().back() !=
           resultElementBits * resultVectorType.getShape().back()) {
  return emitOpError(
      "source/result bitwidth of the minor 1-D vectors must be equal");
}

return success();
4886}

4888OpFoldResult BitCastOp::fold(ArrayRef<Attribute> operands) {
// Nop cast.
if (getSource().getType() == getResult().getType())
  return getSource();

// Canceling bitcasts.
if (auto otherOp = getSource().getDefiningOp<BitCastOp>()) {
  if (getResult().getType() == otherOp.getSource().getType())
    return otherOp.getSource();

  setOperand(otherOp.getSource());
  return getResult();
}

Attribute sourceConstant = operands.front();
if (!sourceConstant)
  return {};

Type srcElemType = getSourceVectorType().getElementType();
Type dstElemType = getResultVectorType().getElementType();

if (auto floatPack = sourceConstant.dyn_cast<DenseFPElementsAttr>()) {
  if (floatPack.isSplat()) {
    auto splat = floatPack.getSplatValue<FloatAttr>();

    // Casting fp16 into fp32.
    if (srcElemType.isF16() && dstElemType.isF32()) {
      uint32_t bits = static_cast<uint32_t>(
          splat.getValue().bitcastToAPInt().getZExtValue());
      // Duplicate the 16-bit pattern.
      bits = (bits << 16) | (bits & 0xffff);
      APInt intBits(32, bits);
      APFloat floatBits(llvm::APFloat::IEEEsingle(), intBits);
      return DenseElementsAttr::get(getResultVectorType(), floatBits);
    }
  }
}

return {};
4927}

4929//===----------------------------------------------------------------------===//
4930// TypeCastOp
4931//===----------------------------------------------------------------------===//

4933static SmallVector<int64_t, 8> extractShape(MemRefType memRefType) {
auto vectorType = memRefType.getElementType().dyn_cast<VectorType>();
SmallVector<int64_t, 8> res(memRefType.getShape().begin(),
                            memRefType.getShape().end());
if (vectorType)
  res.append(vectorType.getShape().begin(), vectorType.getShape().end());
return res;
4940}

4942/// Build the canonical memRefType with a single vector.
4943/// E.g. memref<4 x 5 x vector<6 x f32>> -> memref<vector<4 x 5 x 6 x f32>>.
4944void TypeCastOp::build(OpBuilder &builder, OperationState &result,
                     Value source) {
result.addOperands(source);
MemRefType memRefType = source.getType().cast<MemRefType>();
VectorType vectorType =
    VectorType::get(extractShape(memRefType),
                    getElementTypeOrSelf(getElementTypeOrSelf(memRefType)));
result.addTypes(MemRefType::get({}, vectorType, MemRefLayoutAttrInterface(),
                                memRefType.getMemorySpace()));
4953}

4955LogicalResult TypeCastOp::verify() {
MemRefType canonicalType = canonicalizeStridedLayout(getMemRefType());
if (!canonicalType.getLayout().isIdentity())
  return emitOpError("expects operand to be a memref with identity layout");
if (!getResultMemRefType().getLayout().isIdentity())
  return emitOpError("expects result to be a memref with identity layout");
if (getResultMemRefType().getMemorySpace() !=
    getMemRefType().getMemorySpace())
  return emitOpError("expects result in same memory space");

auto sourceType = getMemRefType();
auto resultType = getResultMemRefType();
if (getElementTypeOrSelf(getElementTypeOrSelf(sourceType)) !=
    getElementTypeOrSelf(getElementTypeOrSelf(resultType)))
  return emitOpError(
             "expects result and operand with same underlying scalar type: ")
         << resultType;
if (extractShape(sourceType) != extractShape(resultType))
  return emitOpError(
             "expects concatenated result and operand shapes to be equal: ")
         << resultType;
return success();
4977}

4979//===----------------------------------------------------------------------===//
4980// TransposeOp
4981//===----------------------------------------------------------------------===//

4983void vector::TransposeOp::build(OpBuilder &builder, OperationState &result,
                              Value vector, ArrayRef<int64_t> transp) {
VectorType vt = vector.getType().cast<VectorType>();
SmallVector<int64_t, 4> transposedShape(vt.getRank());
for (unsigned i = 0; i < transp.size(); ++i)
  transposedShape[i] = vt.getShape()[transp[i]];

result.addOperands(vector);
result.addTypes(VectorType::get(transposedShape, vt.getElementType()));
result.addAttribute(getTranspAttrStrName(), builder.getI64ArrayAttr(transp));
4993}

4995OpFoldResult vector::TransposeOp::fold(ArrayRef<Attribute> operands) {
// Eliminate splat constant transpose ops.
if (auto attr = operands.front().dyn_cast_or_null<DenseElementsAttr>())
  if (attr.isSplat())
    return attr.reshape(getResultType());

// Eliminate identity transpose ops. This happens when the dimensions of the
// input vector remain in their original order after the transpose operation.
SmallVector<int64_t, 4> transp;
getTransp(transp);

// Check if the permutation of the dimensions contains sequential values:
// {0, 1, 2, ...}.
for (int64_t i = 0, e = transp.size(); i < e; i++) {
  if (transp[i] != i)
    return {};
}

return getVector();
5014}

5016LogicalResult vector::TransposeOp::verify() {
VectorType vectorType = getVectorType();
VectorType resultType = getResultType();
int64_t rank = resultType.getRank();
if (vectorType.getRank() != rank)
  return emitOpError("vector result rank mismatch: ") << rank;
// Verify transposition array.
auto transpAttr = getTransp().getValue();
int64_t size = transpAttr.size();
if (rank != size)
  return emitOpError("transposition length mismatch: ") << size;
SmallVector<bool, 8> seen(rank, false);
for (const auto &ta : llvm::enumerate(transpAttr)) {
  int64_t i = ta.value().cast<IntegerAttr>().getInt();
  if (i < 0 || i >= rank)
    return emitOpError("transposition index out of range: ") << i;
  if (seen[i])
    return emitOpError("duplicate position index: ") << i;
  seen[i] = true;
  if (resultType.getDimSize(ta.index()) != vectorType.getDimSize(i))
    return emitOpError("dimension size mismatch at: ") << i;
}
return success();
5039}

5041std::optional<SmallVector<int64_t, 4>> TransposeOp::getShapeForUnroll() {
return llvm::to_vector<4>(getResultType().getShape());
5043}

5045namespace {

5047// Rewrites two back-to-back TransposeOp operations into a single TransposeOp.
5048class TransposeFolder final : public OpRewritePattern<vector::TransposeOp> {
5049public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(vector::TransposeOp transposeOp,
                              PatternRewriter &rewriter) const override {
  // Wrapper around vector::TransposeOp::getTransp() for cleaner code.
  auto getPermutation = [](vector::TransposeOp transpose) {
    SmallVector<int64_t, 4> permutation;
    transpose.getTransp(permutation);
    return permutation;
  };

  // Composes two permutations: result[i] = permutation1[permutation2[i]].
  auto composePermutations = [](ArrayRef<int64_t> permutation1,
                                ArrayRef<int64_t> permutation2) {
    SmallVector<int64_t, 4> result;
    for (auto index : permutation2)
      result.push_back(permutation1[index]);
    return result;
  };

  // Return if the input of 'transposeOp' is not defined by another transpose.
  vector::TransposeOp parentTransposeOp =
      transposeOp.getVector().getDefiningOp<vector::TransposeOp>();
  if (!parentTransposeOp)
    return failure();

  SmallVector<int64_t, 4> permutation = composePermutations(
      getPermutation(parentTransposeOp), getPermutation(transposeOp));
  // Replace 'transposeOp' with a new transpose operation.
  rewriter.replaceOpWithNewOp<vector::TransposeOp>(
      transposeOp, transposeOp.getResult().getType(),
      parentTransposeOp.getVector(),
      vector::getVectorSubscriptAttr(rewriter, permutation));
  return success();
}
5085};

5087// Folds transpose(broadcast(<scalar>)) into brodcast(<scalar>).
5088struct FoldTransposedScalarBroadcast final
  : public OpRewritePattern<vector::TransposeOp> {
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(vector::TransposeOp transposeOp,
                              PatternRewriter &rewriter) const override {
  auto bcastOp = transposeOp.getVector().getDefiningOp<vector::BroadcastOp>();
  if (!bcastOp)
    return failure();

  auto srcVectorType = bcastOp.getSourceType().dyn_cast<VectorType>();
  if (!srcVectorType || srcVectorType.getNumElements() == 1) {
    rewriter.replaceOpWithNewOp<vector::BroadcastOp>(
        transposeOp, transposeOp.getResultType(), bcastOp.getSource());
    return success();
  }

  return failure();
}
5107};

5109// Folds transpose(splat x : src_type) : res_type into splat x : res_type.
5110class FoldTransposeSplat final : public OpRewritePattern<TransposeOp> {
5111public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(TransposeOp transposeOp,
                              PatternRewriter &rewriter) const override {
  auto splatOp = transposeOp.getVector().getDefiningOp<vector::SplatOp>();
  if (!splatOp)
    return failure();

  rewriter.replaceOpWithNewOp<vector::SplatOp>(
      transposeOp, transposeOp.getResultType(), splatOp.getInput());
  return success();
}
5124};

5126} // namespace

5128void vector::TransposeOp::getCanonicalizationPatterns(
  RewritePatternSet &results, MLIRContext *context) {
results
    .add<FoldTransposedScalarBroadcast, TransposeFolder, FoldTransposeSplat>(
        context);
5133}

5135void vector::TransposeOp::getTransp(SmallVectorImpl<int64_t> &results) {
populateFromInt64AttrArray(getTransp(), results);
5137}

5139//===----------------------------------------------------------------------===//
5140// ConstantMaskOp
5141//===----------------------------------------------------------------------===//

5143LogicalResult ConstantMaskOp::verify() {
auto resultType = getResult().getType().cast<VectorType>();
// Check the corner case of 0-D vectors first.
if (resultType.getRank() == 0) {
  if (getMaskDimSizes().size() != 1)
    return emitError("array attr must have length 1 for 0-D vectors");
  auto dim = getMaskDimSizes()[0].cast<IntegerAttr>().getInt();
  if (dim != 0 && dim != 1)
    return emitError("mask dim size must be either 0 or 1 for 0-D vectors");
  return success();
}

// Verify that array attr size matches the rank of the vector result.
if (static_cast<int64_t>(getMaskDimSizes().size()) != resultType.getRank())
  return emitOpError(
      "must specify array attr of size equal vector result rank");
// Verify that each array attr element is in bounds of corresponding vector
// result dimension size.
auto resultShape = resultType.getShape();
SmallVector<int64_t, 4> maskDimSizes;
for (const auto &it : llvm::enumerate(getMaskDimSizes())) {
  int64_t attrValue = it.value().cast<IntegerAttr>().getInt();
  if (attrValue < 0 || attrValue > resultShape[it.index()])
    return emitOpError(
        "array attr of size out of bounds of vector result dimension size");
  maskDimSizes.push_back(attrValue);
}
// Verify that if one mask dim size is zero, they all should be zero (because
// the mask region is a conjunction of each mask dimension interval).
bool anyZeros = llvm::is_contained(maskDimSizes, 0);
bool allZeros = llvm::all_of(maskDimSizes, [](int64_t s) { return s == 0; });
if (anyZeros && !allZeros)
  return emitOpError("expected all mask dim sizes to be zeros, "
                     "as a result of conjunction with zero mask dim");
// Verify that if the mask type is scalable, dimensions should be zero because
// constant scalable masks can only be defined for the "none set" or "all set"
// cases, and there is no VLA way to define an "all set" case for
// `vector.constant_mask`. In the future, a convention could be established
// to decide if a specific dimension value could be considered as "all set".
if (resultType.isScalable() &&
    getMaskDimSizes()[0].cast<IntegerAttr>().getInt() != 0)
  return emitOpError("expected mask dim sizes for scalable masks to be 0");
return success();
5186}

5188//===----------------------------------------------------------------------===//
5189// CreateMaskOp
5190//===----------------------------------------------------------------------===//

5192LogicalResult CreateMaskOp::verify() {
auto vectorType = getResult().getType().cast<VectorType>();
// Verify that an operand was specified for each result vector each dimension.
if (vectorType.getRank() == 0) {
  if (getNumOperands() != 1)
    return emitOpError(
        "must specify exactly one operand for 0-D create_mask");
} else if (getNumOperands() !=
           getResult().getType().cast<VectorType>().getRank()) {
  return emitOpError(
      "must specify an operand for each result vector dimension");
}
return success();
5205}

5207namespace {

5209// Pattern to rewrite a CreateMaskOp with a ConstantMaskOp.
5210class CreateMaskFolder final : public OpRewritePattern<CreateMaskOp> {
5211public:
using OpRewritePattern::OpRewritePattern;

LogicalResult matchAndRewrite(CreateMaskOp createMaskOp,
                              PatternRewriter &rewriter) const override {
  // Return if any of 'createMaskOp' operands are not defined by a constant.
  auto isNotDefByConstant = [](Value operand) {
    return !isa_and_nonnull<arith::ConstantIndexOp>(operand.getDefiningOp());
  };
  if (llvm::any_of(createMaskOp.getOperands(), isNotDefByConstant))
    return failure();

  // CreateMaskOp for scalable vectors can be folded only if all dimensions
  // are negative or zero.
  if (auto vType = createMaskOp.getType().dyn_cast<VectorType>()) {
    if (vType.isScalable())
      for (auto opDim : createMaskOp.getOperands()) {
        APInt intVal;
        if (matchPattern(opDim, m_ConstantInt(&intVal)) &&
            intVal.isStrictlyPositive())
          return failure();
      }
  }

  // Gather constant mask dimension sizes.
  SmallVector<int64_t, 4> maskDimSizes;
  maskDimSizes.reserve(createMaskOp->getNumOperands());
  for (auto [operand, maxDimSize] : llvm::zip_equal(
           createMaskOp.getOperands(), createMaskOp.getType().getShape())) {
    Operation *defOp = operand.getDefiningOp();
    int64_t dimSize = cast<arith::ConstantIndexOp>(defOp).value();
    dimSize = std::min(dimSize, maxDimSize);
    // If one of dim sizes is zero, set all dims to zero.
    if (dimSize <= 0) {
      maskDimSizes.assign(createMaskOp.getType().getRank(), 0);
      break;
    }
    maskDimSizes.push_back(dimSize);
  }
  // Replace 'createMaskOp' with ConstantMaskOp.
  rewriter.replaceOpWithNewOp<ConstantMaskOp>(
      createMaskOp, createMaskOp.getResult().getType(),
      vector::getVectorSubscriptAttr(rewriter, maskDimSizes));
  return success();
}
5256};

5258} // namespace

5260void CreateMaskOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                             MLIRContext *context) {
results.add<CreateMaskFolder>(context);
5263}

5265//===----------------------------------------------------------------------===//
5266// MaskOp
5267//===----------------------------------------------------------------------===//

5269void MaskOp::build(
  OpBuilder &builder, OperationState &result, Value mask,
  function_ref<void(OpBuilder &, Location)> maskRegionBuilder) {
assert(maskRegionBuilder &&(static_cast <bool> (maskRegionBuilder && "builder callback for 'maskRegion' must be present"
) ? void (0) : __assert_fail ("maskRegionBuilder && \"builder callback for 'maskRegion' must be present\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 5273, __extension__
 __PRETTY_FUNCTION__))
       "builder callback for 'maskRegion' must be present")(static_cast <bool> (maskRegionBuilder && "builder callback for 'maskRegion' must be present"
) ? void (0) : __assert_fail ("maskRegionBuilder && \"builder callback for 'maskRegion' must be present\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 5273, __extension__
 __PRETTY_FUNCTION__));

result.addOperands(mask);
OpBuilder::InsertionGuard guard(builder);
Region *maskRegion = result.addRegion();
builder.createBlock(maskRegion);
maskRegionBuilder(builder, result.location);
5280}

5282void MaskOp::build(
  OpBuilder &builder, OperationState &result, Type resultType, Value mask,
  function_ref<void(OpBuilder &, Location)> maskRegionBuilder) {
build(builder, result, resultType, mask, /*passthru=*/Value(),
      maskRegionBuilder);
5287}

5289void MaskOp::build(
  OpBuilder &builder, OperationState &result, Type resultType, Value mask,
  Value passthru,
  function_ref<void(OpBuilder &, Location)> maskRegionBuilder) {
build(builder, result, mask, maskRegionBuilder);
if (passthru)
  result.addOperands(passthru);
result.addTypes(resultType);
5297}

5299ParseResult MaskOp::parse(OpAsmParser &parser, OperationState &result) {
// Create the op region.
result.regions.reserve(1);
Region &maskRegion = *result.addRegion();

auto &builder = parser.getBuilder();

// Parse all the operands.
OpAsmParser::UnresolvedOperand mask;
if (parser.parseOperand(mask))
  return failure();

// Optional passthru operand.
OpAsmParser::UnresolvedOperand passthru;
ParseResult parsePassthru = parser.parseOptionalComma();
if (parsePassthru.succeeded() && parser.parseOperand(passthru))
  return failure();

// Parse op region.
if (parser.parseRegion(maskRegion, /*arguments=*/{}, /*argTypes=*/{}))
  return failure();

MaskOp::ensureTerminator(maskRegion, builder, result.location);

// Parse the optional attribute list.
if (parser.parseOptionalAttrDict(result.attributes))
  return failure();

// Parse all the types.
Type maskType;
if (parser.parseColonType(maskType))
  return failure();

SmallVector<Type> resultTypes;
if (parser.parseOptionalArrowTypeList(resultTypes))
  return failure();
result.types.append(resultTypes);

// Resolve operands.
if (parser.resolveOperand(mask, maskType, result.operands))
  return failure();

if (parsePassthru.succeeded())
  if (parser.resolveOperand(passthru, resultTypes[0], result.operands))
    return failure();

return success();
5346}

5348void mlir::vector::MaskOp::print(OpAsmPrinter &p) {
p << " " << getMask();
if (getPassthru())
  p << ", " << getPassthru();

// Print single masked operation and skip terminator.
p << " { ";
Block *singleBlock = &getMaskRegion().getBlocks().front();
if (singleBlock && singleBlock->getOperations().size() > 1)
  p.printCustomOrGenericOp(&singleBlock->front());
p << " }";

p.printOptionalAttrDict(getOperation()->getAttrs());

p << " : " << getMask().getType();
if (getNumResults() > 0)
  p << " -> " << getResultTypes();
5365}

5367void MaskOp::ensureTerminator(Region &region, Builder &builder, Location loc) {
OpTrait::SingleBlockImplicitTerminator<vector::YieldOp>::Impl<
    MaskOp>::ensureTerminator(region, builder, loc);
// Keep the default yield terminator if the number of masked operations is not
// the expected. This case will trigger a verification failure.
if (region.front().getOperations().size() != 2)
  return;

// Replace default yield terminator with a new one that returns the results
// from the masked operation.
OpBuilder opBuilder(builder.getContext());
Operation *maskedOp = &region.front().front();
Operation *oldYieldOp = &region.front().back();
assert(isa<vector::YieldOp>(oldYieldOp) && "Expected vector::YieldOp")(static_cast <bool> (isa<vector::YieldOp>(oldYieldOp
) && "Expected vector::YieldOp") ? void (0) : __assert_fail
 ("isa<vector::YieldOp>(oldYieldOp) && \"Expected vector::YieldOp\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 5380, __extension__
 __PRETTY_FUNCTION__));

opBuilder.setInsertionPoint(oldYieldOp);
opBuilder.create<vector::YieldOp>(maskedOp->getLoc(), maskedOp->getResults());
oldYieldOp->dropAllReferences();
oldYieldOp->erase();
5386}

5388LogicalResult MaskOp::verify() {
// Structural checks.
Block &block = getMaskRegion().getBlocks().front();
if (block.getOperations().size() < 2)
  return emitOpError("expects an operation to mask");
if (block.getOperations().size() > 2)
  return emitOpError("expects only one operation to mask");

auto maskableOp = dyn_cast<MaskableOpInterface>(block.front());
if (!maskableOp)
  return emitOpError("expects a maskable operation");

// Result checks.
if (maskableOp->getNumResults() != getNumResults())
  return emitOpError("expects number of results to match maskable operation "
                     "number of results");

if (!llvm::equal(maskableOp->getResultTypes(), getResultTypes()))
  return emitOpError(
      "expects result type to match maskable operation result type");

// Mask checks.
Type expectedMaskType = maskableOp.getExpectedMaskType();
if (getMask().getType() != expectedMaskType)
  return emitOpError("expects a ")
         << expectedMaskType << " mask for the maskable operation";

// Passthru checks.
Value passthru = getPassthru();
if (passthru) {
  if (!maskableOp.supportsPassthru())
    return emitOpError(
        "doesn't expect a passthru argument for this maskable operation");

  if (maskableOp->getNumResults() != 1)
    return emitOpError("expects result when passthru argument is provided");

  if (passthru.getType() != maskableOp->getResultTypes()[0])
    return emitOpError("expects passthru type to match result type");
}

return success();
5430}

5432// MaskingOpInterface definitions.

5434/// Returns the operation masked by this 'vector.mask'.
5435Operation *MaskOp::getMaskableOp() { return &getMaskRegion().front().front(); }

5437/// Returns true if 'vector.mask' has a passthru value.
5438bool MaskOp::hasPassthru() { return getPassthru() != Value(); }

5440//===----------------------------------------------------------------------===//
5441// ScanOp
5442//===----------------------------------------------------------------------===//

5444LogicalResult ScanOp::verify() {
VectorType srcType = getSourceType();
VectorType initialType = getInitialValueType();
// Check reduction dimension < rank.
int64_t srcRank = srcType.getRank();
int64_t reductionDim = getReductionDim();
if (reductionDim >= srcRank)
  return emitOpError("reduction dimension ")
         << reductionDim << " has to be less than " << srcRank;

// Check that rank(initial_value) = rank(src) - 1.
int64_t initialValueRank = initialType.getRank();
if (initialValueRank != srcRank - 1)
  return emitOpError("initial value rank ")
         << initialValueRank << " has to be equal to " << srcRank - 1;

// Check shapes of initial value and src.
ArrayRef<int64_t> srcShape = srcType.getShape();
ArrayRef<int64_t> initialValueShapes = initialType.getShape();
SmallVector<int64_t> expectedShape;
for (int i = 0; i < srcRank; i++) {
  if (i != reductionDim)
    expectedShape.push_back(srcShape[i]);
}
if (!llvm::equal(initialValueShapes, expectedShape)) {
  return emitOpError("incompatible input/initial value shapes");
}

// Verify supported reduction kind.
Type eltType = getDestType().getElementType();
if (!isSupportedCombiningKind(getKind(), eltType))
  return emitOpError("unsupported reduction type ")
         << eltType << " for kind '" << stringifyCombiningKind(getKind())
         << "'";

return success();
5480}

5482void mlir::vector::populateVectorToVectorCanonicalizationPatterns(
  RewritePatternSet &patterns, PatternBenefit benefit) {
patterns
    .add<CreateMaskFolder, MaskedLoadFolder, MaskedStoreFolder, GatherFolder,
         ScatterFolder, ExpandLoadFolder, CompressStoreFolder,
         StridedSliceConstantMaskFolder, TransposeFolder>(
        patterns.getContext(), benefit);
5489}

5491//===----------------------------------------------------------------------===//
5492// SplatOp
5493//===----------------------------------------------------------------------===//

5495OpFoldResult SplatOp::fold(ArrayRef<Attribute> operands) {
auto constOperand = operands.front();
if (!constOperand.isa_and_nonnull<IntegerAttr, FloatAttr>())
  return {};

// SplatElementsAttr::get treats single value for second arg as being a splat.
return SplatElementsAttr::get(getType(), {constOperand});
5502}

5504//===----------------------------------------------------------------------===//
5505// WarpExecuteOnLane0Op
5506//===----------------------------------------------------------------------===//

5508void WarpExecuteOnLane0Op::print(OpAsmPrinter &p) {
p << "(" << getLaneid() << ")";

SmallVector<StringRef> coreAttr = {getWarpSizeAttrName()};
auto warpSizeAttr = getOperation()->getAttr(getWarpSizeAttrName());
p << "[" << warpSizeAttr.cast<IntegerAttr>().getInt() << "]";

if (!getArgs().empty())
  p << " args(" << getArgs() << " : " << getArgs().getTypes() << ")";
if (!getResults().empty())
  p << " -> (" << getResults().getTypes() << ')';
p << " ";
p.printRegion(getRegion(),
              /*printEntryBlockArgs=*/true,
              /*printBlockTerminators=*/!getResults().empty());
p.printOptionalAttrDict(getOperation()->getAttrs(), coreAttr);
5524}

5526ParseResult WarpExecuteOnLane0Op::parse(OpAsmParser &parser,
                                      OperationState &result) {
// Create the region.
result.regions.reserve(1);
Region *warpRegion = result.addRegion();

auto &builder = parser.getBuilder();
OpAsmParser::UnresolvedOperand laneId;

// Parse predicate operand.
if (parser.parseLParen() ||
1
Taking false branch→
    parser.parseOperand(laneId, /*allowResultNumber=*/false) ||
    parser.parseRParen())
  return failure();

int64_t warpSize;
2
←
'warpSize' declared without an initial value→
if (parser.parseLSquare() || parser.parseInteger(warpSize) ||
3
←
Calling 'AsmParser::parseInteger'→
11
←
Returning from 'AsmParser::parseInteger'→
12
←
Taking false branch→
    parser.parseRSquare())
  return failure();
result.addAttribute(getWarpSizeAttrName(OperationName(getOperationName(),
                                                      builder.getContext())),
                    builder.getI64IntegerAttr(warpSize));
13
←
1st function call argument is an uninitialized value

if (parser.resolveOperand(laneId, builder.getIndexType(), result.operands))
  return failure();

llvm::SMLoc inputsOperandsLoc;
SmallVector<OpAsmParser::UnresolvedOperand> inputsOperands;
SmallVector<Type> inputTypes;
if (succeeded(parser.parseOptionalKeyword("args"))) {
  if (parser.parseLParen())
    return failure();

  inputsOperandsLoc = parser.getCurrentLocation();
  if (parser.parseOperandList(inputsOperands) ||
      parser.parseColonTypeList(inputTypes) || parser.parseRParen())
    return failure();
}
if (parser.resolveOperands(inputsOperands, inputTypes, inputsOperandsLoc,
                           result.operands))
  return failure();

// Parse optional results type list.
if (parser.parseOptionalArrowTypeList(result.types))
  return failure();
// Parse the region.
if (parser.parseRegion(*warpRegion, /*arguments=*/{},
                       /*argTypes=*/{}))
  return failure();
WarpExecuteOnLane0Op::ensureTerminator(*warpRegion, builder, result.location);

// Parse the optional attribute list.
if (parser.parseOptionalAttrDict(result.attributes))
  return failure();
return success();
5581}

5583void WarpExecuteOnLane0Op::getSuccessorRegions(
  std::optional<unsigned> index, ArrayRef<Attribute> operands,
  SmallVectorImpl<RegionSuccessor> &regions) {
if (index) {
  regions.push_back(RegionSuccessor(getResults()));
  return;
}

// The warp region is always executed
regions.push_back(RegionSuccessor(&getWarpRegion()));
5593}

5595void WarpExecuteOnLane0Op::build(OpBuilder &builder, OperationState &result,
                               TypeRange resultTypes, Value laneId,
                               int64_t warpSize) {
build(builder, result, resultTypes, laneId, warpSize,
      /*operands=*/std::nullopt, /*argTypes=*/std::nullopt);
5600}

5602void WarpExecuteOnLane0Op::build(OpBuilder &builder, OperationState &result,
                               TypeRange resultTypes, Value laneId,
                               int64_t warpSize, ValueRange args,
                               TypeRange blockArgTypes) {
result.addOperands(laneId);
result.addAttribute(getAttributeNames()[0],
                    builder.getI64IntegerAttr(warpSize));
result.addTypes(resultTypes);
result.addOperands(args);
assert(args.size() == blockArgTypes.size())(static_cast <bool> (args.size() == blockArgTypes.size(
)) ? void (0) : __assert_fail ("args.size() == blockArgTypes.size()"
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 5611, __extension__
 __PRETTY_FUNCTION__));
OpBuilder::InsertionGuard guard(builder);
Region *warpRegion = result.addRegion();
Block *block = builder.createBlock(warpRegion);
for (auto [type, arg] : llvm::zip_equal(blockArgTypes, args))
  block->addArgument(type, arg.getLoc());
5617}

5619/// Helper check if the distributed vector type is consistent with the expanded
5620/// type and distributed size.
5621static LogicalResult verifyDistributedType(Type expanded, Type distributed,
                                         int64_t warpSize, Operation *op) {
// If the types matches there is no distribution.
if (expanded == distributed)
  return success();
auto expandedVecType = expanded.dyn_cast<VectorType>();
auto distributedVecType = distributed.dyn_cast<VectorType>();
if (!expandedVecType || !distributedVecType)
  return op->emitOpError("expected vector type for distributed operands.");
if (expandedVecType.getRank() != distributedVecType.getRank() ||
    expandedVecType.getElementType() != distributedVecType.getElementType())
  return op->emitOpError(
      "expected distributed vectors to have same rank and element type.");
bool foundDistributedDim = false;
for (int64_t i = 0, e = expandedVecType.getRank(); i < e; i++) {
  if (expandedVecType.getDimSize(i) == distributedVecType.getDimSize(i))
    continue;
  if (expandedVecType.getDimSize(i) ==
      distributedVecType.getDimSize(i) * warpSize) {
    if (foundDistributedDim)
      return op->emitOpError()
             << "expected only one dimension to be distributed from "
             << expandedVecType << " to " << distributedVecType;
    foundDistributedDim = true;
    continue;
  }
  return op->emitOpError() << "incompatible distribution dimensions from "
                           << expandedVecType << " to " << distributedVecType;
}
return success();
5651}

5653LogicalResult WarpExecuteOnLane0Op::verify() {
if (getArgs().size() != getWarpRegion().getNumArguments())
  return emitOpError(
      "expected same number op arguments and block arguments.");
auto yield =
    cast<YieldOp>(getWarpRegion().getBlocks().begin()->getTerminator());
if (yield.getNumOperands() != getNumResults())
  return emitOpError(
      "expected same number of yield operands and return values.");
int64_t warpSize = getWarpSize();
for (auto [regionArg, arg] :
     llvm::zip_equal(getWarpRegion().getArguments(), getArgs())) {
  if (failed(verifyDistributedType(regionArg.getType(), arg.getType(),
                                   warpSize, getOperation())))
    return failure();
}
for (auto [yieldOperand, result] :
     llvm::zip_equal(yield.getOperands(), getResults())) {
  if (failed(verifyDistributedType(yieldOperand.getType(), result.getType(),
                                   warpSize, getOperation())))
    return failure();
}
return success();
5676}

5678bool WarpExecuteOnLane0Op::areTypesCompatible(Type lhs, Type rhs) {
return succeeded(
    verifyDistributedType(lhs, rhs, getWarpSize(), getOperation()));
5681}

5683Value mlir::vector::makeArithReduction(OpBuilder &b, Location loc,
                                     CombiningKind kind, Value v1, Value v2) {
Type t1 = getElementTypeOrSelf(v1.getType());
Type t2 = getElementTypeOrSelf(v2.getType());
switch (kind) {
case CombiningKind::ADD:
  if (t1.isIntOrIndex() && t2.isIntOrIndex())
    return b.createOrFold<arith::AddIOp>(loc, v1, v2);
  else if (t1.isa<FloatType>() && t2.isa<FloatType>())
    return b.createOrFold<arith::AddFOp>(loc, v1, v2);
  llvm_unreachable("invalid value types for ADD reduction")::llvm::llvm_unreachable_internal("invalid value types for ADD reduction"
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 5693);
case CombiningKind::AND:
  assert(t1.isIntOrIndex() && t2.isIntOrIndex() && "expected int values")(static_cast <bool> (t1.isIntOrIndex() && t2.isIntOrIndex
() && "expected int values") ? void (0) : __assert_fail
 ("t1.isIntOrIndex() && t2.isIntOrIndex() && \"expected int values\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 5695, __extension__
 __PRETTY_FUNCTION__));
  return b.createOrFold<arith::AndIOp>(loc, v1, v2);
case CombiningKind::MAXF:
  assert(t1.isa<FloatType>() && t2.isa<FloatType>() &&(static_cast <bool> (t1.isa<FloatType>() &&
 t2.isa<FloatType>() && "expected float values"
) ? void (0) : __assert_fail ("t1.isa<FloatType>() && t2.isa<FloatType>() && \"expected float values\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 5699, __extension__
 __PRETTY_FUNCTION__))
         "expected float values")(static_cast <bool> (t1.isa<FloatType>() &&
 t2.isa<FloatType>() && "expected float values"
) ? void (0) : __assert_fail ("t1.isa<FloatType>() && t2.isa<FloatType>() && \"expected float values\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 5699, __extension__
 __PRETTY_FUNCTION__));
  return b.createOrFold<arith::MaxFOp>(loc, v1, v2);
case CombiningKind::MINF:
  assert(t1.isa<FloatType>() && t2.isa<FloatType>() &&(static_cast <bool> (t1.isa<FloatType>() &&
 t2.isa<FloatType>() && "expected float values"
) ? void (0) : __assert_fail ("t1.isa<FloatType>() && t2.isa<FloatType>() && \"expected float values\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 5703, __extension__
 __PRETTY_FUNCTION__))
         "expected float values")(static_cast <bool> (t1.isa<FloatType>() &&
 t2.isa<FloatType>() && "expected float values"
) ? void (0) : __assert_fail ("t1.isa<FloatType>() && t2.isa<FloatType>() && \"expected float values\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 5703, __extension__
 __PRETTY_FUNCTION__));
  return b.createOrFold<arith::MinFOp>(loc, v1, v2);
case CombiningKind::MAXSI:
  assert(t1.isIntOrIndex() && t2.isIntOrIndex() && "expected int values")(static_cast <bool> (t1.isIntOrIndex() && t2.isIntOrIndex
() && "expected int values") ? void (0) : __assert_fail
 ("t1.isIntOrIndex() && t2.isIntOrIndex() && \"expected int values\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 5706, __extension__
 __PRETTY_FUNCTION__));
  return b.createOrFold<arith::MaxSIOp>(loc, v1, v2);
case CombiningKind::MINSI:
  assert(t1.isIntOrIndex() && t2.isIntOrIndex() && "expected int values")(static_cast <bool> (t1.isIntOrIndex() && t2.isIntOrIndex
() && "expected int values") ? void (0) : __assert_fail
 ("t1.isIntOrIndex() && t2.isIntOrIndex() && \"expected int values\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 5709, __extension__
 __PRETTY_FUNCTION__));
  return b.createOrFold<arith::MinSIOp>(loc, v1, v2);
case CombiningKind::MAXUI:
  assert(t1.isIntOrIndex() && t2.isIntOrIndex() && "expected int values")(static_cast <bool> (t1.isIntOrIndex() && t2.isIntOrIndex
() && "expected int values") ? void (0) : __assert_fail
 ("t1.isIntOrIndex() && t2.isIntOrIndex() && \"expected int values\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 5712, __extension__
 __PRETTY_FUNCTION__));
  return b.createOrFold<arith::MaxUIOp>(loc, v1, v2);
case CombiningKind::MINUI:
  assert(t1.isIntOrIndex() && t2.isIntOrIndex() && "expected int values")(static_cast <bool> (t1.isIntOrIndex() && t2.isIntOrIndex
() && "expected int values") ? void (0) : __assert_fail
 ("t1.isIntOrIndex() && t2.isIntOrIndex() && \"expected int values\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 5715, __extension__
 __PRETTY_FUNCTION__));
  return b.createOrFold<arith::MinUIOp>(loc, v1, v2);
case CombiningKind::MUL:
  if (t1.isIntOrIndex() && t2.isIntOrIndex())
    return b.createOrFold<arith::MulIOp>(loc, v1, v2);
  else if (t1.isa<FloatType>() && t2.isa<FloatType>())
    return b.createOrFold<arith::MulFOp>(loc, v1, v2);
  llvm_unreachable("invalid value types for MUL reduction")::llvm::llvm_unreachable_internal("invalid value types for MUL reduction"
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 5722);
case CombiningKind::OR:
  assert(t1.isIntOrIndex() && t2.isIntOrIndex() && "expected int values")(static_cast <bool> (t1.isIntOrIndex() && t2.isIntOrIndex
() && "expected int values") ? void (0) : __assert_fail
 ("t1.isIntOrIndex() && t2.isIntOrIndex() && \"expected int values\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 5724, __extension__
 __PRETTY_FUNCTION__));
  return b.createOrFold<arith::OrIOp>(loc, v1, v2);
case CombiningKind::XOR:
  assert(t1.isIntOrIndex() && t2.isIntOrIndex() && "expected int values")(static_cast <bool> (t1.isIntOrIndex() && t2.isIntOrIndex
() && "expected int values") ? void (0) : __assert_fail
 ("t1.isIntOrIndex() && t2.isIntOrIndex() && \"expected int values\""
, "mlir/lib/Dialect/Vector/IR/VectorOps.cpp", 5727, __extension__
 __PRETTY_FUNCTION__));
  return b.createOrFold<arith::XOrIOp>(loc, v1, v2);
};
llvm_unreachable("unknown CombiningKind")::llvm::llvm_unreachable_internal("unknown CombiningKind", "mlir/lib/Dialect/Vector/IR/VectorOps.cpp"
, 5730);
5731}

5733//===----------------------------------------------------------------------===//
5734// TableGen'd op method definitions
5735//===----------------------------------------------------------------------===//

5737#define GET_ATTRDEF_CLASSES
5738#include "mlir/Dialect/Vector/IR/VectorOpsAttrDefs.cpp.inc"

5740#define GET_OP_CLASSES
5741#include "mlir/Dialect/Vector/IR/VectorOps.cpp.inc"

←

/build/source/mlir/include/mlir/IR/OpImplementation.h

1//===- OpImplementation.h - Classes for implementing Op types ---*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This classes used by the implementation details of Op types.
10//
11//===----------------------------------------------------------------------===//

13#ifndef MLIR_IR_OPIMPLEMENTATION_H
14#define MLIR_IR_OPIMPLEMENTATION_H

16#include "mlir/IR/BuiltinTypes.h"
17#include "mlir/IR/DialectInterface.h"
18#include "mlir/IR/OpDefinition.h"
19#include "llvm/ADT/Twine.h"
20#include "llvm/Support/SMLoc.h"

22namespace mlir {
23class AsmParsedResourceEntry;
24class AsmResourceBuilder;
25class Builder;

27//===----------------------------------------------------------------------===//
28// AsmDialectResourceHandle
29//===----------------------------------------------------------------------===//

31/// This class represents an opaque handle to a dialect resource entry.
32class AsmDialectResourceHandle {
33public:
AsmDialectResourceHandle() = default;
AsmDialectResourceHandle(void *resource, TypeID resourceID, Dialect *dialect)
    : resource(resource), opaqueID(resourceID), dialect(dialect) {}
bool operator==(const AsmDialectResourceHandle &other) const {
  return resource == other.resource;
}

/// Return an opaque pointer to the referenced resource.
void *getResource() const { return resource; }

/// Return the type ID of the resource.
TypeID getTypeID() const { return opaqueID; }

/// Return the dialect that owns the resource.
Dialect *getDialect() const { return dialect; }

50private:
/// The opaque handle to the dialect resource.
void *resource = nullptr;
/// The type of the resource referenced.
TypeID opaqueID;
/// The dialect owning the given resource.
Dialect *dialect;
57};

59/// This class represents a CRTP base class for dialect resource handles. It
60/// abstracts away various utilities necessary for defined derived resource
61/// handles.
62template <typename DerivedT, typename ResourceT, typename DialectT>
63class AsmDialectResourceHandleBase : public AsmDialectResourceHandle {
64public:
using Dialect = DialectT;

/// Construct a handle from a pointer to the resource. The given pointer
/// should be guaranteed to live beyond the life of this handle.
AsmDialectResourceHandleBase(ResourceT *resource, DialectT *dialect)
    : AsmDialectResourceHandle(resource, TypeID::get<DerivedT>(), dialect) {}
AsmDialectResourceHandleBase(AsmDialectResourceHandle handle)
    : AsmDialectResourceHandle(handle) {
  assert(handle.getTypeID() == TypeID::get<DerivedT>())(static_cast <bool> (handle.getTypeID() == TypeID::get<
DerivedT>()) ? void (0) : __assert_fail ("handle.getTypeID() == TypeID::get<DerivedT>()"
, "mlir/include/mlir/IR/OpImplementation.h", 73, __extension__
 __PRETTY_FUNCTION__));
}

/// Return the resource referenced by this handle.
ResourceT *getResource() {
  return static_cast<ResourceT *>(AsmDialectResourceHandle::getResource());
}
const ResourceT *getResource() const {
  return const_cast<AsmDialectResourceHandleBase *>(this)->getResource();
}

/// Return the dialect that owns the resource.
DialectT *getDialect() const {
  return static_cast<DialectT *>(AsmDialectResourceHandle::getDialect());
}

/// Support llvm style casting.
static bool classof(const AsmDialectResourceHandle *handle) {
  return handle->getTypeID() == TypeID::get<DerivedT>();
}
93};

95inline llvm::hash_code hash_value(const AsmDialectResourceHandle &param) {
return llvm::hash_value(param.getResource());
97}

99//===----------------------------------------------------------------------===//
100// AsmPrinter
101//===----------------------------------------------------------------------===//

103/// This base class exposes generic asm printer hooks, usable across the various
104/// derived printers.
105class AsmPrinter {
106public:
/// This class contains the internal default implementation of the base
/// printer methods.
class Impl;

/// Initialize the printer with the given internal implementation.
AsmPrinter(Impl &impl) : impl(&impl) {}
virtual ~AsmPrinter();

/// Return the raw output stream used by this printer.
virtual raw_ostream &getStream() const;

/// Print the given floating point value in a stabilized form that can be
/// roundtripped through the IR. This is the companion to the 'parseFloat'
/// hook on the AsmParser.
virtual void printFloat(const APFloat &value);

virtual void printType(Type type);
virtual void printAttribute(Attribute attr);

/// Trait to check if `AttrType` provides a `print` method.
template <typename AttrOrType>
using has_print_method =
    decltype(std::declval<AttrOrType>().print(std::declval<AsmPrinter &>()));
template <typename AttrOrType>
using detect_has_print_method =
    llvm::is_detected<has_print_method, AttrOrType>;

/// Print the provided attribute in the context of an operation custom
/// printer/parser: this will invoke directly the print method on the
/// attribute class and skip the `#dialect.mnemonic` prefix in most cases.
template <typename AttrOrType,
          std::enable_if_t<detect_has_print_method<AttrOrType>::value>
              *sfinae = nullptr>
void printStrippedAttrOrType(AttrOrType attrOrType) {
  if (succeeded(printAlias(attrOrType)))
    return;
  attrOrType.print(*this);
}

/// Print the provided array of attributes or types in the context of an
/// operation custom printer/parser: this will invoke directly the print
/// method on the attribute class and skip the `#dialect.mnemonic` prefix in
/// most cases.
template <typename AttrOrType,
          std::enable_if_t<detect_has_print_method<AttrOrType>::value>
              *sfinae = nullptr>
void printStrippedAttrOrType(ArrayRef<AttrOrType> attrOrTypes) {
  llvm::interleaveComma(
      attrOrTypes, getStream(),
      [this](AttrOrType attrOrType) { printStrippedAttrOrType(attrOrType); });
}

/// SFINAE for printing the provided attribute in the context of an operation
/// custom printer in the case where the attribute does not define a print
/// method.
template <typename AttrOrType,
          std::enable_if_t<!detect_has_print_method<AttrOrType>::value>
              *sfinae = nullptr>
void printStrippedAttrOrType(AttrOrType attrOrType) {
  *this << attrOrType;
}

/// Print the given attribute without its type. The corresponding parser must
/// provide a valid type for the attribute.
virtual void printAttributeWithoutType(Attribute attr);

/// Print the given string as a keyword, or a quoted and escaped string if it
/// has any special or non-printable characters in it.
virtual void printKeywordOrString(StringRef keyword);

/// Print the given string as a symbol reference, i.e. a form representable by
/// a SymbolRefAttr. A symbol reference is represented as a string prefixed
/// with '@'. The reference is surrounded with ""'s and escaped if it has any
/// special or non-printable characters in it.
virtual void printSymbolName(StringRef symbolRef);

/// Print a handle to the given dialect resource.
virtual void printResourceHandle(const AsmDialectResourceHandle &resource);

/// Print an optional arrow followed by a type list.
template <typename TypeRange>
void printOptionalArrowTypeList(TypeRange &&types) {
  if (types.begin() != types.end())
    printArrowTypeList(types);
}
template <typename TypeRange>
void printArrowTypeList(TypeRange &&types) {
  auto &os = getStream() << " -> ";

  bool wrapped = !llvm::hasSingleElement(types) ||
                 (*types.begin()).template isa<FunctionType>();
  if (wrapped)
    os << '(';
  llvm::interleaveComma(types, *this);
  if (wrapped)
    os << ')';
}

/// Print the two given type ranges in a functional form.
template <typename InputRangeT, typename ResultRangeT>
void printFunctionalType(InputRangeT &&inputs, ResultRangeT &&results) {
  auto &os = getStream();
  os << '(';
  llvm::interleaveComma(inputs, *this);
  os << ')';
  printArrowTypeList(results);
}

215protected:
/// Initialize the printer with no internal implementation. In this case, all
/// virtual methods of this class must be overriden.
AsmPrinter() = default;

220private:
AsmPrinter(const AsmPrinter &) = delete;
void operator=(const AsmPrinter &) = delete;

/// Print the alias for the given attribute, return failure if no alias could
/// be printed.
virtual LogicalResult printAlias(Attribute attr);

/// Print the alias for the given type, return failure if no alias could
/// be printed.
virtual LogicalResult printAlias(Type type);

/// The internal implementation of the printer.
Impl *impl{nullptr};
234};

236template <typename AsmPrinterT>
237inline std::enable_if_t<std::is_base_of<AsmPrinter, AsmPrinterT>::value,
                      AsmPrinterT &>
239operator<<(AsmPrinterT &p, Type type) {
p.printType(type);
return p;
242}

244template <typename AsmPrinterT>
245inline std::enable_if_t<std::is_base_of<AsmPrinter, AsmPrinterT>::value,
                      AsmPrinterT &>
247operator<<(AsmPrinterT &p, Attribute attr) {
p.printAttribute(attr);
return p;
250}

252template <typename AsmPrinterT>
253inline std::enable_if_t<std::is_base_of<AsmPrinter, AsmPrinterT>::value,
                      AsmPrinterT &>
255operator<<(AsmPrinterT &p, const APFloat &value) {
p.printFloat(value);
return p;
258}
259template <typename AsmPrinterT>
260inline std::enable_if_t<std::is_base_of<AsmPrinter, AsmPrinterT>::value,
                      AsmPrinterT &>
262operator<<(AsmPrinterT &p, float value) {
return p << APFloat(value);
264}
265template <typename AsmPrinterT>
266inline std::enable_if_t<std::is_base_of<AsmPrinter, AsmPrinterT>::value,
                      AsmPrinterT &>
268operator<<(AsmPrinterT &p, double value) {
return p << APFloat(value);
270}

272// Support printing anything that isn't convertible to one of the other
273// streamable types, even if it isn't exactly one of them. For example, we want
274// to print FunctionType with the Type version above, not have it match this.
275template <typename AsmPrinterT, typename T,
        std::enable_if_t<!std::is_convertible<T &, Value &>::value &&
                             !std::is_convertible<T &, Type &>::value &&
                             !std::is_convertible<T &, Attribute &>::value &&
                             !std::is_convertible<T &, ValueRange>::value &&
                             !std::is_convertible<T &, APFloat &>::value &&
                             !llvm::is_one_of<T, bool, float, double>::value,
                         T> * = nullptr>
283inline std::enable_if_t<std::is_base_of<AsmPrinter, AsmPrinterT>::value,
                      AsmPrinterT &>
285operator<<(AsmPrinterT &p, const T &other) {
p.getStream() << other;
return p;
288}

290template <typename AsmPrinterT>
291inline std::enable_if_t<std::is_base_of<AsmPrinter, AsmPrinterT>::value,
                      AsmPrinterT &>
293operator<<(AsmPrinterT &p, bool value) {
return p << (value ? StringRef("true") : "false");
295}

297template <typename AsmPrinterT, typename ValueRangeT>
298inline std::enable_if_t<std::is_base_of<AsmPrinter, AsmPrinterT>::value,
                      AsmPrinterT &>
300operator<<(AsmPrinterT &p, const ValueTypeRange<ValueRangeT> &types) {
llvm::interleaveComma(types, p);
return p;
303}
304template <typename AsmPrinterT>
305inline std::enable_if_t<std::is_base_of<AsmPrinter, AsmPrinterT>::value,
                      AsmPrinterT &>
307operator<<(AsmPrinterT &p, const TypeRange &types) {
llvm::interleaveComma(types, p);
return p;
310}
311template <typename AsmPrinterT, typename ElementT>
312inline std::enable_if_t<std::is_base_of<AsmPrinter, AsmPrinterT>::value,
                      AsmPrinterT &>
314operator<<(AsmPrinterT &p, ArrayRef<ElementT> types) {
llvm::interleaveComma(types, p);
return p;
317}

319//===----------------------------------------------------------------------===//
320// OpAsmPrinter
321//===----------------------------------------------------------------------===//

323/// This is a pure-virtual base class that exposes the asmprinter hooks
324/// necessary to implement a custom print() method.
325class OpAsmPrinter : public AsmPrinter {
326public:
using AsmPrinter::AsmPrinter;
~OpAsmPrinter() override;

/// Print a loc(...) specifier if printing debug info is enabled.
virtual void printOptionalLocationSpecifier(Location loc) = 0;

/// Print a newline and indent the printer to the start of the current
/// operation.
virtual void printNewline() = 0;

/// Increase indentation.
virtual void increaseIndent() = 0;

/// Decrease indentation.
virtual void decreaseIndent() = 0;

/// Print a block argument in the usual format of:
///   %ssaName : type {attr1=42} loc("here")
/// where location printing is controlled by the standard internal option.
/// You may pass omitType=true to not print a type, and pass an empty
/// attribute list if you don't care for attributes.
virtual void printRegionArgument(BlockArgument arg,
                                 ArrayRef<NamedAttribute> argAttrs = {},
                                 bool omitType = false) = 0;

/// Print implementations for various things an operation contains.
virtual void printOperand(Value value) = 0;
virtual void printOperand(Value value, raw_ostream &os) = 0;

/// Print a comma separated list of operands.
template <typename ContainerType>
void printOperands(const ContainerType &container) {
  printOperands(container.begin(), container.end());
}

/// Print a comma separated list of operands.
template <typename IteratorType>
void printOperands(IteratorType it, IteratorType end) {
  llvm::interleaveComma(llvm::make_range(it, end), getStream(),
                        [this](Value value) { printOperand(value); });
}

/// Print the given successor.
virtual void printSuccessor(Block *successor) = 0;

/// Print the successor and its operands.
virtual void printSuccessorAndUseList(Block *successor,
                                      ValueRange succOperands) = 0;

/// If the specified operation has attributes, print out an attribute
/// dictionary with their values.  elidedAttrs allows the client to ignore
/// specific well known attributes, commonly used if the attribute value is
/// printed some other way (like as a fixed operand).
virtual void printOptionalAttrDict(ArrayRef<NamedAttribute> attrs,
                                   ArrayRef<StringRef> elidedAttrs = {}) = 0;

/// If the specified operation has attributes, print out an attribute
/// dictionary prefixed with 'attributes'.
virtual void
printOptionalAttrDictWithKeyword(ArrayRef<NamedAttribute> attrs,
                                 ArrayRef<StringRef> elidedAttrs = {}) = 0;

/// Prints the entire operation with the custom assembly form, if available,
/// or the generic assembly form, otherwise.
virtual void printCustomOrGenericOp(Operation *op) = 0;

/// Print the entire operation with the default generic assembly form.
/// If `printOpName` is true, then the operation name is printed (the default)
/// otherwise it is omitted and the print will start with the operand list.
virtual void printGenericOp(Operation *op, bool printOpName = true) = 0;

/// Prints a region.
/// If 'printEntryBlockArgs' is false, the arguments of the
/// block are not printed. If 'printBlockTerminator' is false, the terminator
/// operation of the block is not printed. If printEmptyBlock is true, then
/// the block header is printed even if the block is empty.
virtual void printRegion(Region &blocks, bool printEntryBlockArgs = true,
                         bool printBlockTerminators = true,
                         bool printEmptyBlock = false) = 0;

/// Renumber the arguments for the specified region to the same names as the
/// SSA values in namesToUse.  This may only be used for IsolatedFromAbove
/// operations.  If any entry in namesToUse is null, the corresponding
/// argument name is left alone.
virtual void shadowRegionArgs(Region &region, ValueRange namesToUse) = 0;

/// Prints an affine map of SSA ids, where SSA id names are used in place
/// of dims/symbols.
/// Operand values must come from single-result sources, and be valid
/// dimensions/symbol identifiers according to mlir::isValidDim/Symbol.
virtual void printAffineMapOfSSAIds(AffineMapAttr mapAttr,
                                    ValueRange operands) = 0;

/// Prints an affine expression of SSA ids with SSA id names used instead of
/// dims and symbols.
/// Operand values must come from single-result sources, and be valid
/// dimensions/symbol identifiers according to mlir::isValidDim/Symbol.
virtual void printAffineExprOfSSAIds(AffineExpr expr, ValueRange dimOperands,
                                     ValueRange symOperands) = 0;

/// Print the complete type of an operation in functional form.
void printFunctionalType(Operation *op);
using AsmPrinter::printFunctionalType;
430};

432// Make the implementations convenient to use.
433inline OpAsmPrinter &operator<<(OpAsmPrinter &p, Value value) {
p.printOperand(value);
return p;
436}

438template <typename T,
        std::enable_if_t<std::is_convertible<T &, ValueRange>::value &&
                             !std::is_convertible<T &, Value &>::value,
                         T> * = nullptr>
442inline OpAsmPrinter &operator<<(OpAsmPrinter &p, const T &values) {
p.printOperands(values);
return p;
445}

447inline OpAsmPrinter &operator<<(OpAsmPrinter &p, Block *value) {
p.printSuccessor(value);
return p;
450}

452//===----------------------------------------------------------------------===//
453// AsmParser
454//===----------------------------------------------------------------------===//

456/// This base class exposes generic asm parser hooks, usable across the various
457/// derived parsers.
458class AsmParser {
459public:
AsmParser() = default;
virtual ~AsmParser();

MLIRContext *getContext() const;

/// Return the location of the original name token.
virtual SMLoc getNameLoc() const = 0;

//===--------------------------------------------------------------------===//
// Utilities
//===--------------------------------------------------------------------===//

/// Emit a diagnostic at the specified location and return failure.
virtual InFlightDiagnostic emitError(SMLoc loc,
                                     const Twine &message = {}) = 0;

/// Return a builder which provides useful access to MLIRContext, global
/// objects like types and attributes.
virtual Builder &getBuilder() const = 0;

/// Get the location of the next token and store it into the argument.  This
/// always succeeds.
virtual SMLoc getCurrentLocation() = 0;
ParseResult getCurrentLocation(SMLoc *loc) {
  *loc = getCurrentLocation();
  return success();
}

/// Re-encode the given source location as an MLIR location and return it.
/// Note: This method should only be used when a `Location` is necessary, as
/// the encoding process is not efficient.
virtual Location getEncodedSourceLoc(SMLoc loc) = 0;

//===--------------------------------------------------------------------===//
// Token Parsing
//===--------------------------------------------------------------------===//

/// Parse a '->' token.
virtual ParseResult parseArrow() = 0;

/// Parse a '->' token if present
virtual ParseResult parseOptionalArrow() = 0;

/// Parse a `{` token.
virtual ParseResult parseLBrace() = 0;

/// Parse a `{` token if present.
virtual ParseResult parseOptionalLBrace() = 0;

/// Parse a `}` token.
virtual ParseResult parseRBrace() = 0;

/// Parse a `}` token if present.
virtual ParseResult parseOptionalRBrace() = 0;

/// Parse a `:` token.
virtual ParseResult parseColon() = 0;

/// Parse a `:` token if present.
virtual ParseResult parseOptionalColon() = 0;

/// Parse a `,` token.
virtual ParseResult parseComma() = 0;

/// Parse a `,` token if present.
virtual ParseResult parseOptionalComma() = 0;

/// Parse a `=` token.
virtual ParseResult parseEqual() = 0;

/// Parse a `=` token if present.
virtual ParseResult parseOptionalEqual() = 0;

/// Parse a '<' token.
virtual ParseResult parseLess() = 0;

/// Parse a '<' token if present.
virtual ParseResult parseOptionalLess() = 0;

/// Parse a '>' token.
virtual ParseResult parseGreater() = 0;

/// Parse a '>' token if present.
virtual ParseResult parseOptionalGreater() = 0;

/// Parse a '?' token.
virtual ParseResult parseQuestion() = 0;

/// Parse a '?' token if present.
virtual ParseResult parseOptionalQuestion() = 0;

/// Parse a '+' token.
virtual ParseResult parsePlus() = 0;

/// Parse a '+' token if present.
virtual ParseResult parseOptionalPlus() = 0;

/// Parse a '*' token.
virtual ParseResult parseStar() = 0;

/// Parse a '*' token if present.
virtual ParseResult parseOptionalStar() = 0;

/// Parse a '|' token.
virtual ParseResult parseVerticalBar() = 0;

/// Parse a '|' token if present.
virtual ParseResult parseOptionalVerticalBar() = 0;

/// Parse a quoted string token.
ParseResult parseString(std::string *string) {
  auto loc = getCurrentLocation();
  if (parseOptionalString(string))
    return emitError(loc, "expected string");
  return success();
}

/// Parse a quoted string token if present.
virtual ParseResult parseOptionalString(std::string *string) = 0;

/// Parses a Base64 encoded string of bytes.
virtual ParseResult parseBase64Bytes(std::vector<char> *bytes) = 0;

/// Parse a `(` token.
virtual ParseResult parseLParen() = 0;

/// Parse a `(` token if present.
virtual ParseResult parseOptionalLParen() = 0;

/// Parse a `)` token.
virtual ParseResult parseRParen() = 0;

/// Parse a `)` token if present.
virtual ParseResult parseOptionalRParen() = 0;

/// Parse a `[` token.
virtual ParseResult parseLSquare() = 0;

/// Parse a `[` token if present.
virtual ParseResult parseOptionalLSquare() = 0;

/// Parse a `]` token.
virtual ParseResult parseRSquare() = 0;

/// Parse a `]` token if present.
virtual ParseResult parseOptionalRSquare() = 0;

/// Parse a `...` token.
virtual ParseResult parseEllipsis() = 0;

/// Parse a `...` token if present;
virtual ParseResult parseOptionalEllipsis() = 0;

/// Parse a floating point value from the stream.
virtual ParseResult parseFloat(double &result) = 0;

/// Parse an integer value from the stream.
template <typename IntT>
ParseResult parseInteger(IntT &result) {
  auto loc = getCurrentLocation();
  OptionalParseResult parseResult = parseOptionalInteger(result);
4
←
Calling 'AsmParser::parseOptionalInteger'→
8
←
Returning from 'AsmParser::parseOptionalInteger'→
  if (!parseResult.has_value())
9
←
Taking true branch→
    return emitError(loc, "expected integer value");
10
←
Returning without writing to 'result'→
  return *parseResult;
}

/// Parse an optional integer value from the stream.
virtual OptionalParseResult parseOptionalInteger(APInt &result) = 0;

template <typename IntT>
OptionalParseResult parseOptionalInteger(IntT &result) {
  auto loc = getCurrentLocation();

  // Parse the unsigned variant.
  APInt uintResult;
  OptionalParseResult parseResult = parseOptionalInteger(uintResult);
  if (!parseResult.has_value() || failed(*parseResult))
5
←
Assuming the condition is true→
6
←
Taking true branch→
    return parseResult;
7
←
Returning without writing to 'result'→

  // Try to convert to the provided integer type.  sextOrTrunc is correct even
  // for unsigned types because parseOptionalInteger ensures the sign bit is
  // zero for non-negated integers.
  result =
      (IntT)uintResult.sextOrTrunc(sizeof(IntT) * CHAR_BIT8).getLimitedValue();
  if (APInt(uintResult.getBitWidth(), result) != uintResult)
    return emitError(loc, "integer value too large");
  return success();
}

/// These are the supported delimiters around operand lists and region
/// argument lists, used by parseOperandList.
enum class Delimiter {
  /// Zero or more operands with no delimiters.
  None,
  /// Parens surrounding zero or more operands.
  Paren,
  /// Square brackets surrounding zero or more operands.
  Square,
  /// <> brackets surrounding zero or more operands.
  LessGreater,
  /// {} brackets surrounding zero or more operands.
  Braces,
  /// Parens supporting zero or more operands, or nothing.
  OptionalParen,
  /// Square brackets supporting zero or more ops, or nothing.
  OptionalSquare,
  /// <> brackets supporting zero or more ops, or nothing.
  OptionalLessGreater,
  /// {} brackets surrounding zero or more operands, or nothing.
  OptionalBraces,
};

/// Parse a list of comma-separated items with an optional delimiter.  If a
/// delimiter is provided, then an empty list is allowed.  If not, then at
/// least one element will be parsed.
///
/// contextMessage is an optional message appended to "expected '('" sorts of
/// diagnostics when parsing the delimeters.
virtual ParseResult
parseCommaSeparatedList(Delimiter delimiter,
                        function_ref<ParseResult()> parseElementFn,
                        StringRef contextMessage = StringRef()) = 0;

/// Parse a comma separated list of elements that must have at least one entry
/// in it.
ParseResult
parseCommaSeparatedList(function_ref<ParseResult()> parseElementFn) {
  return parseCommaSeparatedList(Delimiter::None, parseElementFn);
}

//===--------------------------------------------------------------------===//
// Keyword Parsing
//===--------------------------------------------------------------------===//

/// This class represents a StringSwitch like class that is useful for parsing
/// expected keywords. On construction, it invokes `parseKeyword` and
/// processes each of the provided cases statements until a match is hit. The
/// provided `ResultT` must be assignable from `failure()`.
template <typename ResultT = ParseResult>
class KeywordSwitch {
public:
  KeywordSwitch(AsmParser &parser)
      : parser(parser), loc(parser.getCurrentLocation()) {
    if (failed(parser.parseKeywordOrCompletion(&keyword)))
      result = failure();
  }

  /// Case that uses the provided value when true.
  KeywordSwitch &Case(StringLiteral str, ResultT value) {
    return Case(str, [&](StringRef, SMLoc) { return std::move(value); });
  }
  KeywordSwitch &Default(ResultT value) {
    return Default([&](StringRef, SMLoc) { return std::move(value); });
  }
  /// Case that invokes the provided functor when true. The parameters passed
  /// to the functor are the keyword, and the location of the keyword (in case
  /// any errors need to be emitted).
  template <typename FnT>
  std::enable_if_t<!std::is_convertible<FnT, ResultT>::value, KeywordSwitch &>
  Case(StringLiteral str, FnT &&fn) {
    if (result)
      return *this;

    // If the word was empty, record this as a completion.
    if (keyword.empty())
      parser.codeCompleteExpectedTokens(str);
    else if (keyword == str)
      result.emplace(std::move(fn(keyword, loc)));
    return *this;
  }
  template <typename FnT>
  std::enable_if_t<!std::is_convertible<FnT, ResultT>::value, KeywordSwitch &>
  Default(FnT &&fn) {
    if (!result)
      result.emplace(fn(keyword, loc));
    return *this;
  }

  /// Returns true if this switch has a value yet.
  bool hasValue() const { return result.has_value(); }

  /// Return the result of the switch.
  [[nodiscard]] operator ResultT() {
    if (!result)
      return parser.emitError(loc, "unexpected keyword: ") << keyword;
    return std::move(*result);
  }

private:
  /// The parser used to construct this switch.
  AsmParser &parser;

  /// The location of the keyword, used to emit errors as necessary.
  SMLoc loc;

  /// The parsed keyword itself.
  StringRef keyword;

  /// The result of the switch statement or none if currently unknown.
  Optional<ResultT> result;
};

/// Parse a given keyword.
ParseResult parseKeyword(StringRef keyword) {
  return parseKeyword(keyword, "");
}
virtual ParseResult parseKeyword(StringRef keyword, const Twine &msg) = 0;

/// Parse a keyword into 'keyword'.
ParseResult parseKeyword(StringRef *keyword) {
  auto loc = getCurrentLocation();
  if (parseOptionalKeyword(keyword))
    return emitError(loc, "expected valid keyword");
  return success();
}

/// Parse the given keyword if present.
virtual ParseResult parseOptionalKeyword(StringRef keyword) = 0;

/// Parse a keyword, if present, into 'keyword'.
virtual ParseResult parseOptionalKeyword(StringRef *keyword) = 0;

/// Parse a keyword, if present, and if one of the 'allowedValues',
/// into 'keyword'
virtual ParseResult
parseOptionalKeyword(StringRef *keyword,
                     ArrayRef<StringRef> allowedValues) = 0;

/// Parse a keyword or a quoted string.
ParseResult parseKeywordOrString(std::string *result) {
  if (failed(parseOptionalKeywordOrString(result)))
    return emitError(getCurrentLocation())
           << "expected valid keyword or string";
  return success();
}

/// Parse an optional keyword or string.
virtual ParseResult parseOptionalKeywordOrString(std::string *result) = 0;

//===--------------------------------------------------------------------===//
// Attribute/Type Parsing
//===--------------------------------------------------------------------===//

/// Invoke the `getChecked` method of the given Attribute or Type class, using
/// the provided location to emit errors in the case of failure. Note that
/// unlike `OpBuilder::getType`, this method does not implicitly insert a
/// context parameter.
template <typename T, typename... ParamsT>
auto getChecked(SMLoc loc, ParamsT &&...params) {
  return T::getChecked([&] { return emitError(loc); },
                       std::forward<ParamsT>(params)...);
}
/// A variant of `getChecked` that uses the result of `getNameLoc` to emit
/// errors.
template <typename T, typename... ParamsT>
auto getChecked(ParamsT &&...params) {
  return T::getChecked([&] { return emitError(getNameLoc()); },
                       std::forward<ParamsT>(params)...);
}

//===--------------------------------------------------------------------===//
// Attribute Parsing
//===--------------------------------------------------------------------===//

/// Parse an arbitrary attribute of a given type and return it in result.
virtual ParseResult parseAttribute(Attribute &result, Type type = {}) = 0;

/// Parse a custom attribute with the provided callback, unless the next
/// token is `#`, in which case the generic parser is invoked.
virtual ParseResult parseCustomAttributeWithFallback(
    Attribute &result, Type type,
    function_ref<ParseResult(Attribute &result, Type type)>
        parseAttribute) = 0;

/// Parse an attribute of a specific kind and type.
template <typename AttrType>
ParseResult parseAttribute(AttrType &result, Type type = {}) {
  SMLoc loc = getCurrentLocation();

  // Parse any kind of attribute.
  Attribute attr;
  if (parseAttribute(attr, type))
    return failure();

  // Check for the right kind of attribute.
  if (!(result = attr.dyn_cast<AttrType>()))
    return emitError(loc, "invalid kind of attribute specified");

  return success();
}

/// Parse an arbitrary attribute and return it in result.  This also adds the
/// attribute to the specified attribute list with the specified name.
ParseResult parseAttribute(Attribute &result, StringRef attrName,
                           NamedAttrList &attrs) {
  return parseAttribute(result, Type(), attrName, attrs);
}

/// Parse an attribute of a specific kind and type.
template <typename AttrType>
ParseResult parseAttribute(AttrType &result, StringRef attrName,
                           NamedAttrList &attrs) {
  return parseAttribute(result, Type(), attrName, attrs);
}

/// Parse an arbitrary attribute of a given type and populate it in `result`.
/// This also adds the attribute to the specified attribute list with the
/// specified name.
template <typename AttrType>
ParseResult parseAttribute(AttrType &result, Type type, StringRef attrName,
                           NamedAttrList &attrs) {
  SMLoc loc = getCurrentLocation();

  // Parse any kind of attribute.
  Attribute attr;
  if (parseAttribute(attr, type))
    return failure();

  // Check for the right kind of attribute.
  result = attr.dyn_cast<AttrType>();
  if (!result)
    return emitError(loc, "invalid kind of attribute specified");

  attrs.append(attrName, result);
  return success();
}

/// Trait to check if `AttrType` provides a `parse` method.
template <typename AttrType>
using has_parse_method = decltype(AttrType::parse(std::declval<AsmParser &>(),
                                                  std::declval<Type>()));
template <typename AttrType>
using detect_has_parse_method = llvm::is_detected<has_parse_method, AttrType>;

/// Parse a custom attribute of a given type unless the next token is `#`, in
/// which case the generic parser is invoked. The parsed attribute is
/// populated in `result` and also added to the specified attribute list with
/// the specified name.
template <typename AttrType>
std::enable_if_t<detect_has_parse_method<AttrType>::value, ParseResult>
parseCustomAttributeWithFallback(AttrType &result, Type type,
                                 StringRef attrName, NamedAttrList &attrs) {
  SMLoc loc = getCurrentLocation();

  // Parse any kind of attribute.
  Attribute attr;
  if (parseCustomAttributeWithFallback(
          attr, type, [&](Attribute &result, Type type) -> ParseResult {
            result = AttrType::parse(*this, type);
            if (!result)
              return failure();
            return success();
          }))
    return failure();

  // Check for the right kind of attribute.
  result = attr.dyn_cast<AttrType>();
  if (!result)
    return emitError(loc, "invalid kind of attribute specified");

  attrs.append(attrName, result);
  return success();
}

/// SFINAE parsing method for Attribute that don't implement a parse method.
template <typename AttrType>
std::enable_if_t<!detect_has_parse_method<AttrType>::value, ParseResult>
parseCustomAttributeWithFallback(AttrType &result, Type type,
                                 StringRef attrName, NamedAttrList &attrs) {
  return parseAttribute(result, type, attrName, attrs);
}

/// Parse a custom attribute of a given type unless the next token is `#`, in
/// which case the generic parser is invoked. The parsed attribute is
/// populated in `result`.
template <typename AttrType>
std::enable_if_t<detect_has_parse_method<AttrType>::value, ParseResult>
parseCustomAttributeWithFallback(AttrType &result) {
  SMLoc loc = getCurrentLocation();

  // Parse any kind of attribute.
  Attribute attr;
  if (parseCustomAttributeWithFallback(
          attr, {}, [&](Attribute &result, Type type) -> ParseResult {
            result = AttrType::parse(*this, type);
            return success(!!result);
          }))
    return failure();

  // Check for the right kind of attribute.
  result = attr.dyn_cast<AttrType>();
  if (!result)
    return emitError(loc, "invalid kind of attribute specified");
  return success();
}

/// SFINAE parsing method for Attribute that don't implement a parse method.
template <typename AttrType>
std::enable_if_t<!detect_has_parse_method<AttrType>::value, ParseResult>
parseCustomAttributeWithFallback(AttrType &result) {
  return parseAttribute(result);
}

/// Parse an arbitrary optional attribute of a given type and return it in
/// result.
virtual OptionalParseResult parseOptionalAttribute(Attribute &result,
                                                   Type type = {}) = 0;

/// Parse an optional array attribute and return it in result.
virtual OptionalParseResult parseOptionalAttribute(ArrayAttr &result,
                                                   Type type = {}) = 0;

/// Parse an optional string attribute and return it in result.
virtual OptionalParseResult parseOptionalAttribute(StringAttr &result,
                                                   Type type = {}) = 0;

/// Parse an optional symbol ref attribute and return it in result.
virtual OptionalParseResult parseOptionalAttribute(SymbolRefAttr &result,
                                                   Type type = {}) = 0;

/// Parse an optional attribute of a specific type and add it to the list with
/// the specified name.
template <typename AttrType>
OptionalParseResult parseOptionalAttribute(AttrType &result,
                                           StringRef attrName,
                                           NamedAttrList &attrs) {
  return parseOptionalAttribute(result, Type(), attrName, attrs);
}

/// Parse an optional attribute of a specific type and add it to the list with
/// the specified name.
template <typename AttrType>
OptionalParseResult parseOptionalAttribute(AttrType &result, Type type,
                                           StringRef attrName,
                                           NamedAttrList &attrs) {
  OptionalParseResult parseResult = parseOptionalAttribute(result, type);
  if (parseResult.has_value() && succeeded(*parseResult))
    attrs.append(attrName, result);
  return parseResult;
}

/// Parse a named dictionary into 'result' if it is present.
virtual ParseResult parseOptionalAttrDict(NamedAttrList &result) = 0;

/// Parse a named dictionary into 'result' if the `attributes` keyword is
/// present.
virtual ParseResult
parseOptionalAttrDictWithKeyword(NamedAttrList &result) = 0;

/// Parse an affine map instance into 'map'.
virtual ParseResult parseAffineMap(AffineMap &map) = 0;

/// Parse an integer set instance into 'set'.
virtual ParseResult printIntegerSet(IntegerSet &set) = 0;

//===--------------------------------------------------------------------===//
// Identifier Parsing
//===--------------------------------------------------------------------===//

/// Parse an @-identifier and store it (without the '@' symbol) in a string
/// attribute.
ParseResult parseSymbolName(StringAttr &result) {
  if (failed(parseOptionalSymbolName(result)))
    return emitError(getCurrentLocation())
           << "expected valid '@'-identifier for symbol name";
  return success();
}

/// Parse an @-identifier and store it (without the '@' symbol) in a string
/// attribute named 'attrName'.
ParseResult parseSymbolName(StringAttr &result, StringRef attrName,
                            NamedAttrList &attrs) {
  if (parseSymbolName(result))
    return failure();
  attrs.append(attrName, result);
  return success();
}

/// Parse an optional @-identifier and store it (without the '@' symbol) in a
/// string attribute.
virtual ParseResult parseOptionalSymbolName(StringAttr &result) = 0;

/// Parse an optional @-identifier and store it (without the '@' symbol) in a
/// string attribute named 'attrName'.
ParseResult parseOptionalSymbolName(StringAttr &result, StringRef attrName,
                                    NamedAttrList &attrs) {
  if (succeeded(parseOptionalSymbolName(result))) {
    attrs.append(attrName, result);
    return success();
  }
  return failure();
}

//===--------------------------------------------------------------------===//
// Resource Parsing
//===--------------------------------------------------------------------===//

/// Parse a handle to a resource within the assembly format.
template <typename ResourceT>
FailureOr<ResourceT> parseResourceHandle() {
  SMLoc handleLoc = getCurrentLocation();

  // Try to load the dialect that owns the handle.
  auto *dialect =
      getContext()->getOrLoadDialect<typename ResourceT::Dialect>();
  if (!dialect) {
    return emitError(handleLoc)
           << "dialect '" << ResourceT::Dialect::getDialectNamespace()
           << "' is unknown";
  }

  FailureOr<AsmDialectResourceHandle> handle = parseResourceHandle(dialect);
  if (failed(handle))
    return failure();
  if (auto *result = dyn_cast<ResourceT>(&*handle))
    return std::move(*result);
  return emitError(handleLoc) << "provided resource handle differs from the "
                                 "expected resource type";
}

//===--------------------------------------------------------------------===//
// Type Parsing
//===--------------------------------------------------------------------===//

/// Parse a type.
virtual ParseResult parseType(Type &result) = 0;

/// Parse a custom type with the provided callback, unless the next
/// token is `#`, in which case the generic parser is invoked.
virtual ParseResult parseCustomTypeWithFallback(
    Type &result, function_ref<ParseResult(Type &result)> parseType) = 0;

/// Parse an optional type.
virtual OptionalParseResult parseOptionalType(Type &result) = 0;

/// Parse a type of a specific type.
template <typename TypeT>
ParseResult parseType(TypeT &result) {
  SMLoc loc = getCurrentLocation();

  // Parse any kind of type.
  Type type;
  if (parseType(type))
    return failure();

  // Check for the right kind of type.
  result = type.dyn_cast<TypeT>();
  if (!result)
    return emitError(loc, "invalid kind of type specified");

  return success();
}

/// Trait to check if `TypeT` provides a `parse` method.
template <typename TypeT>
using type_has_parse_method =
    decltype(TypeT::parse(std::declval<AsmParser &>()));
template <typename TypeT>
using detect_type_has_parse_method =
    llvm::is_detected<type_has_parse_method, TypeT>;

/// Parse a custom Type of a given type unless the next token is `#`, in
/// which case the generic parser is invoked. The parsed Type is
/// populated in `result`.
template <typename TypeT>
std::enable_if_t<detect_type_has_parse_method<TypeT>::value, ParseResult>
parseCustomTypeWithFallback(TypeT &result) {
  SMLoc loc = getCurrentLocation();

  // Parse any kind of Type.
  Type type;
  if (parseCustomTypeWithFallback(type, [&](Type &result) -> ParseResult {
        result = TypeT::parse(*this);
        return success(!!result);
      }))
    return failure();

  // Check for the right kind of Type.
  result = type.dyn_cast<TypeT>();
  if (!result)
    return emitError(loc, "invalid kind of Type specified");
  return success();
}

/// SFINAE parsing method for Type that don't implement a parse method.
template <typename TypeT>
std::enable_if_t<!detect_type_has_parse_method<TypeT>::value, ParseResult>
parseCustomTypeWithFallback(TypeT &result) {
  return parseType(result);
}

/// Parse a type list.
ParseResult parseTypeList(SmallVectorImpl<Type> &result) {
  return parseCommaSeparatedList(
      [&]() { return parseType(result.emplace_back()); });
}

/// Parse an arrow followed by a type list.
virtual ParseResult parseArrowTypeList(SmallVectorImpl<Type> &result) = 0;

/// Parse an optional arrow followed by a type list.
virtual ParseResult
parseOptionalArrowTypeList(SmallVectorImpl<Type> &result) = 0;

/// Parse a colon followed by a type.
virtual ParseResult parseColonType(Type &result) = 0;

/// Parse a colon followed by a type of a specific kind, e.g. a FunctionType.
template <typename TypeType>
ParseResult parseColonType(TypeType &result) {
  SMLoc loc = getCurrentLocation();

  // Parse any kind of type.
  Type type;
  if (parseColonType(type))
    return failure();

  // Check for the right kind of type.
  result = type.dyn_cast<TypeType>();
  if (!result)
    return emitError(loc, "invalid kind of type specified");

  return success();
}

/// Parse a colon followed by a type list, which must have at least one type.
virtual ParseResult parseColonTypeList(SmallVectorImpl<Type> &result) = 0;

/// Parse an optional colon followed by a type list, which if present must
/// have at least one type.
virtual ParseResult
parseOptionalColonTypeList(SmallVectorImpl<Type> &result) = 0;

/// Parse a keyword followed by a type.
ParseResult parseKeywordType(const char *keyword, Type &result) {
  return failure(parseKeyword(keyword) || parseType(result));
}

/// Add the specified type to the end of the specified type list and return
/// success.  This is a helper designed to allow parse methods to be simple
/// and chain through || operators.
ParseResult addTypeToList(Type type, SmallVectorImpl<Type> &result) {
  result.push_back(type);
  return success();
}

/// Add the specified types to the end of the specified type list and return
/// success.  This is a helper designed to allow parse methods to be simple
/// and chain through || operators.
ParseResult addTypesToList(ArrayRef<Type> types,
                           SmallVectorImpl<Type> &result) {
  result.append(types.begin(), types.end());
  return success();
}

/// Parse a dimension list of a tensor or memref type.  This populates the
/// dimension list, using -1 for the `?` dimensions if `allowDynamic` is set
/// and errors out on `?` otherwise. Parsing the trailing `x` is configurable.
///
///   dimension-list ::= eps | dimension (`x` dimension)*
///   dimension-list-with-trailing-x ::= (dimension `x`)*
///   dimension ::= `?` | decimal-literal
///
/// When `allowDynamic` is not set, this is used to parse:
///
///   static-dimension-list ::= eps | decimal-literal (`x` decimal-literal)*
///   static-dimension-list-with-trailing-x ::= (dimension `x`)*
virtual ParseResult parseDimensionList(SmallVectorImpl<int64_t> &dimensions,
                                       bool allowDynamic = true,
                                       bool withTrailingX = true) = 0;

/// Parse an 'x' token in a dimension list, handling the case where the x is
/// juxtaposed with an element type, as in "xf32", leaving the "f32" as the
/// next token.
virtual ParseResult parseXInDimensionList() = 0;

1236protected:
/// Parse a handle to a resource within the assembly format for the given
/// dialect.
virtual FailureOr<AsmDialectResourceHandle>
parseResourceHandle(Dialect *dialect) = 0;

//===--------------------------------------------------------------------===//
// Code Completion
//===--------------------------------------------------------------------===//

/// Parse a keyword, or an empty string if the current location signals a code
/// completion.
virtual ParseResult parseKeywordOrCompletion(StringRef *keyword) = 0;

/// Signal the code completion of a set of expected tokens.
virtual void codeCompleteExpectedTokens(ArrayRef<StringRef> tokens) = 0;

1253private:
AsmParser(const AsmParser &) = delete;
void operator=(const AsmParser &) = delete;
1256};

1258//===----------------------------------------------------------------------===//
1259// OpAsmParser
1260//===----------------------------------------------------------------------===//

1262/// The OpAsmParser has methods for interacting with the asm parser: parsing
1263/// things from it, emitting errors etc.  It has an intentionally high-level API
1264/// that is designed to reduce/constrain syntax innovation in individual
1265/// operations.
1266///
1267/// For example, consider an op like this:
1268///
1269///    %x = load %p[%1, %2] : memref<...>
1270///
1271/// The "%x = load" tokens are already parsed and therefore invisible to the
1272/// custom op parser.  This can be supported by calling `parseOperandList` to
1273/// parse the %p, then calling `parseOperandList` with a `SquareDelimiter` to
1274/// parse the indices, then calling `parseColonTypeList` to parse the result
1275/// type.
1276///
1277class OpAsmParser : public AsmParser {
1278public:
using AsmParser::AsmParser;
~OpAsmParser() override;

/// Parse a loc(...) specifier if present, filling in result if so.
/// Location for BlockArgument and Operation may be deferred with an alias, in
/// which case an OpaqueLoc is set and will be resolved when parsing
/// completes.
virtual ParseResult
parseOptionalLocationSpecifier(Optional<Location> &result) = 0;

/// Return the name of the specified result in the specified syntax, as well
/// as the sub-element in the name.  It returns an empty string and ~0U for
/// invalid result numbers.  For example, in this operation:
///
///  %x, %y:2, %z = foo.op
///
///    getResultName(0) == {"x", 0 }
///    getResultName(1) == {"y", 0 }
///    getResultName(2) == {"y", 1 }
///    getResultName(3) == {"z", 0 }
///    getResultName(4) == {"", ~0U }
virtual std::pair<StringRef, unsigned>
getResultName(unsigned resultNo) const = 0;

/// Return the number of declared SSA results.  This returns 4 for the foo.op
/// example in the comment for `getResultName`.
virtual size_t getNumResults() const = 0;

// These methods emit an error and return failure or success. This allows
// these to be chained together into a linear sequence of || expressions in
// many cases.

/// Parse an operation in its generic form.
/// The parsed operation is parsed in the current context and inserted in the
/// provided block and insertion point. The results produced by this operation
/// aren't mapped to any named value in the parser. Returns nullptr on
/// failure.
virtual Operation *parseGenericOperation(Block *insertBlock,
                                         Block::iterator insertPt) = 0;

/// Parse the name of an operation, in the custom form. On success, return a
/// an object of type 'OperationName'. Otherwise, failure is returned.
virtual FailureOr<OperationName> parseCustomOperationName() = 0;

//===--------------------------------------------------------------------===//
// Operand Parsing
//===--------------------------------------------------------------------===//

/// This is the representation of an operand reference.
struct UnresolvedOperand {
  SMLoc location;  // Location of the token.
  StringRef name;  // Value name, e.g. %42 or %abc
  unsigned number; // Number, e.g. 12 for an operand like %xyz#12
};

/// Parse different components, viz., use-info of operand(s), successor(s),
/// region(s), attribute(s) and function-type, of the generic form of an
/// operation instance and populate the input operation-state 'result' with
/// those components. If any of the components is explicitly provided, then
/// skip parsing that component.
virtual ParseResult parseGenericOperationAfterOpName(
    OperationState &result,
    Optional<ArrayRef<UnresolvedOperand>> parsedOperandType = std::nullopt,
    Optional<ArrayRef<Block *>> parsedSuccessors = std::nullopt,
    Optional<MutableArrayRef<std::unique_ptr<Region>>> parsedRegions =
        std::nullopt,
    Optional<ArrayRef<NamedAttribute>> parsedAttributes = std::nullopt,
    Optional<FunctionType> parsedFnType = std::nullopt) = 0;

/// Parse a single SSA value operand name along with a result number if
/// `allowResultNumber` is true.
virtual ParseResult parseOperand(UnresolvedOperand &result,
                                 bool allowResultNumber = true) = 0;

/// Parse a single operand if present.
virtual OptionalParseResult
parseOptionalOperand(UnresolvedOperand &result,
                     bool allowResultNumber = true) = 0;

/// Parse zero or more SSA comma-separated operand references with a specified
/// surrounding delimiter, and an optional required operand count.
virtual ParseResult
parseOperandList(SmallVectorImpl<UnresolvedOperand> &result,
                 Delimiter delimiter = Delimiter::None,
                 bool allowResultNumber = true,
                 int requiredOperandCount = -1) = 0;

/// Parse a specified number of comma separated operands.
ParseResult parseOperandList(SmallVectorImpl<UnresolvedOperand> &result,
                             int requiredOperandCount,
                             Delimiter delimiter = Delimiter::None) {
  return parseOperandList(result, delimiter,
                          /*allowResultNumber=*/true, requiredOperandCount);
}

/// Parse zero or more trailing SSA comma-separated trailing operand
/// references with a specified surrounding delimiter, and an optional
/// required operand count. A leading comma is expected before the
/// operands.
ParseResult
parseTrailingOperandList(SmallVectorImpl<UnresolvedOperand> &result,
                         Delimiter delimiter = Delimiter::None) {
  if (failed(parseOptionalComma()))
    return success(); // The comma is optional.
  return parseOperandList(result, delimiter);
}

/// Resolve an operand to an SSA value, emitting an error on failure.
virtual ParseResult resolveOperand(const UnresolvedOperand &operand,
                                   Type type,
                                   SmallVectorImpl<Value> &result) = 0;

/// Resolve a list of operands to SSA values, emitting an error on failure, or
/// appending the results to the list on success. This method should be used
/// when all operands have the same type.
template <typename Operands = ArrayRef<UnresolvedOperand>>
ParseResult resolveOperands(Operands &&operands, Type type,
                            SmallVectorImpl<Value> &result) {
  for (const UnresolvedOperand &operand : operands)
    if (resolveOperand(operand, type, result))
      return failure();
  return success();
}
template <typename Operands = ArrayRef<UnresolvedOperand>>
ParseResult resolveOperands(Operands &&operands, Type type, SMLoc loc,
                            SmallVectorImpl<Value> &result) {
  return resolveOperands(std::forward<Operands>(operands), type, result);
}

/// Resolve a list of operands and a list of operand types to SSA values,
/// emitting an error and returning failure, or appending the results
/// to the list on success.
template <typename Operands = ArrayRef<UnresolvedOperand>,
          typename Types = ArrayRef<Type>>
std::enable_if_t<!std::is_convertible<Types, Type>::value, ParseResult>
resolveOperands(Operands &&operands, Types &&types, SMLoc loc,
                SmallVectorImpl<Value> &result) {
  size_t operandSize = std::distance(operands.begin(), operands.end());
  size_t typeSize = std::distance(types.begin(), types.end());
  if (operandSize != typeSize)
    return emitError(loc)
           << operandSize << " operands present, but expected " << typeSize;

  for (auto [operand, type] : llvm::zip(operands, types))
    if (resolveOperand(operand, type, result))
      return failure();
  return success();
}

/// Parses an affine map attribute where dims and symbols are SSA operands.
/// Operand values must come from single-result sources, and be valid
/// dimensions/symbol identifiers according to mlir::isValidDim/Symbol.
virtual ParseResult
parseAffineMapOfSSAIds(SmallVectorImpl<UnresolvedOperand> &operands,
                       Attribute &map, StringRef attrName,
                       NamedAttrList &attrs,
                       Delimiter delimiter = Delimiter::Square) = 0;

/// Parses an affine expression where dims and symbols are SSA operands.
/// Operand values must come from single-result sources, and be valid
/// dimensions/symbol identifiers according to mlir::isValidDim/Symbol.
virtual ParseResult
parseAffineExprOfSSAIds(SmallVectorImpl<UnresolvedOperand> &dimOperands,
                        SmallVectorImpl<UnresolvedOperand> &symbOperands,
                        AffineExpr &expr) = 0;

//===--------------------------------------------------------------------===//
// Argument Parsing
//===--------------------------------------------------------------------===//

struct Argument {
  UnresolvedOperand ssaName;    // SourceLoc, SSA name, result #.
  Type type;                    // Type.
  DictionaryAttr attrs;         // Attributes if present.
  Optional<Location> sourceLoc; // Source location specifier if present.
};

/// Parse a single argument with the following syntax:
///
///   `%ssaName : !type { optionalAttrDict} loc(optionalSourceLoc)`
///
/// If `allowType` is false or `allowAttrs` are false then the respective
/// parts of the grammar are not parsed.
virtual ParseResult parseArgument(Argument &result, bool allowType = false,
                                  bool allowAttrs = false) = 0;

/// Parse a single argument if present.
virtual OptionalParseResult
parseOptionalArgument(Argument &result, bool allowType = false,
                      bool allowAttrs = false) = 0;

/// Parse zero or more arguments with a specified surrounding delimiter.
virtual ParseResult parseArgumentList(SmallVectorImpl<Argument> &result,
                                      Delimiter delimiter = Delimiter::None,
                                      bool allowType = false,
                                      bool allowAttrs = false) = 0;

//===--------------------------------------------------------------------===//
// Region Parsing
//===--------------------------------------------------------------------===//

/// Parses a region. Any parsed blocks are appended to 'region' and must be
/// moved to the op regions after the op is created. The first block of the
/// region takes 'arguments'.
///
/// If 'enableNameShadowing' is set to true, the argument names are allowed to
/// shadow the names of other existing SSA values defined above the region
/// scope. 'enableNameShadowing' can only be set to true for regions attached
/// to operations that are 'IsolatedFromAbove'.
virtual ParseResult parseRegion(Region &region,
                                ArrayRef<Argument> arguments = {},
                                bool enableNameShadowing = false) = 0;

/// Parses a region if present.
virtual OptionalParseResult
parseOptionalRegion(Region &region, ArrayRef<Argument> arguments = {},
                    bool enableNameShadowing = false) = 0;

/// Parses a region if present. If the region is present, a new region is
/// allocated and placed in `region`. If no region is present or on failure,
/// `region` remains untouched.
virtual OptionalParseResult
parseOptionalRegion(std::unique_ptr<Region> &region,
                    ArrayRef<Argument> arguments = {},
                    bool enableNameShadowing = false) = 0;

//===--------------------------------------------------------------------===//
// Successor Parsing
//===--------------------------------------------------------------------===//

/// Parse a single operation successor.
virtual ParseResult parseSuccessor(Block *&dest) = 0;

/// Parse an optional operation successor.
virtual OptionalParseResult parseOptionalSuccessor(Block *&dest) = 0;

/// Parse a single operation successor and its operand list.
virtual ParseResult
parseSuccessorAndUseList(Block *&dest, SmallVectorImpl<Value> &operands) = 0;

//===--------------------------------------------------------------------===//
// Type Parsing
//===--------------------------------------------------------------------===//

/// Parse a list of assignments of the form
///   (%x1 = %y1, %x2 = %y2, ...)
ParseResult parseAssignmentList(SmallVectorImpl<Argument> &lhs,
                                SmallVectorImpl<UnresolvedOperand> &rhs) {
  OptionalParseResult result = parseOptionalAssignmentList(lhs, rhs);
  if (!result.has_value())
    return emitError(getCurrentLocation(), "expected '('");
  return result.value();
}

virtual OptionalParseResult
parseOptionalAssignmentList(SmallVectorImpl<Argument> &lhs,
                            SmallVectorImpl<UnresolvedOperand> &rhs) = 0;
1536};

1538//===--------------------------------------------------------------------===//
1539// Dialect OpAsm interface.
1540//===--------------------------------------------------------------------===//

1542/// A functor used to set the name of the start of a result group of an
1543/// operation. See 'getAsmResultNames' below for more details.
1544using OpAsmSetValueNameFn = function_ref<void(Value, StringRef)>;

1546/// A functor used to set the name of blocks in regions directly nested under
1547/// an operation.
1548using OpAsmSetBlockNameFn = function_ref<void(Block *, StringRef)>;

1550class OpAsmDialectInterface
  : public DialectInterface::Base<OpAsmDialectInterface> {
1552public:
OpAsmDialectInterface(Dialect *dialect) : Base(dialect) {}

//===------------------------------------------------------------------===//
// Aliases
//===------------------------------------------------------------------===//

/// Holds the result of `getAlias` hook call.
enum class AliasResult {
  /// The object (type or attribute) is not supported by the hook
  /// and an alias was not provided.
  NoAlias,
  /// An alias was provided, but it might be overriden by other hook.
  OverridableAlias,
  /// An alias was provided and it should be used
  /// (no other hooks will be checked).
  FinalAlias
};

/// Hooks for getting an alias identifier alias for a given symbol, that is
/// not necessarily a part of this dialect. The identifier is used in place of
/// the symbol when printing textual IR. These aliases must not contain `.` or
/// end with a numeric digit([0-9]+).
virtual AliasResult getAlias(Attribute attr, raw_ostream &os) const {
  return AliasResult::NoAlias;
}
virtual AliasResult getAlias(Type type, raw_ostream &os) const {
  return AliasResult::NoAlias;
}

//===--------------------------------------------------------------------===//
// Resources
//===--------------------------------------------------------------------===//

/// Declare a resource with the given key, returning a handle to use for any
/// references of this resource key within the IR during parsing. The result
/// of `getResourceKey` on the returned handle is permitted to be different
/// than `key`.
virtual FailureOr<AsmDialectResourceHandle>
declareResource(StringRef key) const {
  return failure();
}

/// Return a key to use for the given resource. This key should uniquely
/// identify this resource within the dialect.
virtual std::string
getResourceKey(const AsmDialectResourceHandle &handle) const {
  llvm_unreachable(::llvm::llvm_unreachable_internal("Dialect must implement `getResourceKey` when defining resources"
, "mlir/include/mlir/IR/OpImplementation.h", 1600)
      "Dialect must implement `getResourceKey` when defining resources")::llvm::llvm_unreachable_internal("Dialect must implement `getResourceKey` when defining resources"
, "mlir/include/mlir/IR/OpImplementation.h", 1600);
}

/// Hook for parsing resource entries. Returns failure if the entry was not
/// valid, or could otherwise not be processed correctly. Any necessary errors
/// can be emitted via the provided entry.
virtual LogicalResult parseResource(AsmParsedResourceEntry &entry) const;

/// Hook for building resources to use during printing. The given `op` may be
/// inspected to help determine what information to include.
/// `referencedResources` contains all of the resources detected when printing
/// 'op'.
virtual void
buildResources(Operation *op,
               const SetVector<AsmDialectResourceHandle> &referencedResources,
               AsmResourceBuilder &builder) const {}
1616};
1617} // namespace mlir

1619//===--------------------------------------------------------------------===//
1620// Operation OpAsm interface.
1621//===--------------------------------------------------------------------===//

1623/// The OpAsmOpInterface, see OpAsmInterface.td for more details.
1624#include "mlir/IR/OpAsmInterface.h.inc"

1626namespace llvm {
1627template <>
1628struct DenseMapInfo<mlir::AsmDialectResourceHandle> {
static inline mlir::AsmDialectResourceHandle getEmptyKey() {
  return {DenseMapInfo<void *>::getEmptyKey(),
          DenseMapInfo<mlir::TypeID>::getEmptyKey(), nullptr};
}
static inline mlir::AsmDialectResourceHandle getTombstoneKey() {
  return {DenseMapInfo<void *>::getTombstoneKey(),
          DenseMapInfo<mlir::TypeID>::getTombstoneKey(), nullptr};
}
static unsigned getHashValue(const mlir::AsmDialectResourceHandle &handle) {
  return DenseMapInfo<void *>::getHashValue(handle.getResource());
}
static bool isEqual(const mlir::AsmDialectResourceHandle &lhs,
                    const mlir::AsmDialectResourceHandle &rhs) {
  return lhs.getResource() == rhs.getResource();
}
1644};
1645} // namespace llvm

1647#endif