Bug Summary

File:llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
Warning:line 940, column 31
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LowerMatrixIntrinsics.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-12/lib/clang/12.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/build-llvm/lib/Transforms/Scalar -I /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar -I /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/build-llvm/include -I /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-12/lib/clang/12.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/build-llvm/lib/Transforms/Scalar -fdebug-prefix-map=/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-09-26-161721-17566-1 -x c++ /build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
1//===- LowerMatrixIntrinsics.cpp - Lower matrix intrinsics -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Lower matrix intrinsics to vector operations.
10//
11// TODO:
12// * Improve fusion:
13// * Support more cases, e.g. multiply-add, multiply-sub, operands/results
14// transposed.
15// * Improve cost-modeling, e.g. choose different number of rows/columns
16// columns for tiles, consider cost of copies on alias.
17//
18//===----------------------------------------------------------------------===//
19
20#include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
21#include "llvm/ADT/GraphTraits.h"
22#include "llvm/ADT/PostOrderIterator.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/Analysis/AliasAnalysis.h"
25#include "llvm/Analysis/DomTreeUpdater.h"
26#include "llvm/Analysis/OptimizationRemarkEmitter.h"
27#include "llvm/Analysis/TargetTransformInfo.h"
28#include "llvm/Analysis/ValueTracking.h"
29#include "llvm/Analysis/VectorUtils.h"
30#include "llvm/IR/CFG.h"
31#include "llvm/IR/DataLayout.h"
32#include "llvm/IR/DebugInfoMetadata.h"
33#include "llvm/IR/Function.h"
34#include "llvm/IR/IRBuilder.h"
35#include "llvm/IR/Instructions.h"
36#include "llvm/IR/IntrinsicInst.h"
37#include "llvm/IR/PatternMatch.h"
38#include "llvm/InitializePasses.h"
39#include "llvm/Pass.h"
40#include "llvm/Support/Alignment.h"
41#include "llvm/Support/CommandLine.h"
42#include "llvm/Support/Debug.h"
43#include "llvm/Transforms/Scalar.h"
44#include "llvm/Transforms/Utils/BasicBlockUtils.h"
45#include "llvm/Transforms/Utils/LoopUtils.h"
46#include "llvm/Transforms/Utils/MatrixUtils.h"
47
48using namespace llvm;
49using namespace PatternMatch;
50
51#define DEBUG_TYPE"lower-matrix-intrinsics" "lower-matrix-intrinsics"
52
53static cl::opt<bool> EnableShapePropagation(
54 "matrix-propagate-shape", cl::init(true), cl::Hidden,
55 cl::desc("Enable/disable shape propagation from matrix intrinsics to other "
56 "instructions."));
57
58static cl::opt<bool>
59 FuseMatrix("fuse-matrix", cl::init(true), cl::Hidden,
60 cl::desc("Enable/disable fusing matrix instructions."));
61// TODO: Allow and use non-square tiles.
62static cl::opt<unsigned> TileSize(
63 "fuse-matrix-tile-size", cl::init(4), cl::Hidden,
64 cl::desc(
65 "Tile size for matrix instruction fusion using square-shaped tiles."));
66static cl::opt<bool> TileUseLoops("fuse-matrix-use-loops", cl::init(false),
67 cl::Hidden,
68 cl::desc("Generate loop nest for tiling."));
69static cl::opt<bool> ForceFusion(
70 "force-fuse-matrix", cl::init(false), cl::Hidden,
71 cl::desc("Force matrix instruction fusion even if not profitable."));
72static cl::opt<bool> AllowContractEnabled(
73 "matrix-allow-contract", cl::init(false), cl::Hidden,
74 cl::desc("Allow the use of FMAs if available and profitable. This may "
75 "result in different results, due to less rounding error."));
76
77enum class MatrixLayoutTy { ColumnMajor, RowMajor };
78
79static cl::opt<MatrixLayoutTy> MatrixLayout(
80 "matrix-default-layout", cl::init(MatrixLayoutTy::ColumnMajor),
81 cl::desc("Sets the default matrix layout"),
82 cl::values(clEnumValN(MatrixLayoutTy::ColumnMajor, "column-major",llvm::cl::OptionEnumValue { "column-major", int(MatrixLayoutTy
::ColumnMajor), "Use column-major layout" }
83 "Use column-major layout")llvm::cl::OptionEnumValue { "column-major", int(MatrixLayoutTy
::ColumnMajor), "Use column-major layout" }
,
84 clEnumValN(MatrixLayoutTy::RowMajor, "row-major",llvm::cl::OptionEnumValue { "row-major", int(MatrixLayoutTy::
RowMajor), "Use row-major layout" }
85 "Use row-major layout")llvm::cl::OptionEnumValue { "row-major", int(MatrixLayoutTy::
RowMajor), "Use row-major layout" }
));
86
87/// Helper function to either return Scope, if it is a subprogram or the
88/// attached subprogram for a local scope.
89static DISubprogram *getSubprogram(DIScope *Scope) {
90 if (auto *Subprogram = dyn_cast<DISubprogram>(Scope))
91 return Subprogram;
92 return cast<DILocalScope>(Scope)->getSubprogram();
93}
94
95namespace {
96
97// Given an element pointer \p BasePtr to the start of a (sub) matrix, compute
98// the start address of vector \p VecIdx with type (\p EltType x \p NumElements)
99// assuming \p Stride elements between start two consecutive vectors.
100// \p Stride must be >= \p NumElements.
101// For column-major matrixes, the function computes the address of a column
102// vectors and \p NumElements must be set to the number of elements in a column
103// (= number of rows of the matrix). For row-major matrixes, the function
104// computes the address of a row vector and \p NumElements must be set to the
105// number of elements in a column (= number of columns of the matrix).
106//
107// Consider a 4x4 matrix in column-mjaor layout like below
108//
109// 0 1 2 3
110// 0 v_0_0 v_0_1 v_0_2 v_0_3
111// 1 v_1_0 v_1_1 v_1_2 v_1_3
112// 2 v_2_0 v_2_1 v_2_2 v_2_3
113// 3 v_3_0 v_3_1 v_3_2 v_3_3
114
115// To compute the column addresses for a 2x3 sub-matrix at row 1 and column 1,
116// we need a pointer to the first element of the submatrix as base pointer.
117// Then we can use computeVectorAddr to compute the addresses for the columns
118// of the sub-matrix.
119//
120// Column 0: computeVectorAddr(Base, 0 (column), 4 (stride), 2 (num rows), ..)
121// -> just returns Base
122// Column 1: computeVectorAddr(Base, 1 (column), 4 (stride), 2 (num rows), ..)
123// -> returns Base + (1 * 4)
124// Column 2: computeVectorAddr(Base, 2 (column), 4 (stride), 2 (num rows), ..)
125// -> returns Base + (2 * 4)
126//
127// The graphic below illustrates the number of elements in a column (marked
128// with |) and the number of skipped elements (marked with }).
129//
130// v_0_0 v_0_1 {v_0_2 {v_0_3
131// Base Col 1 Col 2
132// | | |
133// v_1_0 |v_1_1 |v_1_2 |v_1_3
134// v_2_0 |v_2_1 |v_2_2 |v_2_3
135// v_3_0 {v_3_1 {v_3_2 v_3_3
136//
137Value *computeVectorAddr(Value *BasePtr, Value *VecIdx, Value *Stride,
138 unsigned NumElements, Type *EltType,
139 IRBuilder<> &Builder) {
140
141 assert((!isa<ConstantInt>(Stride) ||(((!isa<ConstantInt>(Stride) || cast<ConstantInt>
(Stride)->getZExtValue() >= NumElements) && "Stride must be >= the number of elements in the result vector."
) ? static_cast<void> (0) : __assert_fail ("(!isa<ConstantInt>(Stride) || cast<ConstantInt>(Stride)->getZExtValue() >= NumElements) && \"Stride must be >= the number of elements in the result vector.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 143, __PRETTY_FUNCTION__))
142 cast<ConstantInt>(Stride)->getZExtValue() >= NumElements) &&(((!isa<ConstantInt>(Stride) || cast<ConstantInt>
(Stride)->getZExtValue() >= NumElements) && "Stride must be >= the number of elements in the result vector."
) ? static_cast<void> (0) : __assert_fail ("(!isa<ConstantInt>(Stride) || cast<ConstantInt>(Stride)->getZExtValue() >= NumElements) && \"Stride must be >= the number of elements in the result vector.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 143, __PRETTY_FUNCTION__))
143 "Stride must be >= the number of elements in the result vector.")(((!isa<ConstantInt>(Stride) || cast<ConstantInt>
(Stride)->getZExtValue() >= NumElements) && "Stride must be >= the number of elements in the result vector."
) ? static_cast<void> (0) : __assert_fail ("(!isa<ConstantInt>(Stride) || cast<ConstantInt>(Stride)->getZExtValue() >= NumElements) && \"Stride must be >= the number of elements in the result vector.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 143, __PRETTY_FUNCTION__))
;
144 unsigned AS = cast<PointerType>(BasePtr->getType())->getAddressSpace();
145
146 // Compute the start of the vector with index VecIdx as VecIdx * Stride.
147 Value *VecStart = Builder.CreateMul(VecIdx, Stride, "vec.start");
148
149 // Get pointer to the start of the selected vector. Skip GEP creation,
150 // if we select vector 0.
151 if (isa<ConstantInt>(VecStart) && cast<ConstantInt>(VecStart)->isZero())
152 VecStart = BasePtr;
153 else
154 VecStart = Builder.CreateGEP(EltType, BasePtr, VecStart, "vec.gep");
155
156 // Cast elementwise vector start pointer to a pointer to a vector
157 // (EltType x NumElements)*.
158 auto *VecType = FixedVectorType::get(EltType, NumElements);
159 Type *VecPtrType = PointerType::get(VecType, AS);
160 return Builder.CreatePointerCast(VecStart, VecPtrType, "vec.cast");
161}
162
163/// LowerMatrixIntrinsics contains the methods used to lower matrix intrinsics.
164///
165/// Currently, the lowering for each matrix intrinsic is done as follows:
166/// 1. Propagate the shape information from intrinsics to connected
167/// instructions.
168/// 2. Lower instructions with shape information (assuming column-major layout).
169/// The lowering works similarly using row-major layout.
170/// 2.1. Get column vectors for each argument. If we already lowered the
171/// definition of an argument, use the produced column vectors directly.
172/// If not, split the operand vector containing an embedded matrix into
173/// a set of column vectors,
174/// 2.2. Lower the instruction in terms of column major operations, which
175/// yields a set of column vectors containing result matrix. Note that we
176/// lower all instructions that have shape information. Besides the
177/// intrinsics, this includes stores for example.
178/// 2.3. Update uses of the lowered instruction. If we have shape information
179/// for a user, there is nothing to do, as we will look up the result
180/// column matrix when lowering the user. For other uses, we embed the
181/// result matrix in a flat vector and update the use.
182/// 2.4. Cache the result column matrix for the instruction we lowered
183/// 3. After we lowered all instructions in a function, remove the now
184/// obsolete instructions.
185///
186class LowerMatrixIntrinsics {
187 Function &Func;
188 const DataLayout &DL;
189 const TargetTransformInfo &TTI;
190 AliasAnalysis *AA;
191 DominatorTree *DT;
192 LoopInfo *LI;
193 OptimizationRemarkEmitter *ORE;
194
195 /// Contains estimates of the number of operations (loads, stores, compute) required to lower a matrix operation.
196 struct OpInfoTy {
197 /// Number of stores emitted to generate this matrix.
198 unsigned NumStores = 0;
199 /// Number of loads emitted to generate this matrix.
200 unsigned NumLoads = 0;
201 /// Number of compute operations emitted to generate this matrix.
202 unsigned NumComputeOps = 0;
203
204 OpInfoTy &operator+=(const OpInfoTy &RHS) {
205 NumStores += RHS.NumStores;
206 NumLoads += RHS.NumLoads;
207 NumComputeOps += RHS.NumComputeOps;
208 return *this;
209 }
210 };
211
212 /// Wrapper class representing a matrix as a set of vectors, either in row or
213 /// column major layout. All vectors must have the same vector type.
214 class MatrixTy {
215 SmallVector<Value *, 16> Vectors;
216
217 OpInfoTy OpInfo;
218
219 bool IsColumnMajor = true;
220
221 public:
222 MatrixTy()
223 : Vectors(),
224 IsColumnMajor(MatrixLayout == MatrixLayoutTy::ColumnMajor) {}
225 MatrixTy(ArrayRef<Value *> Vectors)
226 : Vectors(Vectors.begin(), Vectors.end()),
227 IsColumnMajor(MatrixLayout == MatrixLayoutTy::ColumnMajor) {}
228 MatrixTy(unsigned NumRows, unsigned NumColumns, Type *EltTy)
229 : IsColumnMajor(MatrixLayout == MatrixLayoutTy::ColumnMajor) {
230
231 unsigned D = isColumnMajor() ? NumColumns : NumRows;
232 for (unsigned J = 0; J < D; ++J)
233 addVector(UndefValue::get(FixedVectorType::get(
234 EltTy, isColumnMajor() ? NumRows : NumColumns)));
235 }
236
237 Value *getVector(unsigned i) const { return Vectors[i]; }
238 Value *getColumn(unsigned i) const {
239 assert(isColumnMajor() && "only supported for column-major matrixes")((isColumnMajor() && "only supported for column-major matrixes"
) ? static_cast<void> (0) : __assert_fail ("isColumnMajor() && \"only supported for column-major matrixes\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 239, __PRETTY_FUNCTION__))
;
240 return Vectors[i];
241 }
242 Value *getRow(unsigned i) const {
243 assert(!isColumnMajor() && "only supported for row-major matrixes")((!isColumnMajor() && "only supported for row-major matrixes"
) ? static_cast<void> (0) : __assert_fail ("!isColumnMajor() && \"only supported for row-major matrixes\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 243, __PRETTY_FUNCTION__))
;
244 return Vectors[i];
245 }
246
247 void setVector(unsigned i, Value *V) { Vectors[i] = V; }
248
249 Type *getElementType() const { return getVectorTy()->getElementType(); }
250
251 unsigned getNumVectors() const {
252 if (isColumnMajor())
253 return getNumColumns();
254 return getNumRows();
255 }
256
257 unsigned getNumColumns() const {
258 if (isColumnMajor())
259 return Vectors.size();
260 else {
261 assert(Vectors.size() > 0 && "Cannot call getNumRows without columns")((Vectors.size() > 0 && "Cannot call getNumRows without columns"
) ? static_cast<void> (0) : __assert_fail ("Vectors.size() > 0 && \"Cannot call getNumRows without columns\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 261, __PRETTY_FUNCTION__))
;
262 return cast<FixedVectorType>(Vectors[0]->getType())->getNumElements();
263 }
264 }
265 unsigned getNumRows() const {
266 if (isColumnMajor()) {
267 assert(Vectors.size() > 0 && "Cannot call getNumRows without columns")((Vectors.size() > 0 && "Cannot call getNumRows without columns"
) ? static_cast<void> (0) : __assert_fail ("Vectors.size() > 0 && \"Cannot call getNumRows without columns\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 267, __PRETTY_FUNCTION__))
;
268 return cast<FixedVectorType>(Vectors[0]->getType())->getNumElements();
269 } else
270 return Vectors.size();
271 }
272
273 void addVector(Value *V) { Vectors.push_back(V); }
274 VectorType *getColumnTy() {
275 assert(isColumnMajor() && "only supported for column-major matrixes")((isColumnMajor() && "only supported for column-major matrixes"
) ? static_cast<void> (0) : __assert_fail ("isColumnMajor() && \"only supported for column-major matrixes\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 275, __PRETTY_FUNCTION__))
;
276 return getVectorTy();
277 }
278
279 VectorType *getVectorTy() const {
280 return cast<VectorType>(Vectors[0]->getType());
281 }
282
283 iterator_range<SmallVector<Value *, 8>::iterator> columns() {
284 assert(isColumnMajor() &&((isColumnMajor() && "columns() only supported for column-major matrixes"
) ? static_cast<void> (0) : __assert_fail ("isColumnMajor() && \"columns() only supported for column-major matrixes\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 285, __PRETTY_FUNCTION__))
285 "columns() only supported for column-major matrixes")((isColumnMajor() && "columns() only supported for column-major matrixes"
) ? static_cast<void> (0) : __assert_fail ("isColumnMajor() && \"columns() only supported for column-major matrixes\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 285, __PRETTY_FUNCTION__))
;
286 return make_range(Vectors.begin(), Vectors.end());
287 }
288
289 iterator_range<SmallVector<Value *, 8>::iterator> vectors() {
290 return make_range(Vectors.begin(), Vectors.end());
291 }
292
293 /// Embed the vectors of the matrix into a flat vector by concatenating
294 /// them.
295 Value *embedInVector(IRBuilder<> &Builder) const {
296 return Vectors.size() == 1 ? Vectors[0]
297 : concatenateVectors(Builder, Vectors);
298 }
299
300 MatrixTy &addNumLoads(unsigned N) {
301 OpInfo.NumLoads += N;
302 return *this;
303 }
304
305 void setNumLoads(unsigned N) { OpInfo.NumLoads = N; }
306
307 MatrixTy &addNumStores(unsigned N) {
308 OpInfo.NumStores += N;
309 return *this;
310 }
311
312 MatrixTy &addNumComputeOps(unsigned N) {
313 OpInfo.NumComputeOps += N;
314 return *this;
315 }
316
317 unsigned getNumStores() const { return OpInfo.NumStores; }
318 unsigned getNumLoads() const { return OpInfo.NumLoads; }
319 unsigned getNumComputeOps() const { return OpInfo.NumComputeOps; }
320
321 const OpInfoTy &getOpInfo() const { return OpInfo; }
322
323 bool isColumnMajor() const { return IsColumnMajor; }
27
Returning zero, which participates in a condition later
324
325 unsigned getStride() const {
326 if (isColumnMajor())
327 return getNumRows();
328 return getNumColumns();
329 }
330
331 /// Extract a vector of \p NumElts starting at index (\p I, \p J). If the
332 /// matrix is column-major, the result vector is extracted from a column
333 /// vector, otherwise from a row vector.
334 Value *extractVector(unsigned I, unsigned J, unsigned NumElts,
335 IRBuilder<> &Builder) const {
336 Value *Vec = isColumnMajor() ? getColumn(J) : getRow(I);
337 Value *Undef = UndefValue::get(Vec->getType());
338 return Builder.CreateShuffleVector(
339 Vec, Undef, createSequentialMask(isColumnMajor() ? I : J, NumElts, 0),
340 "block");
341 }
342 };
343
344 struct ShapeInfo {
345 unsigned NumRows;
346 unsigned NumColumns;
347
348 bool IsColumnMajor;
349
350 ShapeInfo(unsigned NumRows = 0, unsigned NumColumns = 0)
351 : NumRows(NumRows), NumColumns(NumColumns),
352 IsColumnMajor(MatrixLayout == MatrixLayoutTy::ColumnMajor) {}
353
354 ShapeInfo(Value *NumRows, Value *NumColumns)
355 : ShapeInfo(cast<ConstantInt>(NumRows)->getZExtValue(),
356 cast<ConstantInt>(NumColumns)->getZExtValue()) {}
357
358 bool operator==(const ShapeInfo &other) {
359 return NumRows == other.NumRows && NumColumns == other.NumColumns;
360 }
361 bool operator!=(const ShapeInfo &other) { return !(*this == other); }
362
363 /// Returns true if shape-information is defined, meaning both dimensions
364 /// are != 0.
365 operator bool() const {
366 assert(NumRows == 0 || NumColumns != 0)((NumRows == 0 || NumColumns != 0) ? static_cast<void> (
0) : __assert_fail ("NumRows == 0 || NumColumns != 0", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 366, __PRETTY_FUNCTION__))
;
367 return NumRows != 0;
368 }
369
370 unsigned getStride() const {
371 if (IsColumnMajor)
372 return NumRows;
373 return NumColumns;
374 }
375
376 unsigned getNumVectors() const {
377 if (IsColumnMajor)
378 return NumColumns;
379 return NumRows;
380 }
381 };
382
383 /// Maps instructions to their shape information. The shape information
384 /// describes the shape to be used while lowering. This matches the shape of
385 /// the result value of the instruction, with the only exceptions being store
386 /// instructions and the matrix_column_major_store intrinsics. For those, the
387 /// shape information indicates that those instructions should be lowered
388 /// using shape information as well.
389 DenseMap<Value *, ShapeInfo> ShapeMap;
390
391 /// List of instructions to remove. While lowering, we are not replacing all
392 /// users of a lowered instruction, if shape information is available and
393 /// those need to be removed after we finished lowering.
394 SmallVector<Instruction *, 16> ToRemove;
395
396 /// Map from instructions to their produced column matrix.
397 MapVector<Value *, MatrixTy> Inst2ColumnMatrix;
398
399public:
400 LowerMatrixIntrinsics(Function &F, TargetTransformInfo &TTI,
401 AliasAnalysis *AA, DominatorTree *DT, LoopInfo *LI,
402 OptimizationRemarkEmitter *ORE)
403 : Func(F), DL(F.getParent()->getDataLayout()), TTI(TTI), AA(AA), DT(DT),
404 LI(LI), ORE(ORE) {}
405
406 unsigned getNumOps(Type *VT) {
407 assert(isa<VectorType>(VT) && "Expected vector type")((isa<VectorType>(VT) && "Expected vector type"
) ? static_cast<void> (0) : __assert_fail ("isa<VectorType>(VT) && \"Expected vector type\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 407, __PRETTY_FUNCTION__))
;
408 return getNumOps(VT->getScalarType(),
409 cast<FixedVectorType>(VT)->getNumElements());
410 }
411
412 //
413 /// Return the estimated number of vector ops required for an operation on
414 /// \p VT * N.
415 unsigned getNumOps(Type *ST, unsigned N) {
416 return std::ceil((ST->getPrimitiveSizeInBits() * N).getFixedSize() /
417 double(TTI.getRegisterBitWidth(true)));
418 }
419
420 /// Return the set of vectors that a matrix value is lowered to.
421 ///
422 /// If we lowered \p MatrixVal, just return the cache result matrix. Otherwise
423 /// split the flat vector \p MatrixVal containing a matrix with shape \p SI
424 /// into vectors.
425 MatrixTy getMatrix(Value *MatrixVal, const ShapeInfo &SI,
426 IRBuilder<> &Builder) {
427 VectorType *VType = dyn_cast<VectorType>(MatrixVal->getType());
428 assert(VType && "MatrixVal must be a vector type")((VType && "MatrixVal must be a vector type") ? static_cast
<void> (0) : __assert_fail ("VType && \"MatrixVal must be a vector type\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 428, __PRETTY_FUNCTION__))
;
429 assert(cast<FixedVectorType>(VType)->getNumElements() ==((cast<FixedVectorType>(VType)->getNumElements() == SI
.NumRows * SI.NumColumns && "The vector size must match the number of matrix elements"
) ? static_cast<void> (0) : __assert_fail ("cast<FixedVectorType>(VType)->getNumElements() == SI.NumRows * SI.NumColumns && \"The vector size must match the number of matrix elements\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 431, __PRETTY_FUNCTION__))
430 SI.NumRows * SI.NumColumns &&((cast<FixedVectorType>(VType)->getNumElements() == SI
.NumRows * SI.NumColumns && "The vector size must match the number of matrix elements"
) ? static_cast<void> (0) : __assert_fail ("cast<FixedVectorType>(VType)->getNumElements() == SI.NumRows * SI.NumColumns && \"The vector size must match the number of matrix elements\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 431, __PRETTY_FUNCTION__))
431 "The vector size must match the number of matrix elements")((cast<FixedVectorType>(VType)->getNumElements() == SI
.NumRows * SI.NumColumns && "The vector size must match the number of matrix elements"
) ? static_cast<void> (0) : __assert_fail ("cast<FixedVectorType>(VType)->getNumElements() == SI.NumRows * SI.NumColumns && \"The vector size must match the number of matrix elements\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 431, __PRETTY_FUNCTION__))
;
432
433 // Check if we lowered MatrixVal using shape information. In that case,
434 // return the existing matrix, if it matches the requested shape
435 // information. If there is a mis-match, embed the result in a flat
436 // vector and split it later.
437 auto Found = Inst2ColumnMatrix.find(MatrixVal);
438 if (Found != Inst2ColumnMatrix.end()) {
439 MatrixTy &M = Found->second;
440 // Return the found matrix, if its shape matches the requested shape
441 // information
442 if (SI.NumRows == M.getNumRows() && SI.NumColumns == M.getNumColumns())
443 return M;
444
445 MatrixVal = M.embedInVector(Builder);
446 }
447
448 // Otherwise split MatrixVal.
449 SmallVector<Value *, 16> SplitVecs;
450 Value *Undef = UndefValue::get(VType);
451 for (unsigned MaskStart = 0;
452 MaskStart < cast<FixedVectorType>(VType)->getNumElements();
453 MaskStart += SI.getStride()) {
454 Value *V = Builder.CreateShuffleVector(
455 MatrixVal, Undef, createSequentialMask(MaskStart, SI.getStride(), 0),
456 "split");
457 SplitVecs.push_back(V);
458 }
459
460 return {SplitVecs};
461 }
462
463 /// If \p V already has a known shape return false. Otherwise set the shape
464 /// for instructions that support it.
465 bool setShapeInfo(Value *V, ShapeInfo Shape) {
466 assert(Shape && "Shape not set")((Shape && "Shape not set") ? static_cast<void>
(0) : __assert_fail ("Shape && \"Shape not set\"", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 466, __PRETTY_FUNCTION__))
;
467 if (isa<UndefValue>(V) || !supportsShapeInfo(V))
468 return false;
469
470 auto SIter = ShapeMap.find(V);
471 if (SIter != ShapeMap.end()) {
472 LLVM_DEBUG(dbgs() << " not overriding existing shape: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("lower-matrix-intrinsics")) { dbgs() << " not overriding existing shape: "
<< SIter->second.NumRows << " " << SIter
->second.NumColumns << " for " << *V << "\n"
; } } while (false)
473 << SIter->second.NumRows << " "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("lower-matrix-intrinsics")) { dbgs() << " not overriding existing shape: "
<< SIter->second.NumRows << " " << SIter
->second.NumColumns << " for " << *V << "\n"
; } } while (false)
474 << SIter->second.NumColumns << " for " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("lower-matrix-intrinsics")) { dbgs() << " not overriding existing shape: "
<< SIter->second.NumRows << " " << SIter
->second.NumColumns << " for " << *V << "\n"
; } } while (false)
;
475 return false;
476 }
477
478 ShapeMap.insert({V, Shape});
479 LLVM_DEBUG(dbgs() << " " << Shape.NumRows << " x " << Shape.NumColumnsdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("lower-matrix-intrinsics")) { dbgs() << " " << Shape
.NumRows << " x " << Shape.NumColumns << " for "
<< *V << "\n"; } } while (false)
480 << " for " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("lower-matrix-intrinsics")) { dbgs() << " " << Shape
.NumRows << " x " << Shape.NumColumns << " for "
<< *V << "\n"; } } while (false)
;
481 return true;
482 }
483
484 bool isUniformShape(Value *V) {
485 Instruction *I = dyn_cast<Instruction>(V);
486 if (!I)
487 return true;
488
489 switch (I->getOpcode()) {
490 case Instruction::FAdd:
491 case Instruction::FSub:
492 case Instruction::FMul: // Scalar multiply.
493 case Instruction::Add:
494 case Instruction::Mul:
495 case Instruction::Sub:
496 return true;
497 default:
498 return false;
499 }
500 }
501
502 /// Returns true if shape information can be used for \p V. The supported
503 /// instructions must match the instructions that can be lowered by this pass.
504 bool supportsShapeInfo(Value *V) {
505 Instruction *Inst = dyn_cast<Instruction>(V);
506 if (!Inst)
507 return false;
508
509 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst);
510 if (II)
511 switch (II->getIntrinsicID()) {
512 case Intrinsic::matrix_multiply:
513 case Intrinsic::matrix_transpose:
514 case Intrinsic::matrix_column_major_load:
515 case Intrinsic::matrix_column_major_store:
516 return true;
517 default:
518 return false;
519 }
520 return isUniformShape(V) || isa<StoreInst>(V) || isa<LoadInst>(V);
521 }
522
523 /// Propagate the shape information of instructions to their users.
524 /// The work list contains instructions for which we can compute the shape,
525 /// either based on the information provided by matrix intrinsics or known
526 /// shapes of operands.
527 SmallVector<Instruction *, 32>
528 propagateShapeForward(SmallVectorImpl<Instruction *> &WorkList) {
529 SmallVector<Instruction *, 32> NewWorkList;
530 // Pop an element for which we guaranteed to have at least one of the
531 // operand shapes. Add the shape for this and then add users to the work
532 // list.
533 LLVM_DEBUG(dbgs() << "Forward-propagate shapes:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("lower-matrix-intrinsics")) { dbgs() << "Forward-propagate shapes:\n"
; } } while (false)
;
534 while (!WorkList.empty()) {
535 Instruction *Inst = WorkList.back();
536 WorkList.pop_back();
537
538 // New entry, set the value and insert operands
539 bool Propagate = false;
540
541 Value *MatrixA;
542 Value *MatrixB;
543 Value *M;
544 Value *N;
545 Value *K;
546 if (match(Inst, m_Intrinsic<Intrinsic::matrix_multiply>(
547 m_Value(MatrixA), m_Value(MatrixB), m_Value(M),
548 m_Value(N), m_Value(K)))) {
549 Propagate = setShapeInfo(Inst, {M, K});
550 } else if (match(Inst, m_Intrinsic<Intrinsic::matrix_transpose>(
551 m_Value(MatrixA), m_Value(M), m_Value(N)))) {
552 // Flip dimensions.
553 Propagate = setShapeInfo(Inst, {N, M});
554 } else if (match(Inst, m_Intrinsic<Intrinsic::matrix_column_major_store>(
555 m_Value(MatrixA), m_Value(), m_Value(),
556 m_Value(), m_Value(M), m_Value(N)))) {
557 Propagate = setShapeInfo(Inst, {N, M});
558 } else if (match(Inst, m_Intrinsic<Intrinsic::matrix_column_major_load>(
559 m_Value(), m_Value(), m_Value(), m_Value(M),
560 m_Value(N)))) {
561 Propagate = setShapeInfo(Inst, {M, N});
562 } else if (match(Inst, m_Store(m_Value(MatrixA), m_Value()))) {
563 auto OpShape = ShapeMap.find(MatrixA);
564 if (OpShape != ShapeMap.end())
565 setShapeInfo(Inst, OpShape->second);
566 continue;
567 } else if (isUniformShape(Inst)) {
568 // Find the first operand that has a known shape and use that.
569 for (auto &Op : Inst->operands()) {
570 auto OpShape = ShapeMap.find(Op.get());
571 if (OpShape != ShapeMap.end()) {
572 Propagate |= setShapeInfo(Inst, OpShape->second);
573 break;
574 }
575 }
576 }
577
578 if (Propagate) {
579 NewWorkList.push_back(Inst);
580 for (auto *User : Inst->users())
581 if (ShapeMap.count(User) == 0)
582 WorkList.push_back(cast<Instruction>(User));
583 }
584 }
585
586 return NewWorkList;
587 }
588
589 /// Propagate the shape to operands of instructions with shape information.
590 /// \p Worklist contains the instruction for which we already know the shape.
591 SmallVector<Instruction *, 32>
592 propagateShapeBackward(SmallVectorImpl<Instruction *> &WorkList) {
593 SmallVector<Instruction *, 32> NewWorkList;
594
595 auto pushInstruction = [](Value *V,
596 SmallVectorImpl<Instruction *> &WorkList) {
597 Instruction *I = dyn_cast<Instruction>(V);
598 if (I)
599 WorkList.push_back(I);
600 };
601 // Pop an element with known shape. Traverse the operands, if their shape
602 // derives from the result shape and is unknown, add it and add them to the
603 // worklist.
604 LLVM_DEBUG(dbgs() << "Backward-propagate shapes:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("lower-matrix-intrinsics")) { dbgs() << "Backward-propagate shapes:\n"
; } } while (false)
;
605 while (!WorkList.empty()) {
606 Value *V = WorkList.back();
607 WorkList.pop_back();
608
609 size_t BeforeProcessingV = WorkList.size();
610 if (!isa<Instruction>(V))
611 continue;
612
613 Value *MatrixA;
614 Value *MatrixB;
615 Value *M;
616 Value *N;
617 Value *K;
618 if (match(V, m_Intrinsic<Intrinsic::matrix_multiply>(
619 m_Value(MatrixA), m_Value(MatrixB), m_Value(M),
620 m_Value(N), m_Value(K)))) {
621 if (setShapeInfo(MatrixA, {M, N}))
622 pushInstruction(MatrixA, WorkList);
623
624 if (setShapeInfo(MatrixB, {N, K}))
625 pushInstruction(MatrixB, WorkList);
626
627 } else if (match(V, m_Intrinsic<Intrinsic::matrix_transpose>(
628 m_Value(MatrixA), m_Value(M), m_Value(N)))) {
629 // Flip dimensions.
630 if (setShapeInfo(MatrixA, {M, N}))
631 pushInstruction(MatrixA, WorkList);
632 } else if (match(V, m_Intrinsic<Intrinsic::matrix_column_major_store>(
633 m_Value(MatrixA), m_Value(), m_Value(), m_Value(),
634 m_Value(M), m_Value(N)))) {
635 if (setShapeInfo(MatrixA, {M, N})) {
636 pushInstruction(MatrixA, WorkList);
637 }
638 } else if (isa<LoadInst>(V) ||
639 match(V, m_Intrinsic<Intrinsic::matrix_column_major_load>())) {
640 // Nothing to do, no matrix input.
641 } else if (isa<StoreInst>(V)) {
642 // Nothing to do. We forward-propagated to this so we would just
643 // backward propagate to an instruction with an already known shape.
644 } else if (isUniformShape(V)) {
645 // Propagate to all operands.
646 ShapeInfo Shape = ShapeMap[V];
647 for (Use &U : cast<Instruction>(V)->operands()) {
648 if (setShapeInfo(U.get(), Shape))
649 pushInstruction(U.get(), WorkList);
650 }
651 }
652 // After we discovered new shape info for new instructions in the
653 // worklist, we use their users as seeds for the next round of forward
654 // propagation.
655 for (size_t I = BeforeProcessingV; I != WorkList.size(); I++)
656 for (User *U : WorkList[I]->users())
657 if (isa<Instruction>(U) && V != U)
658 NewWorkList.push_back(cast<Instruction>(U));
659 }
660 return NewWorkList;
661 }
662
663 bool Visit() {
664 if (EnableShapePropagation) {
2
Assuming the condition is false
3
Taking false branch
665 SmallVector<Instruction *, 32> WorkList;
666
667 // Initially only the shape of matrix intrinsics is known.
668 // Initialize the work list with ops carrying shape information.
669 for (BasicBlock &BB : Func)
670 for (Instruction &Inst : BB) {
671 IntrinsicInst *II = dyn_cast<IntrinsicInst>(&Inst);
672 if (!II)
673 continue;
674
675 switch (II->getIntrinsicID()) {
676 case Intrinsic::matrix_multiply:
677 case Intrinsic::matrix_transpose:
678 case Intrinsic::matrix_column_major_load:
679 case Intrinsic::matrix_column_major_store:
680 WorkList.push_back(&Inst);
681 break;
682 default:
683 break;
684 }
685 }
686 // Propagate shapes until nothing changes any longer.
687 while (!WorkList.empty()) {
688 WorkList = propagateShapeForward(WorkList);
689 WorkList = propagateShapeBackward(WorkList);
690 }
691 }
692
693 bool Changed = false;
694 SmallVector<CallInst *, 16> MaybeFusableInsts;
695 SmallVector<Instruction *, 16> MatrixInsts;
696
697 // First, collect all instructions with shape information and candidates for
698 // fusion (currently only matrix multiplies).
699 ReversePostOrderTraversal<Function *> RPOT(&Func);
700 for (auto *BB : RPOT)
701 for (Instruction &I : *BB) {
702 if (ShapeMap.find(&I) == ShapeMap.end())
703 continue;
704 if (match(&I, m_Intrinsic<Intrinsic::matrix_multiply>()))
705 MaybeFusableInsts.push_back(cast<CallInst>(&I));
706 MatrixInsts.push_back(&I);
707 }
708
709 // Second, try to fuse candidates.
710 SmallPtrSet<Instruction *, 16> FusedInsts;
711 for (CallInst *CI : MaybeFusableInsts)
4
Assuming '__begin2' is equal to '__end2'
712 LowerMatrixMultiplyFused(CI, FusedInsts);
713 Changed = !FusedInsts.empty();
714
715 // Third, lower remaining instructions with shape information.
716 for (Instruction *Inst : MatrixInsts) {
5
Assuming '__begin2' is not equal to '__end2'
717 if (FusedInsts.count(Inst))
6
Assuming the condition is false
7
Taking false branch
718 continue;
719
720 IRBuilder<> Builder(Inst);
721
722 if (CallInst *CInst
8.1
'CInst' is non-null
= dyn_cast<CallInst>(Inst))
8
Assuming 'Inst' is a 'CallInst'
9
Taking true branch
723 Changed |= VisitCallInst(CInst);
10
Calling 'LowerMatrixIntrinsics::VisitCallInst'
724
725 Value *Op1;
726 Value *Op2;
727 if (auto *BinOp = dyn_cast<BinaryOperator>(Inst))
728 Changed |= VisitBinaryOperator(BinOp);
729 if (match(Inst, m_Load(m_Value(Op1))))
730 Changed |= VisitLoad(cast<LoadInst>(Inst), Op1, Builder);
731 else if (match(Inst, m_Store(m_Value(Op1), m_Value(Op2))))
732 Changed |= VisitStore(cast<StoreInst>(Inst), Op1, Op2, Builder);
733 }
734
735 if (ORE) {
736 RemarkGenerator RemarkGen(Inst2ColumnMatrix, *ORE, Func);
737 RemarkGen.emitRemarks();
738 }
739
740 for (Instruction *Inst : reverse(ToRemove))
741 Inst->eraseFromParent();
742
743 return Changed;
744 }
745
746 /// Turns \p BasePtr into an elementwise pointer to \p EltType.
747 Value *createElementPtr(Value *BasePtr, Type *EltType, IRBuilder<> &Builder) {
748 unsigned AS = cast<PointerType>(BasePtr->getType())->getAddressSpace();
749 Type *EltPtrType = PointerType::get(EltType, AS);
750 return Builder.CreatePointerCast(BasePtr, EltPtrType);
751 }
752
753 /// Replace intrinsic calls
754 bool VisitCallInst(CallInst *Inst) {
755 if (!Inst->getCalledFunction() || !Inst->getCalledFunction()->isIntrinsic())
11
Assuming the condition is false
12
Taking false branch
756 return false;
757
758 switch (Inst->getCalledFunction()->getIntrinsicID()) {
13
Control jumps to 'case matrix_multiply:' at line 759
759 case Intrinsic::matrix_multiply:
760 LowerMultiply(Inst);
14
Calling 'LowerMatrixIntrinsics::LowerMultiply'
761 break;
762 case Intrinsic::matrix_transpose:
763 LowerTranspose(Inst);
764 break;
765 case Intrinsic::matrix_column_major_load:
766 LowerColumnMajorLoad(Inst);
767 break;
768 case Intrinsic::matrix_column_major_store:
769 LowerColumnMajorStore(Inst);
770 break;
771 default:
772 return false;
773 }
774 return true;
775 }
776
777 /// Compute the alignment for a column/row \p Idx with \p Stride between them.
778 /// The address at \p Idx == 0 has alignment \p A. If \p Stride is a
779 /// ConstantInt, reduce the initial alignment based on the byte offset. For
780 /// non-ConstantInt strides, return the common alignment of the initial
781 /// alignment and the element size in bytes.
782 Align getAlignForIndex(unsigned Idx, Value *Stride, Type *ElementTy,
783 MaybeAlign A) const {
784 Align InitialAlign = DL.getValueOrABITypeAlignment(A, ElementTy);
785 if (Idx == 0)
786 return InitialAlign;
787
788 TypeSize ElementSizeInBits = DL.getTypeSizeInBits(ElementTy);
789 if (auto *ConstStride = dyn_cast<ConstantInt>(Stride)) {
790 uint64_t StrideInBytes =
791 ConstStride->getZExtValue() * ElementSizeInBits / 8;
792 return commonAlignment(InitialAlign, Idx * StrideInBytes);
793 }
794 return commonAlignment(InitialAlign, ElementSizeInBits / 8);
795 }
796
797 /// Load a matrix with \p Shape starting at \p Ptr and using \p Stride between
798 /// vectors.
799 MatrixTy loadMatrix(Type *Ty, Value *Ptr, MaybeAlign MAlign, Value *Stride,
800 bool IsVolatile, ShapeInfo Shape, IRBuilder<> &Builder) {
801 auto VType = cast<VectorType>(Ty);
802 Value *EltPtr = createElementPtr(Ptr, VType->getElementType(), Builder);
803 MatrixTy Result;
804 for (unsigned I = 0, E = Shape.getNumVectors(); I < E; ++I) {
805 Value *GEP = computeVectorAddr(EltPtr, Builder.getInt64(I), Stride,
806 Shape.getStride(), VType->getElementType(),
807 Builder);
808 Value *Vector = Builder.CreateAlignedLoad(
809 GEP, getAlignForIndex(I, Stride, VType->getElementType(), MAlign),
810 IsVolatile, "col.load");
811
812 Result.addVector(Vector);
813 }
814 return Result.addNumLoads(getNumOps(Result.getVectorTy()) *
815 Result.getNumVectors());
816 }
817
818 /// Loads a sub-matrix with shape \p ResultShape from a \p R x \p C matrix,
819 /// starting at \p MatrixPtr[I][J].
820 MatrixTy loadMatrix(Value *MatrixPtr, MaybeAlign Align, bool IsVolatile,
821 ShapeInfo MatrixShape, Value *I, Value *J,
822 ShapeInfo ResultShape, Type *EltTy,
823 IRBuilder<> &Builder) {
824
825 Value *Offset = Builder.CreateAdd(
826 Builder.CreateMul(J, Builder.getInt64(MatrixShape.getStride())), I);
827
828 unsigned AS = cast<PointerType>(MatrixPtr->getType())->getAddressSpace();
829 Value *EltPtr =
830 Builder.CreatePointerCast(MatrixPtr, PointerType::get(EltTy, AS));
831 Value *TileStart = Builder.CreateGEP(EltTy, EltPtr, Offset);
832 auto *TileTy = FixedVectorType::get(EltTy, ResultShape.NumRows *
833 ResultShape.NumColumns);
834 Type *TilePtrTy = PointerType::get(TileTy, AS);
835 Value *TilePtr =
836 Builder.CreatePointerCast(TileStart, TilePtrTy, "col.cast");
837
838 return loadMatrix(TileTy, TilePtr, Align,
839 Builder.getInt64(MatrixShape.getStride()), IsVolatile,
840 ResultShape, Builder);
841 }
842
843 /// Lower a load instruction with shape information.
844 void LowerLoad(Instruction *Inst, Value *Ptr, MaybeAlign Align, Value *Stride,
845 bool IsVolatile, ShapeInfo Shape) {
846 IRBuilder<> Builder(Inst);
847 finalizeLowering(Inst,
848 loadMatrix(Inst->getType(), Ptr, Align, Stride, IsVolatile,
849 Shape, Builder),
850 Builder);
851 }
852
853 /// Lowers llvm.matrix.column.major.load.
854 ///
855 /// The intrinsic loads a matrix from memory using a stride between columns.
856 void LowerColumnMajorLoad(CallInst *Inst) {
857 assert(MatrixLayout == MatrixLayoutTy::ColumnMajor &&((MatrixLayout == MatrixLayoutTy::ColumnMajor && "Intrinsic only supports column-major layout!"
) ? static_cast<void> (0) : __assert_fail ("MatrixLayout == MatrixLayoutTy::ColumnMajor && \"Intrinsic only supports column-major layout!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 858, __PRETTY_FUNCTION__))
858 "Intrinsic only supports column-major layout!")((MatrixLayout == MatrixLayoutTy::ColumnMajor && "Intrinsic only supports column-major layout!"
) ? static_cast<void> (0) : __assert_fail ("MatrixLayout == MatrixLayoutTy::ColumnMajor && \"Intrinsic only supports column-major layout!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 858, __PRETTY_FUNCTION__))
;
859 Value *Ptr = Inst->getArgOperand(0);
860 Value *Stride = Inst->getArgOperand(1);
861 LowerLoad(Inst, Ptr, Inst->getParamAlign(0), Stride,
862 cast<ConstantInt>(Inst->getArgOperand(2))->isOne(),
863 {Inst->getArgOperand(3), Inst->getArgOperand(4)});
864 }
865
866 /// Stores a sub-matrix \p StoreVal into the \p R x \p C matrix starting at \p
867 /// MatrixPtr[I][J].
868 void storeMatrix(const MatrixTy &StoreVal, Value *MatrixPtr,
869 MaybeAlign MAlign, bool IsVolatile, ShapeInfo MatrixShape,
870 Value *I, Value *J, Type *EltTy, IRBuilder<> &Builder) {
871 Value *Offset = Builder.CreateAdd(
872 Builder.CreateMul(J, Builder.getInt64(MatrixShape.getStride())), I);
873
874 unsigned AS = cast<PointerType>(MatrixPtr->getType())->getAddressSpace();
875 Value *EltPtr =
876 Builder.CreatePointerCast(MatrixPtr, PointerType::get(EltTy, AS));
877 Value *TileStart = Builder.CreateGEP(EltTy, EltPtr, Offset);
878 auto *TileTy = FixedVectorType::get(EltTy, StoreVal.getNumRows() *
879 StoreVal.getNumColumns());
880 Type *TilePtrTy = PointerType::get(TileTy, AS);
881 Value *TilePtr =
882 Builder.CreatePointerCast(TileStart, TilePtrTy, "col.cast");
883
884 storeMatrix(TileTy, StoreVal, TilePtr, MAlign,
885 Builder.getInt64(MatrixShape.getStride()), IsVolatile, Builder);
886 }
887
888 /// Store matrix \p StoreVal starting at \p Ptr and using \p Stride between
889 /// vectors.
890 MatrixTy storeMatrix(Type *Ty, MatrixTy StoreVal, Value *Ptr,
891 MaybeAlign MAlign, Value *Stride, bool IsVolatile,
892 IRBuilder<> &Builder) {
893 auto VType = cast<VectorType>(Ty);
894 Value *EltPtr = createElementPtr(Ptr, VType->getElementType(), Builder);
895 for (auto Vec : enumerate(StoreVal.vectors())) {
896 Value *GEP = computeVectorAddr(EltPtr, Builder.getInt64(Vec.index()),
897 Stride, StoreVal.getStride(),
898 VType->getElementType(), Builder);
899 Builder.CreateAlignedStore(Vec.value(), GEP,
900 getAlignForIndex(Vec.index(), Stride,
901 VType->getElementType(),
902 MAlign),
903 IsVolatile);
904 }
905 return MatrixTy().addNumStores(getNumOps(StoreVal.getVectorTy()) *
906 StoreVal.getNumVectors());
907 }
908
909 /// Lower a store instruction with shape information.
910 void LowerStore(Instruction *Inst, Value *Matrix, Value *Ptr, MaybeAlign A,
911 Value *Stride, bool IsVolatile, ShapeInfo Shape) {
912 IRBuilder<> Builder(Inst);
913 auto StoreVal = getMatrix(Matrix, Shape, Builder);
914 finalizeLowering(Inst,
915 storeMatrix(Matrix->getType(), StoreVal, Ptr, A, Stride,
916 IsVolatile, Builder),
917 Builder);
918 }
919
920 /// Lowers llvm.matrix.column.major.store.
921 ///
922 /// The intrinsic store a matrix back memory using a stride between columns.
923 void LowerColumnMajorStore(CallInst *Inst) {
924 assert(MatrixLayout == MatrixLayoutTy::ColumnMajor &&((MatrixLayout == MatrixLayoutTy::ColumnMajor && "Intrinsic only supports column-major layout!"
) ? static_cast<void> (0) : __assert_fail ("MatrixLayout == MatrixLayoutTy::ColumnMajor && \"Intrinsic only supports column-major layout!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 925, __PRETTY_FUNCTION__))
925 "Intrinsic only supports column-major layout!")((MatrixLayout == MatrixLayoutTy::ColumnMajor && "Intrinsic only supports column-major layout!"
) ? static_cast<void> (0) : __assert_fail ("MatrixLayout == MatrixLayoutTy::ColumnMajor && \"Intrinsic only supports column-major layout!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 925, __PRETTY_FUNCTION__))
;
926 Value *Matrix = Inst->getArgOperand(0);
927 Value *Ptr = Inst->getArgOperand(1);
928 Value *Stride = Inst->getArgOperand(2);
929 LowerStore(Inst, Matrix, Ptr, Inst->getParamAlign(1), Stride,
930 cast<ConstantInt>(Inst->getArgOperand(3))->isOne(),
931 {Inst->getArgOperand(4), Inst->getArgOperand(5)});
932 }
933
934 // Set elements I..I+NumElts-1 to Block
935 Value *insertVector(Value *Col, unsigned I, Value *Block,
936 IRBuilder<> &Builder) {
937
938 // First, bring Block to the same size as Col
939 unsigned BlockNumElts =
940 cast<FixedVectorType>(Block->getType())->getNumElements();
41
Called C++ object pointer is null
941 unsigned NumElts = cast<FixedVectorType>(Col->getType())->getNumElements();
942 assert(NumElts >= BlockNumElts && "Too few elements for current block")((NumElts >= BlockNumElts && "Too few elements for current block"
) ? static_cast<void> (0) : __assert_fail ("NumElts >= BlockNumElts && \"Too few elements for current block\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 942, __PRETTY_FUNCTION__))
;
943
944 Value *Undef = UndefValue::get(Block->getType());
945 Block = Builder.CreateShuffleVector(
946 Block, Undef,
947 createSequentialMask(0, BlockNumElts, NumElts - BlockNumElts));
948
949 // If Col is 7 long and I is 2 and BlockNumElts is 2 the mask is: 0, 1, 7,
950 // 8, 4, 5, 6
951 SmallVector<int, 16> Mask;
952 unsigned i;
953 for (i = 0; i < I; i++)
954 Mask.push_back(i);
955
956 unsigned VecNumElts =
957 cast<FixedVectorType>(Col->getType())->getNumElements();
958 for (; i < I + BlockNumElts; i++)
959 Mask.push_back(i - I + VecNumElts);
960
961 for (; i < VecNumElts; i++)
962 Mask.push_back(i);
963
964 return Builder.CreateShuffleVector(Col, Block, Mask);
965 }
966
967 Value *createMulAdd(Value *Sum, Value *A, Value *B, bool UseFPOp,
968 IRBuilder<> &Builder, bool AllowContraction,
969 unsigned &NumComputeOps) {
970 NumComputeOps += getNumOps(A->getType());
971 if (!Sum)
972 return UseFPOp ? Builder.CreateFMul(A, B) : Builder.CreateMul(A, B);
973
974 if (UseFPOp) {
975 if (AllowContraction) {
976 // Use fmuladd for floating point operations and let the backend decide
977 // if that's profitable.
978 Function *FMulAdd = Intrinsic::getDeclaration(
979 Func.getParent(), Intrinsic::fmuladd, A->getType());
980 return Builder.CreateCall(FMulAdd, {A, B, Sum});
981 }
982 NumComputeOps += getNumOps(A->getType());
983 Value *Mul = Builder.CreateFMul(A, B);
984 return Builder.CreateFAdd(Sum, Mul);
985 }
986
987 NumComputeOps += getNumOps(A->getType());
988 Value *Mul = Builder.CreateMul(A, B);
989 return Builder.CreateAdd(Sum, Mul);
990 }
991
992 /// Cache \p Matrix as result of \p Inst and update the uses of \p Inst. For
993 /// users with shape information, there's nothing to do: the will use the
994 /// cached value when they are lowered. For other users, \p Matrix is
995 /// flattened and the uses are updated to use it. Also marks \p Inst for
996 /// deletion.
997 void finalizeLowering(Instruction *Inst, MatrixTy Matrix,
998 IRBuilder<> &Builder) {
999 Inst2ColumnMatrix.insert(std::make_pair(Inst, Matrix));
1000
1001 ToRemove.push_back(Inst);
1002 Value *Flattened = nullptr;
1003 for (auto I = Inst->use_begin(), E = Inst->use_end(); I != E;) {
1004 Use &U = *I++;
1005 if (ShapeMap.find(U.getUser()) == ShapeMap.end()) {
1006 if (!Flattened)
1007 Flattened = Matrix.embedInVector(Builder);
1008 U.set(Flattened);
1009 }
1010 }
1011 }
1012
1013 /// Compute \p Result += \p A * \p B for input matrices with left-associating
1014 /// addition.
1015 void emitMatrixMultiply(MatrixTy &Result, const MatrixTy &A,
1016 const MatrixTy &B, bool AllowContraction,
1017 IRBuilder<> &Builder, bool isTiled) {
1018 const unsigned VF = std::max<unsigned>(
1019 TTI.getRegisterBitWidth(true) /
1020 Result.getElementType()->getPrimitiveSizeInBits().getFixedSize(),
1021 1U);
1022 unsigned R = Result.getNumRows();
1023 unsigned C = Result.getNumColumns();
1024 unsigned M = A.getNumColumns();
1025
1026 bool IsFP = Result.getElementType()->isFloatingPointTy();
1027 assert(A.isColumnMajor() == B.isColumnMajor() &&((A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor
() == A.isColumnMajor() && "operands must agree on matrix layout"
) ? static_cast<void> (0) : __assert_fail ("A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor() == A.isColumnMajor() && \"operands must agree on matrix layout\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1029, __PRETTY_FUNCTION__))
25
'?' condition is true
1028 Result.isColumnMajor() == A.isColumnMajor() &&((A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor
() == A.isColumnMajor() && "operands must agree on matrix layout"
) ? static_cast<void> (0) : __assert_fail ("A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor() == A.isColumnMajor() && \"operands must agree on matrix layout\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1029, __PRETTY_FUNCTION__))
1029 "operands must agree on matrix layout")((A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor
() == A.isColumnMajor() && "operands must agree on matrix layout"
) ? static_cast<void> (0) : __assert_fail ("A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor() == A.isColumnMajor() && \"operands must agree on matrix layout\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1029, __PRETTY_FUNCTION__))
;
1030 unsigned NumComputeOps = 0;
1031 if (A.isColumnMajor()) {
26
Calling 'MatrixTy::isColumnMajor'
28
Returning from 'MatrixTy::isColumnMajor'
29
Taking false branch
1032 // Multiply columns from the first operand with scalars from the second
1033 // operand. Then move along the K axes and accumulate the columns. With
1034 // this the adds can be vectorized without reassociation.
1035 for (unsigned J = 0; J < C; ++J) {
1036 unsigned BlockSize = VF;
1037 // If Result is zero, we don't need to accumulate in the K==0 iteration.
1038 bool isSumZero = isa<ConstantAggregateZero>(Result.getColumn(J));
1039
1040 for (unsigned I = 0; I < R; I += BlockSize) {
1041 // Gradually lower the vectorization factor to cover the remainder.
1042 while (I + BlockSize > R)
1043 BlockSize /= 2;
1044
1045 Value *Sum = isTiled ? Result.extractVector(I, J, BlockSize, Builder)
1046 : nullptr;
1047 for (unsigned K = 0; K < M; ++K) {
1048 Value *L = A.extractVector(I, K, BlockSize, Builder);
1049 Value *RH = Builder.CreateExtractElement(B.getColumn(J), K);
1050 Value *Splat = Builder.CreateVectorSplat(BlockSize, RH, "splat");
1051 Sum = createMulAdd(isSumZero && K == 0 ? nullptr : Sum, L, Splat,
1052 Result.getElementType()->isFloatingPointTy(),
1053 Builder, AllowContraction, NumComputeOps);
1054 }
1055 Result.setVector(J,
1056 insertVector(Result.getVector(J), I, Sum, Builder));
1057 }
1058 }
1059 } else {
1060 // Multiply rows from the second operand with scalars from the first
1061 // operand. Then move along the K axes and accumulate the rows. With this
1062 // the adds can be vectorized without reassociation.
1063 for (unsigned I = 0; I
29.1
'I' is < 'R'
< R; ++I) {
30
Loop condition is true. Entering loop body
1064 unsigned BlockSize = VF;
1065 bool isSumZero = isa<ConstantAggregateZero>(Result.getRow(I));
31
Assuming the object is not a 'ConstantAggregateZero'
1066 for (unsigned J = 0; J < C; J += BlockSize) {
32
Assuming 'J' is < 'C'
33
Loop condition is true. Entering loop body
1067 // Gradually lower the vectorization factor to cover the remainder.
1068 while (J + BlockSize > C)
34
Assuming the condition is false
35
Loop condition is false. Execution continues on line 1071
1069 BlockSize /= 2;
1070
1071 Value *Sum = nullptr;
36
'Sum' initialized to a null pointer value
1072 for (unsigned K = 0; K < M; ++K) {
37
Assuming 'K' is >= 'M'
38
Loop condition is false. Execution continues on line 1079
1073 Value *R = B.extractVector(K, J, BlockSize, Builder);
1074 Value *LH = Builder.CreateExtractElement(A.getVector(I), K);
1075 Value *Splat = Builder.CreateVectorSplat(BlockSize, LH, "splat");
1076 Sum = createMulAdd(isSumZero && K == 0 ? nullptr : Sum, Splat, R,
1077 IsFP, Builder, AllowContraction, NumComputeOps);
1078 }
1079 Result.setVector(I,
1080 insertVector(Result.getVector(I), J, Sum, Builder));
39
Passing null pointer value via 3rd parameter 'Block'
40
Calling 'LowerMatrixIntrinsics::insertVector'
1081 }
1082 }
1083 }
1084 Result.addNumComputeOps(NumComputeOps);
1085 }
1086
1087 /// Ensure that the memory in \p Load does not alias \p Store by potentially
1088 /// copying it to a new location. This new or otherwise the original location
1089 /// is returned.
1090 Value *getNonAliasingPointer(LoadInst *Load, StoreInst *Store,
1091 CallInst *MatMul) {
1092 MemoryLocation StoreLoc = MemoryLocation::get(Store);
1093 MemoryLocation LoadLoc = MemoryLocation::get(Load);
1094
1095 AliasResult LdAliased = AA->alias(LoadLoc, StoreLoc);
1096
1097 // If we can statically determine noalias we're good.
1098 if (!LdAliased)
1099 return Load->getPointerOperand();
1100
1101 // Create code to check if the memory locations of the Load and Store
1102 // overlap and if they do, copy Load's operand to a new buffer.
1103
1104 // First, create new blocks for 2n part of the check and the copy.
1105 BasicBlock *Check0 = MatMul->getParent();
1106 // FIXME: Use lazy DTU and update SplitBlock to accept a DTU instead of a
1107 // DT. Manually collect dominator tree updates, to avoid unnecessary work,
1108 // as we adjust Check0 and Check1's branches.
1109 SmallVector<DominatorTree::UpdateType, 4> DTUpdates;
1110 for (BasicBlock *Succ : successors(Check0))
1111 DTUpdates.push_back({DT->Delete, Check0, Succ});
1112
1113 BasicBlock *Check1 = SplitBlock(MatMul->getParent(), MatMul, nullptr, LI,
1114 nullptr, "alias_cont");
1115 BasicBlock *Copy =
1116 SplitBlock(MatMul->getParent(), MatMul, nullptr, LI, nullptr, "copy");
1117 BasicBlock *Fusion = SplitBlock(MatMul->getParent(), MatMul, nullptr, LI,
1118 nullptr, "no_alias");
1119
1120 // Check if the loaded memory location begins before the end of the store
1121 // location. If the condition holds, they might overlap, otherwise they are
1122 // guaranteed to not overlap.
1123 IRBuilder<> Builder(MatMul);
1124 Check0->getTerminator()->eraseFromParent();
1125 Builder.SetInsertPoint(Check0);
1126 Type *IntPtrTy = Builder.getIntPtrTy(Load->getModule()->getDataLayout());
1127 Value *StoreBegin = Builder.CreatePtrToInt(
1128 const_cast<Value *>(StoreLoc.Ptr), IntPtrTy, "store.begin");
1129 Value *StoreEnd = Builder.CreateAdd(
1130 StoreBegin, ConstantInt::get(IntPtrTy, StoreLoc.Size.getValue()),
1131 "store.end", true, true);
1132 Value *LoadBegin = Builder.CreatePtrToInt(const_cast<Value *>(LoadLoc.Ptr),
1133 IntPtrTy, "load.begin");
1134 Builder.CreateCondBr(Builder.CreateICmpULT(LoadBegin, StoreEnd), Check1,
1135 Fusion);
1136
1137 // Check if the store begins before the end of the load location. If the
1138 // condition holds, they alias, otherwise they are guaranteed to not
1139 // overlap.
1140 Check1->getTerminator()->eraseFromParent();
1141 Builder.SetInsertPoint(Check1, Check1->begin());
1142 Value *LoadEnd = Builder.CreateAdd(
1143 LoadBegin, ConstantInt::get(IntPtrTy, LoadLoc.Size.getValue()),
1144 "load.end", true, true);
1145 Builder.CreateCondBr(Builder.CreateICmpULT(StoreBegin, LoadEnd), Copy,
1146 Fusion);
1147
1148 // Copy load operand to new alloca.
1149 Builder.SetInsertPoint(Copy, Copy->begin());
1150 AllocaInst *NewLd =
1151 Builder.CreateAlloca(Load->getType(), Load->getPointerAddressSpace());
1152 Builder.CreateMemCpy(NewLd, NewLd->getAlign(),
1153 Load->getPointerOperand(), Load->getAlign(),
1154 LoadLoc.Size.getValue());
1155 Builder.SetInsertPoint(Fusion, Fusion->begin());
1156 PHINode *PHI = Builder.CreatePHI(Load->getPointerOperandType(), 3);
1157 PHI->addIncoming(Load->getPointerOperand(), Check0);
1158 PHI->addIncoming(Load->getPointerOperand(), Check1);
1159 PHI->addIncoming(NewLd, Copy);
1160
1161 // Adjust DT.
1162 DTUpdates.push_back({DT->Insert, Check0, Check1});
1163 DTUpdates.push_back({DT->Insert, Check0, Fusion});
1164 DTUpdates.push_back({DT->Insert, Check1, Copy});
1165 DTUpdates.push_back({DT->Insert, Check1, Fusion});
1166 DT->applyUpdates(DTUpdates);
1167 return PHI;
1168 }
1169
1170 bool isFusionProfitable(CallInst *MatMul) {
1171 if (ForceFusion)
1172 return true;
1173
1174 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3));
1175 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4));
1176
1177 const unsigned R = LShape.NumRows;
1178 const unsigned C = RShape.NumColumns;
1179 const unsigned M = LShape.NumColumns;
1180 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
1181
1182 const unsigned VF =
1183 std::max<unsigned>(TTI.getRegisterBitWidth(true) /
1184 EltType->getPrimitiveSizeInBits().getFixedSize(),
1185 1U);
1186
1187 // Cost model for tiling
1188 //
1189 // For tiling to be beneficial, we need reuse either along the R or
1190 // the C axis. We vectorize along the R axis so that means at least
1191 // 3 elements.
1192 // TODO: Also consider cost of copying if operands alias.
1193 if (R <= VF && C == 1)
1194 return false;
1195 // Then we need enough elements to exceed the number of vector
1196 // registers we have. Note that this is an oversimplification since
1197 // fusing also takes some extra loads which may exceed the number of
1198 // reloads necessary.
1199 unsigned Op0Regs = (R + VF - 1) / VF * M;
1200 unsigned Op1Regs = (M + VF - 1) / VF * C;
1201 return Op0Regs + Op1Regs > TTI.getNumberOfRegisters(true);
1202 }
1203
1204 MatrixTy getZeroMatrix(Type *EltType, unsigned R, unsigned C) {
1205 MatrixTy Res;
1206 auto *ColumType = FixedVectorType::get(EltType, R);
1207 for (unsigned I = 0; I < C; ++I)
1208 Res.addVector(ConstantAggregateZero::get(ColumType));
1209 return Res;
1210 }
1211
1212 void createTiledLoops(CallInst *MatMul, Value *LPtr, ShapeInfo LShape,
1213 Value *RPtr, ShapeInfo RShape, StoreInst *Store,
1214 bool AllowContract) {
1215 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
1216
1217 // Create the main tiling loop nest.
1218 TileInfo TI(LShape.NumRows, RShape.NumColumns, LShape.NumColumns, TileSize);
1219 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
1220 Instruction *InsertI = cast<Instruction>(MatMul);
1221 BasicBlock *Start = InsertI->getParent();
1222 BasicBlock *End =
1223 SplitBlock(InsertI->getParent(), InsertI, DT, LI, nullptr, "continue");
1224 IRBuilder<> Builder(MatMul);
1225 BasicBlock *InnerBody = TI.CreateTiledLoops(Start, End, Builder, DTU, *LI);
1226
1227 Type *TileVecTy =
1228 FixedVectorType::get(MatMul->getType()->getScalarType(), TileSize);
1229 MatrixTy TileResult;
1230 // Insert in the inner loop header.
1231 Builder.SetInsertPoint(TI.InnerLoopHeader->getTerminator());
1232 // Create PHI nodes for the result columns to accumulate across iterations.
1233 SmallVector<PHINode *, 4> ColumnPhis;
1234 for (unsigned I = 0; I < TileSize; I++) {
1235 auto *Phi = Builder.CreatePHI(TileVecTy, 2, "result.vec." + Twine(I));
1236 Phi->addIncoming(ConstantAggregateZero::get(TileVecTy),
1237 TI.RowLoopHeader->getSingleSuccessor());
1238 TileResult.addVector(Phi);
1239 ColumnPhis.push_back(Phi);
1240 }
1241
1242 // Insert in the inner loop body, which computes
1243 // Res += Load(CurrentRow, K) * Load(K, CurrentColumn)
1244 Builder.SetInsertPoint(InnerBody->getTerminator());
1245 // Load tiles of the operands.
1246 MatrixTy A = loadMatrix(LPtr, {}, false, LShape, TI.CurrentRow, TI.CurrentK,
1247 {TileSize, TileSize}, EltType, Builder);
1248 MatrixTy B = loadMatrix(RPtr, {}, false, RShape, TI.CurrentK, TI.CurrentCol,
1249 {TileSize, TileSize}, EltType, Builder);
1250 emitMatrixMultiply(TileResult, A, B, AllowContract, Builder, true);
1251 // Store result after the inner loop is done.
1252 Builder.SetInsertPoint(TI.RowLoopLatch->getTerminator());
1253 storeMatrix(TileResult, Store->getPointerOperand(), Store->getAlign(),
1254 Store->isVolatile(), {LShape.NumRows, RShape.NumColumns},
1255 TI.CurrentRow, TI.CurrentCol, EltType, Builder);
1256
1257 for (unsigned I = 0; I < TileResult.getNumVectors(); I++)
1258 ColumnPhis[I]->addIncoming(TileResult.getVector(I), TI.InnerLoopLatch);
1259
1260 // Force unrolling of a few iterations of the inner loop, to make sure there
1261 // is enough work per iteration.
1262 // FIXME: The unroller should make this decision directly instead, but
1263 // currently the cost-model is not up to the task.
1264 unsigned InnerLoopUnrollCount = std::min(10u, LShape.NumColumns / TileSize);
1265 addStringMetadataToLoop(LI->getLoopFor(TI.InnerLoopHeader),
1266 "llvm.loop.unroll.count", InnerLoopUnrollCount);
1267 }
1268
1269 void emitSIMDTiling(CallInst *MatMul, LoadInst *LoadOp0, LoadInst *LoadOp1,
1270 StoreInst *Store,
1271 SmallPtrSetImpl<Instruction *> &FusedInsts) {
1272 assert(MatrixLayout == MatrixLayoutTy::ColumnMajor &&((MatrixLayout == MatrixLayoutTy::ColumnMajor && "Tiling only supported for column-major matrixes at the moment!"
) ? static_cast<void> (0) : __assert_fail ("MatrixLayout == MatrixLayoutTy::ColumnMajor && \"Tiling only supported for column-major matrixes at the moment!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1273, __PRETTY_FUNCTION__))
1273 "Tiling only supported for column-major matrixes at the moment!")((MatrixLayout == MatrixLayoutTy::ColumnMajor && "Tiling only supported for column-major matrixes at the moment!"
) ? static_cast<void> (0) : __assert_fail ("MatrixLayout == MatrixLayoutTy::ColumnMajor && \"Tiling only supported for column-major matrixes at the moment!\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1273, __PRETTY_FUNCTION__))
;
1274 if (!isFusionProfitable(MatMul))
1275 return;
1276
1277 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3));
1278 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4));
1279
1280 const unsigned R = LShape.NumRows;
1281 const unsigned C = RShape.NumColumns;
1282 const unsigned M = LShape.NumColumns;
1283 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
1284
1285 Value *APtr = getNonAliasingPointer(LoadOp0, Store, MatMul);
1286 Value *BPtr = getNonAliasingPointer(LoadOp1, Store, MatMul);
1287 Value *CPtr = Store->getPointerOperand();
1288
1289 bool AllowContract = AllowContractEnabled || (isa<FPMathOperator>(MatMul) &&
1290 MatMul->hasAllowContract());
1291 if (TileUseLoops && (R % TileSize == 0 && C % TileSize == 0))
1292 createTiledLoops(MatMul, APtr, LShape, BPtr, RShape, Store,
1293 AllowContract);
1294 else {
1295 IRBuilder<> Builder(Store);
1296 for (unsigned J = 0; J < C; J += TileSize)
1297 for (unsigned I = 0; I < R; I += TileSize) {
1298 const unsigned TileR = std::min(R - I, unsigned(TileSize));
1299 const unsigned TileC = std::min(C - J, unsigned(TileSize));
1300 MatrixTy Res = getZeroMatrix(EltType, TileR, TileC);
1301
1302 for (unsigned K = 0; K < M; K += TileSize) {
1303 const unsigned TileM = std::min(M - K, unsigned(TileSize));
1304 MatrixTy A =
1305 loadMatrix(APtr, LoadOp0->getAlign(), LoadOp0->isVolatile(),
1306 LShape, Builder.getInt64(I), Builder.getInt64(K),
1307 {TileR, TileM}, EltType, Builder);
1308 MatrixTy B =
1309 loadMatrix(BPtr, LoadOp1->getAlign(), LoadOp1->isVolatile(),
1310 RShape, Builder.getInt64(K), Builder.getInt64(J),
1311 {TileM, TileC}, EltType, Builder);
1312 emitMatrixMultiply(Res, A, B, AllowContract, Builder, true);
1313 }
1314 storeMatrix(Res, CPtr, Store->getAlign(), Store->isVolatile(), {R, M},
1315 Builder.getInt64(I), Builder.getInt64(J), EltType,
1316 Builder);
1317 }
1318 }
1319
1320 // Mark eliminated instructions as fused and remove them.
1321 FusedInsts.insert(Store);
1322 FusedInsts.insert(MatMul);
1323 Store->eraseFromParent();
1324 MatMul->eraseFromParent();
1325 if (LoadOp0->hasNUses(0)) {
1326 FusedInsts.insert(LoadOp0);
1327 LoadOp0->eraseFromParent();
1328 }
1329 if (LoadOp1->hasNUses(0)) {
1330 FusedInsts.insert(LoadOp1);
1331 LoadOp1->eraseFromParent();
1332 }
1333 }
1334
1335 /// Try to lower matrix multiply chains by fusing operations.
1336 ///
1337 /// Currently we only lower {ld, ld} -> matmul -> st chains.
1338 //
1339 /// No need to return a MatrixTy object for the result of the operation, since
1340 /// the single store user will be lowered as part of this. Instructions that
1341 /// are completely eliminated by fusion are added to \p FusedInsts.
1342 void LowerMatrixMultiplyFused(CallInst *MatMul,
1343 SmallPtrSetImpl<Instruction *> &FusedInsts) {
1344 if (!FuseMatrix || !MatMul->hasOneUse() ||
1345 MatrixLayout != MatrixLayoutTy::ColumnMajor || !DT)
1346 return;
1347
1348 auto *LoadOp0 = dyn_cast<LoadInst>(MatMul->getOperand(0));
1349 auto *LoadOp1 = dyn_cast<LoadInst>(MatMul->getOperand(1));
1350 auto *Store = dyn_cast<StoreInst>(*MatMul->user_begin());
1351 if (LoadOp0 && LoadOp1 && Store) {
1352 // The store address must dominate the MatMul instruction, otherwise
1353 // we create invalid IR.
1354 // FIXME: See if we can hoist the store address computation.
1355 auto *AddrI = dyn_cast<Instruction>(Store->getOperand(1));
1356 if (AddrI && (!DT->dominates(AddrI, MatMul)))
1357 return;
1358
1359 emitSIMDTiling(MatMul, LoadOp0, LoadOp1, Store, FusedInsts);
1360 return;
1361 }
1362 }
1363
1364 /// Lowers llvm.matrix.multiply.
1365 void LowerMultiply(CallInst *MatMul) {
1366 IRBuilder<> Builder(MatMul);
1367 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
15
The object is a 'VectorType'
1368 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3));
1369 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4));
1370
1371 const MatrixTy &Lhs = getMatrix(MatMul->getArgOperand(0), LShape, Builder);
1372 const MatrixTy &Rhs = getMatrix(MatMul->getArgOperand(1), RShape, Builder);
1373 assert(Lhs.getElementType() == Rhs.getElementType() &&((Lhs.getElementType() == Rhs.getElementType() && "Matrix multiply argument element types do not match."
) ? static_cast<void> (0) : __assert_fail ("Lhs.getElementType() == Rhs.getElementType() && \"Matrix multiply argument element types do not match.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1374, __PRETTY_FUNCTION__))
16
Assuming the condition is true
17
'?' condition is true
1374 "Matrix multiply argument element types do not match.")((Lhs.getElementType() == Rhs.getElementType() && "Matrix multiply argument element types do not match."
) ? static_cast<void> (0) : __assert_fail ("Lhs.getElementType() == Rhs.getElementType() && \"Matrix multiply argument element types do not match.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1374, __PRETTY_FUNCTION__))
;
1375
1376 const unsigned R = LShape.NumRows;
1377 const unsigned C = RShape.NumColumns;
1378 assert(LShape.NumColumns == RShape.NumRows)((LShape.NumColumns == RShape.NumRows) ? static_cast<void>
(0) : __assert_fail ("LShape.NumColumns == RShape.NumRows", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1378, __PRETTY_FUNCTION__))
;
18
Assuming field 'NumColumns' is equal to field 'NumRows'
19
'?' condition is true
1379
1380 // Initialize the output
1381 MatrixTy Result(R, C, EltType);
1382 assert(Lhs.getElementType() == Result.getElementType() &&((Lhs.getElementType() == Result.getElementType() && "Matrix multiply result element type does not match arguments."
) ? static_cast<void> (0) : __assert_fail ("Lhs.getElementType() == Result.getElementType() && \"Matrix multiply result element type does not match arguments.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1383, __PRETTY_FUNCTION__))
20
Assuming the condition is true
21
'?' condition is true
1383 "Matrix multiply result element type does not match arguments.")((Lhs.getElementType() == Result.getElementType() && "Matrix multiply result element type does not match arguments."
) ? static_cast<void> (0) : __assert_fail ("Lhs.getElementType() == Result.getElementType() && \"Matrix multiply result element type does not match arguments.\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1383, __PRETTY_FUNCTION__))
;
1384
1385 bool AllowContract = AllowContractEnabled || (isa<FPMathOperator>(MatMul) &&
22
Assuming the condition is false
23
Assuming 'MatMul' is not a 'FPMathOperator'
1386 MatMul->hasAllowContract());
1387 emitMatrixMultiply(Result, Lhs, Rhs, AllowContract, Builder, false);
24
Calling 'LowerMatrixIntrinsics::emitMatrixMultiply'
1388 finalizeLowering(MatMul, Result, Builder);
1389 }
1390
1391 /// Lowers llvm.matrix.transpose.
1392 void LowerTranspose(CallInst *Inst) {
1393 MatrixTy Result;
1394 IRBuilder<> Builder(Inst);
1395 Value *InputVal = Inst->getArgOperand(0);
1396 VectorType *VectorTy = cast<VectorType>(InputVal->getType());
1397 ShapeInfo ArgShape(Inst->getArgOperand(1), Inst->getArgOperand(2));
1398 MatrixTy InputMatrix = getMatrix(InputVal, ArgShape, Builder);
1399
1400 const unsigned NewNumVecs =
1401 InputMatrix.isColumnMajor() ? ArgShape.NumRows : ArgShape.NumColumns;
1402 const unsigned NewNumElts =
1403 InputMatrix.isColumnMajor() ? ArgShape.NumColumns : ArgShape.NumRows;
1404
1405 for (unsigned I = 0; I < NewNumVecs; ++I) {
1406 // Build a single result vector. First initialize it.
1407 Value *ResultVector = UndefValue::get(
1408 FixedVectorType::get(VectorTy->getElementType(), NewNumElts));
1409 // Go through the old elements and insert it into the resulting vector.
1410 for (auto J : enumerate(InputMatrix.vectors())) {
1411 Value *Elt = Builder.CreateExtractElement(J.value(), I);
1412 // Row and column indices are transposed.
1413 ResultVector =
1414 Builder.CreateInsertElement(ResultVector, Elt, J.index());
1415 }
1416 Result.addVector(ResultVector);
1417 }
1418
1419 // TODO: Improve estimate of operations needed for transposes. Currently we
1420 // just count the insertelement/extractelement instructions, but do not
1421 // account for later simplifications/combines.
1422 finalizeLowering(
1423 Inst,
1424 Result.addNumComputeOps(2 * ArgShape.NumRows * ArgShape.NumColumns),
1425 Builder);
1426 }
1427
1428 /// Lower load instructions, if shape information is available.
1429 bool VisitLoad(LoadInst *Inst, Value *Ptr, IRBuilder<> &Builder) {
1430 auto I = ShapeMap.find(Inst);
1431 if (I == ShapeMap.end())
1432 return false;
1433
1434 LowerLoad(Inst, Ptr, Inst->getAlign(),
1435 Builder.getInt64(I->second.getStride()), Inst->isVolatile(),
1436 I->second);
1437 return true;
1438 }
1439
1440 bool VisitStore(StoreInst *Inst, Value *StoredVal, Value *Ptr,
1441 IRBuilder<> &Builder) {
1442 auto I = ShapeMap.find(StoredVal);
1443 if (I == ShapeMap.end())
1444 return false;
1445
1446 LowerStore(Inst, StoredVal, Ptr, Inst->getAlign(),
1447 Builder.getInt64(I->second.getStride()), Inst->isVolatile(),
1448 I->second);
1449 return true;
1450 }
1451
1452 /// Lower binary operators, if shape information is available.
1453 bool VisitBinaryOperator(BinaryOperator *Inst) {
1454 auto I = ShapeMap.find(Inst);
1455 if (I == ShapeMap.end())
1456 return false;
1457
1458 Value *Lhs = Inst->getOperand(0);
1459 Value *Rhs = Inst->getOperand(1);
1460
1461 IRBuilder<> Builder(Inst);
1462 ShapeInfo &Shape = I->second;
1463
1464 MatrixTy Result;
1465 MatrixTy A = getMatrix(Lhs, Shape, Builder);
1466 MatrixTy B = getMatrix(Rhs, Shape, Builder);
1467 assert(A.isColumnMajor() == B.isColumnMajor() &&((A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor
() == A.isColumnMajor() && "operands must agree on matrix layout"
) ? static_cast<void> (0) : __assert_fail ("A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor() == A.isColumnMajor() && \"operands must agree on matrix layout\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1469, __PRETTY_FUNCTION__))
1468 Result.isColumnMajor() == A.isColumnMajor() &&((A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor
() == A.isColumnMajor() && "operands must agree on matrix layout"
) ? static_cast<void> (0) : __assert_fail ("A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor() == A.isColumnMajor() && \"operands must agree on matrix layout\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1469, __PRETTY_FUNCTION__))
1469 "operands must agree on matrix layout")((A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor
() == A.isColumnMajor() && "operands must agree on matrix layout"
) ? static_cast<void> (0) : __assert_fail ("A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor() == A.isColumnMajor() && \"operands must agree on matrix layout\""
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1469, __PRETTY_FUNCTION__))
;
1470
1471 // Helper to perform binary op on vectors.
1472 auto BuildVectorOp = [&Builder, Inst](Value *LHS, Value *RHS) {
1473 switch (Inst->getOpcode()) {
1474 case Instruction::Add:
1475 return Builder.CreateAdd(LHS, RHS);
1476 case Instruction::Mul:
1477 return Builder.CreateMul(LHS, RHS);
1478 case Instruction::Sub:
1479 return Builder.CreateSub(LHS, RHS);
1480 case Instruction::FAdd:
1481 return Builder.CreateFAdd(LHS, RHS);
1482 case Instruction::FMul:
1483 return Builder.CreateFMul(LHS, RHS);
1484 case Instruction::FSub:
1485 return Builder.CreateFSub(LHS, RHS);
1486 default:
1487 llvm_unreachable("Unsupported binary operator for matrix")::llvm::llvm_unreachable_internal("Unsupported binary operator for matrix"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1487)
;
1488 }
1489 };
1490
1491 for (unsigned I = 0; I < Shape.getNumVectors(); ++I)
1492 Result.addVector(BuildVectorOp(A.getVector(I), B.getVector(I)));
1493
1494 finalizeLowering(Inst,
1495 Result.addNumComputeOps(getNumOps(Result.getVectorTy()) *
1496 Result.getNumVectors()),
1497 Builder);
1498 return true;
1499 }
1500
1501 /// Helper to linearize a matrix expression tree into a string. Currently
1502 /// matrix expressions are linarized by starting at an expression leaf and
1503 /// linearizing bottom up.
1504 struct ExprLinearizer {
1505 unsigned LengthToBreak = 100;
1506 std::string Str;
1507 raw_string_ostream Stream;
1508 unsigned LineLength = 0;
1509 const DataLayout &DL;
1510
1511 /// Mapping from instructions to matrixes. It is used to identify
1512 /// matrix instructions.
1513 const MapVector<Value *, MatrixTy> &Inst2Matrix;
1514
1515 /// Mapping from values to the leaves of all expressions that the value is
1516 /// part of.
1517 const DenseMap<Value *, SmallPtrSet<Value *, 2>> &Shared;
1518
1519 /// Set of matrix expressions in the scope of a given DISubprogram.
1520 const SmallSetVector<Value *, 32> &ExprsInSubprogram;
1521
1522 /// Leaf node of the expression to linearize.
1523 Value *Leaf;
1524
1525 /// Used to keep track of sub-expressions that get reused while linearizing
1526 /// the expression. Re-used sub-expressions are marked as (reused).
1527 SmallPtrSet<Value *, 8> ReusedExprs;
1528
1529 ExprLinearizer(const DataLayout &DL,
1530 const MapVector<Value *, MatrixTy> &Inst2Matrix,
1531 const DenseMap<Value *, SmallPtrSet<Value *, 2>> &Shared,
1532 const SmallSetVector<Value *, 32> &ExprsInSubprogram,
1533 Value *Leaf)
1534 : Str(), Stream(Str), DL(DL), Inst2Matrix(Inst2Matrix), Shared(Shared),
1535 ExprsInSubprogram(ExprsInSubprogram), Leaf(Leaf) {}
1536
1537 void indent(unsigned N) {
1538 LineLength += N;
1539 for (unsigned i = 0; i < N; i++)
1540 Stream << " ";
1541 }
1542
1543 void lineBreak() {
1544 Stream << "\n";
1545 LineLength = 0;
1546 }
1547
1548 void maybeIndent(unsigned Indent) {
1549 if (LineLength >= LengthToBreak)
1550 lineBreak();
1551
1552 if (LineLength == 0)
1553 indent(Indent);
1554 }
1555
1556 void write(StringRef S) {
1557 LineLength += S.size();
1558 Stream << S;
1559 }
1560
1561 Value *getUnderlyingObjectThroughLoads(Value *V) {
1562 if (Value *Ptr = getPointerOperand(V))
1563 return getUnderlyingObjectThroughLoads(Ptr);
1564 else if (V->getType()->isPointerTy())
1565 return getUnderlyingObject(V);
1566 return V;
1567 }
1568
1569 /// Returns true if \p V is a matrix value in the given subprogram.
1570 bool isMatrix(Value *V) const { return ExprsInSubprogram.count(V); }
1571
1572 /// If \p V is a matrix value, print its shape as as NumRows x NumColumns to
1573 /// \p SS.
1574 void prettyPrintMatrixType(Value *V, raw_string_ostream &SS) {
1575 auto M = Inst2Matrix.find(V);
1576 if (M == Inst2Matrix.end())
1577 SS << "unknown";
1578 else {
1579 SS << M->second.getNumRows();
1580 SS << "x";
1581 SS << M->second.getNumColumns();
1582 }
1583 }
1584
1585 /// Write the called function name. Handles calls to llvm.matrix.*
1586 /// specially: we write the name, followed by the dimensions of the input
1587 /// matrixes, followed by the scalar type name.
1588 void writeFnName(CallInst *CI) {
1589 if (!CI->getCalledFunction())
1590 write("<no called fn>");
1591 else {
1592 StringRef Name = CI->getCalledFunction()->getName();
1593 if (!Name.startswith("llvm.matrix")) {
1594 write(Name);
1595 return;
1596 }
1597 IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI);
1598 write(StringRef(Intrinsic::getName(II->getIntrinsicID(), {}))
1599 .drop_front(StringRef("llvm.matrix.").size()));
1600 write(".");
1601 std::string Tmp = "";
1602 raw_string_ostream SS(Tmp);
1603
1604 switch (II->getIntrinsicID()) {
1605 case Intrinsic::matrix_multiply:
1606 prettyPrintMatrixType(II->getOperand(0), SS);
1607 SS << ".";
1608 prettyPrintMatrixType(II->getOperand(1), SS);
1609 SS << "." << *II->getType()->getScalarType();
1610 break;
1611 case Intrinsic::matrix_transpose:
1612 prettyPrintMatrixType(II->getOperand(0), SS);
1613 SS << "." << *II->getType()->getScalarType();
1614 break;
1615 case Intrinsic::matrix_column_major_load:
1616 prettyPrintMatrixType(II, SS);
1617 SS << "." << *II->getType()->getScalarType();
1618 break;
1619 case Intrinsic::matrix_column_major_store:
1620 prettyPrintMatrixType(II->getOperand(0), SS);
1621 SS << "." << *II->getOperand(0)->getType()->getScalarType();
1622 break;
1623 default:
1624 llvm_unreachable("Unhandled case")::llvm::llvm_unreachable_internal("Unhandled case", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1624)
;
1625 }
1626 SS.flush();
1627 write(Tmp);
1628 }
1629 }
1630
1631 unsigned getNumShapeArgs(CallInst *CI) const {
1632 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI)) {
1633 switch (II->getIntrinsicID()) {
1634 case Intrinsic::matrix_multiply:
1635 return 3;
1636 case Intrinsic::matrix_transpose:
1637 return 2;
1638 case Intrinsic::matrix_column_major_load:
1639 case Intrinsic::matrix_column_major_store:
1640 return 3;
1641 default:
1642 return 0;
1643 }
1644 }
1645 return 0;
1646 }
1647
1648 /// Special printing for values: for pointers, we print if they refer to an
1649 /// (function) external address or a stack address, for other values we
1650 /// either print the constant or "scalar"/"matrix" for other values.
1651 void write(Value *V) {
1652 V = getUnderlyingObjectThroughLoads(V);
1653 if (V->getType()->isPointerTy()) {
1654 if (isa<AllocaInst>(V)) {
1655 Stream << "stack addr";
1656 LineLength += StringRef("stack addr").size();
1657 } else {
1658 Stream << "addr";
1659 LineLength += StringRef("addr").size();
1660 }
1661 if (!V->getName().empty()) {
1662 Stream << " %" << V->getName() << "";
1663 LineLength += V->getName().size() + 2;
1664 }
1665 return;
1666 }
1667
1668 std::string Tmp;
1669 raw_string_ostream TmpStream(Tmp);
1670
1671 if (auto *CI = dyn_cast<ConstantInt>(V))
1672 TmpStream << CI->getValue();
1673 else if (isa<Constant>(V))
1674 TmpStream << "constant";
1675 else {
1676 if (isMatrix(V))
1677 TmpStream << "matrix";
1678 else
1679 TmpStream << "scalar";
1680 }
1681 TmpStream.flush();
1682 Tmp = std::string(StringRef(Tmp).trim());
1683 LineLength += Tmp.size();
1684 Stream << Tmp;
1685 }
1686
1687 /// Linearize expression \p Expr starting at an indentation of \p Indent.
1688 /// Expressions that are re-used multiple times are prefixed with (reused)
1689 /// at the re-used root instruction.
1690 void linearizeExpr(Value *Expr, unsigned Indent, bool ParentReused,
1691 bool ParentShared) {
1692 auto *I = cast<Instruction>(Expr);
1693 maybeIndent(Indent);
1694 SmallVector<Value *, 8> Ops;
1695
1696 // Is Expr shared with other expression leaves?
1697 bool ExprShared = false;
1698
1699 // Deal with shared subtrees. Mark them as shared, if required.
1700 if (!ParentShared) {
1701 auto SI = Shared.find(Expr);
1702 assert(SI != Shared.end() && SI->second.count(Leaf))((SI != Shared.end() && SI->second.count(Leaf)) ? static_cast
<void> (0) : __assert_fail ("SI != Shared.end() && SI->second.count(Leaf)"
, "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1702, __PRETTY_FUNCTION__))
;
1703
1704 for (Value *S : SI->second) {
1705 if (S == Leaf)
1706 continue;
1707 DebugLoc DL = cast<Instruction>(S)->getDebugLoc();
1708 write("shared with remark at line " + std::to_string(DL.getLine()) +
1709 " column " + std::to_string(DL.getCol()) + " (");
1710 }
1711 ExprShared = SI->second.size() > 1;
1712 }
1713
1714 bool Reused = !ReusedExprs.insert(Expr).second;
1715 if (Reused && !ParentReused)
1716 write("(reused) ");
1717
1718 if (auto *CI = dyn_cast<CallInst>(I)) {
1719 writeFnName(CI);
1720
1721 Ops.append(CI->arg_begin(), CI->arg_end() - getNumShapeArgs(CI));
1722 } else if (isa<BitCastInst>(Expr)) {
1723 // Special case bitcasts, which are used to materialize matrixes from
1724 // non-matrix ops.
1725 write("matrix");
1726 return;
1727 } else {
1728 Ops.append(I->value_op_begin(), I->value_op_end());
1729 write(std::string(I->getOpcodeName()));
1730 }
1731
1732 write(std::string("("));
1733
1734 unsigned NumOpsToBreak = 1;
1735 if (match(Expr, m_Intrinsic<Intrinsic::matrix_column_major_load>()))
1736 NumOpsToBreak = 2;
1737
1738 for (Value *Op : Ops) {
1739 if (Ops.size() > NumOpsToBreak)
1740 lineBreak();
1741
1742 maybeIndent(Indent + 1);
1743 if (isMatrix(Op))
1744 linearizeExpr(Op, Indent + 1, Reused, ExprShared);
1745 else
1746 write(Op);
1747 if (Op != Ops.back())
1748 write(", ");
1749 }
1750
1751 write(")");
1752 }
1753
1754 const std::string &getResult() {
1755 Stream.flush();
1756 return Str;
1757 }
1758 };
1759
1760 /// Generate remarks for matrix operations in a function. To generate remarks
1761 /// for matrix expressions, the following approach is used:
1762 /// 1. Use the inlined-at debug information to group matrix operations to the
1763 /// DISubprograms they are contained in.
1764 /// 2. Collect leaves of matrix expressions (done in
1765 /// RemarkGenerator::getExpressionLeaves) for each subprogram - expression
1766 // mapping. Leaves are lowered matrix instructions without other matrix
1767 // users (like stores) in the current subprogram.
1768 /// 3. For each leaf, create a remark containing a linearizied version of the
1769 /// matrix expression. The expression is linearized by a recursive
1770 /// bottom-up traversal of the matrix operands, starting at a leaf. Note
1771 /// that multiple leaves can share sub-expressions. Shared subexpressions
1772 /// are explicitly marked as shared().
1773 struct RemarkGenerator {
1774 const MapVector<Value *, MatrixTy> &Inst2Matrix;
1775 OptimizationRemarkEmitter &ORE;
1776 Function &Func;
1777 const DataLayout &DL;
1778
1779 RemarkGenerator(const MapVector<Value *, MatrixTy> &Inst2Matrix,
1780 OptimizationRemarkEmitter &ORE, Function &Func)
1781 : Inst2Matrix(Inst2Matrix), ORE(ORE), Func(Func),
1782 DL(Func.getParent()->getDataLayout()) {}
1783
1784 /// Return all leaves of the expressions in \p ExprsInSubprogram. Those are
1785 /// instructions in Inst2Matrix returning void or without any users in
1786 /// \p ExprsInSubprogram. Currently that should only include stores.
1787 SmallVector<Value *, 4>
1788 getExpressionLeaves(const SmallSetVector<Value *, 32> &ExprsInSubprogram) {
1789 SmallVector<Value *, 4> Leaves;
1790 for (auto *Expr : ExprsInSubprogram)
1791 if (Expr->getType()->isVoidTy() ||
1792 !any_of(Expr->users(), [&ExprsInSubprogram](User *U) {
1793 return ExprsInSubprogram.count(U);
1794 }))
1795 Leaves.push_back(Expr);
1796 return Leaves;
1797 }
1798
1799 /// Recursively traverse expression \p V starting at \p Leaf and add \p Leaf
1800 /// to all visited expressions in \p Shared. Limit the matrix operations to
1801 /// the ones in \p ExprsInSubprogram.
1802 void collectSharedInfo(Value *Leaf, Value *V,
1803 const SmallSetVector<Value *, 32> &ExprsInSubprogram,
1804 DenseMap<Value *, SmallPtrSet<Value *, 2>> &Shared) {
1805
1806 if (!ExprsInSubprogram.count(V))
1807 return;
1808
1809 auto I = Shared.insert({V, {}});
1810 I.first->second.insert(Leaf);
1811
1812 for (Value *Op : cast<Instruction>(V)->operand_values())
1813 collectSharedInfo(Leaf, Op, ExprsInSubprogram, Shared);
1814 return;
1815 }
1816
1817 /// Calculate the number of exclusive and shared op counts for expression
1818 /// starting at \p V. Expressions used multiple times are counted once.
1819 /// Limit the matrix operations to the ones in \p ExprsInSubprogram.
1820 std::pair<OpInfoTy, OpInfoTy>
1821 sumOpInfos(Value *Root, SmallPtrSetImpl<Value *> &ReusedExprs,
1822 const SmallSetVector<Value *, 32> &ExprsInSubprogram,
1823 DenseMap<Value *, SmallPtrSet<Value *, 2>> &Shared) const {
1824 if (!ExprsInSubprogram.count(Root))
1825 return {};
1826
1827 // Already counted this expression. Stop.
1828 if (!ReusedExprs.insert(Root).second)
1829 return {};
1830
1831 OpInfoTy SharedCount;
1832 OpInfoTy Count;
1833
1834 auto I = Shared.find(Root);
1835 auto CM = Inst2Matrix.find(Root);
1836 if (I->second.size() == 1)
1837 Count = CM->second.getOpInfo();
1838 else
1839 SharedCount = CM->second.getOpInfo();
1840
1841 for (Value *Op : cast<Instruction>(Root)->operand_values()) {
1842 auto C = sumOpInfos(Op, ReusedExprs, ExprsInSubprogram, Shared);
1843 Count += C.first;
1844 SharedCount += C.second;
1845 }
1846 return {Count, SharedCount};
1847 }
1848
1849 void emitRemarks() {
1850 if (!ORE.allowExtraAnalysis(DEBUG_TYPE"lower-matrix-intrinsics"))
1851 return;
1852
1853 // Map matrix operations to their containting subprograms, by traversing
1854 // the inlinedAt chain. If the function does not have a DISubprogram, we
1855 // only map them to the containing function.
1856 MapVector<DISubprogram *, SmallVector<Value *, 8>> Subprog2Exprs;
1857 for (auto &KV : Inst2Matrix) {
1858 if (Func.getSubprogram()) {
1859 auto *I = cast<Instruction>(KV.first);
1860 DILocation *Context = I->getDebugLoc();
1861 while (Context) {
1862 auto I =
1863 Subprog2Exprs.insert({getSubprogram(Context->getScope()), {}});
1864 I.first->second.push_back(KV.first);
1865 Context = DebugLoc(Context).getInlinedAt();
1866 }
1867 } else {
1868 auto I = Subprog2Exprs.insert({nullptr, {}});
1869 I.first->second.push_back(KV.first);
1870 }
1871 }
1872 for (auto &KV : Subprog2Exprs) {
1873 SmallSetVector<Value *, 32> ExprsInSubprogram(KV.second.begin(),
1874 KV.second.end());
1875 auto Leaves = getExpressionLeaves(ExprsInSubprogram);
1876
1877 DenseMap<Value *, SmallPtrSet<Value *, 2>> Shared;
1878 for (Value *Leaf : Leaves)
1879 collectSharedInfo(Leaf, Leaf, ExprsInSubprogram, Shared);
1880
1881 // Generate remarks for each leaf.
1882 for (auto *L : Leaves) {
1883
1884 DebugLoc Loc = cast<Instruction>(L)->getDebugLoc();
1885 DILocation *Context = cast<Instruction>(L)->getDebugLoc();
1886 while (Context) {
1887 if (getSubprogram(Context->getScope()) == KV.first) {
1888 Loc = Context;
1889 break;
1890 }
1891 Context = DebugLoc(Context).getInlinedAt();
1892 }
1893
1894 SmallPtrSet<Value *, 8> ReusedExprs;
1895 OpInfoTy Counts, SharedCounts;
1896 std::tie(Counts, SharedCounts) =
1897 sumOpInfos(L, ReusedExprs, ExprsInSubprogram, Shared);
1898
1899 OptimizationRemark Rem(DEBUG_TYPE"lower-matrix-intrinsics", "matrix-lowered", Loc,
1900 cast<Instruction>(L)->getParent());
1901
1902 Rem << "Lowered with ";
1903 Rem << ore::NV("NumStores", Counts.NumStores) << " stores, "
1904 << ore::NV("NumLoads", Counts.NumLoads) << " loads, "
1905 << ore::NV("NumComputeOps", Counts.NumComputeOps)
1906 << " compute ops";
1907
1908 if (SharedCounts.NumStores > 0 || SharedCounts.NumLoads > 0 ||
1909 SharedCounts.NumComputeOps > 0) {
1910 Rem << ",\nadditionally "
1911 << ore::NV("NumStores", SharedCounts.NumStores) << " stores, "
1912 << ore::NV("NumLoads", SharedCounts.NumLoads) << " loads, "
1913 << ore::NV("NumFPOps", SharedCounts.NumComputeOps)
1914 << " compute ops"
1915 << " are shared with other expressions";
1916 }
1917
1918 Rem << ("\n" + linearize(L, Shared, ExprsInSubprogram, DL));
1919 ORE.emit(Rem);
1920 }
1921 }
1922 }
1923
1924 std::string
1925 linearize(Value *L,
1926 const DenseMap<Value *, SmallPtrSet<Value *, 2>> &Shared,
1927 const SmallSetVector<Value *, 32> &ExprsInSubprogram,
1928 const DataLayout &DL) {
1929 ExprLinearizer Lin(DL, Inst2Matrix, Shared, ExprsInSubprogram, L);
1930 Lin.linearizeExpr(L, 0, false, false);
1931 return Lin.getResult();
1932 }
1933 };
1934};
1935} // namespace
1936
1937PreservedAnalyses LowerMatrixIntrinsicsPass::run(Function &F,
1938 FunctionAnalysisManager &AM) {
1939 auto &TTI = AM.getResult<TargetIRAnalysis>(F);
1940 auto &ORE = AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
1941 auto &AA = AM.getResult<AAManager>(F);
1942 auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
1943 auto &LI = AM.getResult<LoopAnalysis>(F);
1944
1945 LowerMatrixIntrinsics LMT(F, TTI, &AA, &DT, &LI, &ORE);
1946 if (LMT.Visit()) {
1947 PreservedAnalyses PA;
1948 PA.preserveSet<CFGAnalyses>();
1949 return PA;
1950 }
1951 return PreservedAnalyses::all();
1952}
1953
1954namespace {
1955
1956class LowerMatrixIntrinsicsLegacyPass : public FunctionPass {
1957public:
1958 static char ID;
1959
1960 LowerMatrixIntrinsicsLegacyPass() : FunctionPass(ID) {
1961 initializeLowerMatrixIntrinsicsLegacyPassPass(
1962 *PassRegistry::getPassRegistry());
1963 }
1964
1965 bool runOnFunction(Function &F) override {
1966 auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
1967 auto &ORE = getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
1968 auto &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
1969 auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
1970 auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
1971 LowerMatrixIntrinsics LMT(F, TTI, &AA, &DT, &LI, &ORE);
1972 bool C = LMT.Visit();
1973 return C;
1974 }
1975
1976 void getAnalysisUsage(AnalysisUsage &AU) const override {
1977 AU.addRequired<TargetTransformInfoWrapperPass>();
1978 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
1979 AU.addRequired<AAResultsWrapperPass>();
1980 AU.addRequired<DominatorTreeWrapperPass>();
1981 AU.addPreserved<DominatorTreeWrapperPass>();
1982 AU.addRequired<LoopInfoWrapperPass>();
1983 AU.addPreserved<LoopInfoWrapperPass>();
1984 }
1985};
1986} // namespace
1987
1988static const char pass_name[] = "Lower the matrix intrinsics";
1989char LowerMatrixIntrinsicsLegacyPass::ID = 0;
1990INITIALIZE_PASS_BEGIN(LowerMatrixIntrinsicsLegacyPass, DEBUG_TYPE, pass_name,static void *initializeLowerMatrixIntrinsicsLegacyPassPassOnce
(PassRegistry &Registry) {
1991 false, false)static void *initializeLowerMatrixIntrinsicsLegacyPassPassOnce
(PassRegistry &Registry) {
1992INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)initializeOptimizationRemarkEmitterWrapperPassPass(Registry);
1993INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)initializeAAResultsWrapperPassPass(Registry);
1994INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)initializeDominatorTreeWrapperPassPass(Registry);
1995INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)initializeLoopInfoWrapperPassPass(Registry);
1996INITIALIZE_PASS_END(LowerMatrixIntrinsicsLegacyPass, DEBUG_TYPE, pass_name,PassInfo *PI = new PassInfo( pass_name, "lower-matrix-intrinsics"
, &LowerMatrixIntrinsicsLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<LowerMatrixIntrinsicsLegacyPass>), false
, false); Registry.registerPass(*PI, true); return PI; } static
llvm::once_flag InitializeLowerMatrixIntrinsicsLegacyPassPassFlag
; void llvm::initializeLowerMatrixIntrinsicsLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializeLowerMatrixIntrinsicsLegacyPassPassFlag
, initializeLowerMatrixIntrinsicsLegacyPassPassOnce, std::ref
(Registry)); }
1997 false, false)PassInfo *PI = new PassInfo( pass_name, "lower-matrix-intrinsics"
, &LowerMatrixIntrinsicsLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<LowerMatrixIntrinsicsLegacyPass>), false
, false); Registry.registerPass(*PI, true); return PI; } static
llvm::once_flag InitializeLowerMatrixIntrinsicsLegacyPassPassFlag
; void llvm::initializeLowerMatrixIntrinsicsLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializeLowerMatrixIntrinsicsLegacyPassPassFlag
, initializeLowerMatrixIntrinsicsLegacyPassPassOnce, std::ref
(Registry)); }
1998
1999Pass *llvm::createLowerMatrixIntrinsicsPass() {
2000 return new LowerMatrixIntrinsicsLegacyPass();
2001}
2002
2003namespace {
2004
2005/// A lightweight version of the matrix lowering pass that only requires TTI.
2006/// Advanced features that require DT, AA or ORE like tiling are disabled. This
2007/// is used to lower matrix intrinsics if the main lowering pass is not run, for
2008/// example with -O0.
2009class LowerMatrixIntrinsicsMinimalLegacyPass : public FunctionPass {
2010public:
2011 static char ID;
2012
2013 LowerMatrixIntrinsicsMinimalLegacyPass() : FunctionPass(ID) {
2014 initializeLowerMatrixIntrinsicsMinimalLegacyPassPass(
2015 *PassRegistry::getPassRegistry());
2016 }
2017
2018 bool runOnFunction(Function &F) override {
2019 auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
2020 LowerMatrixIntrinsics LMT(F, TTI, nullptr, nullptr, nullptr, nullptr);
2021 bool C = LMT.Visit();
1
Calling 'LowerMatrixIntrinsics::Visit'
2022 return C;
2023 }
2024
2025 void getAnalysisUsage(AnalysisUsage &AU) const override {
2026 AU.addRequired<TargetTransformInfoWrapperPass>();
2027 AU.setPreservesCFG();
2028 }
2029};
2030} // namespace
2031
2032static const char pass_name_minimal[] = "Lower the matrix intrinsics (minimal)";
2033char LowerMatrixIntrinsicsMinimalLegacyPass::ID = 0;
2034INITIALIZE_PASS_BEGIN(LowerMatrixIntrinsicsMinimalLegacyPass,static void *initializeLowerMatrixIntrinsicsMinimalLegacyPassPassOnce
(PassRegistry &Registry) {
2035 "lower-matrix-intrinsics-minimal", pass_name_minimal,static void *initializeLowerMatrixIntrinsicsMinimalLegacyPassPassOnce
(PassRegistry &Registry) {
2036 false, false)static void *initializeLowerMatrixIntrinsicsMinimalLegacyPassPassOnce
(PassRegistry &Registry) {
2037INITIALIZE_PASS_END(LowerMatrixIntrinsicsMinimalLegacyPass,PassInfo *PI = new PassInfo( pass_name_minimal, "lower-matrix-intrinsics-minimal"
, &LowerMatrixIntrinsicsMinimalLegacyPass::ID, PassInfo::
NormalCtor_t(callDefaultCtor<LowerMatrixIntrinsicsMinimalLegacyPass
>), false, false); Registry.registerPass(*PI, true); return
PI; } static llvm::once_flag InitializeLowerMatrixIntrinsicsMinimalLegacyPassPassFlag
; void llvm::initializeLowerMatrixIntrinsicsMinimalLegacyPassPass
(PassRegistry &Registry) { llvm::call_once(InitializeLowerMatrixIntrinsicsMinimalLegacyPassPassFlag
, initializeLowerMatrixIntrinsicsMinimalLegacyPassPassOnce, std
::ref(Registry)); }
2038 "lower-matrix-intrinsics-minimal", pass_name_minimal, false,PassInfo *PI = new PassInfo( pass_name_minimal, "lower-matrix-intrinsics-minimal"
, &LowerMatrixIntrinsicsMinimalLegacyPass::ID, PassInfo::
NormalCtor_t(callDefaultCtor<LowerMatrixIntrinsicsMinimalLegacyPass
>), false, false); Registry.registerPass(*PI, true); return
PI; } static llvm::once_flag InitializeLowerMatrixIntrinsicsMinimalLegacyPassPassFlag
; void llvm::initializeLowerMatrixIntrinsicsMinimalLegacyPassPass
(PassRegistry &Registry) { llvm::call_once(InitializeLowerMatrixIntrinsicsMinimalLegacyPassPassFlag
, initializeLowerMatrixIntrinsicsMinimalLegacyPassPassOnce, std
::ref(Registry)); }
2039 false)PassInfo *PI = new PassInfo( pass_name_minimal, "lower-matrix-intrinsics-minimal"
, &LowerMatrixIntrinsicsMinimalLegacyPass::ID, PassInfo::
NormalCtor_t(callDefaultCtor<LowerMatrixIntrinsicsMinimalLegacyPass
>), false, false); Registry.registerPass(*PI, true); return
PI; } static llvm::once_flag InitializeLowerMatrixIntrinsicsMinimalLegacyPassPassFlag
; void llvm::initializeLowerMatrixIntrinsicsMinimalLegacyPassPass
(PassRegistry &Registry) { llvm::call_once(InitializeLowerMatrixIntrinsicsMinimalLegacyPassPassFlag
, initializeLowerMatrixIntrinsicsMinimalLegacyPassPassOnce, std
::ref(Registry)); }
2040
2041Pass *llvm::createLowerMatrixIntrinsicsMinimalPass() {
2042 return new LowerMatrixIntrinsicsMinimalLegacyPass();
2043}