Bug Summary

File:llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
Warning:line 940, column 31
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LowerMatrixIntrinsics.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-12/lib/clang/12.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/build-llvm/lib/Transforms/Scalar -I /build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar -I /build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/build-llvm/include -I /build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-12/lib/clang/12.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/build-llvm/lib/Transforms/Scalar -fdebug-prefix-map=/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-11-29-190409-37574-1 -x c++ /build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
1//===- LowerMatrixIntrinsics.cpp - Lower matrix intrinsics -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Lower matrix intrinsics to vector operations.
10//
11// TODO:
12// * Improve fusion:
13// * Support more cases, e.g. multiply-add, multiply-sub, operands/results
14// transposed.
15// * Improve cost-modeling, e.g. choose different number of rows/columns
16// columns for tiles, consider cost of copies on alias.
17//
18//===----------------------------------------------------------------------===//
19
20#include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h"
21#include "llvm/ADT/GraphTraits.h"
22#include "llvm/ADT/PostOrderIterator.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/Analysis/AliasAnalysis.h"
25#include "llvm/Analysis/DomTreeUpdater.h"
26#include "llvm/Analysis/OptimizationRemarkEmitter.h"
27#include "llvm/Analysis/TargetTransformInfo.h"
28#include "llvm/Analysis/ValueTracking.h"
29#include "llvm/Analysis/VectorUtils.h"
30#include "llvm/IR/CFG.h"
31#include "llvm/IR/DataLayout.h"
32#include "llvm/IR/DebugInfoMetadata.h"
33#include "llvm/IR/Function.h"
34#include "llvm/IR/IRBuilder.h"
35#include "llvm/IR/Instructions.h"
36#include "llvm/IR/IntrinsicInst.h"
37#include "llvm/IR/PatternMatch.h"
38#include "llvm/InitializePasses.h"
39#include "llvm/Pass.h"
40#include "llvm/Support/Alignment.h"
41#include "llvm/Support/CommandLine.h"
42#include "llvm/Support/Debug.h"
43#include "llvm/Transforms/Scalar.h"
44#include "llvm/Transforms/Utils/BasicBlockUtils.h"
45#include "llvm/Transforms/Utils/LoopUtils.h"
46#include "llvm/Transforms/Utils/MatrixUtils.h"
47
48using namespace llvm;
49using namespace PatternMatch;
50
51#define DEBUG_TYPE"lower-matrix-intrinsics" "lower-matrix-intrinsics"
52
53static cl::opt<bool> EnableShapePropagation(
54 "matrix-propagate-shape", cl::init(true), cl::Hidden,
55 cl::desc("Enable/disable shape propagation from matrix intrinsics to other "
56 "instructions."));
57
58static cl::opt<bool>
59 FuseMatrix("fuse-matrix", cl::init(true), cl::Hidden,
60 cl::desc("Enable/disable fusing matrix instructions."));
61// TODO: Allow and use non-square tiles.
62static cl::opt<unsigned> TileSize(
63 "fuse-matrix-tile-size", cl::init(4), cl::Hidden,
64 cl::desc(
65 "Tile size for matrix instruction fusion using square-shaped tiles."));
66static cl::opt<bool> TileUseLoops("fuse-matrix-use-loops", cl::init(false),
67 cl::Hidden,
68 cl::desc("Generate loop nest for tiling."));
69static cl::opt<bool> ForceFusion(
70 "force-fuse-matrix", cl::init(false), cl::Hidden,
71 cl::desc("Force matrix instruction fusion even if not profitable."));
72static cl::opt<bool> AllowContractEnabled(
73 "matrix-allow-contract", cl::init(false), cl::Hidden,
74 cl::desc("Allow the use of FMAs if available and profitable. This may "
75 "result in different results, due to less rounding error."));
76
77enum class MatrixLayoutTy { ColumnMajor, RowMajor };
78
79static cl::opt<MatrixLayoutTy> MatrixLayout(
80 "matrix-default-layout", cl::init(MatrixLayoutTy::ColumnMajor),
81 cl::desc("Sets the default matrix layout"),
82 cl::values(clEnumValN(MatrixLayoutTy::ColumnMajor, "column-major",llvm::cl::OptionEnumValue { "column-major", int(MatrixLayoutTy
::ColumnMajor), "Use column-major layout" }
83 "Use column-major layout")llvm::cl::OptionEnumValue { "column-major", int(MatrixLayoutTy
::ColumnMajor), "Use column-major layout" }
,
84 clEnumValN(MatrixLayoutTy::RowMajor, "row-major",llvm::cl::OptionEnumValue { "row-major", int(MatrixLayoutTy::
RowMajor), "Use row-major layout" }
85 "Use row-major layout")llvm::cl::OptionEnumValue { "row-major", int(MatrixLayoutTy::
RowMajor), "Use row-major layout" }
));
86
87/// Helper function to either return Scope, if it is a subprogram or the
88/// attached subprogram for a local scope.
89static DISubprogram *getSubprogram(DIScope *Scope) {
90 if (auto *Subprogram = dyn_cast<DISubprogram>(Scope))
91 return Subprogram;
92 return cast<DILocalScope>(Scope)->getSubprogram();
93}
94
95namespace {
96
97// Given an element pointer \p BasePtr to the start of a (sub) matrix, compute
98// the start address of vector \p VecIdx with type (\p EltType x \p NumElements)
99// assuming \p Stride elements between start two consecutive vectors.
100// \p Stride must be >= \p NumElements.
101// For column-major matrixes, the function computes the address of a column
102// vectors and \p NumElements must be set to the number of elements in a column
103// (= number of rows of the matrix). For row-major matrixes, the function
104// computes the address of a row vector and \p NumElements must be set to the
105// number of elements in a column (= number of columns of the matrix).
106//
107// Consider a 4x4 matrix in column-mjaor layout like below
108//
109// 0 1 2 3
110// 0 v_0_0 v_0_1 v_0_2 v_0_3
111// 1 v_1_0 v_1_1 v_1_2 v_1_3
112// 2 v_2_0 v_2_1 v_2_2 v_2_3
113// 3 v_3_0 v_3_1 v_3_2 v_3_3
114
115// To compute the column addresses for a 2x3 sub-matrix at row 1 and column 1,
116// we need a pointer to the first element of the submatrix as base pointer.
117// Then we can use computeVectorAddr to compute the addresses for the columns
118// of the sub-matrix.
119//
120// Column 0: computeVectorAddr(Base, 0 (column), 4 (stride), 2 (num rows), ..)
121// -> just returns Base
122// Column 1: computeVectorAddr(Base, 1 (column), 4 (stride), 2 (num rows), ..)
123// -> returns Base + (1 * 4)
124// Column 2: computeVectorAddr(Base, 2 (column), 4 (stride), 2 (num rows), ..)
125// -> returns Base + (2 * 4)
126//
127// The graphic below illustrates the number of elements in a column (marked
128// with |) and the number of skipped elements (marked with }).
129//
130// v_0_0 v_0_1 {v_0_2 {v_0_3
131// Base Col 1 Col 2
132// | | |
133// v_1_0 |v_1_1 |v_1_2 |v_1_3
134// v_2_0 |v_2_1 |v_2_2 |v_2_3
135// v_3_0 {v_3_1 {v_3_2 v_3_3
136//
137Value *computeVectorAddr(Value *BasePtr, Value *VecIdx, Value *Stride,
138 unsigned NumElements, Type *EltType,
139 IRBuilder<> &Builder) {
140
141 assert((!isa<ConstantInt>(Stride) ||(((!isa<ConstantInt>(Stride) || cast<ConstantInt>
(Stride)->getZExtValue() >= NumElements) && "Stride must be >= the number of elements in the result vector."
) ? static_cast<void> (0) : __assert_fail ("(!isa<ConstantInt>(Stride) || cast<ConstantInt>(Stride)->getZExtValue() >= NumElements) && \"Stride must be >= the number of elements in the result vector.\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 143, __PRETTY_FUNCTION__))
142 cast<ConstantInt>(Stride)->getZExtValue() >= NumElements) &&(((!isa<ConstantInt>(Stride) || cast<ConstantInt>
(Stride)->getZExtValue() >= NumElements) && "Stride must be >= the number of elements in the result vector."
) ? static_cast<void> (0) : __assert_fail ("(!isa<ConstantInt>(Stride) || cast<ConstantInt>(Stride)->getZExtValue() >= NumElements) && \"Stride must be >= the number of elements in the result vector.\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 143, __PRETTY_FUNCTION__))
143 "Stride must be >= the number of elements in the result vector.")(((!isa<ConstantInt>(Stride) || cast<ConstantInt>
(Stride)->getZExtValue() >= NumElements) && "Stride must be >= the number of elements in the result vector."
) ? static_cast<void> (0) : __assert_fail ("(!isa<ConstantInt>(Stride) || cast<ConstantInt>(Stride)->getZExtValue() >= NumElements) && \"Stride must be >= the number of elements in the result vector.\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 143, __PRETTY_FUNCTION__))
;
144 unsigned AS = cast<PointerType>(BasePtr->getType())->getAddressSpace();
145
146 // Compute the start of the vector with index VecIdx as VecIdx * Stride.
147 Value *VecStart = Builder.CreateMul(VecIdx, Stride, "vec.start");
148
149 // Get pointer to the start of the selected vector. Skip GEP creation,
150 // if we select vector 0.
151 if (isa<ConstantInt>(VecStart) && cast<ConstantInt>(VecStart)->isZero())
152 VecStart = BasePtr;
153 else
154 VecStart = Builder.CreateGEP(EltType, BasePtr, VecStart, "vec.gep");
155
156 // Cast elementwise vector start pointer to a pointer to a vector
157 // (EltType x NumElements)*.
158 auto *VecType = FixedVectorType::get(EltType, NumElements);
159 Type *VecPtrType = PointerType::get(VecType, AS);
160 return Builder.CreatePointerCast(VecStart, VecPtrType, "vec.cast");
161}
162
163/// LowerMatrixIntrinsics contains the methods used to lower matrix intrinsics.
164///
165/// Currently, the lowering for each matrix intrinsic is done as follows:
166/// 1. Propagate the shape information from intrinsics to connected
167/// instructions.
168/// 2. Lower instructions with shape information (assuming column-major layout).
169/// The lowering works similarly using row-major layout.
170/// 2.1. Get column vectors for each argument. If we already lowered the
171/// definition of an argument, use the produced column vectors directly.
172/// If not, split the operand vector containing an embedded matrix into
173/// a set of column vectors,
174/// 2.2. Lower the instruction in terms of column major operations, which
175/// yields a set of column vectors containing result matrix. Note that we
176/// lower all instructions that have shape information. Besides the
177/// intrinsics, this includes stores for example.
178/// 2.3. Update uses of the lowered instruction. If we have shape information
179/// for a user, there is nothing to do, as we will look up the result
180/// column matrix when lowering the user. For other uses, we embed the
181/// result matrix in a flat vector and update the use.
182/// 2.4. Cache the result column matrix for the instruction we lowered
183/// 3. After we lowered all instructions in a function, remove the now
184/// obsolete instructions.
185///
186class LowerMatrixIntrinsics {
187 Function &Func;
188 const DataLayout &DL;
189 const TargetTransformInfo &TTI;
190 AliasAnalysis *AA;
191 DominatorTree *DT;
192 LoopInfo *LI;
193 OptimizationRemarkEmitter *ORE;
194
195 /// Contains estimates of the number of operations (loads, stores, compute) required to lower a matrix operation.
196 struct OpInfoTy {
197 /// Number of stores emitted to generate this matrix.
198 unsigned NumStores = 0;
199 /// Number of loads emitted to generate this matrix.
200 unsigned NumLoads = 0;
201 /// Number of compute operations emitted to generate this matrix.
202 unsigned NumComputeOps = 0;
203
204 OpInfoTy &operator+=(const OpInfoTy &RHS) {
205 NumStores += RHS.NumStores;
206 NumLoads += RHS.NumLoads;
207 NumComputeOps += RHS.NumComputeOps;
208 return *this;
209 }
210 };
211
212 /// Wrapper class representing a matrix as a set of vectors, either in row or
213 /// column major layout. All vectors must have the same vector type.
214 class MatrixTy {
215 SmallVector<Value *, 16> Vectors;
216
217 OpInfoTy OpInfo;
218
219 bool IsColumnMajor = true;
220
221 public:
222 MatrixTy()
223 : Vectors(),
224 IsColumnMajor(MatrixLayout == MatrixLayoutTy::ColumnMajor) {}
225 MatrixTy(ArrayRef<Value *> Vectors)
226 : Vectors(Vectors.begin(), Vectors.end()),
227 IsColumnMajor(MatrixLayout == MatrixLayoutTy::ColumnMajor) {}
228 MatrixTy(unsigned NumRows, unsigned NumColumns, Type *EltTy)
229 : IsColumnMajor(MatrixLayout == MatrixLayoutTy::ColumnMajor) {
230
231 unsigned D = isColumnMajor() ? NumColumns : NumRows;
232 for (unsigned J = 0; J < D; ++J)
233 addVector(UndefValue::get(FixedVectorType::get(
234 EltTy, isColumnMajor() ? NumRows : NumColumns)));
235 }
236
237 Value *getVector(unsigned i) const { return Vectors[i]; }
238 Value *getColumn(unsigned i) const {
239 assert(isColumnMajor() && "only supported for column-major matrixes")((isColumnMajor() && "only supported for column-major matrixes"
) ? static_cast<void> (0) : __assert_fail ("isColumnMajor() && \"only supported for column-major matrixes\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 239, __PRETTY_FUNCTION__))
;
240 return Vectors[i];
241 }
242 Value *getRow(unsigned i) const {
243 assert(!isColumnMajor() && "only supported for row-major matrixes")((!isColumnMajor() && "only supported for row-major matrixes"
) ? static_cast<void> (0) : __assert_fail ("!isColumnMajor() && \"only supported for row-major matrixes\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 243, __PRETTY_FUNCTION__))
;
244 return Vectors[i];
245 }
246
247 void setVector(unsigned i, Value *V) { Vectors[i] = V; }
248
249 Type *getElementType() const { return getVectorTy()->getElementType(); }
250
251 unsigned getNumVectors() const {
252 if (isColumnMajor())
253 return getNumColumns();
254 return getNumRows();
255 }
256
257 unsigned getNumColumns() const {
258 if (isColumnMajor())
259 return Vectors.size();
260 else {
261 assert(Vectors.size() > 0 && "Cannot call getNumRows without columns")((Vectors.size() > 0 && "Cannot call getNumRows without columns"
) ? static_cast<void> (0) : __assert_fail ("Vectors.size() > 0 && \"Cannot call getNumRows without columns\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 261, __PRETTY_FUNCTION__))
;
262 return cast<FixedVectorType>(Vectors[0]->getType())->getNumElements();
263 }
264 }
265 unsigned getNumRows() const {
266 if (isColumnMajor()) {
267 assert(Vectors.size() > 0 && "Cannot call getNumRows without columns")((Vectors.size() > 0 && "Cannot call getNumRows without columns"
) ? static_cast<void> (0) : __assert_fail ("Vectors.size() > 0 && \"Cannot call getNumRows without columns\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 267, __PRETTY_FUNCTION__))
;
268 return cast<FixedVectorType>(Vectors[0]->getType())->getNumElements();
269 } else
270 return Vectors.size();
271 }
272
273 void addVector(Value *V) { Vectors.push_back(V); }
274 VectorType *getColumnTy() {
275 assert(isColumnMajor() && "only supported for column-major matrixes")((isColumnMajor() && "only supported for column-major matrixes"
) ? static_cast<void> (0) : __assert_fail ("isColumnMajor() && \"only supported for column-major matrixes\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 275, __PRETTY_FUNCTION__))
;
276 return getVectorTy();
277 }
278
279 VectorType *getVectorTy() const {
280 return cast<VectorType>(Vectors[0]->getType());
281 }
282
283 iterator_range<SmallVector<Value *, 8>::iterator> columns() {
284 assert(isColumnMajor() &&((isColumnMajor() && "columns() only supported for column-major matrixes"
) ? static_cast<void> (0) : __assert_fail ("isColumnMajor() && \"columns() only supported for column-major matrixes\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 285, __PRETTY_FUNCTION__))
285 "columns() only supported for column-major matrixes")((isColumnMajor() && "columns() only supported for column-major matrixes"
) ? static_cast<void> (0) : __assert_fail ("isColumnMajor() && \"columns() only supported for column-major matrixes\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 285, __PRETTY_FUNCTION__))
;
286 return make_range(Vectors.begin(), Vectors.end());
287 }
288
289 iterator_range<SmallVector<Value *, 8>::iterator> vectors() {
290 return make_range(Vectors.begin(), Vectors.end());
291 }
292
293 /// Embed the vectors of the matrix into a flat vector by concatenating
294 /// them.
295 Value *embedInVector(IRBuilder<> &Builder) const {
296 return Vectors.size() == 1 ? Vectors[0]
297 : concatenateVectors(Builder, Vectors);
298 }
299
300 MatrixTy &addNumLoads(unsigned N) {
301 OpInfo.NumLoads += N;
302 return *this;
303 }
304
305 void setNumLoads(unsigned N) { OpInfo.NumLoads = N; }
306
307 MatrixTy &addNumStores(unsigned N) {
308 OpInfo.NumStores += N;
309 return *this;
310 }
311
312 MatrixTy &addNumComputeOps(unsigned N) {
313 OpInfo.NumComputeOps += N;
314 return *this;
315 }
316
317 unsigned getNumStores() const { return OpInfo.NumStores; }
318 unsigned getNumLoads() const { return OpInfo.NumLoads; }
319 unsigned getNumComputeOps() const { return OpInfo.NumComputeOps; }
320
321 const OpInfoTy &getOpInfo() const { return OpInfo; }
322
323 bool isColumnMajor() const { return IsColumnMajor; }
27
Returning zero, which participates in a condition later
324
325 unsigned getStride() const {
326 if (isColumnMajor())
327 return getNumRows();
328 return getNumColumns();
329 }
330
331 /// Extract a vector of \p NumElts starting at index (\p I, \p J). If the
332 /// matrix is column-major, the result vector is extracted from a column
333 /// vector, otherwise from a row vector.
334 Value *extractVector(unsigned I, unsigned J, unsigned NumElts,
335 IRBuilder<> &Builder) const {
336 Value *Vec = isColumnMajor() ? getColumn(J) : getRow(I);
337 Value *Undef = UndefValue::get(Vec->getType());
338 return Builder.CreateShuffleVector(
339 Vec, Undef, createSequentialMask(isColumnMajor() ? I : J, NumElts, 0),
340 "block");
341 }
342 };
343
344 struct ShapeInfo {
345 unsigned NumRows;
346 unsigned NumColumns;
347
348 bool IsColumnMajor;
349
350 ShapeInfo(unsigned NumRows = 0, unsigned NumColumns = 0)
351 : NumRows(NumRows), NumColumns(NumColumns),
352 IsColumnMajor(MatrixLayout == MatrixLayoutTy::ColumnMajor) {}
353
354 ShapeInfo(Value *NumRows, Value *NumColumns)
355 : ShapeInfo(cast<ConstantInt>(NumRows)->getZExtValue(),
356 cast<ConstantInt>(NumColumns)->getZExtValue()) {}
357
358 bool operator==(const ShapeInfo &other) {
359 return NumRows == other.NumRows && NumColumns == other.NumColumns;
360 }
361 bool operator!=(const ShapeInfo &other) { return !(*this == other); }
362
363 /// Returns true if shape-information is defined, meaning both dimensions
364 /// are != 0.
365 operator bool() const {
366 assert(NumRows == 0 || NumColumns != 0)((NumRows == 0 || NumColumns != 0) ? static_cast<void> (
0) : __assert_fail ("NumRows == 0 || NumColumns != 0", "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 366, __PRETTY_FUNCTION__))
;
367 return NumRows != 0;
368 }
369
370 unsigned getStride() const {
371 if (IsColumnMajor)
372 return NumRows;
373 return NumColumns;
374 }
375
376 unsigned getNumVectors() const {
377 if (IsColumnMajor)
378 return NumColumns;
379 return NumRows;
380 }
381 };
382
383 /// Maps instructions to their shape information. The shape information
384 /// describes the shape to be used while lowering. This matches the shape of
385 /// the result value of the instruction, with the only exceptions being store
386 /// instructions and the matrix_column_major_store intrinsics. For those, the
387 /// shape information indicates that those instructions should be lowered
388 /// using shape information as well.
389 DenseMap<Value *, ShapeInfo> ShapeMap;
390
391 /// List of instructions to remove. While lowering, we are not replacing all
392 /// users of a lowered instruction, if shape information is available and
393 /// those need to be removed after we finished lowering.
394 SmallVector<Instruction *, 16> ToRemove;
395
396 /// Map from instructions to their produced column matrix.
397 MapVector<Value *, MatrixTy> Inst2ColumnMatrix;
398
399public:
400 LowerMatrixIntrinsics(Function &F, TargetTransformInfo &TTI,
401 AliasAnalysis *AA, DominatorTree *DT, LoopInfo *LI,
402 OptimizationRemarkEmitter *ORE)
403 : Func(F), DL(F.getParent()->getDataLayout()), TTI(TTI), AA(AA), DT(DT),
404 LI(LI), ORE(ORE) {}
405
406 unsigned getNumOps(Type *VT) {
407 assert(isa<VectorType>(VT) && "Expected vector type")((isa<VectorType>(VT) && "Expected vector type"
) ? static_cast<void> (0) : __assert_fail ("isa<VectorType>(VT) && \"Expected vector type\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 407, __PRETTY_FUNCTION__))
;
408 return getNumOps(VT->getScalarType(),
409 cast<FixedVectorType>(VT)->getNumElements());
410 }
411
412 //
413 /// Return the estimated number of vector ops required for an operation on
414 /// \p VT * N.
415 unsigned getNumOps(Type *ST, unsigned N) {
416 return std::ceil((ST->getPrimitiveSizeInBits() * N).getFixedSize() /
417 double(TTI.getRegisterBitWidth(true)));
418 }
419
420 /// Return the set of vectors that a matrix value is lowered to.
421 ///
422 /// If we lowered \p MatrixVal, just return the cache result matrix. Otherwise
423 /// split the flat vector \p MatrixVal containing a matrix with shape \p SI
424 /// into vectors.
425 MatrixTy getMatrix(Value *MatrixVal, const ShapeInfo &SI,
426 IRBuilder<> &Builder) {
427 VectorType *VType = dyn_cast<VectorType>(MatrixVal->getType());
428 assert(VType && "MatrixVal must be a vector type")((VType && "MatrixVal must be a vector type") ? static_cast
<void> (0) : __assert_fail ("VType && \"MatrixVal must be a vector type\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 428, __PRETTY_FUNCTION__))
;
429 assert(cast<FixedVectorType>(VType)->getNumElements() ==((cast<FixedVectorType>(VType)->getNumElements() == SI
.NumRows * SI.NumColumns && "The vector size must match the number of matrix elements"
) ? static_cast<void> (0) : __assert_fail ("cast<FixedVectorType>(VType)->getNumElements() == SI.NumRows * SI.NumColumns && \"The vector size must match the number of matrix elements\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 431, __PRETTY_FUNCTION__))
430 SI.NumRows * SI.NumColumns &&((cast<FixedVectorType>(VType)->getNumElements() == SI
.NumRows * SI.NumColumns && "The vector size must match the number of matrix elements"
) ? static_cast<void> (0) : __assert_fail ("cast<FixedVectorType>(VType)->getNumElements() == SI.NumRows * SI.NumColumns && \"The vector size must match the number of matrix elements\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 431, __PRETTY_FUNCTION__))
431 "The vector size must match the number of matrix elements")((cast<FixedVectorType>(VType)->getNumElements() == SI
.NumRows * SI.NumColumns && "The vector size must match the number of matrix elements"
) ? static_cast<void> (0) : __assert_fail ("cast<FixedVectorType>(VType)->getNumElements() == SI.NumRows * SI.NumColumns && \"The vector size must match the number of matrix elements\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 431, __PRETTY_FUNCTION__))
;
432
433 // Check if we lowered MatrixVal using shape information. In that case,
434 // return the existing matrix, if it matches the requested shape
435 // information. If there is a mis-match, embed the result in a flat
436 // vector and split it later.
437 auto Found = Inst2ColumnMatrix.find(MatrixVal);
438 if (Found != Inst2ColumnMatrix.end()) {
439 MatrixTy &M = Found->second;
440 // Return the found matrix, if its shape matches the requested shape
441 // information
442 if (SI.NumRows == M.getNumRows() && SI.NumColumns == M.getNumColumns())
443 return M;
444
445 MatrixVal = M.embedInVector(Builder);
446 }
447
448 // Otherwise split MatrixVal.
449 SmallVector<Value *, 16> SplitVecs;
450 Value *Undef = UndefValue::get(VType);
451 for (unsigned MaskStart = 0;
452 MaskStart < cast<FixedVectorType>(VType)->getNumElements();
453 MaskStart += SI.getStride()) {
454 Value *V = Builder.CreateShuffleVector(
455 MatrixVal, Undef, createSequentialMask(MaskStart, SI.getStride(), 0),
456 "split");
457 SplitVecs.push_back(V);
458 }
459
460 return {SplitVecs};
461 }
462
463 /// If \p V already has a known shape return false. Otherwise set the shape
464 /// for instructions that support it.
465 bool setShapeInfo(Value *V, ShapeInfo Shape) {
466 assert(Shape && "Shape not set")((Shape && "Shape not set") ? static_cast<void>
(0) : __assert_fail ("Shape && \"Shape not set\"", "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 466, __PRETTY_FUNCTION__))
;
467 if (isa<UndefValue>(V) || !supportsShapeInfo(V))
468 return false;
469
470 auto SIter = ShapeMap.find(V);
471 if (SIter != ShapeMap.end()) {
472 LLVM_DEBUG(dbgs() << " not overriding existing shape: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("lower-matrix-intrinsics")) { dbgs() << " not overriding existing shape: "
<< SIter->second.NumRows << " " << SIter
->second.NumColumns << " for " << *V << "\n"
; } } while (false)
473 << SIter->second.NumRows << " "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("lower-matrix-intrinsics")) { dbgs() << " not overriding existing shape: "
<< SIter->second.NumRows << " " << SIter
->second.NumColumns << " for " << *V << "\n"
; } } while (false)
474 << SIter->second.NumColumns << " for " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("lower-matrix-intrinsics")) { dbgs() << " not overriding existing shape: "
<< SIter->second.NumRows << " " << SIter
->second.NumColumns << " for " << *V << "\n"
; } } while (false)
;
475 return false;
476 }
477
478 ShapeMap.insert({V, Shape});
479 LLVM_DEBUG(dbgs() << " " << Shape.NumRows << " x " << Shape.NumColumnsdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("lower-matrix-intrinsics")) { dbgs() << " " << Shape
.NumRows << " x " << Shape.NumColumns << " for "
<< *V << "\n"; } } while (false)
480 << " for " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("lower-matrix-intrinsics")) { dbgs() << " " << Shape
.NumRows << " x " << Shape.NumColumns << " for "
<< *V << "\n"; } } while (false)
;
481 return true;
482 }
483
484 bool isUniformShape(Value *V) {
485 Instruction *I = dyn_cast<Instruction>(V);
486 if (!I)
487 return true;
488
489 switch (I->getOpcode()) {
490 case Instruction::FAdd:
491 case Instruction::FSub:
492 case Instruction::FMul: // Scalar multiply.
493 case Instruction::Add:
494 case Instruction::Mul:
495 case Instruction::Sub:
496 return true;
497 default:
498 return false;
499 }
500 }
501
502 /// Returns true if shape information can be used for \p V. The supported
503 /// instructions must match the instructions that can be lowered by this pass.
504 bool supportsShapeInfo(Value *V) {
505 Instruction *Inst = dyn_cast<Instruction>(V);
506 if (!Inst)
507 return false;
508
509 IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst);
510 if (II)
511 switch (II->getIntrinsicID()) {
512 case Intrinsic::matrix_multiply:
513 case Intrinsic::matrix_transpose:
514 case Intrinsic::matrix_column_major_load:
515 case Intrinsic::matrix_column_major_store:
516 return true;
517 default:
518 return false;
519 }
520 return isUniformShape(V) || isa<StoreInst>(V) || isa<LoadInst>(V);
521 }
522
523 /// Propagate the shape information of instructions to their users.
524 /// The work list contains instructions for which we can compute the shape,
525 /// either based on the information provided by matrix intrinsics or known
526 /// shapes of operands.
527 SmallVector<Instruction *, 32>
528 propagateShapeForward(SmallVectorImpl<Instruction *> &WorkList) {
529 SmallVector<Instruction *, 32> NewWorkList;
530 // Pop an element for which we guaranteed to have at least one of the
531 // operand shapes. Add the shape for this and then add users to the work
532 // list.
533 LLVM_DEBUG(dbgs() << "Forward-propagate shapes:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("lower-matrix-intrinsics")) { dbgs() << "Forward-propagate shapes:\n"
; } } while (false)
;
534 while (!WorkList.empty()) {
535 Instruction *Inst = WorkList.back();
536 WorkList.pop_back();
537
538 // New entry, set the value and insert operands
539 bool Propagate = false;
540
541 Value *MatrixA;
542 Value *MatrixB;
543 Value *M;
544 Value *N;
545 Value *K;
546 if (match(Inst, m_Intrinsic<Intrinsic::matrix_multiply>(
547 m_Value(MatrixA), m_Value(MatrixB), m_Value(M),
548 m_Value(N), m_Value(K)))) {
549 Propagate = setShapeInfo(Inst, {M, K});
550 } else if (match(Inst, m_Intrinsic<Intrinsic::matrix_transpose>(
551 m_Value(MatrixA), m_Value(M), m_Value(N)))) {
552 // Flip dimensions.
553 Propagate = setShapeInfo(Inst, {N, M});
554 } else if (match(Inst, m_Intrinsic<Intrinsic::matrix_column_major_store>(
555 m_Value(MatrixA), m_Value(), m_Value(),
556 m_Value(), m_Value(M), m_Value(N)))) {
557 Propagate = setShapeInfo(Inst, {N, M});
558 } else if (match(Inst, m_Intrinsic<Intrinsic::matrix_column_major_load>(
559 m_Value(), m_Value(), m_Value(), m_Value(M),
560 m_Value(N)))) {
561 Propagate = setShapeInfo(Inst, {M, N});
562 } else if (match(Inst, m_Store(m_Value(MatrixA), m_Value()))) {
563 auto OpShape = ShapeMap.find(MatrixA);
564 if (OpShape != ShapeMap.end())
565 setShapeInfo(Inst, OpShape->second);
566 continue;
567 } else if (isUniformShape(Inst)) {
568 // Find the first operand that has a known shape and use that.
569 for (auto &Op : Inst->operands()) {
570 auto OpShape = ShapeMap.find(Op.get());
571 if (OpShape != ShapeMap.end()) {
572 Propagate |= setShapeInfo(Inst, OpShape->second);
573 break;
574 }
575 }
576 }
577
578 if (Propagate) {
579 NewWorkList.push_back(Inst);
580 for (auto *User : Inst->users())
581 if (ShapeMap.count(User) == 0)
582 WorkList.push_back(cast<Instruction>(User));
583 }
584 }
585
586 return NewWorkList;
587 }
588
589 /// Propagate the shape to operands of instructions with shape information.
590 /// \p Worklist contains the instruction for which we already know the shape.
591 SmallVector<Instruction *, 32>
592 propagateShapeBackward(SmallVectorImpl<Instruction *> &WorkList) {
593 SmallVector<Instruction *, 32> NewWorkList;
594
595 auto pushInstruction = [](Value *V,
596 SmallVectorImpl<Instruction *> &WorkList) {
597 Instruction *I = dyn_cast<Instruction>(V);
598 if (I)
599 WorkList.push_back(I);
600 };
601 // Pop an element with known shape. Traverse the operands, if their shape
602 // derives from the result shape and is unknown, add it and add them to the
603 // worklist.
604 LLVM_DEBUG(dbgs() << "Backward-propagate shapes:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("lower-matrix-intrinsics")) { dbgs() << "Backward-propagate shapes:\n"
; } } while (false)
;
605 while (!WorkList.empty()) {
606 Value *V = WorkList.back();
607 WorkList.pop_back();
608
609 size_t BeforeProcessingV = WorkList.size();
610 if (!isa<Instruction>(V))
611 continue;
612
613 Value *MatrixA;
614 Value *MatrixB;
615 Value *M;
616 Value *N;
617 Value *K;
618 if (match(V, m_Intrinsic<Intrinsic::matrix_multiply>(
619 m_Value(MatrixA), m_Value(MatrixB), m_Value(M),
620 m_Value(N), m_Value(K)))) {
621 if (setShapeInfo(MatrixA, {M, N}))
622 pushInstruction(MatrixA, WorkList);
623
624 if (setShapeInfo(MatrixB, {N, K}))
625 pushInstruction(MatrixB, WorkList);
626
627 } else if (match(V, m_Intrinsic<Intrinsic::matrix_transpose>(
628 m_Value(MatrixA), m_Value(M), m_Value(N)))) {
629 // Flip dimensions.
630 if (setShapeInfo(MatrixA, {M, N}))
631 pushInstruction(MatrixA, WorkList);
632 } else if (match(V, m_Intrinsic<Intrinsic::matrix_column_major_store>(
633 m_Value(MatrixA), m_Value(), m_Value(), m_Value(),
634 m_Value(M), m_Value(N)))) {
635 if (setShapeInfo(MatrixA, {M, N})) {
636 pushInstruction(MatrixA, WorkList);
637 }
638 } else if (isa<LoadInst>(V) ||
639 match(V, m_Intrinsic<Intrinsic::matrix_column_major_load>())) {
640 // Nothing to do, no matrix input.
641 } else if (isa<StoreInst>(V)) {
642 // Nothing to do. We forward-propagated to this so we would just
643 // backward propagate to an instruction with an already known shape.
644 } else if (isUniformShape(V)) {
645 // Propagate to all operands.
646 ShapeInfo Shape = ShapeMap[V];
647 for (Use &U : cast<Instruction>(V)->operands()) {
648 if (setShapeInfo(U.get(), Shape))
649 pushInstruction(U.get(), WorkList);
650 }
651 }
652 // After we discovered new shape info for new instructions in the
653 // worklist, we use their users as seeds for the next round of forward
654 // propagation.
655 for (size_t I = BeforeProcessingV; I != WorkList.size(); I++)
656 for (User *U : WorkList[I]->users())
657 if (isa<Instruction>(U) && V != U)
658 NewWorkList.push_back(cast<Instruction>(U));
659 }
660 return NewWorkList;
661 }
662
663 bool Visit() {
664 if (EnableShapePropagation) {
2
Assuming the condition is false
3
Taking false branch
665 SmallVector<Instruction *, 32> WorkList;
666
667 // Initially only the shape of matrix intrinsics is known.
668 // Initialize the work list with ops carrying shape information.
669 for (BasicBlock &BB : Func)
670 for (Instruction &Inst : BB) {
671 IntrinsicInst *II = dyn_cast<IntrinsicInst>(&Inst);
672 if (!II)
673 continue;
674
675 switch (II->getIntrinsicID()) {
676 case Intrinsic::matrix_multiply:
677 case Intrinsic::matrix_transpose:
678 case Intrinsic::matrix_column_major_load:
679 case Intrinsic::matrix_column_major_store:
680 WorkList.push_back(&Inst);
681 break;
682 default:
683 break;
684 }
685 }
686 // Propagate shapes until nothing changes any longer.
687 while (!WorkList.empty()) {
688 WorkList = propagateShapeForward(WorkList);
689 WorkList = propagateShapeBackward(WorkList);
690 }
691 }
692
693 bool Changed = false;
694 SmallVector<CallInst *, 16> MaybeFusableInsts;
695 SmallVector<Instruction *, 16> MatrixInsts;
696
697 // First, collect all instructions with shape information and candidates for
698 // fusion (currently only matrix multiplies).
699 ReversePostOrderTraversal<Function *> RPOT(&Func);
700 for (auto *BB : RPOT)
701 for (Instruction &I : *BB) {
702 if (ShapeMap.find(&I) == ShapeMap.end())
703 continue;
704 if (match(&I, m_Intrinsic<Intrinsic::matrix_multiply>()))
705 MaybeFusableInsts.push_back(cast<CallInst>(&I));
706 MatrixInsts.push_back(&I);
707 }
708
709 // Second, try to fuse candidates.
710 SmallPtrSet<Instruction *, 16> FusedInsts;
711 for (CallInst *CI : MaybeFusableInsts)
4
Assuming '__begin2' is equal to '__end2'
712 LowerMatrixMultiplyFused(CI, FusedInsts);
713 Changed = !FusedInsts.empty();
714
715 // Third, lower remaining instructions with shape information.
716 for (Instruction *Inst : MatrixInsts) {
5
Assuming '__begin2' is not equal to '__end2'
717 if (FusedInsts.count(Inst))
6
Assuming the condition is false
7
Taking false branch
718 continue;
719
720 IRBuilder<> Builder(Inst);
721
722 if (CallInst *CInst
8.1
'CInst' is non-null
= dyn_cast<CallInst>(Inst))
8
Assuming 'Inst' is a 'CallInst'
9
Taking true branch
723 Changed |= VisitCallInst(CInst);
10
Calling 'LowerMatrixIntrinsics::VisitCallInst'
724
725 Value *Op1;
726 Value *Op2;
727 if (auto *BinOp = dyn_cast<BinaryOperator>(Inst))
728 Changed |= VisitBinaryOperator(BinOp);
729 if (match(Inst, m_Load(m_Value(Op1))))
730 Changed |= VisitLoad(cast<LoadInst>(Inst), Op1, Builder);
731 else if (match(Inst, m_Store(m_Value(Op1), m_Value(Op2))))
732 Changed |= VisitStore(cast<StoreInst>(Inst), Op1, Op2, Builder);
733 }
734
735 if (ORE) {
736 RemarkGenerator RemarkGen(Inst2ColumnMatrix, *ORE, Func);
737 RemarkGen.emitRemarks();
738 }
739
740 for (Instruction *Inst : reverse(ToRemove))
741 Inst->eraseFromParent();
742
743 return Changed;
744 }
745
746 /// Turns \p BasePtr into an elementwise pointer to \p EltType.
747 Value *createElementPtr(Value *BasePtr, Type *EltType, IRBuilder<> &Builder) {
748 unsigned AS = cast<PointerType>(BasePtr->getType())->getAddressSpace();
749 Type *EltPtrType = PointerType::get(EltType, AS);
750 return Builder.CreatePointerCast(BasePtr, EltPtrType);
751 }
752
753 /// Replace intrinsic calls
754 bool VisitCallInst(CallInst *Inst) {
755 if (!Inst->getCalledFunction() || !Inst->getCalledFunction()->isIntrinsic())
11
Assuming the condition is false
12
Taking false branch
756 return false;
757
758 switch (Inst->getCalledFunction()->getIntrinsicID()) {
13
Control jumps to 'case matrix_multiply:' at line 759
759 case Intrinsic::matrix_multiply:
760 LowerMultiply(Inst);
14
Calling 'LowerMatrixIntrinsics::LowerMultiply'
761 break;
762 case Intrinsic::matrix_transpose:
763 LowerTranspose(Inst);
764 break;
765 case Intrinsic::matrix_column_major_load:
766 LowerColumnMajorLoad(Inst);
767 break;
768 case Intrinsic::matrix_column_major_store:
769 LowerColumnMajorStore(Inst);
770 break;
771 default:
772 return false;
773 }
774 return true;
775 }
776
777 /// Compute the alignment for a column/row \p Idx with \p Stride between them.
778 /// The address at \p Idx == 0 has alignment \p A. If \p Stride is a
779 /// ConstantInt, reduce the initial alignment based on the byte offset. For
780 /// non-ConstantInt strides, return the common alignment of the initial
781 /// alignment and the element size in bytes.
782 Align getAlignForIndex(unsigned Idx, Value *Stride, Type *ElementTy,
783 MaybeAlign A) const {
784 Align InitialAlign = DL.getValueOrABITypeAlignment(A, ElementTy);
785 if (Idx == 0)
786 return InitialAlign;
787
788 TypeSize ElementSizeInBits = DL.getTypeSizeInBits(ElementTy);
789 if (auto *ConstStride = dyn_cast<ConstantInt>(Stride)) {
790 uint64_t StrideInBytes =
791 ConstStride->getZExtValue() * ElementSizeInBits / 8;
792 return commonAlignment(InitialAlign, Idx * StrideInBytes);
793 }
794 return commonAlignment(InitialAlign, ElementSizeInBits / 8);
795 }
796
797 /// Load a matrix with \p Shape starting at \p Ptr and using \p Stride between
798 /// vectors.
799 MatrixTy loadMatrix(Type *Ty, Value *Ptr, MaybeAlign MAlign, Value *Stride,
800 bool IsVolatile, ShapeInfo Shape, IRBuilder<> &Builder) {
801 auto VType = cast<VectorType>(Ty);
802 Value *EltPtr = createElementPtr(Ptr, VType->getElementType(), Builder);
803 MatrixTy Result;
804 for (unsigned I = 0, E = Shape.getNumVectors(); I < E; ++I) {
805 Value *GEP = computeVectorAddr(EltPtr, Builder.getInt64(I), Stride,
806 Shape.getStride(), VType->getElementType(),
807 Builder);
808 Value *Vector = Builder.CreateAlignedLoad(
809 GEP, getAlignForIndex(I, Stride, VType->getElementType(), MAlign),
810 IsVolatile, "col.load");
811
812 Result.addVector(Vector);
813 }
814 return Result.addNumLoads(getNumOps(Result.getVectorTy()) *
815 Result.getNumVectors());
816 }
817
818 /// Loads a sub-matrix with shape \p ResultShape from a \p R x \p C matrix,
819 /// starting at \p MatrixPtr[I][J].
820 MatrixTy loadMatrix(Value *MatrixPtr, MaybeAlign Align, bool IsVolatile,
821 ShapeInfo MatrixShape, Value *I, Value *J,
822 ShapeInfo ResultShape, Type *EltTy,
823 IRBuilder<> &Builder) {
824
825 Value *Offset = Builder.CreateAdd(
826 Builder.CreateMul(J, Builder.getInt64(MatrixShape.getStride())), I);
827
828 unsigned AS = cast<PointerType>(MatrixPtr->getType())->getAddressSpace();
829 Value *EltPtr =
830 Builder.CreatePointerCast(MatrixPtr, PointerType::get(EltTy, AS));
831 Value *TileStart = Builder.CreateGEP(EltTy, EltPtr, Offset);
832 auto *TileTy = FixedVectorType::get(EltTy, ResultShape.NumRows *
833 ResultShape.NumColumns);
834 Type *TilePtrTy = PointerType::get(TileTy, AS);
835 Value *TilePtr =
836 Builder.CreatePointerCast(TileStart, TilePtrTy, "col.cast");
837
838 return loadMatrix(TileTy, TilePtr, Align,
839 Builder.getInt64(MatrixShape.getStride()), IsVolatile,
840 ResultShape, Builder);
841 }
842
843 /// Lower a load instruction with shape information.
844 void LowerLoad(Instruction *Inst, Value *Ptr, MaybeAlign Align, Value *Stride,
845 bool IsVolatile, ShapeInfo Shape) {
846 IRBuilder<> Builder(Inst);
847 finalizeLowering(Inst,
848 loadMatrix(Inst->getType(), Ptr, Align, Stride, IsVolatile,
849 Shape, Builder),
850 Builder);
851 }
852
853 /// Lowers llvm.matrix.column.major.load.
854 ///
855 /// The intrinsic loads a matrix from memory using a stride between columns.
856 void LowerColumnMajorLoad(CallInst *Inst) {
857 assert(MatrixLayout == MatrixLayoutTy::ColumnMajor &&((MatrixLayout == MatrixLayoutTy::ColumnMajor && "Intrinsic only supports column-major layout!"
) ? static_cast<void> (0) : __assert_fail ("MatrixLayout == MatrixLayoutTy::ColumnMajor && \"Intrinsic only supports column-major layout!\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 858, __PRETTY_FUNCTION__))
858 "Intrinsic only supports column-major layout!")((MatrixLayout == MatrixLayoutTy::ColumnMajor && "Intrinsic only supports column-major layout!"
) ? static_cast<void> (0) : __assert_fail ("MatrixLayout == MatrixLayoutTy::ColumnMajor && \"Intrinsic only supports column-major layout!\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 858, __PRETTY_FUNCTION__))
;
859 Value *Ptr = Inst->getArgOperand(0);
860 Value *Stride = Inst->getArgOperand(1);
861 LowerLoad(Inst, Ptr, Inst->getParamAlign(0), Stride,
862 cast<ConstantInt>(Inst->getArgOperand(2))->isOne(),
863 {Inst->getArgOperand(3), Inst->getArgOperand(4)});
864 }
865
866 /// Stores a sub-matrix \p StoreVal into the \p R x \p C matrix starting at \p
867 /// MatrixPtr[I][J].
868 void storeMatrix(const MatrixTy &StoreVal, Value *MatrixPtr,
869 MaybeAlign MAlign, bool IsVolatile, ShapeInfo MatrixShape,
870 Value *I, Value *J, Type *EltTy, IRBuilder<> &Builder) {
871 Value *Offset = Builder.CreateAdd(
872 Builder.CreateMul(J, Builder.getInt64(MatrixShape.getStride())), I);
873
874 unsigned AS = cast<PointerType>(MatrixPtr->getType())->getAddressSpace();
875 Value *EltPtr =
876 Builder.CreatePointerCast(MatrixPtr, PointerType::get(EltTy, AS));
877 Value *TileStart = Builder.CreateGEP(EltTy, EltPtr, Offset);
878 auto *TileTy = FixedVectorType::get(EltTy, StoreVal.getNumRows() *
879 StoreVal.getNumColumns());
880 Type *TilePtrTy = PointerType::get(TileTy, AS);
881 Value *TilePtr =
882 Builder.CreatePointerCast(TileStart, TilePtrTy, "col.cast");
883
884 storeMatrix(TileTy, StoreVal, TilePtr, MAlign,
885 Builder.getInt64(MatrixShape.getStride()), IsVolatile, Builder);
886 }
887
888 /// Store matrix \p StoreVal starting at \p Ptr and using \p Stride between
889 /// vectors.
890 MatrixTy storeMatrix(Type *Ty, MatrixTy StoreVal, Value *Ptr,
891 MaybeAlign MAlign, Value *Stride, bool IsVolatile,
892 IRBuilder<> &Builder) {
893 auto VType = cast<VectorType>(Ty);
894 Value *EltPtr = createElementPtr(Ptr, VType->getElementType(), Builder);
895 for (auto Vec : enumerate(StoreVal.vectors())) {
896 Value *GEP = computeVectorAddr(EltPtr, Builder.getInt64(Vec.index()),
897 Stride, StoreVal.getStride(),
898 VType->getElementType(), Builder);
899 Builder.CreateAlignedStore(Vec.value(), GEP,
900 getAlignForIndex(Vec.index(), Stride,
901 VType->getElementType(),
902 MAlign),
903 IsVolatile);
904 }
905 return MatrixTy().addNumStores(getNumOps(StoreVal.getVectorTy()) *
906 StoreVal.getNumVectors());
907 }
908
909 /// Lower a store instruction with shape information.
910 void LowerStore(Instruction *Inst, Value *Matrix, Value *Ptr, MaybeAlign A,
911 Value *Stride, bool IsVolatile, ShapeInfo Shape) {
912 IRBuilder<> Builder(Inst);
913 auto StoreVal = getMatrix(Matrix, Shape, Builder);
914 finalizeLowering(Inst,
915 storeMatrix(Matrix->getType(), StoreVal, Ptr, A, Stride,
916 IsVolatile, Builder),
917 Builder);
918 }
919
920 /// Lowers llvm.matrix.column.major.store.
921 ///
922 /// The intrinsic store a matrix back memory using a stride between columns.
923 void LowerColumnMajorStore(CallInst *Inst) {
924 assert(MatrixLayout == MatrixLayoutTy::ColumnMajor &&((MatrixLayout == MatrixLayoutTy::ColumnMajor && "Intrinsic only supports column-major layout!"
) ? static_cast<void> (0) : __assert_fail ("MatrixLayout == MatrixLayoutTy::ColumnMajor && \"Intrinsic only supports column-major layout!\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 925, __PRETTY_FUNCTION__))
925 "Intrinsic only supports column-major layout!")((MatrixLayout == MatrixLayoutTy::ColumnMajor && "Intrinsic only supports column-major layout!"
) ? static_cast<void> (0) : __assert_fail ("MatrixLayout == MatrixLayoutTy::ColumnMajor && \"Intrinsic only supports column-major layout!\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 925, __PRETTY_FUNCTION__))
;
926 Value *Matrix = Inst->getArgOperand(0);
927 Value *Ptr = Inst->getArgOperand(1);
928 Value *Stride = Inst->getArgOperand(2);
929 LowerStore(Inst, Matrix, Ptr, Inst->getParamAlign(1), Stride,
930 cast<ConstantInt>(Inst->getArgOperand(3))->isOne(),
931 {Inst->getArgOperand(4), Inst->getArgOperand(5)});
932 }
933
934 // Set elements I..I+NumElts-1 to Block
935 Value *insertVector(Value *Col, unsigned I, Value *Block,
936 IRBuilder<> &Builder) {
937
938 // First, bring Block to the same size as Col
939 unsigned BlockNumElts =
940 cast<FixedVectorType>(Block->getType())->getNumElements();
41
Called C++ object pointer is null
941 unsigned NumElts = cast<FixedVectorType>(Col->getType())->getNumElements();
942 assert(NumElts >= BlockNumElts && "Too few elements for current block")((NumElts >= BlockNumElts && "Too few elements for current block"
) ? static_cast<void> (0) : __assert_fail ("NumElts >= BlockNumElts && \"Too few elements for current block\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 942, __PRETTY_FUNCTION__))
;
943
944 Value *Undef = UndefValue::get(Block->getType());
945 Block = Builder.CreateShuffleVector(
946 Block, Undef,
947 createSequentialMask(0, BlockNumElts, NumElts - BlockNumElts));
948
949 // If Col is 7 long and I is 2 and BlockNumElts is 2 the mask is: 0, 1, 7,
950 // 8, 4, 5, 6
951 SmallVector<int, 16> Mask;
952 unsigned i;
953 for (i = 0; i < I; i++)
954 Mask.push_back(i);
955
956 unsigned VecNumElts =
957 cast<FixedVectorType>(Col->getType())->getNumElements();
958 for (; i < I + BlockNumElts; i++)
959 Mask.push_back(i - I + VecNumElts);
960
961 for (; i < VecNumElts; i++)
962 Mask.push_back(i);
963
964 return Builder.CreateShuffleVector(Col, Block, Mask);
965 }
966
967 Value *createMulAdd(Value *Sum, Value *A, Value *B, bool UseFPOp,
968 IRBuilder<> &Builder, bool AllowContraction,
969 unsigned &NumComputeOps) {
970 NumComputeOps += getNumOps(A->getType());
971 if (!Sum)
972 return UseFPOp ? Builder.CreateFMul(A, B) : Builder.CreateMul(A, B);
973
974 if (UseFPOp) {
975 if (AllowContraction) {
976 // Use fmuladd for floating point operations and let the backend decide
977 // if that's profitable.
978 Function *FMulAdd = Intrinsic::getDeclaration(
979 Func.getParent(), Intrinsic::fmuladd, A->getType());
980 return Builder.CreateCall(FMulAdd, {A, B, Sum});
981 }
982 NumComputeOps += getNumOps(A->getType());
983 Value *Mul = Builder.CreateFMul(A, B);
984 return Builder.CreateFAdd(Sum, Mul);
985 }
986
987 NumComputeOps += getNumOps(A->getType());
988 Value *Mul = Builder.CreateMul(A, B);
989 return Builder.CreateAdd(Sum, Mul);
990 }
991
992 /// Cache \p Matrix as result of \p Inst and update the uses of \p Inst. For
993 /// users with shape information, there's nothing to do: the will use the
994 /// cached value when they are lowered. For other users, \p Matrix is
995 /// flattened and the uses are updated to use it. Also marks \p Inst for
996 /// deletion.
997 void finalizeLowering(Instruction *Inst, MatrixTy Matrix,
998 IRBuilder<> &Builder) {
999 Inst2ColumnMatrix.insert(std::make_pair(Inst, Matrix));
1000
1001 ToRemove.push_back(Inst);
1002 Value *Flattened = nullptr;
1003 for (auto I = Inst->use_begin(), E = Inst->use_end(); I != E;) {
1004 Use &U = *I++;
1005 if (ShapeMap.find(U.getUser()) == ShapeMap.end()) {
1006 if (!Flattened)
1007 Flattened = Matrix.embedInVector(Builder);
1008 U.set(Flattened);
1009 }
1010 }
1011 }
1012
1013 /// Compute \p Result += \p A * \p B for input matrices with left-associating
1014 /// addition.
1015 void emitMatrixMultiply(MatrixTy &Result, const MatrixTy &A,
1016 const MatrixTy &B, bool AllowContraction,
1017 IRBuilder<> &Builder, bool isTiled) {
1018 const unsigned VF = std::max<unsigned>(
1019 TTI.getRegisterBitWidth(true) /
1020 Result.getElementType()->getPrimitiveSizeInBits().getFixedSize(),
1021 1U);
1022 unsigned R = Result.getNumRows();
1023 unsigned C = Result.getNumColumns();
1024 unsigned M = A.getNumColumns();
1025
1026 bool IsFP = Result.getElementType()->isFloatingPointTy();
1027 assert(A.isColumnMajor() == B.isColumnMajor() &&((A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor
() == A.isColumnMajor() && "operands must agree on matrix layout"
) ? static_cast<void> (0) : __assert_fail ("A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor() == A.isColumnMajor() && \"operands must agree on matrix layout\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1029, __PRETTY_FUNCTION__))
25
'?' condition is true
1028 Result.isColumnMajor() == A.isColumnMajor() &&((A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor
() == A.isColumnMajor() && "operands must agree on matrix layout"
) ? static_cast<void> (0) : __assert_fail ("A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor() == A.isColumnMajor() && \"operands must agree on matrix layout\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1029, __PRETTY_FUNCTION__))
1029 "operands must agree on matrix layout")((A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor
() == A.isColumnMajor() && "operands must agree on matrix layout"
) ? static_cast<void> (0) : __assert_fail ("A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor() == A.isColumnMajor() && \"operands must agree on matrix layout\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1029, __PRETTY_FUNCTION__))
;
1030 unsigned NumComputeOps = 0;
1031 if (A.isColumnMajor()) {
26
Calling 'MatrixTy::isColumnMajor'
28
Returning from 'MatrixTy::isColumnMajor'
29
Taking false branch
1032 // Multiply columns from the first operand with scalars from the second
1033 // operand. Then move along the K axes and accumulate the columns. With
1034 // this the adds can be vectorized without reassociation.
1035 for (unsigned J = 0; J < C; ++J) {
1036 unsigned BlockSize = VF;
1037 // If Result is zero, we don't need to accumulate in the K==0 iteration.
1038 bool isSumZero = isa<ConstantAggregateZero>(Result.getColumn(J));
1039
1040 for (unsigned I = 0; I < R; I += BlockSize) {
1041 // Gradually lower the vectorization factor to cover the remainder.
1042 while (I + BlockSize > R)
1043 BlockSize /= 2;
1044
1045 Value *Sum = isTiled ? Result.extractVector(I, J, BlockSize, Builder)
1046 : nullptr;
1047 for (unsigned K = 0; K < M; ++K) {
1048 Value *L = A.extractVector(I, K, BlockSize, Builder);
1049 Value *RH = Builder.CreateExtractElement(B.getColumn(J), K);
1050 Value *Splat = Builder.CreateVectorSplat(BlockSize, RH, "splat");
1051 Sum = createMulAdd(isSumZero && K == 0 ? nullptr : Sum, L, Splat,
1052 Result.getElementType()->isFloatingPointTy(),
1053 Builder, AllowContraction, NumComputeOps);
1054 }
1055 Result.setVector(J,
1056 insertVector(Result.getVector(J), I, Sum, Builder));
1057 }
1058 }
1059 } else {
1060 // Multiply rows from the second operand with scalars from the first
1061 // operand. Then move along the K axes and accumulate the rows. With this
1062 // the adds can be vectorized without reassociation.
1063 for (unsigned I = 0; I
29.1
'I' is < 'R'
< R; ++I) {
30
Loop condition is true. Entering loop body
1064 unsigned BlockSize = VF;
1065 bool isSumZero = isa<ConstantAggregateZero>(Result.getRow(I));
31
Assuming the object is not a 'ConstantAggregateZero'
1066 for (unsigned J = 0; J < C; J += BlockSize) {
32
Assuming 'J' is < 'C'
33
Loop condition is true. Entering loop body
1067 // Gradually lower the vectorization factor to cover the remainder.
1068 while (J + BlockSize > C)
34
Assuming the condition is false
35
Loop condition is false. Execution continues on line 1071
1069 BlockSize /= 2;
1070
1071 Value *Sum = nullptr;
36
'Sum' initialized to a null pointer value
1072 for (unsigned K = 0; K < M; ++K) {
37
Assuming 'K' is >= 'M'
38
Loop condition is false. Execution continues on line 1079
1073 Value *R = B.extractVector(K, J, BlockSize, Builder);
1074 Value *LH = Builder.CreateExtractElement(A.getVector(I), K);
1075 Value *Splat = Builder.CreateVectorSplat(BlockSize, LH, "splat");
1076 Sum = createMulAdd(isSumZero && K == 0 ? nullptr : Sum, Splat, R,
1077 IsFP, Builder, AllowContraction, NumComputeOps);
1078 }
1079 Result.setVector(I,
1080 insertVector(Result.getVector(I), J, Sum, Builder));
39
Passing null pointer value via 3rd parameter 'Block'
40
Calling 'LowerMatrixIntrinsics::insertVector'
1081 }
1082 }
1083 }
1084 Result.addNumComputeOps(NumComputeOps);
1085 }
1086
1087 /// Ensure that the memory in \p Load does not alias \p Store by potentially
1088 /// copying it to a new location. This new or otherwise the original location
1089 /// is returned.
1090 Value *getNonAliasingPointer(LoadInst *Load, StoreInst *Store,
1091 CallInst *MatMul) {
1092 MemoryLocation StoreLoc = MemoryLocation::get(Store);
1093 MemoryLocation LoadLoc = MemoryLocation::get(Load);
1094
1095 AliasResult LdAliased = AA->alias(LoadLoc, StoreLoc);
1096
1097 // If we can statically determine noalias we're good.
1098 if (!LdAliased)
1099 return Load->getPointerOperand();
1100
1101 // Create code to check if the memory locations of the Load and Store
1102 // overlap and if they do, copy Load's operand to a new buffer.
1103
1104 // First, create new blocks for 2n part of the check and the copy.
1105 BasicBlock *Check0 = MatMul->getParent();
1106 // FIXME: Use lazy DTU and update SplitBlock to accept a DTU instead of a
1107 // DT. Manually collect dominator tree updates, to avoid unnecessary work,
1108 // as we adjust Check0 and Check1's branches.
1109 SmallVector<DominatorTree::UpdateType, 4> DTUpdates;
1110 for (BasicBlock *Succ : successors(Check0))
1111 DTUpdates.push_back({DT->Delete, Check0, Succ});
1112
1113 BasicBlock *Check1 = SplitBlock(MatMul->getParent(), MatMul, nullptr, LI,
1114 nullptr, "alias_cont");
1115 BasicBlock *Copy =
1116 SplitBlock(MatMul->getParent(), MatMul, nullptr, LI, nullptr, "copy");
1117 BasicBlock *Fusion = SplitBlock(MatMul->getParent(), MatMul, nullptr, LI,
1118 nullptr, "no_alias");
1119
1120 // Check if the loaded memory location begins before the end of the store
1121 // location. If the condition holds, they might overlap, otherwise they are
1122 // guaranteed to not overlap.
1123 IRBuilder<> Builder(MatMul);
1124 Check0->getTerminator()->eraseFromParent();
1125 Builder.SetInsertPoint(Check0);
1126 Type *IntPtrTy = Builder.getIntPtrTy(Load->getModule()->getDataLayout());
1127 Value *StoreBegin = Builder.CreatePtrToInt(
1128 const_cast<Value *>(StoreLoc.Ptr), IntPtrTy, "store.begin");
1129 Value *StoreEnd = Builder.CreateAdd(
1130 StoreBegin, ConstantInt::get(IntPtrTy, StoreLoc.Size.getValue()),
1131 "store.end", true, true);
1132 Value *LoadBegin = Builder.CreatePtrToInt(const_cast<Value *>(LoadLoc.Ptr),
1133 IntPtrTy, "load.begin");
1134 Builder.CreateCondBr(Builder.CreateICmpULT(LoadBegin, StoreEnd), Check1,
1135 Fusion);
1136
1137 // Check if the store begins before the end of the load location. If the
1138 // condition holds, they alias, otherwise they are guaranteed to not
1139 // overlap.
1140 Check1->getTerminator()->eraseFromParent();
1141 Builder.SetInsertPoint(Check1, Check1->begin());
1142 Value *LoadEnd = Builder.CreateAdd(
1143 LoadBegin, ConstantInt::get(IntPtrTy, LoadLoc.Size.getValue()),
1144 "load.end", true, true);
1145 Builder.CreateCondBr(Builder.CreateICmpULT(StoreBegin, LoadEnd), Copy,
1146 Fusion);
1147
1148 // Copy load operand to new alloca.
1149 Builder.SetInsertPoint(Copy, Copy->begin());
1150 AllocaInst *NewLd =
1151 Builder.CreateAlloca(Load->getType(), Load->getPointerAddressSpace());
1152 Builder.CreateMemCpy(NewLd, NewLd->getAlign(),
1153 Load->getPointerOperand(), Load->getAlign(),
1154 LoadLoc.Size.getValue());
1155 Builder.SetInsertPoint(Fusion, Fusion->begin());
1156 PHINode *PHI = Builder.CreatePHI(Load->getPointerOperandType(), 3);
1157 PHI->addIncoming(Load->getPointerOperand(), Check0);
1158 PHI->addIncoming(Load->getPointerOperand(), Check1);
1159 PHI->addIncoming(NewLd, Copy);
1160
1161 // Adjust DT.
1162 DTUpdates.push_back({DT->Insert, Check0, Check1});
1163 DTUpdates.push_back({DT->Insert, Check0, Fusion});
1164 DTUpdates.push_back({DT->Insert, Check1, Copy});
1165 DTUpdates.push_back({DT->Insert, Check1, Fusion});
1166 DT->applyUpdates(DTUpdates);
1167 return PHI;
1168 }
1169
1170 bool isFusionProfitable(CallInst *MatMul) {
1171 if (ForceFusion)
1172 return true;
1173
1174 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3));
1175 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4));
1176
1177 const unsigned R = LShape.NumRows;
1178 const unsigned C = RShape.NumColumns;
1179 const unsigned M = LShape.NumColumns;
1180 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
1181
1182 const unsigned VF =
1183 std::max<unsigned>(TTI.getRegisterBitWidth(true) /
1184 EltType->getPrimitiveSizeInBits().getFixedSize(),
1185 1U);
1186
1187 // Cost model for tiling
1188 //
1189 // For tiling to be beneficial, we need reuse either along the R or
1190 // the C axis. We vectorize along the R axis so that means at least
1191 // 3 elements.
1192 // TODO: Also consider cost of copying if operands alias.
1193 if (R <= VF && C == 1)
1194 return false;
1195 // Then we need enough elements to exceed the number of vector
1196 // registers we have. Note that this is an oversimplification since
1197 // fusing also takes some extra loads which may exceed the number of
1198 // reloads necessary.
1199 unsigned Op0Regs = (R + VF - 1) / VF * M;
1200 unsigned Op1Regs = (M + VF - 1) / VF * C;
1201 return Op0Regs + Op1Regs > TTI.getNumberOfRegisters(true);
1202 }
1203
1204 MatrixTy getZeroMatrix(Type *EltType, unsigned R, unsigned C) {
1205 MatrixTy Res;
1206 auto *ColumType = FixedVectorType::get(EltType, R);
1207 for (unsigned I = 0; I < C; ++I)
1208 Res.addVector(ConstantAggregateZero::get(ColumType));
1209 return Res;
1210 }
1211
1212 void createTiledLoops(CallInst *MatMul, Value *LPtr, ShapeInfo LShape,
1213 Value *RPtr, ShapeInfo RShape, StoreInst *Store,
1214 bool AllowContract) {
1215 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
1216
1217 // Create the main tiling loop nest.
1218 TileInfo TI(LShape.NumRows, RShape.NumColumns, LShape.NumColumns, TileSize);
1219 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
1220 Instruction *InsertI = cast<Instruction>(MatMul);
1221 BasicBlock *Start = InsertI->getParent();
1222 BasicBlock *End =
1223 SplitBlock(InsertI->getParent(), InsertI, DT, LI, nullptr, "continue");
1224 IRBuilder<> Builder(MatMul);
1225 BasicBlock *InnerBody = TI.CreateTiledLoops(Start, End, Builder, DTU, *LI);
1226
1227 Type *TileVecTy =
1228 FixedVectorType::get(MatMul->getType()->getScalarType(), TileSize);
1229 MatrixTy TileResult;
1230 // Insert in the inner loop header.
1231 Builder.SetInsertPoint(TI.InnerLoopHeader->getTerminator());
1232 // Create PHI nodes for the result columns to accumulate across iterations.
1233 SmallVector<PHINode *, 4> ColumnPhis;
1234 for (unsigned I = 0; I < TileSize; I++) {
1235 auto *Phi = Builder.CreatePHI(TileVecTy, 2, "result.vec." + Twine(I));
1236 Phi->addIncoming(ConstantAggregateZero::get(TileVecTy),
1237 TI.RowLoopHeader->getSingleSuccessor());
1238 TileResult.addVector(Phi);
1239 ColumnPhis.push_back(Phi);
1240 }
1241
1242 // Insert in the inner loop body, which computes
1243 // Res += Load(CurrentRow, K) * Load(K, CurrentColumn)
1244 Builder.SetInsertPoint(InnerBody->getTerminator());
1245 // Load tiles of the operands.
1246 MatrixTy A = loadMatrix(LPtr, {}, false, LShape, TI.CurrentRow, TI.CurrentK,
1247 {TileSize, TileSize}, EltType, Builder);
1248 MatrixTy B = loadMatrix(RPtr, {}, false, RShape, TI.CurrentK, TI.CurrentCol,
1249 {TileSize, TileSize}, EltType, Builder);
1250 emitMatrixMultiply(TileResult, A, B, AllowContract, Builder, true);
1251 // Store result after the inner loop is done.
1252 Builder.SetInsertPoint(TI.RowLoopLatch->getTerminator());
1253 storeMatrix(TileResult, Store->getPointerOperand(), Store->getAlign(),
1254 Store->isVolatile(), {LShape.NumRows, RShape.NumColumns},
1255 TI.CurrentRow, TI.CurrentCol, EltType, Builder);
1256
1257 for (unsigned I = 0; I < TileResult.getNumVectors(); I++)
1258 ColumnPhis[I]->addIncoming(TileResult.getVector(I), TI.InnerLoopLatch);
1259
1260 // Force unrolling of a few iterations of the inner loop, to make sure there
1261 // is enough work per iteration.
1262 // FIXME: The unroller should make this decision directly instead, but
1263 // currently the cost-model is not up to the task.
1264 unsigned InnerLoopUnrollCount = std::min(10u, LShape.NumColumns / TileSize);
1265 addStringMetadataToLoop(LI->getLoopFor(TI.InnerLoopHeader),
1266 "llvm.loop.unroll.count", InnerLoopUnrollCount);
1267 }
1268
1269 void emitSIMDTiling(CallInst *MatMul, LoadInst *LoadOp0, LoadInst *LoadOp1,
1270 StoreInst *Store,
1271 SmallPtrSetImpl<Instruction *> &FusedInsts) {
1272 assert(MatrixLayout == MatrixLayoutTy::ColumnMajor &&((MatrixLayout == MatrixLayoutTy::ColumnMajor && "Tiling only supported for column-major matrixes at the moment!"
) ? static_cast<void> (0) : __assert_fail ("MatrixLayout == MatrixLayoutTy::ColumnMajor && \"Tiling only supported for column-major matrixes at the moment!\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1273, __PRETTY_FUNCTION__))
1273 "Tiling only supported for column-major matrixes at the moment!")((MatrixLayout == MatrixLayoutTy::ColumnMajor && "Tiling only supported for column-major matrixes at the moment!"
) ? static_cast<void> (0) : __assert_fail ("MatrixLayout == MatrixLayoutTy::ColumnMajor && \"Tiling only supported for column-major matrixes at the moment!\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1273, __PRETTY_FUNCTION__))
;
1274 if (!isFusionProfitable(MatMul))
1275 return;
1276
1277 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3));
1278 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4));
1279
1280 const unsigned R = LShape.NumRows;
1281 const unsigned C = RShape.NumColumns;
1282 const unsigned M = LShape.NumColumns;
1283 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
1284
1285 Value *APtr = getNonAliasingPointer(LoadOp0, Store, MatMul);
1286 Value *BPtr = getNonAliasingPointer(LoadOp1, Store, MatMul);
1287 Value *CPtr = Store->getPointerOperand();
1288
1289 bool AllowContract = AllowContractEnabled || (isa<FPMathOperator>(MatMul) &&
1290 MatMul->hasAllowContract());
1291 if (TileUseLoops && (R % TileSize == 0 && C % TileSize == 0))
1292 createTiledLoops(MatMul, APtr, LShape, BPtr, RShape, Store,
1293 AllowContract);
1294 else {
1295 IRBuilder<> Builder(Store);
1296 for (unsigned J = 0; J < C; J += TileSize)
1297 for (unsigned I = 0; I < R; I += TileSize) {
1298 const unsigned TileR = std::min(R - I, unsigned(TileSize));
1299 const unsigned TileC = std::min(C - J, unsigned(TileSize));
1300 MatrixTy Res = getZeroMatrix(EltType, TileR, TileC);
1301
1302 for (unsigned K = 0; K < M; K += TileSize) {
1303 const unsigned TileM = std::min(M - K, unsigned(TileSize));
1304 MatrixTy A =
1305 loadMatrix(APtr, LoadOp0->getAlign(), LoadOp0->isVolatile(),
1306 LShape, Builder.getInt64(I), Builder.getInt64(K),
1307 {TileR, TileM}, EltType, Builder);
1308 MatrixTy B =
1309 loadMatrix(BPtr, LoadOp1->getAlign(), LoadOp1->isVolatile(),
1310 RShape, Builder.getInt64(K), Builder.getInt64(J),
1311 {TileM, TileC}, EltType, Builder);
1312 emitMatrixMultiply(Res, A, B, AllowContract, Builder, true);
1313 }
1314 storeMatrix(Res, CPtr, Store->getAlign(), Store->isVolatile(), {R, M},
1315 Builder.getInt64(I), Builder.getInt64(J), EltType,
1316 Builder);
1317 }
1318 }
1319
1320 // Mark eliminated instructions as fused and remove them.
1321 FusedInsts.insert(Store);
1322 FusedInsts.insert(MatMul);
1323 Store->eraseFromParent();
1324 MatMul->eraseFromParent();
1325 if (LoadOp0->hasNUses(0)) {
1326 FusedInsts.insert(LoadOp0);
1327 LoadOp0->eraseFromParent();
1328 }
1329 if (LoadOp1->hasNUses(0)) {
1330 FusedInsts.insert(LoadOp1);
1331 LoadOp1->eraseFromParent();
1332 }
1333 }
1334
1335 /// Try to lower matrix multiply chains by fusing operations.
1336 ///
1337 /// Currently we only lower {ld, ld} -> matmul -> st chains.
1338 //
1339 /// No need to return a MatrixTy object for the result of the operation, since
1340 /// the single store user will be lowered as part of this. Instructions that
1341 /// are completely eliminated by fusion are added to \p FusedInsts.
1342 void LowerMatrixMultiplyFused(CallInst *MatMul,
1343 SmallPtrSetImpl<Instruction *> &FusedInsts) {
1344 if (!FuseMatrix || !MatMul->hasOneUse() ||
1345 MatrixLayout != MatrixLayoutTy::ColumnMajor || !DT)
1346 return;
1347
1348 assert(AA && LI && "Analyses should be available")((AA && LI && "Analyses should be available")
? static_cast<void> (0) : __assert_fail ("AA && LI && \"Analyses should be available\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1348, __PRETTY_FUNCTION__))
;
1349
1350 auto *LoadOp0 = dyn_cast<LoadInst>(MatMul->getOperand(0));
1351 auto *LoadOp1 = dyn_cast<LoadInst>(MatMul->getOperand(1));
1352 auto *Store = dyn_cast<StoreInst>(*MatMul->user_begin());
1353 if (LoadOp0 && LoadOp1 && Store) {
1354 // The store address must dominate the MatMul instruction, otherwise
1355 // we create invalid IR.
1356 // FIXME: See if we can hoist the store address computation.
1357 auto *AddrI = dyn_cast<Instruction>(Store->getOperand(1));
1358 if (AddrI && (!DT->dominates(AddrI, MatMul)))
1359 return;
1360
1361 emitSIMDTiling(MatMul, LoadOp0, LoadOp1, Store, FusedInsts);
1362 return;
1363 }
1364 }
1365
1366 /// Lowers llvm.matrix.multiply.
1367 void LowerMultiply(CallInst *MatMul) {
1368 IRBuilder<> Builder(MatMul);
1369 auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
15
The object is a 'VectorType'
1370 ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3));
1371 ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4));
1372
1373 const MatrixTy &Lhs = getMatrix(MatMul->getArgOperand(0), LShape, Builder);
1374 const MatrixTy &Rhs = getMatrix(MatMul->getArgOperand(1), RShape, Builder);
1375 assert(Lhs.getElementType() == Rhs.getElementType() &&((Lhs.getElementType() == Rhs.getElementType() && "Matrix multiply argument element types do not match."
) ? static_cast<void> (0) : __assert_fail ("Lhs.getElementType() == Rhs.getElementType() && \"Matrix multiply argument element types do not match.\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1376, __PRETTY_FUNCTION__))
16
Assuming the condition is true
17
'?' condition is true
1376 "Matrix multiply argument element types do not match.")((Lhs.getElementType() == Rhs.getElementType() && "Matrix multiply argument element types do not match."
) ? static_cast<void> (0) : __assert_fail ("Lhs.getElementType() == Rhs.getElementType() && \"Matrix multiply argument element types do not match.\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1376, __PRETTY_FUNCTION__))
;
1377
1378 const unsigned R = LShape.NumRows;
1379 const unsigned C = RShape.NumColumns;
1380 assert(LShape.NumColumns == RShape.NumRows)((LShape.NumColumns == RShape.NumRows) ? static_cast<void>
(0) : __assert_fail ("LShape.NumColumns == RShape.NumRows", "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1380, __PRETTY_FUNCTION__))
;
18
Assuming field 'NumColumns' is equal to field 'NumRows'
19
'?' condition is true
1381
1382 // Initialize the output
1383 MatrixTy Result(R, C, EltType);
1384 assert(Lhs.getElementType() == Result.getElementType() &&((Lhs.getElementType() == Result.getElementType() && "Matrix multiply result element type does not match arguments."
) ? static_cast<void> (0) : __assert_fail ("Lhs.getElementType() == Result.getElementType() && \"Matrix multiply result element type does not match arguments.\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1385, __PRETTY_FUNCTION__))
20
Assuming the condition is true
21
'?' condition is true
1385 "Matrix multiply result element type does not match arguments.")((Lhs.getElementType() == Result.getElementType() && "Matrix multiply result element type does not match arguments."
) ? static_cast<void> (0) : __assert_fail ("Lhs.getElementType() == Result.getElementType() && \"Matrix multiply result element type does not match arguments.\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1385, __PRETTY_FUNCTION__))
;
1386
1387 bool AllowContract = AllowContractEnabled || (isa<FPMathOperator>(MatMul) &&
22
Assuming the condition is false
23
Assuming 'MatMul' is not a 'FPMathOperator'
1388 MatMul->hasAllowContract());
1389 emitMatrixMultiply(Result, Lhs, Rhs, AllowContract, Builder, false);
24
Calling 'LowerMatrixIntrinsics::emitMatrixMultiply'
1390 finalizeLowering(MatMul, Result, Builder);
1391 }
1392
1393 /// Lowers llvm.matrix.transpose.
1394 void LowerTranspose(CallInst *Inst) {
1395 MatrixTy Result;
1396 IRBuilder<> Builder(Inst);
1397 Value *InputVal = Inst->getArgOperand(0);
1398 VectorType *VectorTy = cast<VectorType>(InputVal->getType());
1399 ShapeInfo ArgShape(Inst->getArgOperand(1), Inst->getArgOperand(2));
1400 MatrixTy InputMatrix = getMatrix(InputVal, ArgShape, Builder);
1401
1402 const unsigned NewNumVecs =
1403 InputMatrix.isColumnMajor() ? ArgShape.NumRows : ArgShape.NumColumns;
1404 const unsigned NewNumElts =
1405 InputMatrix.isColumnMajor() ? ArgShape.NumColumns : ArgShape.NumRows;
1406
1407 for (unsigned I = 0; I < NewNumVecs; ++I) {
1408 // Build a single result vector. First initialize it.
1409 Value *ResultVector = UndefValue::get(
1410 FixedVectorType::get(VectorTy->getElementType(), NewNumElts));
1411 // Go through the old elements and insert it into the resulting vector.
1412 for (auto J : enumerate(InputMatrix.vectors())) {
1413 Value *Elt = Builder.CreateExtractElement(J.value(), I);
1414 // Row and column indices are transposed.
1415 ResultVector =
1416 Builder.CreateInsertElement(ResultVector, Elt, J.index());
1417 }
1418 Result.addVector(ResultVector);
1419 }
1420
1421 // TODO: Improve estimate of operations needed for transposes. Currently we
1422 // just count the insertelement/extractelement instructions, but do not
1423 // account for later simplifications/combines.
1424 finalizeLowering(
1425 Inst,
1426 Result.addNumComputeOps(2 * ArgShape.NumRows * ArgShape.NumColumns),
1427 Builder);
1428 }
1429
1430 /// Lower load instructions, if shape information is available.
1431 bool VisitLoad(LoadInst *Inst, Value *Ptr, IRBuilder<> &Builder) {
1432 auto I = ShapeMap.find(Inst);
1433 if (I == ShapeMap.end())
1434 return false;
1435
1436 LowerLoad(Inst, Ptr, Inst->getAlign(),
1437 Builder.getInt64(I->second.getStride()), Inst->isVolatile(),
1438 I->second);
1439 return true;
1440 }
1441
1442 bool VisitStore(StoreInst *Inst, Value *StoredVal, Value *Ptr,
1443 IRBuilder<> &Builder) {
1444 auto I = ShapeMap.find(StoredVal);
1445 if (I == ShapeMap.end())
1446 return false;
1447
1448 LowerStore(Inst, StoredVal, Ptr, Inst->getAlign(),
1449 Builder.getInt64(I->second.getStride()), Inst->isVolatile(),
1450 I->second);
1451 return true;
1452 }
1453
1454 /// Lower binary operators, if shape information is available.
1455 bool VisitBinaryOperator(BinaryOperator *Inst) {
1456 auto I = ShapeMap.find(Inst);
1457 if (I == ShapeMap.end())
1458 return false;
1459
1460 Value *Lhs = Inst->getOperand(0);
1461 Value *Rhs = Inst->getOperand(1);
1462
1463 IRBuilder<> Builder(Inst);
1464 ShapeInfo &Shape = I->second;
1465
1466 MatrixTy Result;
1467 MatrixTy A = getMatrix(Lhs, Shape, Builder);
1468 MatrixTy B = getMatrix(Rhs, Shape, Builder);
1469 assert(A.isColumnMajor() == B.isColumnMajor() &&((A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor
() == A.isColumnMajor() && "operands must agree on matrix layout"
) ? static_cast<void> (0) : __assert_fail ("A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor() == A.isColumnMajor() && \"operands must agree on matrix layout\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1471, __PRETTY_FUNCTION__))
1470 Result.isColumnMajor() == A.isColumnMajor() &&((A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor
() == A.isColumnMajor() && "operands must agree on matrix layout"
) ? static_cast<void> (0) : __assert_fail ("A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor() == A.isColumnMajor() && \"operands must agree on matrix layout\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1471, __PRETTY_FUNCTION__))
1471 "operands must agree on matrix layout")((A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor
() == A.isColumnMajor() && "operands must agree on matrix layout"
) ? static_cast<void> (0) : __assert_fail ("A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor() == A.isColumnMajor() && \"operands must agree on matrix layout\""
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1471, __PRETTY_FUNCTION__))
;
1472
1473 // Helper to perform binary op on vectors.
1474 auto BuildVectorOp = [&Builder, Inst](Value *LHS, Value *RHS) {
1475 switch (Inst->getOpcode()) {
1476 case Instruction::Add:
1477 return Builder.CreateAdd(LHS, RHS);
1478 case Instruction::Mul:
1479 return Builder.CreateMul(LHS, RHS);
1480 case Instruction::Sub:
1481 return Builder.CreateSub(LHS, RHS);
1482 case Instruction::FAdd:
1483 return Builder.CreateFAdd(LHS, RHS);
1484 case Instruction::FMul:
1485 return Builder.CreateFMul(LHS, RHS);
1486 case Instruction::FSub:
1487 return Builder.CreateFSub(LHS, RHS);
1488 default:
1489 llvm_unreachable("Unsupported binary operator for matrix")::llvm::llvm_unreachable_internal("Unsupported binary operator for matrix"
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1489)
;
1490 }
1491 };
1492
1493 for (unsigned I = 0; I < Shape.getNumVectors(); ++I)
1494 Result.addVector(BuildVectorOp(A.getVector(I), B.getVector(I)));
1495
1496 finalizeLowering(Inst,
1497 Result.addNumComputeOps(getNumOps(Result.getVectorTy()) *
1498 Result.getNumVectors()),
1499 Builder);
1500 return true;
1501 }
1502
1503 /// Helper to linearize a matrix expression tree into a string. Currently
1504 /// matrix expressions are linarized by starting at an expression leaf and
1505 /// linearizing bottom up.
1506 struct ExprLinearizer {
1507 unsigned LengthToBreak = 100;
1508 std::string Str;
1509 raw_string_ostream Stream;
1510 unsigned LineLength = 0;
1511 const DataLayout &DL;
1512
1513 /// Mapping from instructions to matrixes. It is used to identify
1514 /// matrix instructions.
1515 const MapVector<Value *, MatrixTy> &Inst2Matrix;
1516
1517 /// Mapping from values to the leaves of all expressions that the value is
1518 /// part of.
1519 const DenseMap<Value *, SmallPtrSet<Value *, 2>> &Shared;
1520
1521 /// Set of matrix expressions in the scope of a given DISubprogram.
1522 const SmallSetVector<Value *, 32> &ExprsInSubprogram;
1523
1524 /// Leaf node of the expression to linearize.
1525 Value *Leaf;
1526
1527 /// Used to keep track of sub-expressions that get reused while linearizing
1528 /// the expression. Re-used sub-expressions are marked as (reused).
1529 SmallPtrSet<Value *, 8> ReusedExprs;
1530
1531 ExprLinearizer(const DataLayout &DL,
1532 const MapVector<Value *, MatrixTy> &Inst2Matrix,
1533 const DenseMap<Value *, SmallPtrSet<Value *, 2>> &Shared,
1534 const SmallSetVector<Value *, 32> &ExprsInSubprogram,
1535 Value *Leaf)
1536 : Str(), Stream(Str), DL(DL), Inst2Matrix(Inst2Matrix), Shared(Shared),
1537 ExprsInSubprogram(ExprsInSubprogram), Leaf(Leaf) {}
1538
1539 void indent(unsigned N) {
1540 LineLength += N;
1541 for (unsigned i = 0; i < N; i++)
1542 Stream << " ";
1543 }
1544
1545 void lineBreak() {
1546 Stream << "\n";
1547 LineLength = 0;
1548 }
1549
1550 void maybeIndent(unsigned Indent) {
1551 if (LineLength >= LengthToBreak)
1552 lineBreak();
1553
1554 if (LineLength == 0)
1555 indent(Indent);
1556 }
1557
1558 void write(StringRef S) {
1559 LineLength += S.size();
1560 Stream << S;
1561 }
1562
1563 Value *getUnderlyingObjectThroughLoads(Value *V) {
1564 if (Value *Ptr = getPointerOperand(V))
1565 return getUnderlyingObjectThroughLoads(Ptr);
1566 else if (V->getType()->isPointerTy())
1567 return getUnderlyingObject(V);
1568 return V;
1569 }
1570
1571 /// Returns true if \p V is a matrix value in the given subprogram.
1572 bool isMatrix(Value *V) const { return ExprsInSubprogram.count(V); }
1573
1574 /// If \p V is a matrix value, print its shape as as NumRows x NumColumns to
1575 /// \p SS.
1576 void prettyPrintMatrixType(Value *V, raw_string_ostream &SS) {
1577 auto M = Inst2Matrix.find(V);
1578 if (M == Inst2Matrix.end())
1579 SS << "unknown";
1580 else {
1581 SS << M->second.getNumRows();
1582 SS << "x";
1583 SS << M->second.getNumColumns();
1584 }
1585 }
1586
1587 /// Write the called function name. Handles calls to llvm.matrix.*
1588 /// specially: we write the name, followed by the dimensions of the input
1589 /// matrixes, followed by the scalar type name.
1590 void writeFnName(CallInst *CI) {
1591 if (!CI->getCalledFunction())
1592 write("<no called fn>");
1593 else {
1594 StringRef Name = CI->getCalledFunction()->getName();
1595 if (!Name.startswith("llvm.matrix")) {
1596 write(Name);
1597 return;
1598 }
1599 IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI);
1600 write(StringRef(Intrinsic::getName(II->getIntrinsicID(), {}))
1601 .drop_front(StringRef("llvm.matrix.").size()));
1602 write(".");
1603 std::string Tmp = "";
1604 raw_string_ostream SS(Tmp);
1605
1606 switch (II->getIntrinsicID()) {
1607 case Intrinsic::matrix_multiply:
1608 prettyPrintMatrixType(II->getOperand(0), SS);
1609 SS << ".";
1610 prettyPrintMatrixType(II->getOperand(1), SS);
1611 SS << "." << *II->getType()->getScalarType();
1612 break;
1613 case Intrinsic::matrix_transpose:
1614 prettyPrintMatrixType(II->getOperand(0), SS);
1615 SS << "." << *II->getType()->getScalarType();
1616 break;
1617 case Intrinsic::matrix_column_major_load:
1618 prettyPrintMatrixType(II, SS);
1619 SS << "." << *II->getType()->getScalarType();
1620 break;
1621 case Intrinsic::matrix_column_major_store:
1622 prettyPrintMatrixType(II->getOperand(0), SS);
1623 SS << "." << *II->getOperand(0)->getType()->getScalarType();
1624 break;
1625 default:
1626 llvm_unreachable("Unhandled case")::llvm::llvm_unreachable_internal("Unhandled case", "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1626)
;
1627 }
1628 SS.flush();
1629 write(Tmp);
1630 }
1631 }
1632
1633 unsigned getNumShapeArgs(CallInst *CI) const {
1634 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI)) {
1635 switch (II->getIntrinsicID()) {
1636 case Intrinsic::matrix_multiply:
1637 return 3;
1638 case Intrinsic::matrix_transpose:
1639 return 2;
1640 case Intrinsic::matrix_column_major_load:
1641 case Intrinsic::matrix_column_major_store:
1642 return 3;
1643 default:
1644 return 0;
1645 }
1646 }
1647 return 0;
1648 }
1649
1650 /// Special printing for values: for pointers, we print if they refer to an
1651 /// (function) external address or a stack address, for other values we
1652 /// either print the constant or "scalar"/"matrix" for other values.
1653 void write(Value *V) {
1654 V = getUnderlyingObjectThroughLoads(V);
1655 if (V->getType()->isPointerTy()) {
1656 if (isa<AllocaInst>(V)) {
1657 Stream << "stack addr";
1658 LineLength += StringRef("stack addr").size();
1659 } else {
1660 Stream << "addr";
1661 LineLength += StringRef("addr").size();
1662 }
1663 if (!V->getName().empty()) {
1664 Stream << " %" << V->getName() << "";
1665 LineLength += V->getName().size() + 2;
1666 }
1667 return;
1668 }
1669
1670 std::string Tmp;
1671 raw_string_ostream TmpStream(Tmp);
1672
1673 if (auto *CI = dyn_cast<ConstantInt>(V))
1674 TmpStream << CI->getValue();
1675 else if (isa<Constant>(V))
1676 TmpStream << "constant";
1677 else {
1678 if (isMatrix(V))
1679 TmpStream << "matrix";
1680 else
1681 TmpStream << "scalar";
1682 }
1683 TmpStream.flush();
1684 Tmp = std::string(StringRef(Tmp).trim());
1685 LineLength += Tmp.size();
1686 Stream << Tmp;
1687 }
1688
1689 /// Linearize expression \p Expr starting at an indentation of \p Indent.
1690 /// Expressions that are re-used multiple times are prefixed with (reused)
1691 /// at the re-used root instruction.
1692 void linearizeExpr(Value *Expr, unsigned Indent, bool ParentReused,
1693 bool ParentShared) {
1694 auto *I = cast<Instruction>(Expr);
1695 maybeIndent(Indent);
1696 SmallVector<Value *, 8> Ops;
1697
1698 // Is Expr shared with other expression leaves?
1699 bool ExprShared = false;
1700
1701 // Deal with shared subtrees. Mark them as shared, if required.
1702 if (!ParentShared) {
1703 auto SI = Shared.find(Expr);
1704 assert(SI != Shared.end() && SI->second.count(Leaf))((SI != Shared.end() && SI->second.count(Leaf)) ? static_cast
<void> (0) : __assert_fail ("SI != Shared.end() && SI->second.count(Leaf)"
, "/build/llvm-toolchain-snapshot-12~++20201129111111+e987fbdd85d/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp"
, 1704, __PRETTY_FUNCTION__))
;
1705
1706 for (Value *S : SI->second) {
1707 if (S == Leaf)
1708 continue;
1709 DebugLoc DL = cast<Instruction>(S)->getDebugLoc();
1710 write("shared with remark at line " + std::to_string(DL.getLine()) +
1711 " column " + std::to_string(DL.getCol()) + " (");
1712 }
1713 ExprShared = SI->second.size() > 1;
1714 }
1715
1716 bool Reused = !ReusedExprs.insert(Expr).second;
1717 if (Reused && !ParentReused)
1718 write("(reused) ");
1719
1720 if (auto *CI = dyn_cast<CallInst>(I)) {
1721 writeFnName(CI);
1722
1723 Ops.append(CI->arg_begin(), CI->arg_end() - getNumShapeArgs(CI));
1724 } else if (isa<BitCastInst>(Expr)) {
1725 // Special case bitcasts, which are used to materialize matrixes from
1726 // non-matrix ops.
1727 write("matrix");
1728 return;
1729 } else {
1730 Ops.append(I->value_op_begin(), I->value_op_end());
1731 write(std::string(I->getOpcodeName()));
1732 }
1733
1734 write(std::string("("));
1735
1736 unsigned NumOpsToBreak = 1;
1737 if (match(Expr, m_Intrinsic<Intrinsic::matrix_column_major_load>()))
1738 NumOpsToBreak = 2;
1739
1740 for (Value *Op : Ops) {
1741 if (Ops.size() > NumOpsToBreak)
1742 lineBreak();
1743
1744 maybeIndent(Indent + 1);
1745 if (isMatrix(Op))
1746 linearizeExpr(Op, Indent + 1, Reused, ExprShared);
1747 else
1748 write(Op);
1749 if (Op != Ops.back())
1750 write(", ");
1751 }
1752
1753 write(")");
1754 }
1755
1756 const std::string &getResult() {
1757 Stream.flush();
1758 return Str;
1759 }
1760 };
1761
1762 /// Generate remarks for matrix operations in a function. To generate remarks
1763 /// for matrix expressions, the following approach is used:
1764 /// 1. Use the inlined-at debug information to group matrix operations to the
1765 /// DISubprograms they are contained in.
1766 /// 2. Collect leaves of matrix expressions (done in
1767 /// RemarkGenerator::getExpressionLeaves) for each subprogram - expression
1768 // mapping. Leaves are lowered matrix instructions without other matrix
1769 // users (like stores) in the current subprogram.
1770 /// 3. For each leaf, create a remark containing a linearizied version of the
1771 /// matrix expression. The expression is linearized by a recursive
1772 /// bottom-up traversal of the matrix operands, starting at a leaf. Note
1773 /// that multiple leaves can share sub-expressions. Shared subexpressions
1774 /// are explicitly marked as shared().
1775 struct RemarkGenerator {
1776 const MapVector<Value *, MatrixTy> &Inst2Matrix;
1777 OptimizationRemarkEmitter &ORE;
1778 Function &Func;
1779 const DataLayout &DL;
1780
1781 RemarkGenerator(const MapVector<Value *, MatrixTy> &Inst2Matrix,
1782 OptimizationRemarkEmitter &ORE, Function &Func)
1783 : Inst2Matrix(Inst2Matrix), ORE(ORE), Func(Func),
1784 DL(Func.getParent()->getDataLayout()) {}
1785
1786 /// Return all leaves of the expressions in \p ExprsInSubprogram. Those are
1787 /// instructions in Inst2Matrix returning void or without any users in
1788 /// \p ExprsInSubprogram. Currently that should only include stores.
1789 SmallVector<Value *, 4>
1790 getExpressionLeaves(const SmallSetVector<Value *, 32> &ExprsInSubprogram) {
1791 SmallVector<Value *, 4> Leaves;
1792 for (auto *Expr : ExprsInSubprogram)
1793 if (Expr->getType()->isVoidTy() ||
1794 !any_of(Expr->users(), [&ExprsInSubprogram](User *U) {
1795 return ExprsInSubprogram.count(U);
1796 }))
1797 Leaves.push_back(Expr);
1798 return Leaves;
1799 }
1800
1801 /// Recursively traverse expression \p V starting at \p Leaf and add \p Leaf
1802 /// to all visited expressions in \p Shared. Limit the matrix operations to
1803 /// the ones in \p ExprsInSubprogram.
1804 void collectSharedInfo(Value *Leaf, Value *V,
1805 const SmallSetVector<Value *, 32> &ExprsInSubprogram,
1806 DenseMap<Value *, SmallPtrSet<Value *, 2>> &Shared) {
1807
1808 if (!ExprsInSubprogram.count(V))
1809 return;
1810
1811 auto I = Shared.insert({V, {}});
1812 I.first->second.insert(Leaf);
1813
1814 for (Value *Op : cast<Instruction>(V)->operand_values())
1815 collectSharedInfo(Leaf, Op, ExprsInSubprogram, Shared);
1816 return;
1817 }
1818
1819 /// Calculate the number of exclusive and shared op counts for expression
1820 /// starting at \p V. Expressions used multiple times are counted once.
1821 /// Limit the matrix operations to the ones in \p ExprsInSubprogram.
1822 std::pair<OpInfoTy, OpInfoTy>
1823 sumOpInfos(Value *Root, SmallPtrSetImpl<Value *> &ReusedExprs,
1824 const SmallSetVector<Value *, 32> &ExprsInSubprogram,
1825 DenseMap<Value *, SmallPtrSet<Value *, 2>> &Shared) const {
1826 if (!ExprsInSubprogram.count(Root))
1827 return {};
1828
1829 // Already counted this expression. Stop.
1830 if (!ReusedExprs.insert(Root).second)
1831 return {};
1832
1833 OpInfoTy SharedCount;
1834 OpInfoTy Count;
1835
1836 auto I = Shared.find(Root);
1837 auto CM = Inst2Matrix.find(Root);
1838 if (I->second.size() == 1)
1839 Count = CM->second.getOpInfo();
1840 else
1841 SharedCount = CM->second.getOpInfo();
1842
1843 for (Value *Op : cast<Instruction>(Root)->operand_values()) {
1844 auto C = sumOpInfos(Op, ReusedExprs, ExprsInSubprogram, Shared);
1845 Count += C.first;
1846 SharedCount += C.second;
1847 }
1848 return {Count, SharedCount};
1849 }
1850
1851 void emitRemarks() {
1852 if (!ORE.allowExtraAnalysis(DEBUG_TYPE"lower-matrix-intrinsics"))
1853 return;
1854
1855 // Map matrix operations to their containting subprograms, by traversing
1856 // the inlinedAt chain. If the function does not have a DISubprogram, we
1857 // only map them to the containing function.
1858 MapVector<DISubprogram *, SmallVector<Value *, 8>> Subprog2Exprs;
1859 for (auto &KV : Inst2Matrix) {
1860 if (Func.getSubprogram()) {
1861 auto *I = cast<Instruction>(KV.first);
1862 DILocation *Context = I->getDebugLoc();
1863 while (Context) {
1864 auto I =
1865 Subprog2Exprs.insert({getSubprogram(Context->getScope()), {}});
1866 I.first->second.push_back(KV.first);
1867 Context = DebugLoc(Context).getInlinedAt();
1868 }
1869 } else {
1870 auto I = Subprog2Exprs.insert({nullptr, {}});
1871 I.first->second.push_back(KV.first);
1872 }
1873 }
1874 for (auto &KV : Subprog2Exprs) {
1875 SmallSetVector<Value *, 32> ExprsInSubprogram(KV.second.begin(),
1876 KV.second.end());
1877 auto Leaves = getExpressionLeaves(ExprsInSubprogram);
1878
1879 DenseMap<Value *, SmallPtrSet<Value *, 2>> Shared;
1880 for (Value *Leaf : Leaves)
1881 collectSharedInfo(Leaf, Leaf, ExprsInSubprogram, Shared);
1882
1883 // Generate remarks for each leaf.
1884 for (auto *L : Leaves) {
1885
1886 DebugLoc Loc = cast<Instruction>(L)->getDebugLoc();
1887 DILocation *Context = cast<Instruction>(L)->getDebugLoc();
1888 while (Context) {
1889 if (getSubprogram(Context->getScope()) == KV.first) {
1890 Loc = Context;
1891 break;
1892 }
1893 Context = DebugLoc(Context).getInlinedAt();
1894 }
1895
1896 SmallPtrSet<Value *, 8> ReusedExprs;
1897 OpInfoTy Counts, SharedCounts;
1898 std::tie(Counts, SharedCounts) =
1899 sumOpInfos(L, ReusedExprs, ExprsInSubprogram, Shared);
1900
1901 OptimizationRemark Rem(DEBUG_TYPE"lower-matrix-intrinsics", "matrix-lowered", Loc,
1902 cast<Instruction>(L)->getParent());
1903
1904 Rem << "Lowered with ";
1905 Rem << ore::NV("NumStores", Counts.NumStores) << " stores, "
1906 << ore::NV("NumLoads", Counts.NumLoads) << " loads, "
1907 << ore::NV("NumComputeOps", Counts.NumComputeOps)
1908 << " compute ops";
1909
1910 if (SharedCounts.NumStores > 0 || SharedCounts.NumLoads > 0 ||
1911 SharedCounts.NumComputeOps > 0) {
1912 Rem << ",\nadditionally "
1913 << ore::NV("NumStores", SharedCounts.NumStores) << " stores, "
1914 << ore::NV("NumLoads", SharedCounts.NumLoads) << " loads, "
1915 << ore::NV("NumFPOps", SharedCounts.NumComputeOps)
1916 << " compute ops"
1917 << " are shared with other expressions";
1918 }
1919
1920 Rem << ("\n" + linearize(L, Shared, ExprsInSubprogram, DL));
1921 ORE.emit(Rem);
1922 }
1923 }
1924 }
1925
1926 std::string
1927 linearize(Value *L,
1928 const DenseMap<Value *, SmallPtrSet<Value *, 2>> &Shared,
1929 const SmallSetVector<Value *, 32> &ExprsInSubprogram,
1930 const DataLayout &DL) {
1931 ExprLinearizer Lin(DL, Inst2Matrix, Shared, ExprsInSubprogram, L);
1932 Lin.linearizeExpr(L, 0, false, false);
1933 return Lin.getResult();
1934 }
1935 };
1936};
1937} // namespace
1938
1939PreservedAnalyses LowerMatrixIntrinsicsPass::run(Function &F,
1940 FunctionAnalysisManager &AM) {
1941 auto &TTI = AM.getResult<TargetIRAnalysis>(F);
1942 OptimizationRemarkEmitter *ORE = nullptr;
1943 AAResults *AA = nullptr;
1944 DominatorTree *DT = nullptr;
1945 LoopInfo *LI = nullptr;
1946
1947 if (!Minimal) {
1948 ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
1949 AA = &AM.getResult<AAManager>(F);
1950 DT = &AM.getResult<DominatorTreeAnalysis>(F);
1951 LI = &AM.getResult<LoopAnalysis>(F);
1952 }
1953
1954 LowerMatrixIntrinsics LMT(F, TTI, AA, DT, LI, ORE);
1955 if (LMT.Visit()) {
1956 PreservedAnalyses PA;
1957 if (!Minimal) {
1958 PA.preserve<LoopAnalysis>();
1959 PA.preserve<DominatorTreeAnalysis>();
1960 }
1961 return PA;
1962 }
1963 return PreservedAnalyses::all();
1964}
1965
1966namespace {
1967
1968class LowerMatrixIntrinsicsLegacyPass : public FunctionPass {
1969public:
1970 static char ID;
1971
1972 LowerMatrixIntrinsicsLegacyPass() : FunctionPass(ID) {
1973 initializeLowerMatrixIntrinsicsLegacyPassPass(
1974 *PassRegistry::getPassRegistry());
1975 }
1976
1977 bool runOnFunction(Function &F) override {
1978 auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
1979 auto &ORE = getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
1980 auto &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
1981 auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
1982 auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
1983 LowerMatrixIntrinsics LMT(F, TTI, &AA, &DT, &LI, &ORE);
1984 bool C = LMT.Visit();
1985 return C;
1986 }
1987
1988 void getAnalysisUsage(AnalysisUsage &AU) const override {
1989 AU.addRequired<TargetTransformInfoWrapperPass>();
1990 AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
1991 AU.addRequired<AAResultsWrapperPass>();
1992 AU.addRequired<DominatorTreeWrapperPass>();
1993 AU.addPreserved<DominatorTreeWrapperPass>();
1994 AU.addRequired<LoopInfoWrapperPass>();
1995 AU.addPreserved<LoopInfoWrapperPass>();
1996 }
1997};
1998} // namespace
1999
2000static const char pass_name[] = "Lower the matrix intrinsics";
2001char LowerMatrixIntrinsicsLegacyPass::ID = 0;
2002INITIALIZE_PASS_BEGIN(LowerMatrixIntrinsicsLegacyPass, DEBUG_TYPE, pass_name,static void *initializeLowerMatrixIntrinsicsLegacyPassPassOnce
(PassRegistry &Registry) {
2003 false, false)static void *initializeLowerMatrixIntrinsicsLegacyPassPassOnce
(PassRegistry &Registry) {
2004INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)initializeOptimizationRemarkEmitterWrapperPassPass(Registry);
2005INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)initializeAAResultsWrapperPassPass(Registry);
2006INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)initializeDominatorTreeWrapperPassPass(Registry);
2007INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)initializeLoopInfoWrapperPassPass(Registry);
2008INITIALIZE_PASS_END(LowerMatrixIntrinsicsLegacyPass, DEBUG_TYPE, pass_name,PassInfo *PI = new PassInfo( pass_name, "lower-matrix-intrinsics"
, &LowerMatrixIntrinsicsLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<LowerMatrixIntrinsicsLegacyPass>), false
, false); Registry.registerPass(*PI, true); return PI; } static
llvm::once_flag InitializeLowerMatrixIntrinsicsLegacyPassPassFlag
; void llvm::initializeLowerMatrixIntrinsicsLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializeLowerMatrixIntrinsicsLegacyPassPassFlag
, initializeLowerMatrixIntrinsicsLegacyPassPassOnce, std::ref
(Registry)); }
2009 false, false)PassInfo *PI = new PassInfo( pass_name, "lower-matrix-intrinsics"
, &LowerMatrixIntrinsicsLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<LowerMatrixIntrinsicsLegacyPass>), false
, false); Registry.registerPass(*PI, true); return PI; } static
llvm::once_flag InitializeLowerMatrixIntrinsicsLegacyPassPassFlag
; void llvm::initializeLowerMatrixIntrinsicsLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializeLowerMatrixIntrinsicsLegacyPassPassFlag
, initializeLowerMatrixIntrinsicsLegacyPassPassOnce, std::ref
(Registry)); }
2010
2011Pass *llvm::createLowerMatrixIntrinsicsPass() {
2012 return new LowerMatrixIntrinsicsLegacyPass();
2013}
2014
2015namespace {
2016
2017/// A lightweight version of the matrix lowering pass that only requires TTI.
2018/// Advanced features that require DT, AA or ORE like tiling are disabled. This
2019/// is used to lower matrix intrinsics if the main lowering pass is not run, for
2020/// example with -O0.
2021class LowerMatrixIntrinsicsMinimalLegacyPass : public FunctionPass {
2022public:
2023 static char ID;
2024
2025 LowerMatrixIntrinsicsMinimalLegacyPass() : FunctionPass(ID) {
2026 initializeLowerMatrixIntrinsicsMinimalLegacyPassPass(
2027 *PassRegistry::getPassRegistry());
2028 }
2029
2030 bool runOnFunction(Function &F) override {
2031 auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
2032 LowerMatrixIntrinsics LMT(F, TTI, nullptr, nullptr, nullptr, nullptr);
2033 bool C = LMT.Visit();
1
Calling 'LowerMatrixIntrinsics::Visit'
2034 return C;
2035 }
2036
2037 void getAnalysisUsage(AnalysisUsage &AU) const override {
2038 AU.addRequired<TargetTransformInfoWrapperPass>();
2039 AU.setPreservesCFG();
2040 }
2041};
2042} // namespace
2043
2044static const char pass_name_minimal[] = "Lower the matrix intrinsics (minimal)";
2045char LowerMatrixIntrinsicsMinimalLegacyPass::ID = 0;
2046INITIALIZE_PASS_BEGIN(LowerMatrixIntrinsicsMinimalLegacyPass,static void *initializeLowerMatrixIntrinsicsMinimalLegacyPassPassOnce
(PassRegistry &Registry) {
2047 "lower-matrix-intrinsics-minimal", pass_name_minimal,static void *initializeLowerMatrixIntrinsicsMinimalLegacyPassPassOnce
(PassRegistry &Registry) {
2048 false, false)static void *initializeLowerMatrixIntrinsicsMinimalLegacyPassPassOnce
(PassRegistry &Registry) {
2049INITIALIZE_PASS_END(LowerMatrixIntrinsicsMinimalLegacyPass,PassInfo *PI = new PassInfo( pass_name_minimal, "lower-matrix-intrinsics-minimal"
, &LowerMatrixIntrinsicsMinimalLegacyPass::ID, PassInfo::
NormalCtor_t(callDefaultCtor<LowerMatrixIntrinsicsMinimalLegacyPass
>), false, false); Registry.registerPass(*PI, true); return
PI; } static llvm::once_flag InitializeLowerMatrixIntrinsicsMinimalLegacyPassPassFlag
; void llvm::initializeLowerMatrixIntrinsicsMinimalLegacyPassPass
(PassRegistry &Registry) { llvm::call_once(InitializeLowerMatrixIntrinsicsMinimalLegacyPassPassFlag
, initializeLowerMatrixIntrinsicsMinimalLegacyPassPassOnce, std
::ref(Registry)); }
2050 "lower-matrix-intrinsics-minimal", pass_name_minimal, false,PassInfo *PI = new PassInfo( pass_name_minimal, "lower-matrix-intrinsics-minimal"
, &LowerMatrixIntrinsicsMinimalLegacyPass::ID, PassInfo::
NormalCtor_t(callDefaultCtor<LowerMatrixIntrinsicsMinimalLegacyPass
>), false, false); Registry.registerPass(*PI, true); return
PI; } static llvm::once_flag InitializeLowerMatrixIntrinsicsMinimalLegacyPassPassFlag
; void llvm::initializeLowerMatrixIntrinsicsMinimalLegacyPassPass
(PassRegistry &Registry) { llvm::call_once(InitializeLowerMatrixIntrinsicsMinimalLegacyPassPassFlag
, initializeLowerMatrixIntrinsicsMinimalLegacyPassPassOnce, std
::ref(Registry)); }
2051 false)PassInfo *PI = new PassInfo( pass_name_minimal, "lower-matrix-intrinsics-minimal"
, &LowerMatrixIntrinsicsMinimalLegacyPass::ID, PassInfo::
NormalCtor_t(callDefaultCtor<LowerMatrixIntrinsicsMinimalLegacyPass
>), false, false); Registry.registerPass(*PI, true); return
PI; } static llvm::once_flag InitializeLowerMatrixIntrinsicsMinimalLegacyPassPassFlag
; void llvm::initializeLowerMatrixIntrinsicsMinimalLegacyPassPass
(PassRegistry &Registry) { llvm::call_once(InitializeLowerMatrixIntrinsicsMinimalLegacyPassPassFlag
, initializeLowerMatrixIntrinsicsMinimalLegacyPassPassOnce, std
::ref(Registry)); }
2052
2053Pass *llvm::createLowerMatrixIntrinsicsMinimalPass() {
2054 return new LowerMatrixIntrinsicsMinimalLegacyPass();
2055}