| File: | llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp |
| Warning: | line 940, column 31 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | //===- LowerMatrixIntrinsics.cpp - Lower matrix intrinsics -----*- C++ -*-===// | |||
| 2 | // | |||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | |||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
| 6 | // | |||
| 7 | //===----------------------------------------------------------------------===// | |||
| 8 | // | |||
| 9 | // Lower matrix intrinsics to vector operations. | |||
| 10 | // | |||
| 11 | // TODO: | |||
| 12 | // * Improve fusion: | |||
| 13 | // * Support more cases, e.g. multiply-add, multiply-sub, operands/results | |||
| 14 | // transposed. | |||
| 15 | // * Improve cost-modeling, e.g. choose different number of rows/columns | |||
| 16 | // columns for tiles, consider cost of copies on alias. | |||
| 17 | // | |||
| 18 | //===----------------------------------------------------------------------===// | |||
| 19 | ||||
| 20 | #include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h" | |||
| 21 | #include "llvm/ADT/GraphTraits.h" | |||
| 22 | #include "llvm/ADT/PostOrderIterator.h" | |||
| 23 | #include "llvm/ADT/SmallVector.h" | |||
| 24 | #include "llvm/Analysis/AliasAnalysis.h" | |||
| 25 | #include "llvm/Analysis/DomTreeUpdater.h" | |||
| 26 | #include "llvm/Analysis/OptimizationRemarkEmitter.h" | |||
| 27 | #include "llvm/Analysis/TargetTransformInfo.h" | |||
| 28 | #include "llvm/Analysis/ValueTracking.h" | |||
| 29 | #include "llvm/Analysis/VectorUtils.h" | |||
| 30 | #include "llvm/IR/CFG.h" | |||
| 31 | #include "llvm/IR/DataLayout.h" | |||
| 32 | #include "llvm/IR/DebugInfoMetadata.h" | |||
| 33 | #include "llvm/IR/Function.h" | |||
| 34 | #include "llvm/IR/IRBuilder.h" | |||
| 35 | #include "llvm/IR/Instructions.h" | |||
| 36 | #include "llvm/IR/IntrinsicInst.h" | |||
| 37 | #include "llvm/IR/PatternMatch.h" | |||
| 38 | #include "llvm/InitializePasses.h" | |||
| 39 | #include "llvm/Pass.h" | |||
| 40 | #include "llvm/Support/Alignment.h" | |||
| 41 | #include "llvm/Support/CommandLine.h" | |||
| 42 | #include "llvm/Support/Debug.h" | |||
| 43 | #include "llvm/Transforms/Scalar.h" | |||
| 44 | #include "llvm/Transforms/Utils/BasicBlockUtils.h" | |||
| 45 | #include "llvm/Transforms/Utils/LoopUtils.h" | |||
| 46 | #include "llvm/Transforms/Utils/MatrixUtils.h" | |||
| 47 | ||||
| 48 | using namespace llvm; | |||
| 49 | using namespace PatternMatch; | |||
| 50 | ||||
| 51 | #define DEBUG_TYPE"lower-matrix-intrinsics" "lower-matrix-intrinsics" | |||
| 52 | ||||
| 53 | static cl::opt<bool> EnableShapePropagation( | |||
| 54 | "matrix-propagate-shape", cl::init(true), cl::Hidden, | |||
| 55 | cl::desc("Enable/disable shape propagation from matrix intrinsics to other " | |||
| 56 | "instructions.")); | |||
| 57 | ||||
| 58 | static cl::opt<bool> | |||
| 59 | FuseMatrix("fuse-matrix", cl::init(true), cl::Hidden, | |||
| 60 | cl::desc("Enable/disable fusing matrix instructions.")); | |||
| 61 | // TODO: Allow and use non-square tiles. | |||
| 62 | static cl::opt<unsigned> TileSize( | |||
| 63 | "fuse-matrix-tile-size", cl::init(4), cl::Hidden, | |||
| 64 | cl::desc( | |||
| 65 | "Tile size for matrix instruction fusion using square-shaped tiles.")); | |||
| 66 | static cl::opt<bool> TileUseLoops("fuse-matrix-use-loops", cl::init(false), | |||
| 67 | cl::Hidden, | |||
| 68 | cl::desc("Generate loop nest for tiling.")); | |||
| 69 | static cl::opt<bool> ForceFusion( | |||
| 70 | "force-fuse-matrix", cl::init(false), cl::Hidden, | |||
| 71 | cl::desc("Force matrix instruction fusion even if not profitable.")); | |||
| 72 | static cl::opt<bool> AllowContractEnabled( | |||
| 73 | "matrix-allow-contract", cl::init(false), cl::Hidden, | |||
| 74 | cl::desc("Allow the use of FMAs if available and profitable. This may " | |||
| 75 | "result in different results, due to less rounding error.")); | |||
| 76 | ||||
| 77 | enum class MatrixLayoutTy { ColumnMajor, RowMajor }; | |||
| 78 | ||||
| 79 | static cl::opt<MatrixLayoutTy> MatrixLayout( | |||
| 80 | "matrix-default-layout", cl::init(MatrixLayoutTy::ColumnMajor), | |||
| 81 | cl::desc("Sets the default matrix layout"), | |||
| 82 | cl::values(clEnumValN(MatrixLayoutTy::ColumnMajor, "column-major",llvm::cl::OptionEnumValue { "column-major", int(MatrixLayoutTy ::ColumnMajor), "Use column-major layout" } | |||
| 83 | "Use column-major layout")llvm::cl::OptionEnumValue { "column-major", int(MatrixLayoutTy ::ColumnMajor), "Use column-major layout" }, | |||
| 84 | clEnumValN(MatrixLayoutTy::RowMajor, "row-major",llvm::cl::OptionEnumValue { "row-major", int(MatrixLayoutTy:: RowMajor), "Use row-major layout" } | |||
| 85 | "Use row-major layout")llvm::cl::OptionEnumValue { "row-major", int(MatrixLayoutTy:: RowMajor), "Use row-major layout" })); | |||
| 86 | ||||
| 87 | /// Helper function to either return Scope, if it is a subprogram or the | |||
| 88 | /// attached subprogram for a local scope. | |||
| 89 | static DISubprogram *getSubprogram(DIScope *Scope) { | |||
| 90 | if (auto *Subprogram = dyn_cast<DISubprogram>(Scope)) | |||
| 91 | return Subprogram; | |||
| 92 | return cast<DILocalScope>(Scope)->getSubprogram(); | |||
| 93 | } | |||
| 94 | ||||
| 95 | namespace { | |||
| 96 | ||||
| 97 | // Given an element pointer \p BasePtr to the start of a (sub) matrix, compute | |||
| 98 | // the start address of vector \p VecIdx with type (\p EltType x \p NumElements) | |||
| 99 | // assuming \p Stride elements between start two consecutive vectors. | |||
| 100 | // \p Stride must be >= \p NumElements. | |||
| 101 | // For column-major matrixes, the function computes the address of a column | |||
| 102 | // vectors and \p NumElements must be set to the number of elements in a column | |||
| 103 | // (= number of rows of the matrix). For row-major matrixes, the function | |||
| 104 | // computes the address of a row vector and \p NumElements must be set to the | |||
| 105 | // number of elements in a column (= number of columns of the matrix). | |||
| 106 | // | |||
| 107 | // Consider a 4x4 matrix in column-mjaor layout like below | |||
| 108 | // | |||
| 109 | // 0 1 2 3 | |||
| 110 | // 0 v_0_0 v_0_1 v_0_2 v_0_3 | |||
| 111 | // 1 v_1_0 v_1_1 v_1_2 v_1_3 | |||
| 112 | // 2 v_2_0 v_2_1 v_2_2 v_2_3 | |||
| 113 | // 3 v_3_0 v_3_1 v_3_2 v_3_3 | |||
| 114 | ||||
| 115 | // To compute the column addresses for a 2x3 sub-matrix at row 1 and column 1, | |||
| 116 | // we need a pointer to the first element of the submatrix as base pointer. | |||
| 117 | // Then we can use computeVectorAddr to compute the addresses for the columns | |||
| 118 | // of the sub-matrix. | |||
| 119 | // | |||
| 120 | // Column 0: computeVectorAddr(Base, 0 (column), 4 (stride), 2 (num rows), ..) | |||
| 121 | // -> just returns Base | |||
| 122 | // Column 1: computeVectorAddr(Base, 1 (column), 4 (stride), 2 (num rows), ..) | |||
| 123 | // -> returns Base + (1 * 4) | |||
| 124 | // Column 2: computeVectorAddr(Base, 2 (column), 4 (stride), 2 (num rows), ..) | |||
| 125 | // -> returns Base + (2 * 4) | |||
| 126 | // | |||
| 127 | // The graphic below illustrates the number of elements in a column (marked | |||
| 128 | // with |) and the number of skipped elements (marked with }). | |||
| 129 | // | |||
| 130 | // v_0_0 v_0_1 {v_0_2 {v_0_3 | |||
| 131 | // Base Col 1 Col 2 | |||
| 132 | // | | | | |||
| 133 | // v_1_0 |v_1_1 |v_1_2 |v_1_3 | |||
| 134 | // v_2_0 |v_2_1 |v_2_2 |v_2_3 | |||
| 135 | // v_3_0 {v_3_1 {v_3_2 v_3_3 | |||
| 136 | // | |||
| 137 | Value *computeVectorAddr(Value *BasePtr, Value *VecIdx, Value *Stride, | |||
| 138 | unsigned NumElements, Type *EltType, | |||
| 139 | IRBuilder<> &Builder) { | |||
| 140 | ||||
| 141 | assert((!isa<ConstantInt>(Stride) ||(((!isa<ConstantInt>(Stride) || cast<ConstantInt> (Stride)->getZExtValue() >= NumElements) && "Stride must be >= the number of elements in the result vector." ) ? static_cast<void> (0) : __assert_fail ("(!isa<ConstantInt>(Stride) || cast<ConstantInt>(Stride)->getZExtValue() >= NumElements) && \"Stride must be >= the number of elements in the result vector.\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 143, __PRETTY_FUNCTION__)) | |||
| 142 | cast<ConstantInt>(Stride)->getZExtValue() >= NumElements) &&(((!isa<ConstantInt>(Stride) || cast<ConstantInt> (Stride)->getZExtValue() >= NumElements) && "Stride must be >= the number of elements in the result vector." ) ? static_cast<void> (0) : __assert_fail ("(!isa<ConstantInt>(Stride) || cast<ConstantInt>(Stride)->getZExtValue() >= NumElements) && \"Stride must be >= the number of elements in the result vector.\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 143, __PRETTY_FUNCTION__)) | |||
| 143 | "Stride must be >= the number of elements in the result vector.")(((!isa<ConstantInt>(Stride) || cast<ConstantInt> (Stride)->getZExtValue() >= NumElements) && "Stride must be >= the number of elements in the result vector." ) ? static_cast<void> (0) : __assert_fail ("(!isa<ConstantInt>(Stride) || cast<ConstantInt>(Stride)->getZExtValue() >= NumElements) && \"Stride must be >= the number of elements in the result vector.\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 143, __PRETTY_FUNCTION__)); | |||
| 144 | unsigned AS = cast<PointerType>(BasePtr->getType())->getAddressSpace(); | |||
| 145 | ||||
| 146 | // Compute the start of the vector with index VecIdx as VecIdx * Stride. | |||
| 147 | Value *VecStart = Builder.CreateMul(VecIdx, Stride, "vec.start"); | |||
| 148 | ||||
| 149 | // Get pointer to the start of the selected vector. Skip GEP creation, | |||
| 150 | // if we select vector 0. | |||
| 151 | if (isa<ConstantInt>(VecStart) && cast<ConstantInt>(VecStart)->isZero()) | |||
| 152 | VecStart = BasePtr; | |||
| 153 | else | |||
| 154 | VecStart = Builder.CreateGEP(EltType, BasePtr, VecStart, "vec.gep"); | |||
| 155 | ||||
| 156 | // Cast elementwise vector start pointer to a pointer to a vector | |||
| 157 | // (EltType x NumElements)*. | |||
| 158 | auto *VecType = FixedVectorType::get(EltType, NumElements); | |||
| 159 | Type *VecPtrType = PointerType::get(VecType, AS); | |||
| 160 | return Builder.CreatePointerCast(VecStart, VecPtrType, "vec.cast"); | |||
| 161 | } | |||
| 162 | ||||
| 163 | /// LowerMatrixIntrinsics contains the methods used to lower matrix intrinsics. | |||
| 164 | /// | |||
| 165 | /// Currently, the lowering for each matrix intrinsic is done as follows: | |||
| 166 | /// 1. Propagate the shape information from intrinsics to connected | |||
| 167 | /// instructions. | |||
| 168 | /// 2. Lower instructions with shape information (assuming column-major layout). | |||
| 169 | /// The lowering works similarly using row-major layout. | |||
| 170 | /// 2.1. Get column vectors for each argument. If we already lowered the | |||
| 171 | /// definition of an argument, use the produced column vectors directly. | |||
| 172 | /// If not, split the operand vector containing an embedded matrix into | |||
| 173 | /// a set of column vectors, | |||
| 174 | /// 2.2. Lower the instruction in terms of column major operations, which | |||
| 175 | /// yields a set of column vectors containing result matrix. Note that we | |||
| 176 | /// lower all instructions that have shape information. Besides the | |||
| 177 | /// intrinsics, this includes stores for example. | |||
| 178 | /// 2.3. Update uses of the lowered instruction. If we have shape information | |||
| 179 | /// for a user, there is nothing to do, as we will look up the result | |||
| 180 | /// column matrix when lowering the user. For other uses, we embed the | |||
| 181 | /// result matrix in a flat vector and update the use. | |||
| 182 | /// 2.4. Cache the result column matrix for the instruction we lowered | |||
| 183 | /// 3. After we lowered all instructions in a function, remove the now | |||
| 184 | /// obsolete instructions. | |||
| 185 | /// | |||
| 186 | class LowerMatrixIntrinsics { | |||
| 187 | Function &Func; | |||
| 188 | const DataLayout &DL; | |||
| 189 | const TargetTransformInfo &TTI; | |||
| 190 | AliasAnalysis *AA; | |||
| 191 | DominatorTree *DT; | |||
| 192 | LoopInfo *LI; | |||
| 193 | OptimizationRemarkEmitter *ORE; | |||
| 194 | ||||
| 195 | /// Contains estimates of the number of operations (loads, stores, compute) required to lower a matrix operation. | |||
| 196 | struct OpInfoTy { | |||
| 197 | /// Number of stores emitted to generate this matrix. | |||
| 198 | unsigned NumStores = 0; | |||
| 199 | /// Number of loads emitted to generate this matrix. | |||
| 200 | unsigned NumLoads = 0; | |||
| 201 | /// Number of compute operations emitted to generate this matrix. | |||
| 202 | unsigned NumComputeOps = 0; | |||
| 203 | ||||
| 204 | OpInfoTy &operator+=(const OpInfoTy &RHS) { | |||
| 205 | NumStores += RHS.NumStores; | |||
| 206 | NumLoads += RHS.NumLoads; | |||
| 207 | NumComputeOps += RHS.NumComputeOps; | |||
| 208 | return *this; | |||
| 209 | } | |||
| 210 | }; | |||
| 211 | ||||
| 212 | /// Wrapper class representing a matrix as a set of vectors, either in row or | |||
| 213 | /// column major layout. All vectors must have the same vector type. | |||
| 214 | class MatrixTy { | |||
| 215 | SmallVector<Value *, 16> Vectors; | |||
| 216 | ||||
| 217 | OpInfoTy OpInfo; | |||
| 218 | ||||
| 219 | bool IsColumnMajor = true; | |||
| 220 | ||||
| 221 | public: | |||
| 222 | MatrixTy() | |||
| 223 | : Vectors(), | |||
| 224 | IsColumnMajor(MatrixLayout == MatrixLayoutTy::ColumnMajor) {} | |||
| 225 | MatrixTy(ArrayRef<Value *> Vectors) | |||
| 226 | : Vectors(Vectors.begin(), Vectors.end()), | |||
| 227 | IsColumnMajor(MatrixLayout == MatrixLayoutTy::ColumnMajor) {} | |||
| 228 | MatrixTy(unsigned NumRows, unsigned NumColumns, Type *EltTy) | |||
| 229 | : IsColumnMajor(MatrixLayout == MatrixLayoutTy::ColumnMajor) { | |||
| 230 | ||||
| 231 | unsigned D = isColumnMajor() ? NumColumns : NumRows; | |||
| 232 | for (unsigned J = 0; J < D; ++J) | |||
| 233 | addVector(UndefValue::get(FixedVectorType::get( | |||
| 234 | EltTy, isColumnMajor() ? NumRows : NumColumns))); | |||
| 235 | } | |||
| 236 | ||||
| 237 | Value *getVector(unsigned i) const { return Vectors[i]; } | |||
| 238 | Value *getColumn(unsigned i) const { | |||
| 239 | assert(isColumnMajor() && "only supported for column-major matrixes")((isColumnMajor() && "only supported for column-major matrixes" ) ? static_cast<void> (0) : __assert_fail ("isColumnMajor() && \"only supported for column-major matrixes\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 239, __PRETTY_FUNCTION__)); | |||
| 240 | return Vectors[i]; | |||
| 241 | } | |||
| 242 | Value *getRow(unsigned i) const { | |||
| 243 | assert(!isColumnMajor() && "only supported for row-major matrixes")((!isColumnMajor() && "only supported for row-major matrixes" ) ? static_cast<void> (0) : __assert_fail ("!isColumnMajor() && \"only supported for row-major matrixes\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 243, __PRETTY_FUNCTION__)); | |||
| 244 | return Vectors[i]; | |||
| 245 | } | |||
| 246 | ||||
| 247 | void setVector(unsigned i, Value *V) { Vectors[i] = V; } | |||
| 248 | ||||
| 249 | Type *getElementType() const { return getVectorTy()->getElementType(); } | |||
| 250 | ||||
| 251 | unsigned getNumVectors() const { | |||
| 252 | if (isColumnMajor()) | |||
| 253 | return getNumColumns(); | |||
| 254 | return getNumRows(); | |||
| 255 | } | |||
| 256 | ||||
| 257 | unsigned getNumColumns() const { | |||
| 258 | if (isColumnMajor()) | |||
| 259 | return Vectors.size(); | |||
| 260 | else { | |||
| 261 | assert(Vectors.size() > 0 && "Cannot call getNumRows without columns")((Vectors.size() > 0 && "Cannot call getNumRows without columns" ) ? static_cast<void> (0) : __assert_fail ("Vectors.size() > 0 && \"Cannot call getNumRows without columns\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 261, __PRETTY_FUNCTION__)); | |||
| 262 | return cast<FixedVectorType>(Vectors[0]->getType())->getNumElements(); | |||
| 263 | } | |||
| 264 | } | |||
| 265 | unsigned getNumRows() const { | |||
| 266 | if (isColumnMajor()) { | |||
| 267 | assert(Vectors.size() > 0 && "Cannot call getNumRows without columns")((Vectors.size() > 0 && "Cannot call getNumRows without columns" ) ? static_cast<void> (0) : __assert_fail ("Vectors.size() > 0 && \"Cannot call getNumRows without columns\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 267, __PRETTY_FUNCTION__)); | |||
| 268 | return cast<FixedVectorType>(Vectors[0]->getType())->getNumElements(); | |||
| 269 | } else | |||
| 270 | return Vectors.size(); | |||
| 271 | } | |||
| 272 | ||||
| 273 | void addVector(Value *V) { Vectors.push_back(V); } | |||
| 274 | VectorType *getColumnTy() { | |||
| 275 | assert(isColumnMajor() && "only supported for column-major matrixes")((isColumnMajor() && "only supported for column-major matrixes" ) ? static_cast<void> (0) : __assert_fail ("isColumnMajor() && \"only supported for column-major matrixes\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 275, __PRETTY_FUNCTION__)); | |||
| 276 | return getVectorTy(); | |||
| 277 | } | |||
| 278 | ||||
| 279 | VectorType *getVectorTy() const { | |||
| 280 | return cast<VectorType>(Vectors[0]->getType()); | |||
| 281 | } | |||
| 282 | ||||
| 283 | iterator_range<SmallVector<Value *, 8>::iterator> columns() { | |||
| 284 | assert(isColumnMajor() &&((isColumnMajor() && "columns() only supported for column-major matrixes" ) ? static_cast<void> (0) : __assert_fail ("isColumnMajor() && \"columns() only supported for column-major matrixes\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 285, __PRETTY_FUNCTION__)) | |||
| 285 | "columns() only supported for column-major matrixes")((isColumnMajor() && "columns() only supported for column-major matrixes" ) ? static_cast<void> (0) : __assert_fail ("isColumnMajor() && \"columns() only supported for column-major matrixes\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 285, __PRETTY_FUNCTION__)); | |||
| 286 | return make_range(Vectors.begin(), Vectors.end()); | |||
| 287 | } | |||
| 288 | ||||
| 289 | iterator_range<SmallVector<Value *, 8>::iterator> vectors() { | |||
| 290 | return make_range(Vectors.begin(), Vectors.end()); | |||
| 291 | } | |||
| 292 | ||||
| 293 | /// Embed the vectors of the matrix into a flat vector by concatenating | |||
| 294 | /// them. | |||
| 295 | Value *embedInVector(IRBuilder<> &Builder) const { | |||
| 296 | return Vectors.size() == 1 ? Vectors[0] | |||
| 297 | : concatenateVectors(Builder, Vectors); | |||
| 298 | } | |||
| 299 | ||||
| 300 | MatrixTy &addNumLoads(unsigned N) { | |||
| 301 | OpInfo.NumLoads += N; | |||
| 302 | return *this; | |||
| 303 | } | |||
| 304 | ||||
| 305 | void setNumLoads(unsigned N) { OpInfo.NumLoads = N; } | |||
| 306 | ||||
| 307 | MatrixTy &addNumStores(unsigned N) { | |||
| 308 | OpInfo.NumStores += N; | |||
| 309 | return *this; | |||
| 310 | } | |||
| 311 | ||||
| 312 | MatrixTy &addNumComputeOps(unsigned N) { | |||
| 313 | OpInfo.NumComputeOps += N; | |||
| 314 | return *this; | |||
| 315 | } | |||
| 316 | ||||
| 317 | unsigned getNumStores() const { return OpInfo.NumStores; } | |||
| 318 | unsigned getNumLoads() const { return OpInfo.NumLoads; } | |||
| 319 | unsigned getNumComputeOps() const { return OpInfo.NumComputeOps; } | |||
| 320 | ||||
| 321 | const OpInfoTy &getOpInfo() const { return OpInfo; } | |||
| 322 | ||||
| 323 | bool isColumnMajor() const { return IsColumnMajor; } | |||
| 324 | ||||
| 325 | unsigned getStride() const { | |||
| 326 | if (isColumnMajor()) | |||
| 327 | return getNumRows(); | |||
| 328 | return getNumColumns(); | |||
| 329 | } | |||
| 330 | ||||
| 331 | /// Extract a vector of \p NumElts starting at index (\p I, \p J). If the | |||
| 332 | /// matrix is column-major, the result vector is extracted from a column | |||
| 333 | /// vector, otherwise from a row vector. | |||
| 334 | Value *extractVector(unsigned I, unsigned J, unsigned NumElts, | |||
| 335 | IRBuilder<> &Builder) const { | |||
| 336 | Value *Vec = isColumnMajor() ? getColumn(J) : getRow(I); | |||
| 337 | Value *Undef = UndefValue::get(Vec->getType()); | |||
| 338 | return Builder.CreateShuffleVector( | |||
| 339 | Vec, Undef, createSequentialMask(isColumnMajor() ? I : J, NumElts, 0), | |||
| 340 | "block"); | |||
| 341 | } | |||
| 342 | }; | |||
| 343 | ||||
| 344 | struct ShapeInfo { | |||
| 345 | unsigned NumRows; | |||
| 346 | unsigned NumColumns; | |||
| 347 | ||||
| 348 | bool IsColumnMajor; | |||
| 349 | ||||
| 350 | ShapeInfo(unsigned NumRows = 0, unsigned NumColumns = 0) | |||
| 351 | : NumRows(NumRows), NumColumns(NumColumns), | |||
| 352 | IsColumnMajor(MatrixLayout == MatrixLayoutTy::ColumnMajor) {} | |||
| 353 | ||||
| 354 | ShapeInfo(Value *NumRows, Value *NumColumns) | |||
| 355 | : ShapeInfo(cast<ConstantInt>(NumRows)->getZExtValue(), | |||
| 356 | cast<ConstantInt>(NumColumns)->getZExtValue()) {} | |||
| 357 | ||||
| 358 | bool operator==(const ShapeInfo &other) { | |||
| 359 | return NumRows == other.NumRows && NumColumns == other.NumColumns; | |||
| 360 | } | |||
| 361 | bool operator!=(const ShapeInfo &other) { return !(*this == other); } | |||
| 362 | ||||
| 363 | /// Returns true if shape-information is defined, meaning both dimensions | |||
| 364 | /// are != 0. | |||
| 365 | operator bool() const { | |||
| 366 | assert(NumRows == 0 || NumColumns != 0)((NumRows == 0 || NumColumns != 0) ? static_cast<void> ( 0) : __assert_fail ("NumRows == 0 || NumColumns != 0", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 366, __PRETTY_FUNCTION__)); | |||
| 367 | return NumRows != 0; | |||
| 368 | } | |||
| 369 | ||||
| 370 | unsigned getStride() const { | |||
| 371 | if (IsColumnMajor) | |||
| 372 | return NumRows; | |||
| 373 | return NumColumns; | |||
| 374 | } | |||
| 375 | ||||
| 376 | unsigned getNumVectors() const { | |||
| 377 | if (IsColumnMajor) | |||
| 378 | return NumColumns; | |||
| 379 | return NumRows; | |||
| 380 | } | |||
| 381 | }; | |||
| 382 | ||||
| 383 | /// Maps instructions to their shape information. The shape information | |||
| 384 | /// describes the shape to be used while lowering. This matches the shape of | |||
| 385 | /// the result value of the instruction, with the only exceptions being store | |||
| 386 | /// instructions and the matrix_column_major_store intrinsics. For those, the | |||
| 387 | /// shape information indicates that those instructions should be lowered | |||
| 388 | /// using shape information as well. | |||
| 389 | DenseMap<Value *, ShapeInfo> ShapeMap; | |||
| 390 | ||||
| 391 | /// List of instructions to remove. While lowering, we are not replacing all | |||
| 392 | /// users of a lowered instruction, if shape information is available and | |||
| 393 | /// those need to be removed after we finished lowering. | |||
| 394 | SmallVector<Instruction *, 16> ToRemove; | |||
| 395 | ||||
| 396 | /// Map from instructions to their produced column matrix. | |||
| 397 | MapVector<Value *, MatrixTy> Inst2ColumnMatrix; | |||
| 398 | ||||
| 399 | public: | |||
| 400 | LowerMatrixIntrinsics(Function &F, TargetTransformInfo &TTI, | |||
| 401 | AliasAnalysis *AA, DominatorTree *DT, LoopInfo *LI, | |||
| 402 | OptimizationRemarkEmitter *ORE) | |||
| 403 | : Func(F), DL(F.getParent()->getDataLayout()), TTI(TTI), AA(AA), DT(DT), | |||
| 404 | LI(LI), ORE(ORE) {} | |||
| 405 | ||||
| 406 | unsigned getNumOps(Type *VT) { | |||
| 407 | assert(isa<VectorType>(VT) && "Expected vector type")((isa<VectorType>(VT) && "Expected vector type" ) ? static_cast<void> (0) : __assert_fail ("isa<VectorType>(VT) && \"Expected vector type\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 407, __PRETTY_FUNCTION__)); | |||
| 408 | return getNumOps(VT->getScalarType(), | |||
| 409 | cast<FixedVectorType>(VT)->getNumElements()); | |||
| 410 | } | |||
| 411 | ||||
| 412 | // | |||
| 413 | /// Return the estimated number of vector ops required for an operation on | |||
| 414 | /// \p VT * N. | |||
| 415 | unsigned getNumOps(Type *ST, unsigned N) { | |||
| 416 | return std::ceil((ST->getPrimitiveSizeInBits() * N).getFixedSize() / | |||
| 417 | double(TTI.getRegisterBitWidth(true))); | |||
| 418 | } | |||
| 419 | ||||
| 420 | /// Return the set of vectors that a matrix value is lowered to. | |||
| 421 | /// | |||
| 422 | /// If we lowered \p MatrixVal, just return the cache result matrix. Otherwise | |||
| 423 | /// split the flat vector \p MatrixVal containing a matrix with shape \p SI | |||
| 424 | /// into vectors. | |||
| 425 | MatrixTy getMatrix(Value *MatrixVal, const ShapeInfo &SI, | |||
| 426 | IRBuilder<> &Builder) { | |||
| 427 | VectorType *VType = dyn_cast<VectorType>(MatrixVal->getType()); | |||
| 428 | assert(VType && "MatrixVal must be a vector type")((VType && "MatrixVal must be a vector type") ? static_cast <void> (0) : __assert_fail ("VType && \"MatrixVal must be a vector type\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 428, __PRETTY_FUNCTION__)); | |||
| 429 | assert(cast<FixedVectorType>(VType)->getNumElements() ==((cast<FixedVectorType>(VType)->getNumElements() == SI .NumRows * SI.NumColumns && "The vector size must match the number of matrix elements" ) ? static_cast<void> (0) : __assert_fail ("cast<FixedVectorType>(VType)->getNumElements() == SI.NumRows * SI.NumColumns && \"The vector size must match the number of matrix elements\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 431, __PRETTY_FUNCTION__)) | |||
| 430 | SI.NumRows * SI.NumColumns &&((cast<FixedVectorType>(VType)->getNumElements() == SI .NumRows * SI.NumColumns && "The vector size must match the number of matrix elements" ) ? static_cast<void> (0) : __assert_fail ("cast<FixedVectorType>(VType)->getNumElements() == SI.NumRows * SI.NumColumns && \"The vector size must match the number of matrix elements\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 431, __PRETTY_FUNCTION__)) | |||
| 431 | "The vector size must match the number of matrix elements")((cast<FixedVectorType>(VType)->getNumElements() == SI .NumRows * SI.NumColumns && "The vector size must match the number of matrix elements" ) ? static_cast<void> (0) : __assert_fail ("cast<FixedVectorType>(VType)->getNumElements() == SI.NumRows * SI.NumColumns && \"The vector size must match the number of matrix elements\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 431, __PRETTY_FUNCTION__)); | |||
| 432 | ||||
| 433 | // Check if we lowered MatrixVal using shape information. In that case, | |||
| 434 | // return the existing matrix, if it matches the requested shape | |||
| 435 | // information. If there is a mis-match, embed the result in a flat | |||
| 436 | // vector and split it later. | |||
| 437 | auto Found = Inst2ColumnMatrix.find(MatrixVal); | |||
| 438 | if (Found != Inst2ColumnMatrix.end()) { | |||
| 439 | MatrixTy &M = Found->second; | |||
| 440 | // Return the found matrix, if its shape matches the requested shape | |||
| 441 | // information | |||
| 442 | if (SI.NumRows == M.getNumRows() && SI.NumColumns == M.getNumColumns()) | |||
| 443 | return M; | |||
| 444 | ||||
| 445 | MatrixVal = M.embedInVector(Builder); | |||
| 446 | } | |||
| 447 | ||||
| 448 | // Otherwise split MatrixVal. | |||
| 449 | SmallVector<Value *, 16> SplitVecs; | |||
| 450 | Value *Undef = UndefValue::get(VType); | |||
| 451 | for (unsigned MaskStart = 0; | |||
| 452 | MaskStart < cast<FixedVectorType>(VType)->getNumElements(); | |||
| 453 | MaskStart += SI.getStride()) { | |||
| 454 | Value *V = Builder.CreateShuffleVector( | |||
| 455 | MatrixVal, Undef, createSequentialMask(MaskStart, SI.getStride(), 0), | |||
| 456 | "split"); | |||
| 457 | SplitVecs.push_back(V); | |||
| 458 | } | |||
| 459 | ||||
| 460 | return {SplitVecs}; | |||
| 461 | } | |||
| 462 | ||||
| 463 | /// If \p V already has a known shape return false. Otherwise set the shape | |||
| 464 | /// for instructions that support it. | |||
| 465 | bool setShapeInfo(Value *V, ShapeInfo Shape) { | |||
| 466 | assert(Shape && "Shape not set")((Shape && "Shape not set") ? static_cast<void> (0) : __assert_fail ("Shape && \"Shape not set\"", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 466, __PRETTY_FUNCTION__)); | |||
| 467 | if (isa<UndefValue>(V) || !supportsShapeInfo(V)) | |||
| 468 | return false; | |||
| 469 | ||||
| 470 | auto SIter = ShapeMap.find(V); | |||
| 471 | if (SIter != ShapeMap.end()) { | |||
| 472 | LLVM_DEBUG(dbgs() << " not overriding existing shape: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("lower-matrix-intrinsics")) { dbgs() << " not overriding existing shape: " << SIter->second.NumRows << " " << SIter ->second.NumColumns << " for " << *V << "\n" ; } } while (false) | |||
| 473 | << SIter->second.NumRows << " "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("lower-matrix-intrinsics")) { dbgs() << " not overriding existing shape: " << SIter->second.NumRows << " " << SIter ->second.NumColumns << " for " << *V << "\n" ; } } while (false) | |||
| 474 | << SIter->second.NumColumns << " for " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("lower-matrix-intrinsics")) { dbgs() << " not overriding existing shape: " << SIter->second.NumRows << " " << SIter ->second.NumColumns << " for " << *V << "\n" ; } } while (false); | |||
| 475 | return false; | |||
| 476 | } | |||
| 477 | ||||
| 478 | ShapeMap.insert({V, Shape}); | |||
| 479 | LLVM_DEBUG(dbgs() << " " << Shape.NumRows << " x " << Shape.NumColumnsdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("lower-matrix-intrinsics")) { dbgs() << " " << Shape .NumRows << " x " << Shape.NumColumns << " for " << *V << "\n"; } } while (false) | |||
| 480 | << " for " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("lower-matrix-intrinsics")) { dbgs() << " " << Shape .NumRows << " x " << Shape.NumColumns << " for " << *V << "\n"; } } while (false); | |||
| 481 | return true; | |||
| 482 | } | |||
| 483 | ||||
| 484 | bool isUniformShape(Value *V) { | |||
| 485 | Instruction *I = dyn_cast<Instruction>(V); | |||
| 486 | if (!I) | |||
| 487 | return true; | |||
| 488 | ||||
| 489 | switch (I->getOpcode()) { | |||
| 490 | case Instruction::FAdd: | |||
| 491 | case Instruction::FSub: | |||
| 492 | case Instruction::FMul: // Scalar multiply. | |||
| 493 | case Instruction::Add: | |||
| 494 | case Instruction::Mul: | |||
| 495 | case Instruction::Sub: | |||
| 496 | return true; | |||
| 497 | default: | |||
| 498 | return false; | |||
| 499 | } | |||
| 500 | } | |||
| 501 | ||||
| 502 | /// Returns true if shape information can be used for \p V. The supported | |||
| 503 | /// instructions must match the instructions that can be lowered by this pass. | |||
| 504 | bool supportsShapeInfo(Value *V) { | |||
| 505 | Instruction *Inst = dyn_cast<Instruction>(V); | |||
| 506 | if (!Inst) | |||
| 507 | return false; | |||
| 508 | ||||
| 509 | IntrinsicInst *II = dyn_cast<IntrinsicInst>(Inst); | |||
| 510 | if (II) | |||
| 511 | switch (II->getIntrinsicID()) { | |||
| 512 | case Intrinsic::matrix_multiply: | |||
| 513 | case Intrinsic::matrix_transpose: | |||
| 514 | case Intrinsic::matrix_column_major_load: | |||
| 515 | case Intrinsic::matrix_column_major_store: | |||
| 516 | return true; | |||
| 517 | default: | |||
| 518 | return false; | |||
| 519 | } | |||
| 520 | return isUniformShape(V) || isa<StoreInst>(V) || isa<LoadInst>(V); | |||
| 521 | } | |||
| 522 | ||||
| 523 | /// Propagate the shape information of instructions to their users. | |||
| 524 | /// The work list contains instructions for which we can compute the shape, | |||
| 525 | /// either based on the information provided by matrix intrinsics or known | |||
| 526 | /// shapes of operands. | |||
| 527 | SmallVector<Instruction *, 32> | |||
| 528 | propagateShapeForward(SmallVectorImpl<Instruction *> &WorkList) { | |||
| 529 | SmallVector<Instruction *, 32> NewWorkList; | |||
| 530 | // Pop an element for which we guaranteed to have at least one of the | |||
| 531 | // operand shapes. Add the shape for this and then add users to the work | |||
| 532 | // list. | |||
| 533 | LLVM_DEBUG(dbgs() << "Forward-propagate shapes:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("lower-matrix-intrinsics")) { dbgs() << "Forward-propagate shapes:\n" ; } } while (false); | |||
| 534 | while (!WorkList.empty()) { | |||
| 535 | Instruction *Inst = WorkList.back(); | |||
| 536 | WorkList.pop_back(); | |||
| 537 | ||||
| 538 | // New entry, set the value and insert operands | |||
| 539 | bool Propagate = false; | |||
| 540 | ||||
| 541 | Value *MatrixA; | |||
| 542 | Value *MatrixB; | |||
| 543 | Value *M; | |||
| 544 | Value *N; | |||
| 545 | Value *K; | |||
| 546 | if (match(Inst, m_Intrinsic<Intrinsic::matrix_multiply>( | |||
| 547 | m_Value(MatrixA), m_Value(MatrixB), m_Value(M), | |||
| 548 | m_Value(N), m_Value(K)))) { | |||
| 549 | Propagate = setShapeInfo(Inst, {M, K}); | |||
| 550 | } else if (match(Inst, m_Intrinsic<Intrinsic::matrix_transpose>( | |||
| 551 | m_Value(MatrixA), m_Value(M), m_Value(N)))) { | |||
| 552 | // Flip dimensions. | |||
| 553 | Propagate = setShapeInfo(Inst, {N, M}); | |||
| 554 | } else if (match(Inst, m_Intrinsic<Intrinsic::matrix_column_major_store>( | |||
| 555 | m_Value(MatrixA), m_Value(), m_Value(), | |||
| 556 | m_Value(), m_Value(M), m_Value(N)))) { | |||
| 557 | Propagate = setShapeInfo(Inst, {N, M}); | |||
| 558 | } else if (match(Inst, m_Intrinsic<Intrinsic::matrix_column_major_load>( | |||
| 559 | m_Value(), m_Value(), m_Value(), m_Value(M), | |||
| 560 | m_Value(N)))) { | |||
| 561 | Propagate = setShapeInfo(Inst, {M, N}); | |||
| 562 | } else if (match(Inst, m_Store(m_Value(MatrixA), m_Value()))) { | |||
| 563 | auto OpShape = ShapeMap.find(MatrixA); | |||
| 564 | if (OpShape != ShapeMap.end()) | |||
| 565 | setShapeInfo(Inst, OpShape->second); | |||
| 566 | continue; | |||
| 567 | } else if (isUniformShape(Inst)) { | |||
| 568 | // Find the first operand that has a known shape and use that. | |||
| 569 | for (auto &Op : Inst->operands()) { | |||
| 570 | auto OpShape = ShapeMap.find(Op.get()); | |||
| 571 | if (OpShape != ShapeMap.end()) { | |||
| 572 | Propagate |= setShapeInfo(Inst, OpShape->second); | |||
| 573 | break; | |||
| 574 | } | |||
| 575 | } | |||
| 576 | } | |||
| 577 | ||||
| 578 | if (Propagate) { | |||
| 579 | NewWorkList.push_back(Inst); | |||
| 580 | for (auto *User : Inst->users()) | |||
| 581 | if (ShapeMap.count(User) == 0) | |||
| 582 | WorkList.push_back(cast<Instruction>(User)); | |||
| 583 | } | |||
| 584 | } | |||
| 585 | ||||
| 586 | return NewWorkList; | |||
| 587 | } | |||
| 588 | ||||
| 589 | /// Propagate the shape to operands of instructions with shape information. | |||
| 590 | /// \p Worklist contains the instruction for which we already know the shape. | |||
| 591 | SmallVector<Instruction *, 32> | |||
| 592 | propagateShapeBackward(SmallVectorImpl<Instruction *> &WorkList) { | |||
| 593 | SmallVector<Instruction *, 32> NewWorkList; | |||
| 594 | ||||
| 595 | auto pushInstruction = [](Value *V, | |||
| 596 | SmallVectorImpl<Instruction *> &WorkList) { | |||
| 597 | Instruction *I = dyn_cast<Instruction>(V); | |||
| 598 | if (I) | |||
| 599 | WorkList.push_back(I); | |||
| 600 | }; | |||
| 601 | // Pop an element with known shape. Traverse the operands, if their shape | |||
| 602 | // derives from the result shape and is unknown, add it and add them to the | |||
| 603 | // worklist. | |||
| 604 | LLVM_DEBUG(dbgs() << "Backward-propagate shapes:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("lower-matrix-intrinsics")) { dbgs() << "Backward-propagate shapes:\n" ; } } while (false); | |||
| 605 | while (!WorkList.empty()) { | |||
| 606 | Value *V = WorkList.back(); | |||
| 607 | WorkList.pop_back(); | |||
| 608 | ||||
| 609 | size_t BeforeProcessingV = WorkList.size(); | |||
| 610 | if (!isa<Instruction>(V)) | |||
| 611 | continue; | |||
| 612 | ||||
| 613 | Value *MatrixA; | |||
| 614 | Value *MatrixB; | |||
| 615 | Value *M; | |||
| 616 | Value *N; | |||
| 617 | Value *K; | |||
| 618 | if (match(V, m_Intrinsic<Intrinsic::matrix_multiply>( | |||
| 619 | m_Value(MatrixA), m_Value(MatrixB), m_Value(M), | |||
| 620 | m_Value(N), m_Value(K)))) { | |||
| 621 | if (setShapeInfo(MatrixA, {M, N})) | |||
| 622 | pushInstruction(MatrixA, WorkList); | |||
| 623 | ||||
| 624 | if (setShapeInfo(MatrixB, {N, K})) | |||
| 625 | pushInstruction(MatrixB, WorkList); | |||
| 626 | ||||
| 627 | } else if (match(V, m_Intrinsic<Intrinsic::matrix_transpose>( | |||
| 628 | m_Value(MatrixA), m_Value(M), m_Value(N)))) { | |||
| 629 | // Flip dimensions. | |||
| 630 | if (setShapeInfo(MatrixA, {M, N})) | |||
| 631 | pushInstruction(MatrixA, WorkList); | |||
| 632 | } else if (match(V, m_Intrinsic<Intrinsic::matrix_column_major_store>( | |||
| 633 | m_Value(MatrixA), m_Value(), m_Value(), m_Value(), | |||
| 634 | m_Value(M), m_Value(N)))) { | |||
| 635 | if (setShapeInfo(MatrixA, {M, N})) { | |||
| 636 | pushInstruction(MatrixA, WorkList); | |||
| 637 | } | |||
| 638 | } else if (isa<LoadInst>(V) || | |||
| 639 | match(V, m_Intrinsic<Intrinsic::matrix_column_major_load>())) { | |||
| 640 | // Nothing to do, no matrix input. | |||
| 641 | } else if (isa<StoreInst>(V)) { | |||
| 642 | // Nothing to do. We forward-propagated to this so we would just | |||
| 643 | // backward propagate to an instruction with an already known shape. | |||
| 644 | } else if (isUniformShape(V)) { | |||
| 645 | // Propagate to all operands. | |||
| 646 | ShapeInfo Shape = ShapeMap[V]; | |||
| 647 | for (Use &U : cast<Instruction>(V)->operands()) { | |||
| 648 | if (setShapeInfo(U.get(), Shape)) | |||
| 649 | pushInstruction(U.get(), WorkList); | |||
| 650 | } | |||
| 651 | } | |||
| 652 | // After we discovered new shape info for new instructions in the | |||
| 653 | // worklist, we use their users as seeds for the next round of forward | |||
| 654 | // propagation. | |||
| 655 | for (size_t I = BeforeProcessingV; I != WorkList.size(); I++) | |||
| 656 | for (User *U : WorkList[I]->users()) | |||
| 657 | if (isa<Instruction>(U) && V != U) | |||
| 658 | NewWorkList.push_back(cast<Instruction>(U)); | |||
| 659 | } | |||
| 660 | return NewWorkList; | |||
| 661 | } | |||
| 662 | ||||
| 663 | bool Visit() { | |||
| 664 | if (EnableShapePropagation) { | |||
| 665 | SmallVector<Instruction *, 32> WorkList; | |||
| 666 | ||||
| 667 | // Initially only the shape of matrix intrinsics is known. | |||
| 668 | // Initialize the work list with ops carrying shape information. | |||
| 669 | for (BasicBlock &BB : Func) | |||
| 670 | for (Instruction &Inst : BB) { | |||
| 671 | IntrinsicInst *II = dyn_cast<IntrinsicInst>(&Inst); | |||
| 672 | if (!II) | |||
| 673 | continue; | |||
| 674 | ||||
| 675 | switch (II->getIntrinsicID()) { | |||
| 676 | case Intrinsic::matrix_multiply: | |||
| 677 | case Intrinsic::matrix_transpose: | |||
| 678 | case Intrinsic::matrix_column_major_load: | |||
| 679 | case Intrinsic::matrix_column_major_store: | |||
| 680 | WorkList.push_back(&Inst); | |||
| 681 | break; | |||
| 682 | default: | |||
| 683 | break; | |||
| 684 | } | |||
| 685 | } | |||
| 686 | // Propagate shapes until nothing changes any longer. | |||
| 687 | while (!WorkList.empty()) { | |||
| 688 | WorkList = propagateShapeForward(WorkList); | |||
| 689 | WorkList = propagateShapeBackward(WorkList); | |||
| 690 | } | |||
| 691 | } | |||
| 692 | ||||
| 693 | bool Changed = false; | |||
| 694 | SmallVector<CallInst *, 16> MaybeFusableInsts; | |||
| 695 | SmallVector<Instruction *, 16> MatrixInsts; | |||
| 696 | ||||
| 697 | // First, collect all instructions with shape information and candidates for | |||
| 698 | // fusion (currently only matrix multiplies). | |||
| 699 | ReversePostOrderTraversal<Function *> RPOT(&Func); | |||
| 700 | for (auto *BB : RPOT) | |||
| 701 | for (Instruction &I : *BB) { | |||
| 702 | if (ShapeMap.find(&I) == ShapeMap.end()) | |||
| 703 | continue; | |||
| 704 | if (match(&I, m_Intrinsic<Intrinsic::matrix_multiply>())) | |||
| 705 | MaybeFusableInsts.push_back(cast<CallInst>(&I)); | |||
| 706 | MatrixInsts.push_back(&I); | |||
| 707 | } | |||
| 708 | ||||
| 709 | // Second, try to fuse candidates. | |||
| 710 | SmallPtrSet<Instruction *, 16> FusedInsts; | |||
| 711 | for (CallInst *CI : MaybeFusableInsts) | |||
| 712 | LowerMatrixMultiplyFused(CI, FusedInsts); | |||
| 713 | Changed = !FusedInsts.empty(); | |||
| 714 | ||||
| 715 | // Third, lower remaining instructions with shape information. | |||
| 716 | for (Instruction *Inst : MatrixInsts) { | |||
| 717 | if (FusedInsts.count(Inst)) | |||
| 718 | continue; | |||
| 719 | ||||
| 720 | IRBuilder<> Builder(Inst); | |||
| 721 | ||||
| 722 | if (CallInst *CInst
| |||
| 723 | Changed |= VisitCallInst(CInst); | |||
| 724 | ||||
| 725 | Value *Op1; | |||
| 726 | Value *Op2; | |||
| 727 | if (auto *BinOp = dyn_cast<BinaryOperator>(Inst)) | |||
| 728 | Changed |= VisitBinaryOperator(BinOp); | |||
| 729 | if (match(Inst, m_Load(m_Value(Op1)))) | |||
| 730 | Changed |= VisitLoad(cast<LoadInst>(Inst), Op1, Builder); | |||
| 731 | else if (match(Inst, m_Store(m_Value(Op1), m_Value(Op2)))) | |||
| 732 | Changed |= VisitStore(cast<StoreInst>(Inst), Op1, Op2, Builder); | |||
| 733 | } | |||
| 734 | ||||
| 735 | if (ORE) { | |||
| 736 | RemarkGenerator RemarkGen(Inst2ColumnMatrix, *ORE, Func); | |||
| 737 | RemarkGen.emitRemarks(); | |||
| 738 | } | |||
| 739 | ||||
| 740 | for (Instruction *Inst : reverse(ToRemove)) | |||
| 741 | Inst->eraseFromParent(); | |||
| 742 | ||||
| 743 | return Changed; | |||
| 744 | } | |||
| 745 | ||||
| 746 | /// Turns \p BasePtr into an elementwise pointer to \p EltType. | |||
| 747 | Value *createElementPtr(Value *BasePtr, Type *EltType, IRBuilder<> &Builder) { | |||
| 748 | unsigned AS = cast<PointerType>(BasePtr->getType())->getAddressSpace(); | |||
| 749 | Type *EltPtrType = PointerType::get(EltType, AS); | |||
| 750 | return Builder.CreatePointerCast(BasePtr, EltPtrType); | |||
| 751 | } | |||
| 752 | ||||
| 753 | /// Replace intrinsic calls | |||
| 754 | bool VisitCallInst(CallInst *Inst) { | |||
| 755 | if (!Inst->getCalledFunction() || !Inst->getCalledFunction()->isIntrinsic()) | |||
| 756 | return false; | |||
| 757 | ||||
| 758 | switch (Inst->getCalledFunction()->getIntrinsicID()) { | |||
| 759 | case Intrinsic::matrix_multiply: | |||
| 760 | LowerMultiply(Inst); | |||
| 761 | break; | |||
| 762 | case Intrinsic::matrix_transpose: | |||
| 763 | LowerTranspose(Inst); | |||
| 764 | break; | |||
| 765 | case Intrinsic::matrix_column_major_load: | |||
| 766 | LowerColumnMajorLoad(Inst); | |||
| 767 | break; | |||
| 768 | case Intrinsic::matrix_column_major_store: | |||
| 769 | LowerColumnMajorStore(Inst); | |||
| 770 | break; | |||
| 771 | default: | |||
| 772 | return false; | |||
| 773 | } | |||
| 774 | return true; | |||
| 775 | } | |||
| 776 | ||||
| 777 | /// Compute the alignment for a column/row \p Idx with \p Stride between them. | |||
| 778 | /// The address at \p Idx == 0 has alignment \p A. If \p Stride is a | |||
| 779 | /// ConstantInt, reduce the initial alignment based on the byte offset. For | |||
| 780 | /// non-ConstantInt strides, return the common alignment of the initial | |||
| 781 | /// alignment and the element size in bytes. | |||
| 782 | Align getAlignForIndex(unsigned Idx, Value *Stride, Type *ElementTy, | |||
| 783 | MaybeAlign A) const { | |||
| 784 | Align InitialAlign = DL.getValueOrABITypeAlignment(A, ElementTy); | |||
| 785 | if (Idx == 0) | |||
| 786 | return InitialAlign; | |||
| 787 | ||||
| 788 | TypeSize ElementSizeInBits = DL.getTypeSizeInBits(ElementTy); | |||
| 789 | if (auto *ConstStride = dyn_cast<ConstantInt>(Stride)) { | |||
| 790 | uint64_t StrideInBytes = | |||
| 791 | ConstStride->getZExtValue() * ElementSizeInBits / 8; | |||
| 792 | return commonAlignment(InitialAlign, Idx * StrideInBytes); | |||
| 793 | } | |||
| 794 | return commonAlignment(InitialAlign, ElementSizeInBits / 8); | |||
| 795 | } | |||
| 796 | ||||
| 797 | /// Load a matrix with \p Shape starting at \p Ptr and using \p Stride between | |||
| 798 | /// vectors. | |||
| 799 | MatrixTy loadMatrix(Type *Ty, Value *Ptr, MaybeAlign MAlign, Value *Stride, | |||
| 800 | bool IsVolatile, ShapeInfo Shape, IRBuilder<> &Builder) { | |||
| 801 | auto VType = cast<VectorType>(Ty); | |||
| 802 | Value *EltPtr = createElementPtr(Ptr, VType->getElementType(), Builder); | |||
| 803 | MatrixTy Result; | |||
| 804 | for (unsigned I = 0, E = Shape.getNumVectors(); I < E; ++I) { | |||
| 805 | Value *GEP = computeVectorAddr(EltPtr, Builder.getInt64(I), Stride, | |||
| 806 | Shape.getStride(), VType->getElementType(), | |||
| 807 | Builder); | |||
| 808 | Value *Vector = Builder.CreateAlignedLoad( | |||
| 809 | GEP, getAlignForIndex(I, Stride, VType->getElementType(), MAlign), | |||
| 810 | IsVolatile, "col.load"); | |||
| 811 | ||||
| 812 | Result.addVector(Vector); | |||
| 813 | } | |||
| 814 | return Result.addNumLoads(getNumOps(Result.getVectorTy()) * | |||
| 815 | Result.getNumVectors()); | |||
| 816 | } | |||
| 817 | ||||
| 818 | /// Loads a sub-matrix with shape \p ResultShape from a \p R x \p C matrix, | |||
| 819 | /// starting at \p MatrixPtr[I][J]. | |||
| 820 | MatrixTy loadMatrix(Value *MatrixPtr, MaybeAlign Align, bool IsVolatile, | |||
| 821 | ShapeInfo MatrixShape, Value *I, Value *J, | |||
| 822 | ShapeInfo ResultShape, Type *EltTy, | |||
| 823 | IRBuilder<> &Builder) { | |||
| 824 | ||||
| 825 | Value *Offset = Builder.CreateAdd( | |||
| 826 | Builder.CreateMul(J, Builder.getInt64(MatrixShape.getStride())), I); | |||
| 827 | ||||
| 828 | unsigned AS = cast<PointerType>(MatrixPtr->getType())->getAddressSpace(); | |||
| 829 | Value *EltPtr = | |||
| 830 | Builder.CreatePointerCast(MatrixPtr, PointerType::get(EltTy, AS)); | |||
| 831 | Value *TileStart = Builder.CreateGEP(EltTy, EltPtr, Offset); | |||
| 832 | auto *TileTy = FixedVectorType::get(EltTy, ResultShape.NumRows * | |||
| 833 | ResultShape.NumColumns); | |||
| 834 | Type *TilePtrTy = PointerType::get(TileTy, AS); | |||
| 835 | Value *TilePtr = | |||
| 836 | Builder.CreatePointerCast(TileStart, TilePtrTy, "col.cast"); | |||
| 837 | ||||
| 838 | return loadMatrix(TileTy, TilePtr, Align, | |||
| 839 | Builder.getInt64(MatrixShape.getStride()), IsVolatile, | |||
| 840 | ResultShape, Builder); | |||
| 841 | } | |||
| 842 | ||||
| 843 | /// Lower a load instruction with shape information. | |||
| 844 | void LowerLoad(Instruction *Inst, Value *Ptr, MaybeAlign Align, Value *Stride, | |||
| 845 | bool IsVolatile, ShapeInfo Shape) { | |||
| 846 | IRBuilder<> Builder(Inst); | |||
| 847 | finalizeLowering(Inst, | |||
| 848 | loadMatrix(Inst->getType(), Ptr, Align, Stride, IsVolatile, | |||
| 849 | Shape, Builder), | |||
| 850 | Builder); | |||
| 851 | } | |||
| 852 | ||||
| 853 | /// Lowers llvm.matrix.column.major.load. | |||
| 854 | /// | |||
| 855 | /// The intrinsic loads a matrix from memory using a stride between columns. | |||
| 856 | void LowerColumnMajorLoad(CallInst *Inst) { | |||
| 857 | assert(MatrixLayout == MatrixLayoutTy::ColumnMajor &&((MatrixLayout == MatrixLayoutTy::ColumnMajor && "Intrinsic only supports column-major layout!" ) ? static_cast<void> (0) : __assert_fail ("MatrixLayout == MatrixLayoutTy::ColumnMajor && \"Intrinsic only supports column-major layout!\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 858, __PRETTY_FUNCTION__)) | |||
| 858 | "Intrinsic only supports column-major layout!")((MatrixLayout == MatrixLayoutTy::ColumnMajor && "Intrinsic only supports column-major layout!" ) ? static_cast<void> (0) : __assert_fail ("MatrixLayout == MatrixLayoutTy::ColumnMajor && \"Intrinsic only supports column-major layout!\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 858, __PRETTY_FUNCTION__)); | |||
| 859 | Value *Ptr = Inst->getArgOperand(0); | |||
| 860 | Value *Stride = Inst->getArgOperand(1); | |||
| 861 | LowerLoad(Inst, Ptr, Inst->getParamAlign(0), Stride, | |||
| 862 | cast<ConstantInt>(Inst->getArgOperand(2))->isOne(), | |||
| 863 | {Inst->getArgOperand(3), Inst->getArgOperand(4)}); | |||
| 864 | } | |||
| 865 | ||||
| 866 | /// Stores a sub-matrix \p StoreVal into the \p R x \p C matrix starting at \p | |||
| 867 | /// MatrixPtr[I][J]. | |||
| 868 | void storeMatrix(const MatrixTy &StoreVal, Value *MatrixPtr, | |||
| 869 | MaybeAlign MAlign, bool IsVolatile, ShapeInfo MatrixShape, | |||
| 870 | Value *I, Value *J, Type *EltTy, IRBuilder<> &Builder) { | |||
| 871 | Value *Offset = Builder.CreateAdd( | |||
| 872 | Builder.CreateMul(J, Builder.getInt64(MatrixShape.getStride())), I); | |||
| 873 | ||||
| 874 | unsigned AS = cast<PointerType>(MatrixPtr->getType())->getAddressSpace(); | |||
| 875 | Value *EltPtr = | |||
| 876 | Builder.CreatePointerCast(MatrixPtr, PointerType::get(EltTy, AS)); | |||
| 877 | Value *TileStart = Builder.CreateGEP(EltTy, EltPtr, Offset); | |||
| 878 | auto *TileTy = FixedVectorType::get(EltTy, StoreVal.getNumRows() * | |||
| 879 | StoreVal.getNumColumns()); | |||
| 880 | Type *TilePtrTy = PointerType::get(TileTy, AS); | |||
| 881 | Value *TilePtr = | |||
| 882 | Builder.CreatePointerCast(TileStart, TilePtrTy, "col.cast"); | |||
| 883 | ||||
| 884 | storeMatrix(TileTy, StoreVal, TilePtr, MAlign, | |||
| 885 | Builder.getInt64(MatrixShape.getStride()), IsVolatile, Builder); | |||
| 886 | } | |||
| 887 | ||||
| 888 | /// Store matrix \p StoreVal starting at \p Ptr and using \p Stride between | |||
| 889 | /// vectors. | |||
| 890 | MatrixTy storeMatrix(Type *Ty, MatrixTy StoreVal, Value *Ptr, | |||
| 891 | MaybeAlign MAlign, Value *Stride, bool IsVolatile, | |||
| 892 | IRBuilder<> &Builder) { | |||
| 893 | auto VType = cast<VectorType>(Ty); | |||
| 894 | Value *EltPtr = createElementPtr(Ptr, VType->getElementType(), Builder); | |||
| 895 | for (auto Vec : enumerate(StoreVal.vectors())) { | |||
| 896 | Value *GEP = computeVectorAddr(EltPtr, Builder.getInt64(Vec.index()), | |||
| 897 | Stride, StoreVal.getStride(), | |||
| 898 | VType->getElementType(), Builder); | |||
| 899 | Builder.CreateAlignedStore(Vec.value(), GEP, | |||
| 900 | getAlignForIndex(Vec.index(), Stride, | |||
| 901 | VType->getElementType(), | |||
| 902 | MAlign), | |||
| 903 | IsVolatile); | |||
| 904 | } | |||
| 905 | return MatrixTy().addNumStores(getNumOps(StoreVal.getVectorTy()) * | |||
| 906 | StoreVal.getNumVectors()); | |||
| 907 | } | |||
| 908 | ||||
| 909 | /// Lower a store instruction with shape information. | |||
| 910 | void LowerStore(Instruction *Inst, Value *Matrix, Value *Ptr, MaybeAlign A, | |||
| 911 | Value *Stride, bool IsVolatile, ShapeInfo Shape) { | |||
| 912 | IRBuilder<> Builder(Inst); | |||
| 913 | auto StoreVal = getMatrix(Matrix, Shape, Builder); | |||
| 914 | finalizeLowering(Inst, | |||
| 915 | storeMatrix(Matrix->getType(), StoreVal, Ptr, A, Stride, | |||
| 916 | IsVolatile, Builder), | |||
| 917 | Builder); | |||
| 918 | } | |||
| 919 | ||||
| 920 | /// Lowers llvm.matrix.column.major.store. | |||
| 921 | /// | |||
| 922 | /// The intrinsic store a matrix back memory using a stride between columns. | |||
| 923 | void LowerColumnMajorStore(CallInst *Inst) { | |||
| 924 | assert(MatrixLayout == MatrixLayoutTy::ColumnMajor &&((MatrixLayout == MatrixLayoutTy::ColumnMajor && "Intrinsic only supports column-major layout!" ) ? static_cast<void> (0) : __assert_fail ("MatrixLayout == MatrixLayoutTy::ColumnMajor && \"Intrinsic only supports column-major layout!\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 925, __PRETTY_FUNCTION__)) | |||
| 925 | "Intrinsic only supports column-major layout!")((MatrixLayout == MatrixLayoutTy::ColumnMajor && "Intrinsic only supports column-major layout!" ) ? static_cast<void> (0) : __assert_fail ("MatrixLayout == MatrixLayoutTy::ColumnMajor && \"Intrinsic only supports column-major layout!\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 925, __PRETTY_FUNCTION__)); | |||
| 926 | Value *Matrix = Inst->getArgOperand(0); | |||
| 927 | Value *Ptr = Inst->getArgOperand(1); | |||
| 928 | Value *Stride = Inst->getArgOperand(2); | |||
| 929 | LowerStore(Inst, Matrix, Ptr, Inst->getParamAlign(1), Stride, | |||
| 930 | cast<ConstantInt>(Inst->getArgOperand(3))->isOne(), | |||
| 931 | {Inst->getArgOperand(4), Inst->getArgOperand(5)}); | |||
| 932 | } | |||
| 933 | ||||
| 934 | // Set elements I..I+NumElts-1 to Block | |||
| 935 | Value *insertVector(Value *Col, unsigned I, Value *Block, | |||
| 936 | IRBuilder<> &Builder) { | |||
| 937 | ||||
| 938 | // First, bring Block to the same size as Col | |||
| 939 | unsigned BlockNumElts = | |||
| 940 | cast<FixedVectorType>(Block->getType())->getNumElements(); | |||
| ||||
| 941 | unsigned NumElts = cast<FixedVectorType>(Col->getType())->getNumElements(); | |||
| 942 | assert(NumElts >= BlockNumElts && "Too few elements for current block")((NumElts >= BlockNumElts && "Too few elements for current block" ) ? static_cast<void> (0) : __assert_fail ("NumElts >= BlockNumElts && \"Too few elements for current block\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 942, __PRETTY_FUNCTION__)); | |||
| 943 | ||||
| 944 | Value *Undef = UndefValue::get(Block->getType()); | |||
| 945 | Block = Builder.CreateShuffleVector( | |||
| 946 | Block, Undef, | |||
| 947 | createSequentialMask(0, BlockNumElts, NumElts - BlockNumElts)); | |||
| 948 | ||||
| 949 | // If Col is 7 long and I is 2 and BlockNumElts is 2 the mask is: 0, 1, 7, | |||
| 950 | // 8, 4, 5, 6 | |||
| 951 | SmallVector<int, 16> Mask; | |||
| 952 | unsigned i; | |||
| 953 | for (i = 0; i < I; i++) | |||
| 954 | Mask.push_back(i); | |||
| 955 | ||||
| 956 | unsigned VecNumElts = | |||
| 957 | cast<FixedVectorType>(Col->getType())->getNumElements(); | |||
| 958 | for (; i < I + BlockNumElts; i++) | |||
| 959 | Mask.push_back(i - I + VecNumElts); | |||
| 960 | ||||
| 961 | for (; i < VecNumElts; i++) | |||
| 962 | Mask.push_back(i); | |||
| 963 | ||||
| 964 | return Builder.CreateShuffleVector(Col, Block, Mask); | |||
| 965 | } | |||
| 966 | ||||
| 967 | Value *createMulAdd(Value *Sum, Value *A, Value *B, bool UseFPOp, | |||
| 968 | IRBuilder<> &Builder, bool AllowContraction, | |||
| 969 | unsigned &NumComputeOps) { | |||
| 970 | NumComputeOps += getNumOps(A->getType()); | |||
| 971 | if (!Sum) | |||
| 972 | return UseFPOp ? Builder.CreateFMul(A, B) : Builder.CreateMul(A, B); | |||
| 973 | ||||
| 974 | if (UseFPOp) { | |||
| 975 | if (AllowContraction) { | |||
| 976 | // Use fmuladd for floating point operations and let the backend decide | |||
| 977 | // if that's profitable. | |||
| 978 | Function *FMulAdd = Intrinsic::getDeclaration( | |||
| 979 | Func.getParent(), Intrinsic::fmuladd, A->getType()); | |||
| 980 | return Builder.CreateCall(FMulAdd, {A, B, Sum}); | |||
| 981 | } | |||
| 982 | NumComputeOps += getNumOps(A->getType()); | |||
| 983 | Value *Mul = Builder.CreateFMul(A, B); | |||
| 984 | return Builder.CreateFAdd(Sum, Mul); | |||
| 985 | } | |||
| 986 | ||||
| 987 | NumComputeOps += getNumOps(A->getType()); | |||
| 988 | Value *Mul = Builder.CreateMul(A, B); | |||
| 989 | return Builder.CreateAdd(Sum, Mul); | |||
| 990 | } | |||
| 991 | ||||
| 992 | /// Cache \p Matrix as result of \p Inst and update the uses of \p Inst. For | |||
| 993 | /// users with shape information, there's nothing to do: the will use the | |||
| 994 | /// cached value when they are lowered. For other users, \p Matrix is | |||
| 995 | /// flattened and the uses are updated to use it. Also marks \p Inst for | |||
| 996 | /// deletion. | |||
| 997 | void finalizeLowering(Instruction *Inst, MatrixTy Matrix, | |||
| 998 | IRBuilder<> &Builder) { | |||
| 999 | Inst2ColumnMatrix.insert(std::make_pair(Inst, Matrix)); | |||
| 1000 | ||||
| 1001 | ToRemove.push_back(Inst); | |||
| 1002 | Value *Flattened = nullptr; | |||
| 1003 | for (auto I = Inst->use_begin(), E = Inst->use_end(); I != E;) { | |||
| 1004 | Use &U = *I++; | |||
| 1005 | if (ShapeMap.find(U.getUser()) == ShapeMap.end()) { | |||
| 1006 | if (!Flattened) | |||
| 1007 | Flattened = Matrix.embedInVector(Builder); | |||
| 1008 | U.set(Flattened); | |||
| 1009 | } | |||
| 1010 | } | |||
| 1011 | } | |||
| 1012 | ||||
| 1013 | /// Compute \p Result += \p A * \p B for input matrices with left-associating | |||
| 1014 | /// addition. | |||
| 1015 | void emitMatrixMultiply(MatrixTy &Result, const MatrixTy &A, | |||
| 1016 | const MatrixTy &B, bool AllowContraction, | |||
| 1017 | IRBuilder<> &Builder, bool isTiled) { | |||
| 1018 | const unsigned VF = std::max<unsigned>( | |||
| 1019 | TTI.getRegisterBitWidth(true) / | |||
| 1020 | Result.getElementType()->getPrimitiveSizeInBits().getFixedSize(), | |||
| 1021 | 1U); | |||
| 1022 | unsigned R = Result.getNumRows(); | |||
| 1023 | unsigned C = Result.getNumColumns(); | |||
| 1024 | unsigned M = A.getNumColumns(); | |||
| 1025 | ||||
| 1026 | bool IsFP = Result.getElementType()->isFloatingPointTy(); | |||
| 1027 | assert(A.isColumnMajor() == B.isColumnMajor() &&((A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor () == A.isColumnMajor() && "operands must agree on matrix layout" ) ? static_cast<void> (0) : __assert_fail ("A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor() == A.isColumnMajor() && \"operands must agree on matrix layout\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 1029, __PRETTY_FUNCTION__)) | |||
| 1028 | Result.isColumnMajor() == A.isColumnMajor() &&((A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor () == A.isColumnMajor() && "operands must agree on matrix layout" ) ? static_cast<void> (0) : __assert_fail ("A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor() == A.isColumnMajor() && \"operands must agree on matrix layout\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 1029, __PRETTY_FUNCTION__)) | |||
| 1029 | "operands must agree on matrix layout")((A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor () == A.isColumnMajor() && "operands must agree on matrix layout" ) ? static_cast<void> (0) : __assert_fail ("A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor() == A.isColumnMajor() && \"operands must agree on matrix layout\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 1029, __PRETTY_FUNCTION__)); | |||
| 1030 | unsigned NumComputeOps = 0; | |||
| 1031 | if (A.isColumnMajor()) { | |||
| 1032 | // Multiply columns from the first operand with scalars from the second | |||
| 1033 | // operand. Then move along the K axes and accumulate the columns. With | |||
| 1034 | // this the adds can be vectorized without reassociation. | |||
| 1035 | for (unsigned J = 0; J < C; ++J) { | |||
| 1036 | unsigned BlockSize = VF; | |||
| 1037 | // If Result is zero, we don't need to accumulate in the K==0 iteration. | |||
| 1038 | bool isSumZero = isa<ConstantAggregateZero>(Result.getColumn(J)); | |||
| 1039 | ||||
| 1040 | for (unsigned I = 0; I < R; I += BlockSize) { | |||
| 1041 | // Gradually lower the vectorization factor to cover the remainder. | |||
| 1042 | while (I + BlockSize > R) | |||
| 1043 | BlockSize /= 2; | |||
| 1044 | ||||
| 1045 | Value *Sum = isTiled ? Result.extractVector(I, J, BlockSize, Builder) | |||
| 1046 | : nullptr; | |||
| 1047 | for (unsigned K = 0; K < M; ++K) { | |||
| 1048 | Value *L = A.extractVector(I, K, BlockSize, Builder); | |||
| 1049 | Value *RH = Builder.CreateExtractElement(B.getColumn(J), K); | |||
| 1050 | Value *Splat = Builder.CreateVectorSplat(BlockSize, RH, "splat"); | |||
| 1051 | Sum = createMulAdd(isSumZero && K == 0 ? nullptr : Sum, L, Splat, | |||
| 1052 | Result.getElementType()->isFloatingPointTy(), | |||
| 1053 | Builder, AllowContraction, NumComputeOps); | |||
| 1054 | } | |||
| 1055 | Result.setVector(J, | |||
| 1056 | insertVector(Result.getVector(J), I, Sum, Builder)); | |||
| 1057 | } | |||
| 1058 | } | |||
| 1059 | } else { | |||
| 1060 | // Multiply rows from the second operand with scalars from the first | |||
| 1061 | // operand. Then move along the K axes and accumulate the rows. With this | |||
| 1062 | // the adds can be vectorized without reassociation. | |||
| 1063 | for (unsigned I = 0; I
| |||
| 1064 | unsigned BlockSize = VF; | |||
| 1065 | bool isSumZero = isa<ConstantAggregateZero>(Result.getRow(I)); | |||
| 1066 | for (unsigned J = 0; J < C; J += BlockSize) { | |||
| 1067 | // Gradually lower the vectorization factor to cover the remainder. | |||
| 1068 | while (J + BlockSize > C) | |||
| 1069 | BlockSize /= 2; | |||
| 1070 | ||||
| 1071 | Value *Sum = nullptr; | |||
| 1072 | for (unsigned K = 0; K < M; ++K) { | |||
| 1073 | Value *R = B.extractVector(K, J, BlockSize, Builder); | |||
| 1074 | Value *LH = Builder.CreateExtractElement(A.getVector(I), K); | |||
| 1075 | Value *Splat = Builder.CreateVectorSplat(BlockSize, LH, "splat"); | |||
| 1076 | Sum = createMulAdd(isSumZero && K == 0 ? nullptr : Sum, Splat, R, | |||
| 1077 | IsFP, Builder, AllowContraction, NumComputeOps); | |||
| 1078 | } | |||
| 1079 | Result.setVector(I, | |||
| 1080 | insertVector(Result.getVector(I), J, Sum, Builder)); | |||
| 1081 | } | |||
| 1082 | } | |||
| 1083 | } | |||
| 1084 | Result.addNumComputeOps(NumComputeOps); | |||
| 1085 | } | |||
| 1086 | ||||
| 1087 | /// Ensure that the memory in \p Load does not alias \p Store by potentially | |||
| 1088 | /// copying it to a new location. This new or otherwise the original location | |||
| 1089 | /// is returned. | |||
| 1090 | Value *getNonAliasingPointer(LoadInst *Load, StoreInst *Store, | |||
| 1091 | CallInst *MatMul) { | |||
| 1092 | MemoryLocation StoreLoc = MemoryLocation::get(Store); | |||
| 1093 | MemoryLocation LoadLoc = MemoryLocation::get(Load); | |||
| 1094 | ||||
| 1095 | AliasResult LdAliased = AA->alias(LoadLoc, StoreLoc); | |||
| 1096 | ||||
| 1097 | // If we can statically determine noalias we're good. | |||
| 1098 | if (!LdAliased) | |||
| 1099 | return Load->getPointerOperand(); | |||
| 1100 | ||||
| 1101 | // Create code to check if the memory locations of the Load and Store | |||
| 1102 | // overlap and if they do, copy Load's operand to a new buffer. | |||
| 1103 | ||||
| 1104 | // First, create new blocks for 2n part of the check and the copy. | |||
| 1105 | BasicBlock *Check0 = MatMul->getParent(); | |||
| 1106 | // FIXME: Use lazy DTU and update SplitBlock to accept a DTU instead of a | |||
| 1107 | // DT. Manually collect dominator tree updates, to avoid unnecessary work, | |||
| 1108 | // as we adjust Check0 and Check1's branches. | |||
| 1109 | SmallVector<DominatorTree::UpdateType, 4> DTUpdates; | |||
| 1110 | for (BasicBlock *Succ : successors(Check0)) | |||
| 1111 | DTUpdates.push_back({DT->Delete, Check0, Succ}); | |||
| 1112 | ||||
| 1113 | BasicBlock *Check1 = SplitBlock(MatMul->getParent(), MatMul, nullptr, LI, | |||
| 1114 | nullptr, "alias_cont"); | |||
| 1115 | BasicBlock *Copy = | |||
| 1116 | SplitBlock(MatMul->getParent(), MatMul, nullptr, LI, nullptr, "copy"); | |||
| 1117 | BasicBlock *Fusion = SplitBlock(MatMul->getParent(), MatMul, nullptr, LI, | |||
| 1118 | nullptr, "no_alias"); | |||
| 1119 | ||||
| 1120 | // Check if the loaded memory location begins before the end of the store | |||
| 1121 | // location. If the condition holds, they might overlap, otherwise they are | |||
| 1122 | // guaranteed to not overlap. | |||
| 1123 | IRBuilder<> Builder(MatMul); | |||
| 1124 | Check0->getTerminator()->eraseFromParent(); | |||
| 1125 | Builder.SetInsertPoint(Check0); | |||
| 1126 | Type *IntPtrTy = Builder.getIntPtrTy(Load->getModule()->getDataLayout()); | |||
| 1127 | Value *StoreBegin = Builder.CreatePtrToInt( | |||
| 1128 | const_cast<Value *>(StoreLoc.Ptr), IntPtrTy, "store.begin"); | |||
| 1129 | Value *StoreEnd = Builder.CreateAdd( | |||
| 1130 | StoreBegin, ConstantInt::get(IntPtrTy, StoreLoc.Size.getValue()), | |||
| 1131 | "store.end", true, true); | |||
| 1132 | Value *LoadBegin = Builder.CreatePtrToInt(const_cast<Value *>(LoadLoc.Ptr), | |||
| 1133 | IntPtrTy, "load.begin"); | |||
| 1134 | Builder.CreateCondBr(Builder.CreateICmpULT(LoadBegin, StoreEnd), Check1, | |||
| 1135 | Fusion); | |||
| 1136 | ||||
| 1137 | // Check if the store begins before the end of the load location. If the | |||
| 1138 | // condition holds, they alias, otherwise they are guaranteed to not | |||
| 1139 | // overlap. | |||
| 1140 | Check1->getTerminator()->eraseFromParent(); | |||
| 1141 | Builder.SetInsertPoint(Check1, Check1->begin()); | |||
| 1142 | Value *LoadEnd = Builder.CreateAdd( | |||
| 1143 | LoadBegin, ConstantInt::get(IntPtrTy, LoadLoc.Size.getValue()), | |||
| 1144 | "load.end", true, true); | |||
| 1145 | Builder.CreateCondBr(Builder.CreateICmpULT(StoreBegin, LoadEnd), Copy, | |||
| 1146 | Fusion); | |||
| 1147 | ||||
| 1148 | // Copy load operand to new alloca. | |||
| 1149 | Builder.SetInsertPoint(Copy, Copy->begin()); | |||
| 1150 | AllocaInst *NewLd = | |||
| 1151 | Builder.CreateAlloca(Load->getType(), Load->getPointerAddressSpace()); | |||
| 1152 | Builder.CreateMemCpy(NewLd, NewLd->getAlign(), | |||
| 1153 | Load->getPointerOperand(), Load->getAlign(), | |||
| 1154 | LoadLoc.Size.getValue()); | |||
| 1155 | Builder.SetInsertPoint(Fusion, Fusion->begin()); | |||
| 1156 | PHINode *PHI = Builder.CreatePHI(Load->getPointerOperandType(), 3); | |||
| 1157 | PHI->addIncoming(Load->getPointerOperand(), Check0); | |||
| 1158 | PHI->addIncoming(Load->getPointerOperand(), Check1); | |||
| 1159 | PHI->addIncoming(NewLd, Copy); | |||
| 1160 | ||||
| 1161 | // Adjust DT. | |||
| 1162 | DTUpdates.push_back({DT->Insert, Check0, Check1}); | |||
| 1163 | DTUpdates.push_back({DT->Insert, Check0, Fusion}); | |||
| 1164 | DTUpdates.push_back({DT->Insert, Check1, Copy}); | |||
| 1165 | DTUpdates.push_back({DT->Insert, Check1, Fusion}); | |||
| 1166 | DT->applyUpdates(DTUpdates); | |||
| 1167 | return PHI; | |||
| 1168 | } | |||
| 1169 | ||||
| 1170 | bool isFusionProfitable(CallInst *MatMul) { | |||
| 1171 | if (ForceFusion) | |||
| 1172 | return true; | |||
| 1173 | ||||
| 1174 | ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); | |||
| 1175 | ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); | |||
| 1176 | ||||
| 1177 | const unsigned R = LShape.NumRows; | |||
| 1178 | const unsigned C = RShape.NumColumns; | |||
| 1179 | const unsigned M = LShape.NumColumns; | |||
| 1180 | auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); | |||
| 1181 | ||||
| 1182 | const unsigned VF = | |||
| 1183 | std::max<unsigned>(TTI.getRegisterBitWidth(true) / | |||
| 1184 | EltType->getPrimitiveSizeInBits().getFixedSize(), | |||
| 1185 | 1U); | |||
| 1186 | ||||
| 1187 | // Cost model for tiling | |||
| 1188 | // | |||
| 1189 | // For tiling to be beneficial, we need reuse either along the R or | |||
| 1190 | // the C axis. We vectorize along the R axis so that means at least | |||
| 1191 | // 3 elements. | |||
| 1192 | // TODO: Also consider cost of copying if operands alias. | |||
| 1193 | if (R <= VF && C == 1) | |||
| 1194 | return false; | |||
| 1195 | // Then we need enough elements to exceed the number of vector | |||
| 1196 | // registers we have. Note that this is an oversimplification since | |||
| 1197 | // fusing also takes some extra loads which may exceed the number of | |||
| 1198 | // reloads necessary. | |||
| 1199 | unsigned Op0Regs = (R + VF - 1) / VF * M; | |||
| 1200 | unsigned Op1Regs = (M + VF - 1) / VF * C; | |||
| 1201 | return Op0Regs + Op1Regs > TTI.getNumberOfRegisters(true); | |||
| 1202 | } | |||
| 1203 | ||||
| 1204 | MatrixTy getZeroMatrix(Type *EltType, unsigned R, unsigned C) { | |||
| 1205 | MatrixTy Res; | |||
| 1206 | auto *ColumType = FixedVectorType::get(EltType, R); | |||
| 1207 | for (unsigned I = 0; I < C; ++I) | |||
| 1208 | Res.addVector(ConstantAggregateZero::get(ColumType)); | |||
| 1209 | return Res; | |||
| 1210 | } | |||
| 1211 | ||||
| 1212 | void createTiledLoops(CallInst *MatMul, Value *LPtr, ShapeInfo LShape, | |||
| 1213 | Value *RPtr, ShapeInfo RShape, StoreInst *Store, | |||
| 1214 | bool AllowContract) { | |||
| 1215 | auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); | |||
| 1216 | ||||
| 1217 | // Create the main tiling loop nest. | |||
| 1218 | TileInfo TI(LShape.NumRows, RShape.NumColumns, LShape.NumColumns, TileSize); | |||
| 1219 | DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy); | |||
| 1220 | Instruction *InsertI = cast<Instruction>(MatMul); | |||
| 1221 | BasicBlock *Start = InsertI->getParent(); | |||
| 1222 | BasicBlock *End = | |||
| 1223 | SplitBlock(InsertI->getParent(), InsertI, DT, LI, nullptr, "continue"); | |||
| 1224 | IRBuilder<> Builder(MatMul); | |||
| 1225 | BasicBlock *InnerBody = TI.CreateTiledLoops(Start, End, Builder, DTU, *LI); | |||
| 1226 | ||||
| 1227 | Type *TileVecTy = | |||
| 1228 | FixedVectorType::get(MatMul->getType()->getScalarType(), TileSize); | |||
| 1229 | MatrixTy TileResult; | |||
| 1230 | // Insert in the inner loop header. | |||
| 1231 | Builder.SetInsertPoint(TI.InnerLoopHeader->getTerminator()); | |||
| 1232 | // Create PHI nodes for the result columns to accumulate across iterations. | |||
| 1233 | SmallVector<PHINode *, 4> ColumnPhis; | |||
| 1234 | for (unsigned I = 0; I < TileSize; I++) { | |||
| 1235 | auto *Phi = Builder.CreatePHI(TileVecTy, 2, "result.vec." + Twine(I)); | |||
| 1236 | Phi->addIncoming(ConstantAggregateZero::get(TileVecTy), | |||
| 1237 | TI.RowLoopHeader->getSingleSuccessor()); | |||
| 1238 | TileResult.addVector(Phi); | |||
| 1239 | ColumnPhis.push_back(Phi); | |||
| 1240 | } | |||
| 1241 | ||||
| 1242 | // Insert in the inner loop body, which computes | |||
| 1243 | // Res += Load(CurrentRow, K) * Load(K, CurrentColumn) | |||
| 1244 | Builder.SetInsertPoint(InnerBody->getTerminator()); | |||
| 1245 | // Load tiles of the operands. | |||
| 1246 | MatrixTy A = loadMatrix(LPtr, {}, false, LShape, TI.CurrentRow, TI.CurrentK, | |||
| 1247 | {TileSize, TileSize}, EltType, Builder); | |||
| 1248 | MatrixTy B = loadMatrix(RPtr, {}, false, RShape, TI.CurrentK, TI.CurrentCol, | |||
| 1249 | {TileSize, TileSize}, EltType, Builder); | |||
| 1250 | emitMatrixMultiply(TileResult, A, B, AllowContract, Builder, true); | |||
| 1251 | // Store result after the inner loop is done. | |||
| 1252 | Builder.SetInsertPoint(TI.RowLoopLatch->getTerminator()); | |||
| 1253 | storeMatrix(TileResult, Store->getPointerOperand(), Store->getAlign(), | |||
| 1254 | Store->isVolatile(), {LShape.NumRows, RShape.NumColumns}, | |||
| 1255 | TI.CurrentRow, TI.CurrentCol, EltType, Builder); | |||
| 1256 | ||||
| 1257 | for (unsigned I = 0; I < TileResult.getNumVectors(); I++) | |||
| 1258 | ColumnPhis[I]->addIncoming(TileResult.getVector(I), TI.InnerLoopLatch); | |||
| 1259 | ||||
| 1260 | // Force unrolling of a few iterations of the inner loop, to make sure there | |||
| 1261 | // is enough work per iteration. | |||
| 1262 | // FIXME: The unroller should make this decision directly instead, but | |||
| 1263 | // currently the cost-model is not up to the task. | |||
| 1264 | unsigned InnerLoopUnrollCount = std::min(10u, LShape.NumColumns / TileSize); | |||
| 1265 | addStringMetadataToLoop(LI->getLoopFor(TI.InnerLoopHeader), | |||
| 1266 | "llvm.loop.unroll.count", InnerLoopUnrollCount); | |||
| 1267 | } | |||
| 1268 | ||||
| 1269 | void emitSIMDTiling(CallInst *MatMul, LoadInst *LoadOp0, LoadInst *LoadOp1, | |||
| 1270 | StoreInst *Store, | |||
| 1271 | SmallPtrSetImpl<Instruction *> &FusedInsts) { | |||
| 1272 | assert(MatrixLayout == MatrixLayoutTy::ColumnMajor &&((MatrixLayout == MatrixLayoutTy::ColumnMajor && "Tiling only supported for column-major matrixes at the moment!" ) ? static_cast<void> (0) : __assert_fail ("MatrixLayout == MatrixLayoutTy::ColumnMajor && \"Tiling only supported for column-major matrixes at the moment!\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 1273, __PRETTY_FUNCTION__)) | |||
| 1273 | "Tiling only supported for column-major matrixes at the moment!")((MatrixLayout == MatrixLayoutTy::ColumnMajor && "Tiling only supported for column-major matrixes at the moment!" ) ? static_cast<void> (0) : __assert_fail ("MatrixLayout == MatrixLayoutTy::ColumnMajor && \"Tiling only supported for column-major matrixes at the moment!\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 1273, __PRETTY_FUNCTION__)); | |||
| 1274 | if (!isFusionProfitable(MatMul)) | |||
| 1275 | return; | |||
| 1276 | ||||
| 1277 | ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); | |||
| 1278 | ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); | |||
| 1279 | ||||
| 1280 | const unsigned R = LShape.NumRows; | |||
| 1281 | const unsigned C = RShape.NumColumns; | |||
| 1282 | const unsigned M = LShape.NumColumns; | |||
| 1283 | auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); | |||
| 1284 | ||||
| 1285 | Value *APtr = getNonAliasingPointer(LoadOp0, Store, MatMul); | |||
| 1286 | Value *BPtr = getNonAliasingPointer(LoadOp1, Store, MatMul); | |||
| 1287 | Value *CPtr = Store->getPointerOperand(); | |||
| 1288 | ||||
| 1289 | bool AllowContract = AllowContractEnabled || (isa<FPMathOperator>(MatMul) && | |||
| 1290 | MatMul->hasAllowContract()); | |||
| 1291 | if (TileUseLoops && (R % TileSize == 0 && C % TileSize == 0)) | |||
| 1292 | createTiledLoops(MatMul, APtr, LShape, BPtr, RShape, Store, | |||
| 1293 | AllowContract); | |||
| 1294 | else { | |||
| 1295 | IRBuilder<> Builder(Store); | |||
| 1296 | for (unsigned J = 0; J < C; J += TileSize) | |||
| 1297 | for (unsigned I = 0; I < R; I += TileSize) { | |||
| 1298 | const unsigned TileR = std::min(R - I, unsigned(TileSize)); | |||
| 1299 | const unsigned TileC = std::min(C - J, unsigned(TileSize)); | |||
| 1300 | MatrixTy Res = getZeroMatrix(EltType, TileR, TileC); | |||
| 1301 | ||||
| 1302 | for (unsigned K = 0; K < M; K += TileSize) { | |||
| 1303 | const unsigned TileM = std::min(M - K, unsigned(TileSize)); | |||
| 1304 | MatrixTy A = | |||
| 1305 | loadMatrix(APtr, LoadOp0->getAlign(), LoadOp0->isVolatile(), | |||
| 1306 | LShape, Builder.getInt64(I), Builder.getInt64(K), | |||
| 1307 | {TileR, TileM}, EltType, Builder); | |||
| 1308 | MatrixTy B = | |||
| 1309 | loadMatrix(BPtr, LoadOp1->getAlign(), LoadOp1->isVolatile(), | |||
| 1310 | RShape, Builder.getInt64(K), Builder.getInt64(J), | |||
| 1311 | {TileM, TileC}, EltType, Builder); | |||
| 1312 | emitMatrixMultiply(Res, A, B, AllowContract, Builder, true); | |||
| 1313 | } | |||
| 1314 | storeMatrix(Res, CPtr, Store->getAlign(), Store->isVolatile(), {R, M}, | |||
| 1315 | Builder.getInt64(I), Builder.getInt64(J), EltType, | |||
| 1316 | Builder); | |||
| 1317 | } | |||
| 1318 | } | |||
| 1319 | ||||
| 1320 | // Mark eliminated instructions as fused and remove them. | |||
| 1321 | FusedInsts.insert(Store); | |||
| 1322 | FusedInsts.insert(MatMul); | |||
| 1323 | Store->eraseFromParent(); | |||
| 1324 | MatMul->eraseFromParent(); | |||
| 1325 | if (LoadOp0->hasNUses(0)) { | |||
| 1326 | FusedInsts.insert(LoadOp0); | |||
| 1327 | LoadOp0->eraseFromParent(); | |||
| 1328 | } | |||
| 1329 | if (LoadOp1->hasNUses(0)) { | |||
| 1330 | FusedInsts.insert(LoadOp1); | |||
| 1331 | LoadOp1->eraseFromParent(); | |||
| 1332 | } | |||
| 1333 | } | |||
| 1334 | ||||
| 1335 | /// Try to lower matrix multiply chains by fusing operations. | |||
| 1336 | /// | |||
| 1337 | /// Currently we only lower {ld, ld} -> matmul -> st chains. | |||
| 1338 | // | |||
| 1339 | /// No need to return a MatrixTy object for the result of the operation, since | |||
| 1340 | /// the single store user will be lowered as part of this. Instructions that | |||
| 1341 | /// are completely eliminated by fusion are added to \p FusedInsts. | |||
| 1342 | void LowerMatrixMultiplyFused(CallInst *MatMul, | |||
| 1343 | SmallPtrSetImpl<Instruction *> &FusedInsts) { | |||
| 1344 | if (!FuseMatrix || !MatMul->hasOneUse() || | |||
| 1345 | MatrixLayout != MatrixLayoutTy::ColumnMajor || !DT) | |||
| 1346 | return; | |||
| 1347 | ||||
| 1348 | auto *LoadOp0 = dyn_cast<LoadInst>(MatMul->getOperand(0)); | |||
| 1349 | auto *LoadOp1 = dyn_cast<LoadInst>(MatMul->getOperand(1)); | |||
| 1350 | auto *Store = dyn_cast<StoreInst>(*MatMul->user_begin()); | |||
| 1351 | if (LoadOp0 && LoadOp1 && Store) { | |||
| 1352 | // The store address must dominate the MatMul instruction, otherwise | |||
| 1353 | // we create invalid IR. | |||
| 1354 | // FIXME: See if we can hoist the store address computation. | |||
| 1355 | auto *AddrI = dyn_cast<Instruction>(Store->getOperand(1)); | |||
| 1356 | if (AddrI && (!DT->dominates(AddrI, MatMul))) | |||
| 1357 | return; | |||
| 1358 | ||||
| 1359 | emitSIMDTiling(MatMul, LoadOp0, LoadOp1, Store, FusedInsts); | |||
| 1360 | return; | |||
| 1361 | } | |||
| 1362 | } | |||
| 1363 | ||||
| 1364 | /// Lowers llvm.matrix.multiply. | |||
| 1365 | void LowerMultiply(CallInst *MatMul) { | |||
| 1366 | IRBuilder<> Builder(MatMul); | |||
| 1367 | auto *EltType = cast<VectorType>(MatMul->getType())->getElementType(); | |||
| 1368 | ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3)); | |||
| 1369 | ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4)); | |||
| 1370 | ||||
| 1371 | const MatrixTy &Lhs = getMatrix(MatMul->getArgOperand(0), LShape, Builder); | |||
| 1372 | const MatrixTy &Rhs = getMatrix(MatMul->getArgOperand(1), RShape, Builder); | |||
| 1373 | assert(Lhs.getElementType() == Rhs.getElementType() &&((Lhs.getElementType() == Rhs.getElementType() && "Matrix multiply argument element types do not match." ) ? static_cast<void> (0) : __assert_fail ("Lhs.getElementType() == Rhs.getElementType() && \"Matrix multiply argument element types do not match.\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 1374, __PRETTY_FUNCTION__)) | |||
| 1374 | "Matrix multiply argument element types do not match.")((Lhs.getElementType() == Rhs.getElementType() && "Matrix multiply argument element types do not match." ) ? static_cast<void> (0) : __assert_fail ("Lhs.getElementType() == Rhs.getElementType() && \"Matrix multiply argument element types do not match.\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 1374, __PRETTY_FUNCTION__)); | |||
| 1375 | ||||
| 1376 | const unsigned R = LShape.NumRows; | |||
| 1377 | const unsigned C = RShape.NumColumns; | |||
| 1378 | assert(LShape.NumColumns == RShape.NumRows)((LShape.NumColumns == RShape.NumRows) ? static_cast<void> (0) : __assert_fail ("LShape.NumColumns == RShape.NumRows", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 1378, __PRETTY_FUNCTION__)); | |||
| 1379 | ||||
| 1380 | // Initialize the output | |||
| 1381 | MatrixTy Result(R, C, EltType); | |||
| 1382 | assert(Lhs.getElementType() == Result.getElementType() &&((Lhs.getElementType() == Result.getElementType() && "Matrix multiply result element type does not match arguments." ) ? static_cast<void> (0) : __assert_fail ("Lhs.getElementType() == Result.getElementType() && \"Matrix multiply result element type does not match arguments.\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 1383, __PRETTY_FUNCTION__)) | |||
| 1383 | "Matrix multiply result element type does not match arguments.")((Lhs.getElementType() == Result.getElementType() && "Matrix multiply result element type does not match arguments." ) ? static_cast<void> (0) : __assert_fail ("Lhs.getElementType() == Result.getElementType() && \"Matrix multiply result element type does not match arguments.\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 1383, __PRETTY_FUNCTION__)); | |||
| 1384 | ||||
| 1385 | bool AllowContract = AllowContractEnabled || (isa<FPMathOperator>(MatMul) && | |||
| 1386 | MatMul->hasAllowContract()); | |||
| 1387 | emitMatrixMultiply(Result, Lhs, Rhs, AllowContract, Builder, false); | |||
| 1388 | finalizeLowering(MatMul, Result, Builder); | |||
| 1389 | } | |||
| 1390 | ||||
| 1391 | /// Lowers llvm.matrix.transpose. | |||
| 1392 | void LowerTranspose(CallInst *Inst) { | |||
| 1393 | MatrixTy Result; | |||
| 1394 | IRBuilder<> Builder(Inst); | |||
| 1395 | Value *InputVal = Inst->getArgOperand(0); | |||
| 1396 | VectorType *VectorTy = cast<VectorType>(InputVal->getType()); | |||
| 1397 | ShapeInfo ArgShape(Inst->getArgOperand(1), Inst->getArgOperand(2)); | |||
| 1398 | MatrixTy InputMatrix = getMatrix(InputVal, ArgShape, Builder); | |||
| 1399 | ||||
| 1400 | const unsigned NewNumVecs = | |||
| 1401 | InputMatrix.isColumnMajor() ? ArgShape.NumRows : ArgShape.NumColumns; | |||
| 1402 | const unsigned NewNumElts = | |||
| 1403 | InputMatrix.isColumnMajor() ? ArgShape.NumColumns : ArgShape.NumRows; | |||
| 1404 | ||||
| 1405 | for (unsigned I = 0; I < NewNumVecs; ++I) { | |||
| 1406 | // Build a single result vector. First initialize it. | |||
| 1407 | Value *ResultVector = UndefValue::get( | |||
| 1408 | FixedVectorType::get(VectorTy->getElementType(), NewNumElts)); | |||
| 1409 | // Go through the old elements and insert it into the resulting vector. | |||
| 1410 | for (auto J : enumerate(InputMatrix.vectors())) { | |||
| 1411 | Value *Elt = Builder.CreateExtractElement(J.value(), I); | |||
| 1412 | // Row and column indices are transposed. | |||
| 1413 | ResultVector = | |||
| 1414 | Builder.CreateInsertElement(ResultVector, Elt, J.index()); | |||
| 1415 | } | |||
| 1416 | Result.addVector(ResultVector); | |||
| 1417 | } | |||
| 1418 | ||||
| 1419 | // TODO: Improve estimate of operations needed for transposes. Currently we | |||
| 1420 | // just count the insertelement/extractelement instructions, but do not | |||
| 1421 | // account for later simplifications/combines. | |||
| 1422 | finalizeLowering( | |||
| 1423 | Inst, | |||
| 1424 | Result.addNumComputeOps(2 * ArgShape.NumRows * ArgShape.NumColumns), | |||
| 1425 | Builder); | |||
| 1426 | } | |||
| 1427 | ||||
| 1428 | /// Lower load instructions, if shape information is available. | |||
| 1429 | bool VisitLoad(LoadInst *Inst, Value *Ptr, IRBuilder<> &Builder) { | |||
| 1430 | auto I = ShapeMap.find(Inst); | |||
| 1431 | if (I == ShapeMap.end()) | |||
| 1432 | return false; | |||
| 1433 | ||||
| 1434 | LowerLoad(Inst, Ptr, Inst->getAlign(), | |||
| 1435 | Builder.getInt64(I->second.getStride()), Inst->isVolatile(), | |||
| 1436 | I->second); | |||
| 1437 | return true; | |||
| 1438 | } | |||
| 1439 | ||||
| 1440 | bool VisitStore(StoreInst *Inst, Value *StoredVal, Value *Ptr, | |||
| 1441 | IRBuilder<> &Builder) { | |||
| 1442 | auto I = ShapeMap.find(StoredVal); | |||
| 1443 | if (I == ShapeMap.end()) | |||
| 1444 | return false; | |||
| 1445 | ||||
| 1446 | LowerStore(Inst, StoredVal, Ptr, Inst->getAlign(), | |||
| 1447 | Builder.getInt64(I->second.getStride()), Inst->isVolatile(), | |||
| 1448 | I->second); | |||
| 1449 | return true; | |||
| 1450 | } | |||
| 1451 | ||||
| 1452 | /// Lower binary operators, if shape information is available. | |||
| 1453 | bool VisitBinaryOperator(BinaryOperator *Inst) { | |||
| 1454 | auto I = ShapeMap.find(Inst); | |||
| 1455 | if (I == ShapeMap.end()) | |||
| 1456 | return false; | |||
| 1457 | ||||
| 1458 | Value *Lhs = Inst->getOperand(0); | |||
| 1459 | Value *Rhs = Inst->getOperand(1); | |||
| 1460 | ||||
| 1461 | IRBuilder<> Builder(Inst); | |||
| 1462 | ShapeInfo &Shape = I->second; | |||
| 1463 | ||||
| 1464 | MatrixTy Result; | |||
| 1465 | MatrixTy A = getMatrix(Lhs, Shape, Builder); | |||
| 1466 | MatrixTy B = getMatrix(Rhs, Shape, Builder); | |||
| 1467 | assert(A.isColumnMajor() == B.isColumnMajor() &&((A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor () == A.isColumnMajor() && "operands must agree on matrix layout" ) ? static_cast<void> (0) : __assert_fail ("A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor() == A.isColumnMajor() && \"operands must agree on matrix layout\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 1469, __PRETTY_FUNCTION__)) | |||
| 1468 | Result.isColumnMajor() == A.isColumnMajor() &&((A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor () == A.isColumnMajor() && "operands must agree on matrix layout" ) ? static_cast<void> (0) : __assert_fail ("A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor() == A.isColumnMajor() && \"operands must agree on matrix layout\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 1469, __PRETTY_FUNCTION__)) | |||
| 1469 | "operands must agree on matrix layout")((A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor () == A.isColumnMajor() && "operands must agree on matrix layout" ) ? static_cast<void> (0) : __assert_fail ("A.isColumnMajor() == B.isColumnMajor() && Result.isColumnMajor() == A.isColumnMajor() && \"operands must agree on matrix layout\"" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 1469, __PRETTY_FUNCTION__)); | |||
| 1470 | ||||
| 1471 | // Helper to perform binary op on vectors. | |||
| 1472 | auto BuildVectorOp = [&Builder, Inst](Value *LHS, Value *RHS) { | |||
| 1473 | switch (Inst->getOpcode()) { | |||
| 1474 | case Instruction::Add: | |||
| 1475 | return Builder.CreateAdd(LHS, RHS); | |||
| 1476 | case Instruction::Mul: | |||
| 1477 | return Builder.CreateMul(LHS, RHS); | |||
| 1478 | case Instruction::Sub: | |||
| 1479 | return Builder.CreateSub(LHS, RHS); | |||
| 1480 | case Instruction::FAdd: | |||
| 1481 | return Builder.CreateFAdd(LHS, RHS); | |||
| 1482 | case Instruction::FMul: | |||
| 1483 | return Builder.CreateFMul(LHS, RHS); | |||
| 1484 | case Instruction::FSub: | |||
| 1485 | return Builder.CreateFSub(LHS, RHS); | |||
| 1486 | default: | |||
| 1487 | llvm_unreachable("Unsupported binary operator for matrix")::llvm::llvm_unreachable_internal("Unsupported binary operator for matrix" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 1487); | |||
| 1488 | } | |||
| 1489 | }; | |||
| 1490 | ||||
| 1491 | for (unsigned I = 0; I < Shape.getNumVectors(); ++I) | |||
| 1492 | Result.addVector(BuildVectorOp(A.getVector(I), B.getVector(I))); | |||
| 1493 | ||||
| 1494 | finalizeLowering(Inst, | |||
| 1495 | Result.addNumComputeOps(getNumOps(Result.getVectorTy()) * | |||
| 1496 | Result.getNumVectors()), | |||
| 1497 | Builder); | |||
| 1498 | return true; | |||
| 1499 | } | |||
| 1500 | ||||
| 1501 | /// Helper to linearize a matrix expression tree into a string. Currently | |||
| 1502 | /// matrix expressions are linarized by starting at an expression leaf and | |||
| 1503 | /// linearizing bottom up. | |||
| 1504 | struct ExprLinearizer { | |||
| 1505 | unsigned LengthToBreak = 100; | |||
| 1506 | std::string Str; | |||
| 1507 | raw_string_ostream Stream; | |||
| 1508 | unsigned LineLength = 0; | |||
| 1509 | const DataLayout &DL; | |||
| 1510 | ||||
| 1511 | /// Mapping from instructions to matrixes. It is used to identify | |||
| 1512 | /// matrix instructions. | |||
| 1513 | const MapVector<Value *, MatrixTy> &Inst2Matrix; | |||
| 1514 | ||||
| 1515 | /// Mapping from values to the leaves of all expressions that the value is | |||
| 1516 | /// part of. | |||
| 1517 | const DenseMap<Value *, SmallPtrSet<Value *, 2>> &Shared; | |||
| 1518 | ||||
| 1519 | /// Set of matrix expressions in the scope of a given DISubprogram. | |||
| 1520 | const SmallSetVector<Value *, 32> &ExprsInSubprogram; | |||
| 1521 | ||||
| 1522 | /// Leaf node of the expression to linearize. | |||
| 1523 | Value *Leaf; | |||
| 1524 | ||||
| 1525 | /// Used to keep track of sub-expressions that get reused while linearizing | |||
| 1526 | /// the expression. Re-used sub-expressions are marked as (reused). | |||
| 1527 | SmallPtrSet<Value *, 8> ReusedExprs; | |||
| 1528 | ||||
| 1529 | ExprLinearizer(const DataLayout &DL, | |||
| 1530 | const MapVector<Value *, MatrixTy> &Inst2Matrix, | |||
| 1531 | const DenseMap<Value *, SmallPtrSet<Value *, 2>> &Shared, | |||
| 1532 | const SmallSetVector<Value *, 32> &ExprsInSubprogram, | |||
| 1533 | Value *Leaf) | |||
| 1534 | : Str(), Stream(Str), DL(DL), Inst2Matrix(Inst2Matrix), Shared(Shared), | |||
| 1535 | ExprsInSubprogram(ExprsInSubprogram), Leaf(Leaf) {} | |||
| 1536 | ||||
| 1537 | void indent(unsigned N) { | |||
| 1538 | LineLength += N; | |||
| 1539 | for (unsigned i = 0; i < N; i++) | |||
| 1540 | Stream << " "; | |||
| 1541 | } | |||
| 1542 | ||||
| 1543 | void lineBreak() { | |||
| 1544 | Stream << "\n"; | |||
| 1545 | LineLength = 0; | |||
| 1546 | } | |||
| 1547 | ||||
| 1548 | void maybeIndent(unsigned Indent) { | |||
| 1549 | if (LineLength >= LengthToBreak) | |||
| 1550 | lineBreak(); | |||
| 1551 | ||||
| 1552 | if (LineLength == 0) | |||
| 1553 | indent(Indent); | |||
| 1554 | } | |||
| 1555 | ||||
| 1556 | void write(StringRef S) { | |||
| 1557 | LineLength += S.size(); | |||
| 1558 | Stream << S; | |||
| 1559 | } | |||
| 1560 | ||||
| 1561 | Value *getUnderlyingObjectThroughLoads(Value *V) { | |||
| 1562 | if (Value *Ptr = getPointerOperand(V)) | |||
| 1563 | return getUnderlyingObjectThroughLoads(Ptr); | |||
| 1564 | else if (V->getType()->isPointerTy()) | |||
| 1565 | return getUnderlyingObject(V); | |||
| 1566 | return V; | |||
| 1567 | } | |||
| 1568 | ||||
| 1569 | /// Returns true if \p V is a matrix value in the given subprogram. | |||
| 1570 | bool isMatrix(Value *V) const { return ExprsInSubprogram.count(V); } | |||
| 1571 | ||||
| 1572 | /// If \p V is a matrix value, print its shape as as NumRows x NumColumns to | |||
| 1573 | /// \p SS. | |||
| 1574 | void prettyPrintMatrixType(Value *V, raw_string_ostream &SS) { | |||
| 1575 | auto M = Inst2Matrix.find(V); | |||
| 1576 | if (M == Inst2Matrix.end()) | |||
| 1577 | SS << "unknown"; | |||
| 1578 | else { | |||
| 1579 | SS << M->second.getNumRows(); | |||
| 1580 | SS << "x"; | |||
| 1581 | SS << M->second.getNumColumns(); | |||
| 1582 | } | |||
| 1583 | } | |||
| 1584 | ||||
| 1585 | /// Write the called function name. Handles calls to llvm.matrix.* | |||
| 1586 | /// specially: we write the name, followed by the dimensions of the input | |||
| 1587 | /// matrixes, followed by the scalar type name. | |||
| 1588 | void writeFnName(CallInst *CI) { | |||
| 1589 | if (!CI->getCalledFunction()) | |||
| 1590 | write("<no called fn>"); | |||
| 1591 | else { | |||
| 1592 | StringRef Name = CI->getCalledFunction()->getName(); | |||
| 1593 | if (!Name.startswith("llvm.matrix")) { | |||
| 1594 | write(Name); | |||
| 1595 | return; | |||
| 1596 | } | |||
| 1597 | IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI); | |||
| 1598 | write(StringRef(Intrinsic::getName(II->getIntrinsicID(), {})) | |||
| 1599 | .drop_front(StringRef("llvm.matrix.").size())); | |||
| 1600 | write("."); | |||
| 1601 | std::string Tmp = ""; | |||
| 1602 | raw_string_ostream SS(Tmp); | |||
| 1603 | ||||
| 1604 | switch (II->getIntrinsicID()) { | |||
| 1605 | case Intrinsic::matrix_multiply: | |||
| 1606 | prettyPrintMatrixType(II->getOperand(0), SS); | |||
| 1607 | SS << "."; | |||
| 1608 | prettyPrintMatrixType(II->getOperand(1), SS); | |||
| 1609 | SS << "." << *II->getType()->getScalarType(); | |||
| 1610 | break; | |||
| 1611 | case Intrinsic::matrix_transpose: | |||
| 1612 | prettyPrintMatrixType(II->getOperand(0), SS); | |||
| 1613 | SS << "." << *II->getType()->getScalarType(); | |||
| 1614 | break; | |||
| 1615 | case Intrinsic::matrix_column_major_load: | |||
| 1616 | prettyPrintMatrixType(II, SS); | |||
| 1617 | SS << "." << *II->getType()->getScalarType(); | |||
| 1618 | break; | |||
| 1619 | case Intrinsic::matrix_column_major_store: | |||
| 1620 | prettyPrintMatrixType(II->getOperand(0), SS); | |||
| 1621 | SS << "." << *II->getOperand(0)->getType()->getScalarType(); | |||
| 1622 | break; | |||
| 1623 | default: | |||
| 1624 | llvm_unreachable("Unhandled case")::llvm::llvm_unreachable_internal("Unhandled case", "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 1624); | |||
| 1625 | } | |||
| 1626 | SS.flush(); | |||
| 1627 | write(Tmp); | |||
| 1628 | } | |||
| 1629 | } | |||
| 1630 | ||||
| 1631 | unsigned getNumShapeArgs(CallInst *CI) const { | |||
| 1632 | if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI)) { | |||
| 1633 | switch (II->getIntrinsicID()) { | |||
| 1634 | case Intrinsic::matrix_multiply: | |||
| 1635 | return 3; | |||
| 1636 | case Intrinsic::matrix_transpose: | |||
| 1637 | return 2; | |||
| 1638 | case Intrinsic::matrix_column_major_load: | |||
| 1639 | case Intrinsic::matrix_column_major_store: | |||
| 1640 | return 3; | |||
| 1641 | default: | |||
| 1642 | return 0; | |||
| 1643 | } | |||
| 1644 | } | |||
| 1645 | return 0; | |||
| 1646 | } | |||
| 1647 | ||||
| 1648 | /// Special printing for values: for pointers, we print if they refer to an | |||
| 1649 | /// (function) external address or a stack address, for other values we | |||
| 1650 | /// either print the constant or "scalar"/"matrix" for other values. | |||
| 1651 | void write(Value *V) { | |||
| 1652 | V = getUnderlyingObjectThroughLoads(V); | |||
| 1653 | if (V->getType()->isPointerTy()) { | |||
| 1654 | if (isa<AllocaInst>(V)) { | |||
| 1655 | Stream << "stack addr"; | |||
| 1656 | LineLength += StringRef("stack addr").size(); | |||
| 1657 | } else { | |||
| 1658 | Stream << "addr"; | |||
| 1659 | LineLength += StringRef("addr").size(); | |||
| 1660 | } | |||
| 1661 | if (!V->getName().empty()) { | |||
| 1662 | Stream << " %" << V->getName() << ""; | |||
| 1663 | LineLength += V->getName().size() + 2; | |||
| 1664 | } | |||
| 1665 | return; | |||
| 1666 | } | |||
| 1667 | ||||
| 1668 | std::string Tmp; | |||
| 1669 | raw_string_ostream TmpStream(Tmp); | |||
| 1670 | ||||
| 1671 | if (auto *CI = dyn_cast<ConstantInt>(V)) | |||
| 1672 | TmpStream << CI->getValue(); | |||
| 1673 | else if (isa<Constant>(V)) | |||
| 1674 | TmpStream << "constant"; | |||
| 1675 | else { | |||
| 1676 | if (isMatrix(V)) | |||
| 1677 | TmpStream << "matrix"; | |||
| 1678 | else | |||
| 1679 | TmpStream << "scalar"; | |||
| 1680 | } | |||
| 1681 | TmpStream.flush(); | |||
| 1682 | Tmp = std::string(StringRef(Tmp).trim()); | |||
| 1683 | LineLength += Tmp.size(); | |||
| 1684 | Stream << Tmp; | |||
| 1685 | } | |||
| 1686 | ||||
| 1687 | /// Linearize expression \p Expr starting at an indentation of \p Indent. | |||
| 1688 | /// Expressions that are re-used multiple times are prefixed with (reused) | |||
| 1689 | /// at the re-used root instruction. | |||
| 1690 | void linearizeExpr(Value *Expr, unsigned Indent, bool ParentReused, | |||
| 1691 | bool ParentShared) { | |||
| 1692 | auto *I = cast<Instruction>(Expr); | |||
| 1693 | maybeIndent(Indent); | |||
| 1694 | SmallVector<Value *, 8> Ops; | |||
| 1695 | ||||
| 1696 | // Is Expr shared with other expression leaves? | |||
| 1697 | bool ExprShared = false; | |||
| 1698 | ||||
| 1699 | // Deal with shared subtrees. Mark them as shared, if required. | |||
| 1700 | if (!ParentShared) { | |||
| 1701 | auto SI = Shared.find(Expr); | |||
| 1702 | assert(SI != Shared.end() && SI->second.count(Leaf))((SI != Shared.end() && SI->second.count(Leaf)) ? static_cast <void> (0) : __assert_fail ("SI != Shared.end() && SI->second.count(Leaf)" , "/build/llvm-toolchain-snapshot-12~++20200926111128+c6c5629f2fb/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp" , 1702, __PRETTY_FUNCTION__)); | |||
| 1703 | ||||
| 1704 | for (Value *S : SI->second) { | |||
| 1705 | if (S == Leaf) | |||
| 1706 | continue; | |||
| 1707 | DebugLoc DL = cast<Instruction>(S)->getDebugLoc(); | |||
| 1708 | write("shared with remark at line " + std::to_string(DL.getLine()) + | |||
| 1709 | " column " + std::to_string(DL.getCol()) + " ("); | |||
| 1710 | } | |||
| 1711 | ExprShared = SI->second.size() > 1; | |||
| 1712 | } | |||
| 1713 | ||||
| 1714 | bool Reused = !ReusedExprs.insert(Expr).second; | |||
| 1715 | if (Reused && !ParentReused) | |||
| 1716 | write("(reused) "); | |||
| 1717 | ||||
| 1718 | if (auto *CI = dyn_cast<CallInst>(I)) { | |||
| 1719 | writeFnName(CI); | |||
| 1720 | ||||
| 1721 | Ops.append(CI->arg_begin(), CI->arg_end() - getNumShapeArgs(CI)); | |||
| 1722 | } else if (isa<BitCastInst>(Expr)) { | |||
| 1723 | // Special case bitcasts, which are used to materialize matrixes from | |||
| 1724 | // non-matrix ops. | |||
| 1725 | write("matrix"); | |||
| 1726 | return; | |||
| 1727 | } else { | |||
| 1728 | Ops.append(I->value_op_begin(), I->value_op_end()); | |||
| 1729 | write(std::string(I->getOpcodeName())); | |||
| 1730 | } | |||
| 1731 | ||||
| 1732 | write(std::string("(")); | |||
| 1733 | ||||
| 1734 | unsigned NumOpsToBreak = 1; | |||
| 1735 | if (match(Expr, m_Intrinsic<Intrinsic::matrix_column_major_load>())) | |||
| 1736 | NumOpsToBreak = 2; | |||
| 1737 | ||||
| 1738 | for (Value *Op : Ops) { | |||
| 1739 | if (Ops.size() > NumOpsToBreak) | |||
| 1740 | lineBreak(); | |||
| 1741 | ||||
| 1742 | maybeIndent(Indent + 1); | |||
| 1743 | if (isMatrix(Op)) | |||
| 1744 | linearizeExpr(Op, Indent + 1, Reused, ExprShared); | |||
| 1745 | else | |||
| 1746 | write(Op); | |||
| 1747 | if (Op != Ops.back()) | |||
| 1748 | write(", "); | |||
| 1749 | } | |||
| 1750 | ||||
| 1751 | write(")"); | |||
| 1752 | } | |||
| 1753 | ||||
| 1754 | const std::string &getResult() { | |||
| 1755 | Stream.flush(); | |||
| 1756 | return Str; | |||
| 1757 | } | |||
| 1758 | }; | |||
| 1759 | ||||
| 1760 | /// Generate remarks for matrix operations in a function. To generate remarks | |||
| 1761 | /// for matrix expressions, the following approach is used: | |||
| 1762 | /// 1. Use the inlined-at debug information to group matrix operations to the | |||
| 1763 | /// DISubprograms they are contained in. | |||
| 1764 | /// 2. Collect leaves of matrix expressions (done in | |||
| 1765 | /// RemarkGenerator::getExpressionLeaves) for each subprogram - expression | |||
| 1766 | // mapping. Leaves are lowered matrix instructions without other matrix | |||
| 1767 | // users (like stores) in the current subprogram. | |||
| 1768 | /// 3. For each leaf, create a remark containing a linearizied version of the | |||
| 1769 | /// matrix expression. The expression is linearized by a recursive | |||
| 1770 | /// bottom-up traversal of the matrix operands, starting at a leaf. Note | |||
| 1771 | /// that multiple leaves can share sub-expressions. Shared subexpressions | |||
| 1772 | /// are explicitly marked as shared(). | |||
| 1773 | struct RemarkGenerator { | |||
| 1774 | const MapVector<Value *, MatrixTy> &Inst2Matrix; | |||
| 1775 | OptimizationRemarkEmitter &ORE; | |||
| 1776 | Function &Func; | |||
| 1777 | const DataLayout &DL; | |||
| 1778 | ||||
| 1779 | RemarkGenerator(const MapVector<Value *, MatrixTy> &Inst2Matrix, | |||
| 1780 | OptimizationRemarkEmitter &ORE, Function &Func) | |||
| 1781 | : Inst2Matrix(Inst2Matrix), ORE(ORE), Func(Func), | |||
| 1782 | DL(Func.getParent()->getDataLayout()) {} | |||
| 1783 | ||||
| 1784 | /// Return all leaves of the expressions in \p ExprsInSubprogram. Those are | |||
| 1785 | /// instructions in Inst2Matrix returning void or without any users in | |||
| 1786 | /// \p ExprsInSubprogram. Currently that should only include stores. | |||
| 1787 | SmallVector<Value *, 4> | |||
| 1788 | getExpressionLeaves(const SmallSetVector<Value *, 32> &ExprsInSubprogram) { | |||
| 1789 | SmallVector<Value *, 4> Leaves; | |||
| 1790 | for (auto *Expr : ExprsInSubprogram) | |||
| 1791 | if (Expr->getType()->isVoidTy() || | |||
| 1792 | !any_of(Expr->users(), [&ExprsInSubprogram](User *U) { | |||
| 1793 | return ExprsInSubprogram.count(U); | |||
| 1794 | })) | |||
| 1795 | Leaves.push_back(Expr); | |||
| 1796 | return Leaves; | |||
| 1797 | } | |||
| 1798 | ||||
| 1799 | /// Recursively traverse expression \p V starting at \p Leaf and add \p Leaf | |||
| 1800 | /// to all visited expressions in \p Shared. Limit the matrix operations to | |||
| 1801 | /// the ones in \p ExprsInSubprogram. | |||
| 1802 | void collectSharedInfo(Value *Leaf, Value *V, | |||
| 1803 | const SmallSetVector<Value *, 32> &ExprsInSubprogram, | |||
| 1804 | DenseMap<Value *, SmallPtrSet<Value *, 2>> &Shared) { | |||
| 1805 | ||||
| 1806 | if (!ExprsInSubprogram.count(V)) | |||
| 1807 | return; | |||
| 1808 | ||||
| 1809 | auto I = Shared.insert({V, {}}); | |||
| 1810 | I.first->second.insert(Leaf); | |||
| 1811 | ||||
| 1812 | for (Value *Op : cast<Instruction>(V)->operand_values()) | |||
| 1813 | collectSharedInfo(Leaf, Op, ExprsInSubprogram, Shared); | |||
| 1814 | return; | |||
| 1815 | } | |||
| 1816 | ||||
| 1817 | /// Calculate the number of exclusive and shared op counts for expression | |||
| 1818 | /// starting at \p V. Expressions used multiple times are counted once. | |||
| 1819 | /// Limit the matrix operations to the ones in \p ExprsInSubprogram. | |||
| 1820 | std::pair<OpInfoTy, OpInfoTy> | |||
| 1821 | sumOpInfos(Value *Root, SmallPtrSetImpl<Value *> &ReusedExprs, | |||
| 1822 | const SmallSetVector<Value *, 32> &ExprsInSubprogram, | |||
| 1823 | DenseMap<Value *, SmallPtrSet<Value *, 2>> &Shared) const { | |||
| 1824 | if (!ExprsInSubprogram.count(Root)) | |||
| 1825 | return {}; | |||
| 1826 | ||||
| 1827 | // Already counted this expression. Stop. | |||
| 1828 | if (!ReusedExprs.insert(Root).second) | |||
| 1829 | return {}; | |||
| 1830 | ||||
| 1831 | OpInfoTy SharedCount; | |||
| 1832 | OpInfoTy Count; | |||
| 1833 | ||||
| 1834 | auto I = Shared.find(Root); | |||
| 1835 | auto CM = Inst2Matrix.find(Root); | |||
| 1836 | if (I->second.size() == 1) | |||
| 1837 | Count = CM->second.getOpInfo(); | |||
| 1838 | else | |||
| 1839 | SharedCount = CM->second.getOpInfo(); | |||
| 1840 | ||||
| 1841 | for (Value *Op : cast<Instruction>(Root)->operand_values()) { | |||
| 1842 | auto C = sumOpInfos(Op, ReusedExprs, ExprsInSubprogram, Shared); | |||
| 1843 | Count += C.first; | |||
| 1844 | SharedCount += C.second; | |||
| 1845 | } | |||
| 1846 | return {Count, SharedCount}; | |||
| 1847 | } | |||
| 1848 | ||||
| 1849 | void emitRemarks() { | |||
| 1850 | if (!ORE.allowExtraAnalysis(DEBUG_TYPE"lower-matrix-intrinsics")) | |||
| 1851 | return; | |||
| 1852 | ||||
| 1853 | // Map matrix operations to their containting subprograms, by traversing | |||
| 1854 | // the inlinedAt chain. If the function does not have a DISubprogram, we | |||
| 1855 | // only map them to the containing function. | |||
| 1856 | MapVector<DISubprogram *, SmallVector<Value *, 8>> Subprog2Exprs; | |||
| 1857 | for (auto &KV : Inst2Matrix) { | |||
| 1858 | if (Func.getSubprogram()) { | |||
| 1859 | auto *I = cast<Instruction>(KV.first); | |||
| 1860 | DILocation *Context = I->getDebugLoc(); | |||
| 1861 | while (Context) { | |||
| 1862 | auto I = | |||
| 1863 | Subprog2Exprs.insert({getSubprogram(Context->getScope()), {}}); | |||
| 1864 | I.first->second.push_back(KV.first); | |||
| 1865 | Context = DebugLoc(Context).getInlinedAt(); | |||
| 1866 | } | |||
| 1867 | } else { | |||
| 1868 | auto I = Subprog2Exprs.insert({nullptr, {}}); | |||
| 1869 | I.first->second.push_back(KV.first); | |||
| 1870 | } | |||
| 1871 | } | |||
| 1872 | for (auto &KV : Subprog2Exprs) { | |||
| 1873 | SmallSetVector<Value *, 32> ExprsInSubprogram(KV.second.begin(), | |||
| 1874 | KV.second.end()); | |||
| 1875 | auto Leaves = getExpressionLeaves(ExprsInSubprogram); | |||
| 1876 | ||||
| 1877 | DenseMap<Value *, SmallPtrSet<Value *, 2>> Shared; | |||
| 1878 | for (Value *Leaf : Leaves) | |||
| 1879 | collectSharedInfo(Leaf, Leaf, ExprsInSubprogram, Shared); | |||
| 1880 | ||||
| 1881 | // Generate remarks for each leaf. | |||
| 1882 | for (auto *L : Leaves) { | |||
| 1883 | ||||
| 1884 | DebugLoc Loc = cast<Instruction>(L)->getDebugLoc(); | |||
| 1885 | DILocation *Context = cast<Instruction>(L)->getDebugLoc(); | |||
| 1886 | while (Context) { | |||
| 1887 | if (getSubprogram(Context->getScope()) == KV.first) { | |||
| 1888 | Loc = Context; | |||
| 1889 | break; | |||
| 1890 | } | |||
| 1891 | Context = DebugLoc(Context).getInlinedAt(); | |||
| 1892 | } | |||
| 1893 | ||||
| 1894 | SmallPtrSet<Value *, 8> ReusedExprs; | |||
| 1895 | OpInfoTy Counts, SharedCounts; | |||
| 1896 | std::tie(Counts, SharedCounts) = | |||
| 1897 | sumOpInfos(L, ReusedExprs, ExprsInSubprogram, Shared); | |||
| 1898 | ||||
| 1899 | OptimizationRemark Rem(DEBUG_TYPE"lower-matrix-intrinsics", "matrix-lowered", Loc, | |||
| 1900 | cast<Instruction>(L)->getParent()); | |||
| 1901 | ||||
| 1902 | Rem << "Lowered with "; | |||
| 1903 | Rem << ore::NV("NumStores", Counts.NumStores) << " stores, " | |||
| 1904 | << ore::NV("NumLoads", Counts.NumLoads) << " loads, " | |||
| 1905 | << ore::NV("NumComputeOps", Counts.NumComputeOps) | |||
| 1906 | << " compute ops"; | |||
| 1907 | ||||
| 1908 | if (SharedCounts.NumStores > 0 || SharedCounts.NumLoads > 0 || | |||
| 1909 | SharedCounts.NumComputeOps > 0) { | |||
| 1910 | Rem << ",\nadditionally " | |||
| 1911 | << ore::NV("NumStores", SharedCounts.NumStores) << " stores, " | |||
| 1912 | << ore::NV("NumLoads", SharedCounts.NumLoads) << " loads, " | |||
| 1913 | << ore::NV("NumFPOps", SharedCounts.NumComputeOps) | |||
| 1914 | << " compute ops" | |||
| 1915 | << " are shared with other expressions"; | |||
| 1916 | } | |||
| 1917 | ||||
| 1918 | Rem << ("\n" + linearize(L, Shared, ExprsInSubprogram, DL)); | |||
| 1919 | ORE.emit(Rem); | |||
| 1920 | } | |||
| 1921 | } | |||
| 1922 | } | |||
| 1923 | ||||
| 1924 | std::string | |||
| 1925 | linearize(Value *L, | |||
| 1926 | const DenseMap<Value *, SmallPtrSet<Value *, 2>> &Shared, | |||
| 1927 | const SmallSetVector<Value *, 32> &ExprsInSubprogram, | |||
| 1928 | const DataLayout &DL) { | |||
| 1929 | ExprLinearizer Lin(DL, Inst2Matrix, Shared, ExprsInSubprogram, L); | |||
| 1930 | Lin.linearizeExpr(L, 0, false, false); | |||
| 1931 | return Lin.getResult(); | |||
| 1932 | } | |||
| 1933 | }; | |||
| 1934 | }; | |||
| 1935 | } // namespace | |||
| 1936 | ||||
| 1937 | PreservedAnalyses LowerMatrixIntrinsicsPass::run(Function &F, | |||
| 1938 | FunctionAnalysisManager &AM) { | |||
| 1939 | auto &TTI = AM.getResult<TargetIRAnalysis>(F); | |||
| 1940 | auto &ORE = AM.getResult<OptimizationRemarkEmitterAnalysis>(F); | |||
| 1941 | auto &AA = AM.getResult<AAManager>(F); | |||
| 1942 | auto &DT = AM.getResult<DominatorTreeAnalysis>(F); | |||
| 1943 | auto &LI = AM.getResult<LoopAnalysis>(F); | |||
| 1944 | ||||
| 1945 | LowerMatrixIntrinsics LMT(F, TTI, &AA, &DT, &LI, &ORE); | |||
| 1946 | if (LMT.Visit()) { | |||
| 1947 | PreservedAnalyses PA; | |||
| 1948 | PA.preserveSet<CFGAnalyses>(); | |||
| 1949 | return PA; | |||
| 1950 | } | |||
| 1951 | return PreservedAnalyses::all(); | |||
| 1952 | } | |||
| 1953 | ||||
| 1954 | namespace { | |||
| 1955 | ||||
| 1956 | class LowerMatrixIntrinsicsLegacyPass : public FunctionPass { | |||
| 1957 | public: | |||
| 1958 | static char ID; | |||
| 1959 | ||||
| 1960 | LowerMatrixIntrinsicsLegacyPass() : FunctionPass(ID) { | |||
| 1961 | initializeLowerMatrixIntrinsicsLegacyPassPass( | |||
| 1962 | *PassRegistry::getPassRegistry()); | |||
| 1963 | } | |||
| 1964 | ||||
| 1965 | bool runOnFunction(Function &F) override { | |||
| 1966 | auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); | |||
| 1967 | auto &ORE = getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE(); | |||
| 1968 | auto &AA = getAnalysis<AAResultsWrapperPass>().getAAResults(); | |||
| 1969 | auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree(); | |||
| 1970 | auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo(); | |||
| 1971 | LowerMatrixIntrinsics LMT(F, TTI, &AA, &DT, &LI, &ORE); | |||
| 1972 | bool C = LMT.Visit(); | |||
| 1973 | return C; | |||
| 1974 | } | |||
| 1975 | ||||
| 1976 | void getAnalysisUsage(AnalysisUsage &AU) const override { | |||
| 1977 | AU.addRequired<TargetTransformInfoWrapperPass>(); | |||
| 1978 | AU.addRequired<OptimizationRemarkEmitterWrapperPass>(); | |||
| 1979 | AU.addRequired<AAResultsWrapperPass>(); | |||
| 1980 | AU.addRequired<DominatorTreeWrapperPass>(); | |||
| 1981 | AU.addPreserved<DominatorTreeWrapperPass>(); | |||
| 1982 | AU.addRequired<LoopInfoWrapperPass>(); | |||
| 1983 | AU.addPreserved<LoopInfoWrapperPass>(); | |||
| 1984 | } | |||
| 1985 | }; | |||
| 1986 | } // namespace | |||
| 1987 | ||||
| 1988 | static const char pass_name[] = "Lower the matrix intrinsics"; | |||
| 1989 | char LowerMatrixIntrinsicsLegacyPass::ID = 0; | |||
| 1990 | INITIALIZE_PASS_BEGIN(LowerMatrixIntrinsicsLegacyPass, DEBUG_TYPE, pass_name,static void *initializeLowerMatrixIntrinsicsLegacyPassPassOnce (PassRegistry &Registry) { | |||
| 1991 | false, false)static void *initializeLowerMatrixIntrinsicsLegacyPassPassOnce (PassRegistry &Registry) { | |||
| 1992 | INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)initializeOptimizationRemarkEmitterWrapperPassPass(Registry); | |||
| 1993 | INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)initializeAAResultsWrapperPassPass(Registry); | |||
| 1994 | INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)initializeDominatorTreeWrapperPassPass(Registry); | |||
| 1995 | INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)initializeLoopInfoWrapperPassPass(Registry); | |||
| 1996 | INITIALIZE_PASS_END(LowerMatrixIntrinsicsLegacyPass, DEBUG_TYPE, pass_name,PassInfo *PI = new PassInfo( pass_name, "lower-matrix-intrinsics" , &LowerMatrixIntrinsicsLegacyPass::ID, PassInfo::NormalCtor_t (callDefaultCtor<LowerMatrixIntrinsicsLegacyPass>), false , false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeLowerMatrixIntrinsicsLegacyPassPassFlag ; void llvm::initializeLowerMatrixIntrinsicsLegacyPassPass(PassRegistry &Registry) { llvm::call_once(InitializeLowerMatrixIntrinsicsLegacyPassPassFlag , initializeLowerMatrixIntrinsicsLegacyPassPassOnce, std::ref (Registry)); } | |||
| 1997 | false, false)PassInfo *PI = new PassInfo( pass_name, "lower-matrix-intrinsics" , &LowerMatrixIntrinsicsLegacyPass::ID, PassInfo::NormalCtor_t (callDefaultCtor<LowerMatrixIntrinsicsLegacyPass>), false , false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeLowerMatrixIntrinsicsLegacyPassPassFlag ; void llvm::initializeLowerMatrixIntrinsicsLegacyPassPass(PassRegistry &Registry) { llvm::call_once(InitializeLowerMatrixIntrinsicsLegacyPassPassFlag , initializeLowerMatrixIntrinsicsLegacyPassPassOnce, std::ref (Registry)); } | |||
| 1998 | ||||
| 1999 | Pass *llvm::createLowerMatrixIntrinsicsPass() { | |||
| 2000 | return new LowerMatrixIntrinsicsLegacyPass(); | |||
| 2001 | } | |||
| 2002 | ||||
| 2003 | namespace { | |||
| 2004 | ||||
| 2005 | /// A lightweight version of the matrix lowering pass that only requires TTI. | |||
| 2006 | /// Advanced features that require DT, AA or ORE like tiling are disabled. This | |||
| 2007 | /// is used to lower matrix intrinsics if the main lowering pass is not run, for | |||
| 2008 | /// example with -O0. | |||
| 2009 | class LowerMatrixIntrinsicsMinimalLegacyPass : public FunctionPass { | |||
| 2010 | public: | |||
| 2011 | static char ID; | |||
| 2012 | ||||
| 2013 | LowerMatrixIntrinsicsMinimalLegacyPass() : FunctionPass(ID) { | |||
| 2014 | initializeLowerMatrixIntrinsicsMinimalLegacyPassPass( | |||
| 2015 | *PassRegistry::getPassRegistry()); | |||
| 2016 | } | |||
| 2017 | ||||
| 2018 | bool runOnFunction(Function &F) override { | |||
| 2019 | auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F); | |||
| 2020 | LowerMatrixIntrinsics LMT(F, TTI, nullptr, nullptr, nullptr, nullptr); | |||
| 2021 | bool C = LMT.Visit(); | |||
| ||||
| 2022 | return C; | |||
| 2023 | } | |||
| 2024 | ||||
| 2025 | void getAnalysisUsage(AnalysisUsage &AU) const override { | |||
| 2026 | AU.addRequired<TargetTransformInfoWrapperPass>(); | |||
| 2027 | AU.setPreservesCFG(); | |||
| 2028 | } | |||
| 2029 | }; | |||
| 2030 | } // namespace | |||
| 2031 | ||||
| 2032 | static const char pass_name_minimal[] = "Lower the matrix intrinsics (minimal)"; | |||
| 2033 | char LowerMatrixIntrinsicsMinimalLegacyPass::ID = 0; | |||
| 2034 | INITIALIZE_PASS_BEGIN(LowerMatrixIntrinsicsMinimalLegacyPass,static void *initializeLowerMatrixIntrinsicsMinimalLegacyPassPassOnce (PassRegistry &Registry) { | |||
| 2035 | "lower-matrix-intrinsics-minimal", pass_name_minimal,static void *initializeLowerMatrixIntrinsicsMinimalLegacyPassPassOnce (PassRegistry &Registry) { | |||
| 2036 | false, false)static void *initializeLowerMatrixIntrinsicsMinimalLegacyPassPassOnce (PassRegistry &Registry) { | |||
| 2037 | INITIALIZE_PASS_END(LowerMatrixIntrinsicsMinimalLegacyPass,PassInfo *PI = new PassInfo( pass_name_minimal, "lower-matrix-intrinsics-minimal" , &LowerMatrixIntrinsicsMinimalLegacyPass::ID, PassInfo:: NormalCtor_t(callDefaultCtor<LowerMatrixIntrinsicsMinimalLegacyPass >), false, false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeLowerMatrixIntrinsicsMinimalLegacyPassPassFlag ; void llvm::initializeLowerMatrixIntrinsicsMinimalLegacyPassPass (PassRegistry &Registry) { llvm::call_once(InitializeLowerMatrixIntrinsicsMinimalLegacyPassPassFlag , initializeLowerMatrixIntrinsicsMinimalLegacyPassPassOnce, std ::ref(Registry)); } | |||
| 2038 | "lower-matrix-intrinsics-minimal", pass_name_minimal, false,PassInfo *PI = new PassInfo( pass_name_minimal, "lower-matrix-intrinsics-minimal" , &LowerMatrixIntrinsicsMinimalLegacyPass::ID, PassInfo:: NormalCtor_t(callDefaultCtor<LowerMatrixIntrinsicsMinimalLegacyPass >), false, false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeLowerMatrixIntrinsicsMinimalLegacyPassPassFlag ; void llvm::initializeLowerMatrixIntrinsicsMinimalLegacyPassPass (PassRegistry &Registry) { llvm::call_once(InitializeLowerMatrixIntrinsicsMinimalLegacyPassPassFlag , initializeLowerMatrixIntrinsicsMinimalLegacyPassPassOnce, std ::ref(Registry)); } | |||
| 2039 | false)PassInfo *PI = new PassInfo( pass_name_minimal, "lower-matrix-intrinsics-minimal" , &LowerMatrixIntrinsicsMinimalLegacyPass::ID, PassInfo:: NormalCtor_t(callDefaultCtor<LowerMatrixIntrinsicsMinimalLegacyPass >), false, false); Registry.registerPass(*PI, true); return PI; } static llvm::once_flag InitializeLowerMatrixIntrinsicsMinimalLegacyPassPassFlag ; void llvm::initializeLowerMatrixIntrinsicsMinimalLegacyPassPass (PassRegistry &Registry) { llvm::call_once(InitializeLowerMatrixIntrinsicsMinimalLegacyPassPassFlag , initializeLowerMatrixIntrinsicsMinimalLegacyPassPassOnce, std ::ref(Registry)); } | |||
| 2040 | ||||
| 2041 | Pass *llvm::createLowerMatrixIntrinsicsMinimalPass() { | |||
| 2042 | return new LowerMatrixIntrinsicsMinimalLegacyPass(); | |||
| 2043 | } |