59#define DEBUG_TYPE "lower-matrix-intrinsics"
61STATISTIC(FlattenedMatrices,
"Number of matrix flattenings");
62STATISTIC(ReshapedMatrices,
"Number of matrix reshapes");
77 return SV->isZeroEltSplat();
82template <
typename LTy,
typename RTy>
83static auto m_AnyMul(
const LTy &L,
const RTy &R) {
88template <
typename LTy,
typename RTy>
89static auto m_AnyAdd(
const LTy &L,
const RTy &R) {
134 unsigned NumElements,
Type *EltType,
139 "Stride must be >= the number of elements in the result vector.");
142 Value *VecStart = Builder.CreateMul(VecIdx, Stride,
"vec.start");
149 VecStart = Builder.CreateInBoundsGEP(EltType, BasePtr, VecStart,
"vec.gep");
159 ShapeInfo(
unsigned NumRows = 0,
unsigned NumColumns = 0)
160 : NumRows(NumRows), NumColumns(NumColumns) {}
163 : ShapeInfo(
cast<ConstantInt>(NumRows)->getZExtValue(),
164 cast<ConstantInt>(NumColumns)->getZExtValue()) {}
167 return NumRows == other.NumRows && NumColumns == other.NumColumns;
169 bool operator!=(
const ShapeInfo &other) {
return !(*
this == other); }
173 operator bool()
const {
174 assert(NumRows == 0 || NumColumns != 0);
179 ShapeInfo t()
const {
return ShapeInfo(NumColumns, NumRows); }
181 friend raw_ostream &
operator<<(raw_ostream &OS, ShapeInfo SI);
187 return OS <<
SI.NumRows <<
'x' <<
SI.NumColumns;
204 switch (Cast->getOpcode()) {
205 case llvm::Instruction::Trunc:
206 case llvm::Instruction::ZExt:
207 case llvm::Instruction::SExt:
208 case llvm::Instruction::FPToUI:
209 case llvm::Instruction::FPToSI:
210 case llvm::Instruction::UIToFP:
211 case llvm::Instruction::SIToFP:
212 case llvm::Instruction::FPTrunc:
213 case llvm::Instruction::FPExt:
215 case llvm::Instruction::AddrSpaceCast:
216 case CastInst::PtrToAddr:
217 case CastInst::PtrToInt:
218 case CastInst::IntToPtr:
220 case CastInst::BitCast: {
223 return SrcVTy->getNumElements() == DestVTy->getNumElements();
226 case llvm::Instruction::CastOpsEnd:
233 switch (
II->getIntrinsicID()) {
235 case Intrinsic::fabs:
241 switch (
I->getOpcode()) {
242 case Instruction::PHI:
243 case Instruction::FNeg:
254 "Can't retrieve shaped operands for an instruction that does not "
255 "preserve shape information");
256 auto Ops =
I->operands();
261static std::optional<ShapeInfo>
269 return ShapeInfo(M,
K);
273 return ShapeInfo(
N, M);
278 return ShapeInfo(
N, M);
281 return ShapeInfo(M,
N);
284 auto OpShape = ShapeMap.
find(MatrixA);
285 if (OpShape != ShapeMap.
end())
286 return OpShape->second;
292 for (
auto &
Op : ShapedOps) {
293 auto OpShape = ShapeMap.
find(
Op.get());
294 if (OpShape != ShapeMap.
end())
295 return OpShape->second;
326class LowerMatrixIntrinsics {
327 const ScalarOptions &Opts;
329 const DataLayout &DL;
330 const TargetTransformInfo &TTI;
333 DominatorTree *DT =
nullptr;
334 LoopInfo *LI =
nullptr;
335 OptimizationRemarkEmitter *ORE =
nullptr;
341 unsigned NumStores = 0;
343 unsigned NumLoads = 0;
345 unsigned NumComputeOps = 0;
349 unsigned NumExposedTransposes = 0;
352 NumStores +=
RHS.NumStores;
353 NumLoads +=
RHS.NumLoads;
354 NumComputeOps +=
RHS.NumComputeOps;
355 NumExposedTransposes +=
RHS.NumExposedTransposes;
363 SmallVector<Value *, 16> Vectors;
367 bool IsColumnMajor =
true;
370 MatrixTy() =
default;
371 explicit MatrixTy(
bool IsColumnMajor) : IsColumnMajor(IsColumnMajor) {}
373 : Vectors(Vectors), IsColumnMajor(IsColumnMajor) {}
374 MatrixTy(
bool IsColumnMajor,
unsigned NumRows,
unsigned NumColumns,
376 : IsColumnMajor(IsColumnMajor) {
378 unsigned D = isColumnMajor() ? NumColumns : NumRows;
379 for (
unsigned J = 0; J <
D; ++J)
381 EltTy, isColumnMajor() ? NumRows : NumColumns)));
384 Value *getVector(
unsigned i)
const {
return Vectors[i]; }
385 Value *getColumn(
unsigned i)
const {
386 assert(isColumnMajor() &&
"only supported for column-major matrixes");
389 Value *getRow(
unsigned i)
const {
390 assert(!isColumnMajor() &&
"only supported for row-major matrixes");
394 void setVector(
unsigned i,
Value *
V) { Vectors[i] =
V; }
396 Type *getElementType()
const {
return getVectorTy()->getElementType(); }
398 unsigned getNumVectors()
const {
400 return getNumColumns();
404 unsigned getNumColumns()
const {
406 return Vectors.size();
408 assert(Vectors.size() > 0 &&
"Cannot call getNumRows without columns");
409 return getVectorTy()->getNumElements();
412 unsigned getNumRows()
const {
413 if (isColumnMajor()) {
414 assert(Vectors.size() > 0 &&
"Cannot call getNumRows without columns");
415 return getVectorTy()->getNumElements();
417 return Vectors.size();
420 void addVector(
Value *
V) { Vectors.push_back(
V); }
421 FixedVectorType *getColumnTy() {
422 assert(isColumnMajor() &&
"only supported for column-major matrixes");
423 return getVectorTy();
426 FixedVectorType *getVectorTy()
const {
430 iterator_range<SmallVector<Value *, 8>::iterator> columns() {
432 "columns() only supported for column-major matrixes");
433 return make_range(Vectors.begin(), Vectors.end());
436 iterator_range<SmallVector<Value *, 8>::iterator>
vectors() {
437 return make_range(Vectors.begin(), Vectors.end());
443 return Vectors.size() == 1 ? Vectors[0]
447 MatrixTy &addNumLoads(
unsigned N) {
448 OpInfo.NumLoads +=
N;
452 void setNumLoads(
unsigned N) { OpInfo.NumLoads =
N; }
454 MatrixTy &addNumStores(
unsigned N) {
455 OpInfo.NumStores +=
N;
459 MatrixTy &addNumExposedTransposes(
unsigned N) {
460 OpInfo.NumExposedTransposes +=
N;
464 MatrixTy &addNumComputeOps(
unsigned N) {
465 OpInfo.NumComputeOps +=
N;
469 unsigned getNumStores()
const {
return OpInfo.NumStores; }
470 unsigned getNumLoads()
const {
return OpInfo.NumLoads; }
471 unsigned getNumComputeOps()
const {
return OpInfo.NumComputeOps; }
473 const OpInfoTy &getOpInfo()
const {
return OpInfo; }
475 bool isColumnMajor()
const {
return IsColumnMajor; }
477 unsigned getStride()
const {
480 return getNumColumns();
483 ShapeInfo shape()
const {
return {getNumRows(), getNumColumns()}; }
490 Value *Vec = isColumnMajor() ? getColumn(J) : getRow(
I);
493 "Extracted vector will contain poison values");
514 DenseMap<Value *, ShapeInfo> ShapeMap;
519 SmallVector<Instruction *, 16> ToRemove;
522 MapVector<Value *, MatrixTy> Inst2ColumnMatrix;
536 bool isColumnMajor()
const {
537 return Opts.matrix_default_layout == MatrixLayoutTy::ColumnMajor;
540 unsigned getStride(
const ShapeInfo &SI)
const {
541 return isColumnMajor() ?
SI.NumRows :
SI.NumColumns;
544 unsigned getNumVectors(
const ShapeInfo &SI)
const {
545 return isColumnMajor() ?
SI.NumColumns :
SI.NumRows;
549 LowerMatrixIntrinsics(
const ScalarOptions &Opts,
Function &
F,
551 : Opts(Opts), Func(
F), DL(
F.getDataLayout()), TTI(TTI), AM(AM) {}
553 unsigned getNumOps(
Type *VT) {
560 bool isMinimal()
const {
566 unsigned getNumOps(
Type *ST,
unsigned N) {
567 return std::ceil((
ST->getPrimitiveSizeInBits() *
N).getFixedValue() /
568 double(TTI.getRegisterBitWidth(
582 unsigned getNumNativeVectorOps(
Type *EltType,
unsigned R,
unsigned M,
584 unsigned NumFMAs =
C * getNumOps(EltType, R) *
M;
585 unsigned NumALoads = getNumOps(EltType, R) *
M;
586 unsigned NumBLoads = getNumOps(EltType, M) *
C;
587 unsigned NumStores = getNumOps(EltType, R) *
C;
588 return NumFMAs + NumALoads + NumBLoads + NumStores;
596 MatrixTy getMatrix(
Value *MatrixVal,
const ShapeInfo &SI,
600 "The vector size must match the number of matrix elements");
606 auto Found = Inst2ColumnMatrix.find(MatrixVal);
607 if (Found != Inst2ColumnMatrix.end()) {
608 MatrixTy &
M = Found->second;
611 if (
SI.NumRows ==
M.getNumRows() &&
SI.NumColumns ==
M.getNumColumns())
614 MatrixVal =
M.embedInVector(Builder);
618 SmallVector<Value *, 16> SplitVecs;
620 MaskStart += getStride(SI)) {
628 if (Found != Inst2ColumnMatrix.end()) {
631 LLVM_DEBUG(
dbgs() <<
"matrix reshape from " << Found->second.shape()
632 <<
" to " << SI <<
" using at least "
633 << SplitVecs.
size() <<
" shuffles on behalf of:\n"
636 }
else if (!ShapeMap.contains(MatrixVal)) {
639 <<
"splitting a " << SI <<
" matrix with " << SplitVecs.
size()
640 <<
" shuffles beacuse we do not have a shape-aware lowering for "
652 return {isColumnMajor(), SplitVecs};
657 bool setShapeInfo(
Value *
V, ShapeInfo Shape) {
658 assert(Shape &&
"Shape not set");
662 auto SIter = ShapeMap.find(
V);
663 if (SIter != ShapeMap.end()) {
664 if (Opts.verify_matrix_shapes &&
665 (SIter->second.NumRows != Shape.NumRows ||
666 SIter->second.NumColumns != Shape.NumColumns)) {
667 errs() <<
"Conflicting shapes (" << SIter->second.NumRows <<
"x"
668 << SIter->second.NumColumns <<
" vs " << Shape.NumRows <<
"x"
669 << Shape.NumColumns <<
") for " << *
V <<
"\n";
671 "Matrix shape verification failed, compilation aborted!");
675 << SIter->second.NumRows <<
" "
676 << SIter->second.NumColumns <<
" for " << *
V <<
"\n");
680 ShapeMap.insert({
V, Shape});
681 LLVM_DEBUG(
dbgs() <<
" " << Shape.NumRows <<
" x " << Shape.NumColumns
682 <<
" for " << *
V <<
"\n");
688 bool supportsShapeInfo(
Value *
V) {
695 switch (
II->getIntrinsicID()) {
696 case Intrinsic::matrix_multiply:
697 case Intrinsic::matrix_transpose:
698 case Intrinsic::matrix_column_major_load:
699 case Intrinsic::matrix_column_major_store:
712 propagateShapeForward(SmallVectorImpl<Instruction *> &WorkList) {
718 while (!WorkList.
empty()) {
722 bool Propagate =
false;
724 Propagate = setShapeInfo(Inst, *SI);
728 for (
auto *User : Inst->
users())
729 if (ShapeMap.count(User) == 0)
740 propagateShapeBackward(SmallVectorImpl<Instruction *> &WorkList) {
743 auto pushInstruction = [](
Value *
V,
744 SmallVectorImpl<Instruction *> &WorkList) {
753 while (!WorkList.
empty()) {
756 size_t BeforeProcessingV = WorkList.
size();
768 if (setShapeInfo(MatrixA, {
M,
N}))
769 pushInstruction(MatrixA, WorkList);
771 if (setShapeInfo(MatrixB, {
N,
K}))
772 pushInstruction(MatrixB, WorkList);
777 if (setShapeInfo(MatrixA, {
M,
N}))
778 pushInstruction(MatrixA, WorkList);
782 if (setShapeInfo(MatrixA, {
M,
N})) {
783 pushInstruction(MatrixA, WorkList);
794 ShapeInfo Shape = ShapeMap[
V];
795 for (Use &U : ShapedOps) {
796 if (setShapeInfo(
U.get(), Shape))
797 pushInstruction(
U.get(), WorkList);
803 for (
size_t I = BeforeProcessingV;
I != WorkList.
size();
I++)
804 for (User *U : WorkList[
I]->
users())
815 Value *Op0, ShapeInfo Shape0,
Value *Op1, ShapeInfo Shape1,
816 MatrixBuilder &Builder,
817 function_ref<Instruction *(
Value *, ShapeInfo,
Value *, ShapeInfo)>
820 Op0, Shape0.NumRows, Shape0.NumColumns, Op0->
getName() +
"_t");
823 setShapeInfo(T0, Shape0.t());
825 Op1, Shape1.NumRows, Shape1.NumColumns, Op1->
getName() +
"_t");
826 setShapeInfo(
T1, Shape1.t());
832 void eraseFromParentAndRemoveFromShapeMap(Instruction *Inst) {
833 ShapeMap.erase(Inst);
845 if (
II != BB.
rend() && Inst == &*
II)
847 eraseFromParentAndRemoveFromShapeMap(Inst);
852 void updateShapeAndReplaceAllUsesWith(Instruction &Old,
Value *New) {
856 auto S = ShapeMap.find(&Old);
857 if (S != ShapeMap.end()) {
858 ShapeInfo Shape = S->second;
860 if (supportsShapeInfo(New))
861 ShapeMap.insert({
New, Shape});
874 MatrixBuilder Builder(IB);
877 ConstantInt *
R, *
K, *
C;
886 updateShapeAndReplaceAllUsesWith(
I, TATA);
887 eraseFromParentAndMove(&
I,
II, BB);
888 eraseFromParentAndMove(TA,
II, BB);
895 updateShapeAndReplaceAllUsesWith(
I, TA);
896 eraseFromParentAndMove(&
I,
II, BB);
906 auto NewInst = distributeTransposes(
907 TAMB, {
K,
C}, TAMA, {
R,
K}, Builder,
908 [&](
Value *T0, ShapeInfo Shape0,
Value *
T1, ShapeInfo Shape1) {
911 Shape1.NumColumns,
"mmul");
913 updateShapeAndReplaceAllUsesWith(
I, NewInst);
914 eraseFromParentAndMove(&
I,
II, BB);
915 eraseFromParentAndMove(TA,
II, BB);
929 auto NewInst = distributeTransposes(
930 TAMA, {
R,
C}, TAMB, {
R,
C}, Builder,
931 [&](
Value *T0, ShapeInfo Shape0,
Value *
T1, ShapeInfo Shape1) {
932 bool IsFP =
I.getType()->isFPOrFPVectorTy();
933 auto *
Mul = IsFP ? LocalBuilder.CreateFMul(T0,
T1,
"mmul")
934 : LocalBuilder.CreateMul(T0,
T1,
"mmul");
936 setShapeInfo(Result, Shape0);
939 updateShapeAndReplaceAllUsesWith(
I, NewInst);
940 eraseFromParentAndMove(&
I,
II, BB);
941 eraseFromParentAndMove(TA,
II, BB);
950 auto NewInst = distributeTransposes(
951 TAMA, {
R,
C}, TAMB, {
R,
C}, Builder,
952 [&](
Value *T0, ShapeInfo Shape0,
Value *
T1, ShapeInfo Shape1) {
953 bool IsFP =
I.getType()->isFPOrFPVectorTy();
954 auto *
Add = IsFP ? LocalBuilder.CreateFAdd(T0,
T1,
"madd")
955 : LocalBuilder.CreateAdd(T0,
T1,
"madd");
958 setShapeInfo(Result, Shape0);
961 updateShapeAndReplaceAllUsesWith(
I, NewInst);
962 eraseFromParentAndMove(&
I,
II, BB);
963 eraseFromParentAndMove(TA,
II, BB);
971 bool liftTranspose(Instruction &
I) {
975 eraseFromParentAndRemoveFromShapeMap(&
T);
978 if (
A !=
B &&
B->use_empty())
983 ConstantInt *
R, *
K, *
C;
991 MatrixBuilder Builder(IB);
993 BT, AT,
C->getZExtValue(),
K->getZExtValue(),
R->getZExtValue());
994 setShapeInfo(M, {
C,
R});
997 updateShapeAndReplaceAllUsesWith(
I, NewInst);
998 CleanupBinOp(
I,
A,
B);
1010 auto *
Add = Builder.CreateFAdd(AT,
BT,
"mfadd");
1011 MatrixBuilder MBuilder(Builder);
1012 Instruction *NewInst = MBuilder.CreateMatrixTranspose(
1013 Add,
R->getZExtValue(),
C->getZExtValue(),
"mfadd_t");
1014 updateShapeAndReplaceAllUsesWith(
I, NewInst);
1017 "Shape of new instruction doesn't match original shape.");
1018 CleanupBinOp(
I,
A,
B);
1020 setShapeInfo(AddI, {
R,
C});
1024 "Shape of updated addition doesn't match cached shape.");
1032 bool optimizeTransposes() {
1036 for (BasicBlock &BB :
reverse(Func)) {
1041 if (Instruction *NewInst = sinkTranspose(
I,
II,
Changed))
1048 for (BasicBlock &BB : Func) {
1061 for (BasicBlock &BB : Func)
1062 for (Instruction &Inst : BB) {
1067 switch (
II->getIntrinsicID()) {
1068 case Intrinsic::matrix_multiply:
1069 case Intrinsic::matrix_transpose:
1070 case Intrinsic::matrix_column_major_load:
1071 case Intrinsic::matrix_column_major_store:
1080 if (WorkList.
empty())
1084 ORE = &AM->getResult<OptimizationRemarkEmitterAnalysis>(Func);
1085 AA = &AM->getResult<AAManager>(Func);
1086 DT = &AM->getResult<DominatorTreeAnalysis>(Func);
1087 LI = &AM->getResult<LoopAnalysis>(Func);
1091 while (!WorkList.
empty()) {
1092 WorkList = propagateShapeForward(WorkList);
1093 WorkList = propagateShapeBackward(WorkList);
1098 Changed |= optimizeTransposes();
1099 if (Opts.matrix_print_after_transpose_opt) {
1100 dbgs() <<
"Dump after matrix transpose optimization:\n";
1106 SmallVector<Instruction *, 16> MatrixInsts;
1111 ReversePostOrderTraversal<Function *> RPOT(&Func);
1112 for (
auto *BB : RPOT)
1113 for (Instruction &
I : *BB) {
1116 if (!ShapeMap.contains(&
I))
1124 SmallPtrSet<Instruction *, 16> FusedInsts;
1125 for (CallInst *CI : MaybeFusableInsts)
1129 for (CallInst *CI : MaybeFusableInsts)
1131 LowerMatrixMultiplyFused(CI, FusedInsts, LifetimeEnds);
1137 for (Instruction *Inst : MatrixInsts) {
1138 if (FusedInsts.
count(Inst))
1145 const ShapeInfo &
SI = ShapeMap.at(Inst);
1147 MatrixTy PhiM(isColumnMajor(),
SI.NumRows,
SI.NumColumns, EltTy);
1150 for (
unsigned VI = 0, VE = PhiM.getNumVectors(); VI != VE; ++VI)
1151 PhiM.setVector(VI, Builder.CreatePHI(PhiM.getVectorTy(),
1152 PHI->getNumIncomingValues(),
1154 assert(!Inst2ColumnMatrix.contains(
PHI) &&
"map already contains phi?");
1155 Inst2ColumnMatrix[
PHI] = PhiM;
1159 for (Instruction *Inst : MatrixInsts) {
1160 if (FusedInsts.
count(Inst))
1163 const ShapeInfo &
SI = ShapeMap.at(Inst);
1169 Builder.disableGEPCanonicalization();
1171 Result = VisitBinaryOperator(BinOp, SI, Builder);
1173 Result = VisitCastInstruction(Cast, SI, Builder);
1175 Result = VisitUnaryOperator(UnOp, SI, Builder);
1177 Result = VisitIntrinsicInst(Intr, SI, Builder);
1189 finalizeLowering(Inst, Result, Builder);
1194 RemarkGenerator RemarkGen(Inst2ColumnMatrix, *ORE, Func);
1195 RemarkGen.emitRemarks();
1207 SmallPtrSet<Instruction *, 16> PoisonedInsts;
1208 for (
auto *Inst :
reverse(ToRemove)) {
1211 PoisonedInsts.
insert(Poisoned);
1215 PoisonedInsts.
erase(Inst);
1217 if (!PoisonedInsts.
empty()) {
1219 dbgs() <<
"Poisoned but present instructions:\n";
1220 for (
auto *
I : PoisonedInsts)
1221 dbgs() << *
I <<
"\n";
1229 MatrixTy VisitIntrinsicInst(IntrinsicInst *Inst,
const ShapeInfo &SI,
1235 case Intrinsic::matrix_multiply:
1236 return LowerMultiply(Inst, Builder);
1237 case Intrinsic::matrix_transpose:
1238 return LowerTranspose(Inst, Builder);
1239 case Intrinsic::matrix_column_major_load:
1240 return LowerColumnMajorLoad(Inst, Builder);
1241 case Intrinsic::matrix_column_major_store:
1242 return LowerColumnMajorStore(Inst, Builder);
1243 case Intrinsic::abs:
1244 case Intrinsic::fabs: {
1245 MatrixTy
Result(isColumnMajor());
1246 MatrixTy
M = getMatrix(Inst->
getOperand(0), SI, Builder);
1249 for (
auto *
Vector :
M.vectors()) {
1251 case Intrinsic::abs:
1255 case Intrinsic::fabs:
1264 return Result.addNumComputeOps(getNumOps(
Result.getVectorTy()) *
1271 "only intrinsics supporting shape info should be seen here");
1279 Align getAlignForIndex(
unsigned Idx,
Value *Stride,
Type *ElementTy,
1280 MaybeAlign
A)
const {
1281 Align InitialAlign = DL.getValueOrABITypeAlignment(
A, ElementTy);
1283 return InitialAlign;
1285 TypeSize ElementSizeInBits = DL.getTypeSizeInBits(ElementTy);
1288 ConstStride->getZExtValue() * ElementSizeInBits / 8;
1304 "Attempted to cast non-integral type to integer index");
1309 V->getName() +
".cast");
1314 MatrixTy loadMatrix(
Type *Ty,
Value *Ptr, MaybeAlign MAlign,
Value *Stride,
1315 bool IsVolatile, ShapeInfo Shape,
IRBuilder<> &Builder) {
1319 Value *EltPtr = Ptr;
1320 MatrixTy
Result(isColumnMajor());
1321 Stride = castToIndexType(Ptr, Stride, Builder);
1322 for (
unsigned I = 0,
E = getNumVectors(Shape);
I <
E; ++
I) {
1325 Stride, getStride(Shape), EltTy, Builder);
1327 VecTy,
GEP, getAlignForIndex(
I, Stride, EltTy, MAlign),
1328 IsVolatile,
"col.load");
1332 return Result.addNumLoads(getNumOps(
Result.getVectorTy()) *
1338 MatrixTy loadMatrix(
Value *MatrixPtr, MaybeAlign Align,
bool IsVolatile,
1340 ShapeInfo ResultShape,
Type *EltTy,
1343 Builder.
CreateMul(J, getIndex(MatrixPtr, getStride(MatrixShape))),
I);
1347 ResultShape.NumColumns);
1349 return loadMatrix(TileTy, TileStart, Align,
1350 getIndex(MatrixPtr, getStride(MatrixShape)), IsVolatile,
1351 ResultShape, Builder);
1355 MatrixTy
LowerLoad(Instruction *Inst,
Value *Ptr, MaybeAlign Align,
1356 Value *Stride,
bool IsVolatile, ShapeInfo Shape,
1358 return loadMatrix(Inst->
getType(), Ptr, Align, Stride, IsVolatile, Shape,
1365 MatrixTy LowerColumnMajorLoad(CallInst *Inst,
IRBuilder<> &Builder) {
1366 assert(Opts.matrix_default_layout == MatrixLayoutTy::ColumnMajor &&
1367 "Intrinsic only supports column-major layout!");
1372 {Inst->getArgOperand(3), Inst->getArgOperand(4)}, Builder);
1377 void storeMatrix(
const MatrixTy &StoreVal,
Value *MatrixPtr,
1378 MaybeAlign MAlign,
bool IsVolatile, ShapeInfo MatrixShape,
1381 Builder.
CreateMul(J, getIndex(MatrixPtr, getStride(MatrixShape))),
I);
1385 StoreVal.getNumColumns());
1387 storeMatrix(TileTy, StoreVal, TileStart, MAlign,
1388 getIndex(MatrixPtr, getStride(MatrixShape)), IsVolatile,
1394 MatrixTy storeMatrix(
Type *Ty, MatrixTy StoreVal,
Value *Ptr,
1395 MaybeAlign MAlign,
Value *Stride,
bool IsVolatile,
1398 Value *EltPtr = Ptr;
1399 Stride = castToIndexType(Ptr, Stride, Builder);
1400 for (
auto Vec :
enumerate(StoreVal.vectors())) {
1407 getAlignForIndex(Vec.index(), Stride,
1412 return MatrixTy(isColumnMajor())
1413 .addNumStores(getNumOps(StoreVal.getVectorTy()) *
1414 StoreVal.getNumVectors());
1419 MaybeAlign
A,
Value *Stride,
bool IsVolatile,
1421 auto StoreVal = getMatrix(
Matrix, Shape, Builder);
1422 return storeMatrix(
Matrix->getType(), StoreVal, Ptr,
A, Stride, IsVolatile,
1429 MatrixTy LowerColumnMajorStore(CallInst *Inst,
IRBuilder<> &Builder) {
1430 assert(Opts.matrix_default_layout == MatrixLayoutTy::ColumnMajor &&
1431 "Intrinsic only supports column-major layout!");
1437 {Inst->getArgOperand(4), Inst->getArgOperand(5)},
1446 unsigned BlockNumElts =
1449 assert(NumElts >= BlockNumElts &&
"Too few elements for current block");
1456 SmallVector<int, 16>
Mask;
1458 for (i = 0; i <
I; i++)
1461 unsigned VecNumElts =
1463 for (; i <
I + BlockNumElts; i++)
1464 Mask.push_back(i -
I + VecNumElts);
1466 for (; i < VecNumElts; i++)
1474 unsigned &NumComputeOps) {
1475 NumComputeOps += getNumOps(
A->getType());
1480 if (AllowContraction) {
1486 NumComputeOps += getNumOps(
A->getType());
1491 NumComputeOps += getNumOps(
A->getType());
1501 void finalizeLowering(Instruction *Inst, MatrixTy
Matrix,
1503 auto inserted = Inst2ColumnMatrix.insert(std::make_pair(Inst,
Matrix));
1506 "multiple matrix lowering mapping");
1508 ToRemove.push_back(Inst);
1509 Value *Flattened =
nullptr;
1511 if (ShapeMap.contains(
U.getUser()))
1515 Flattened =
Matrix.embedInVector(Builder);
1518 <<
"flattening a " <<
Matrix.shape() <<
" matrix:\n"
1520 <<
"\nbecause we do not have a shape-aware lowering for its "
1523 FlattenedMatrices++;
1532 void lowerDotProduct(CallInst *MatMul,
1533 SmallPtrSet<Instruction *, 16> &FusedInsts,
1534 FastMathFlags FMF) {
1536 Opts.matrix_default_layout != MatrixLayoutTy::ColumnMajor)
1541 if (LShape.NumRows != 1 || RShape.NumColumns != 1)
1556 auto CanBeFlattened = [](
Value *
Op) {
1569 auto GetCostForArg = [
this, &CanBeFlattened, CostKind](
Value *
Op,
1571 if (!ShapeMap.contains(
Op))
1580 if (!CanBeFlattened(
Op)) {
1583 for (
unsigned I = 1;
I <
N; ++
I)
1597 return NewCost - OriginalCost;
1605 for (
unsigned I = 1;
I <
N; ++
I)
1616 return TTI.getMemoryOpCost(Instruction::Load, VecTy,
Align(1), 0,
1618 N * TTI.getMemoryOpCost(Instruction::Load, EltTy,
Align(1), 0,
1625 SmallPtrSet<Value *, 4> Seen;
1630 while (!WorkList.
empty()) {
1636 if (OpCost + LHSCost >= LHSCost)
1642 WorkList.
append(
I->op_begin(),
I->op_end());
1646 int AddOpCode = IsIntVec ? Instruction::Add : Instruction::FAdd;
1647 int MulOpCode = IsIntVec ? Instruction::Mul : Instruction::FMul;
1649 TTI.getArithmeticReductionCost(
1651 IsIntVec ? std::nullopt : std::optional(FMF), CostKind) +
1652 TTI.getArithmeticInstrCost(MulOpCode,
LHS->
getType(), CostKind);
1654 TTI.getArithmeticInstrCost(AddOpCode, ElementType, CostKind) *
1655 (LShape.NumColumns - 1) +
1656 TTI.getArithmeticInstrCost(MulOpCode, ElementType, CostKind) *
1657 (LShape.NumColumns);
1658 if ((LHSCost + ReductionCost - SequentialAddCost) >
InstructionCost(0))
1661 FusedInsts.
insert(MatMul);
1663 auto FlattenArg = [&Builder, &FusedInsts, &CanBeFlattened,
1668 if (!CanBeFlattened(
Op))
1672 auto It = ShapeMap.find(
Op);
1673 if (It != ShapeMap.end()) {
1674 It->second = It->second.t();
1688 0, MatLoad->getArgOperand(1),
1690 MatLoad->getParamAlign(0));
1693 Op->replaceAllUsesWith(NewLoad);
1699 Op->replaceAllUsesWith(Arg);
1704 for (
auto *
V : ToFlatten)
1726 FusedInsts.insert(MatMul);
1727 ToRemove.push_back(MatMul);
1733 unsigned capBlockSize(
unsigned BlockSize,
unsigned Remainder,
Type *EltType) {
1739 if (TTI.isTypeLegal(VecTy))
1745 Opts.matrix_split_matmul_remainder_over_threshold)
1763 void emitMatrixMultiply(MatrixTy &Result,
const MatrixTy &
A,
1765 bool IsScalarMatrixTransposed, FastMathFlags FMF) {
1766 const unsigned VF = std::max<unsigned>(
1769 Result.getElementType()->getPrimitiveSizeInBits().getFixedValue(),
1771 unsigned R =
Result.getNumRows();
1772 unsigned C =
Result.getNumColumns();
1773 unsigned M =
A.getNumColumns();
1775 bool IsFP =
Result.getElementType()->isFloatingPointTy();
1776 assert(
A.isColumnMajor() ==
B.isColumnMajor() &&
1777 Result.isColumnMajor() ==
A.isColumnMajor() &&
1778 "operands must agree on matrix layout");
1779 unsigned NumComputeOps = 0;
1783 if (
A.isColumnMajor()) {
1787 for (
unsigned J = 0; J <
C; ++J) {
1797 for (
unsigned K = 0;
K <
M; ++
K) {
1800 B.getColumn(IsScalarMatrixTransposed ? K : J),
1801 IsScalarMatrixTransposed ? J : K);
1804 createMulAdd(isSumZero && K == 0 ?
nullptr : Sum, L,
Splat,
1815 for (
unsigned I = 0;
I <
R; ++
I) {
1818 for (
unsigned J = 0; J <
C; J +=
BlockSize) {
1822 Value *Sum =
nullptr;
1823 for (
unsigned K = 0;
K <
M; ++
K) {
1826 A.getVector(IsScalarMatrixTransposed ? K :
I),
1827 IsScalarMatrixTransposed ?
I : K);
1830 createMulAdd(isSumZero && K == 0 ?
nullptr : Sum,
Splat, R,
1838 Result.addNumComputeOps(NumComputeOps);
1844 std::pair<Value *, AllocaInst *>
1845 getNonAliasingPointer(LoadInst *
Load, StoreInst *
Store, CallInst *MatMul) {
1850 if (AA->isNoAlias(LoadLoc, StoreLoc))
1851 return {
Load->getPointerOperand(),
nullptr};
1856 auto *ArrayTy = ArrayType::get(VT->getElementType(), VT->getNumElements());
1857 IRBuilder<> AllocaBuilder(&Func.getEntryBlock().front());
1858 AllocaInst *Alloca =
1859 AllocaBuilder.CreateAlloca(ArrayTy,
Load->getPointerAddressSpace());
1865 if (
Load->getPointerAddressSpace() !=
Store->getPointerAddressSpace()) {
1869 Load->getPointerOperand(),
Load->getAlign(),
1871 return {Alloca, Alloca};
1884 DTUpdates.
push_back({DT->Delete, Check0, Succ});
1888 nullptr,
"alias_cont");
1894 nullptr,
"no_alias");
1902 Value *StoreBegin =
Store->getPointerOperand();
1906 Value *LoadBegin =
Load->getPointerOperand();
1908 Builder.
CreateICmpULT(LoadBegin, StoreEnd), Check1, Fusion);
1932 PHI->addIncoming(
Load->getPointerOperand(), Check1);
1933 PHI->addIncoming(Alloca, Copy);
1936 DTUpdates.
push_back({DT->Insert, Check0, Check1});
1937 DTUpdates.
push_back({DT->Insert, Check0, Fusion});
1939 DTUpdates.
push_back({DT->Insert, Check1, Fusion});
1940 DT->applyUpdates(DTUpdates);
1941 return {
PHI, Alloca};
1944 bool isFusionProfitable(CallInst *MatMul) {
1945 if (Opts.force_fuse_matrix)
1951 const unsigned R = LShape.NumRows;
1952 const unsigned C = RShape.NumColumns;
1953 const unsigned M = LShape.NumColumns;
1956 const unsigned VF = std::max<unsigned>(
1968 if (R <= VF &&
C == 1)
1974 unsigned Op0Regs = (
R + VF - 1) / VF * M;
1975 unsigned Op1Regs = (
M + VF - 1) / VF *
C;
1976 return Op0Regs + Op1Regs >
1977 TTI.getNumberOfRegisters(TTI.getRegisterClassForType(
true));
1980 MatrixTy getZeroMatrix(
Type *EltType,
unsigned R,
unsigned C) {
1981 MatrixTy Res(isColumnMajor());
1983 for (
unsigned I = 0;
I <
C; ++
I)
1988 void createTiledLoops(CallInst *MatMul,
Value *LPtr, ShapeInfo LShape,
1989 Value *RPtr, ShapeInfo RShape, StoreInst *
Store) {
1993 TileInfo TI(LShape.NumRows, RShape.NumColumns, LShape.NumColumns,
1994 Opts.fuse_matrix_tile_size);
1995 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
2002 BasicBlock *InnerBody = TI.CreateTiledLoops(Start,
End, Builder, DTU, *LI);
2005 Opts.fuse_matrix_tile_size);
2006 MatrixTy TileResult(isColumnMajor());
2011 :
Constant::getNullValue(TileVecTy);
2014 for (
unsigned I = 0;
I < Opts.fuse_matrix_tile_size;
I++) {
2015 auto *
Phi = Builder.
CreatePHI(TileVecTy, 2,
"result.vec." + Twine(
I));
2016 Phi->addIncoming(Init, TI.RowLoop.Header->getSingleSuccessor());
2017 TileResult.addVector(Phi);
2026 loadMatrix(LPtr, {},
false, LShape, TI.RowLoop.Index, TI.KLoop.Index,
2027 {Opts.fuse_matrix_tile_size, Opts.fuse_matrix_tile_size},
2030 loadMatrix(RPtr, {},
false, RShape, TI.KLoop.Index, TI.ColumnLoop.Index,
2031 {Opts.fuse_matrix_tile_size, Opts.fuse_matrix_tile_size},
2033 emitMatrixMultiply(TileResult,
A,
B, Builder,
true,
false,
2037 storeMatrix(TileResult,
Store->getPointerOperand(),
Store->getAlign(),
2038 Store->isVolatile(), {LShape.NumRows, RShape.NumColumns},
2039 TI.RowLoop.Index, TI.ColumnLoop.Index, EltType, Builder);
2041 for (
unsigned I = 0;
I < TileResult.getNumVectors();
I++)
2042 ColumnPhis[
I]->addIncoming(TileResult.getVector(
I), TI.KLoop.Latch);
2048 unsigned InnerLoopUnrollCount =
2049 std::min(10u, LShape.NumColumns / Opts.fuse_matrix_tile_size);
2051 "llvm.loop.unroll.count", InnerLoopUnrollCount);
2054 void emitSIMDTiling(CallInst *MatMul, LoadInst *LoadOp0, LoadInst *LoadOp1,
2056 SmallPtrSetImpl<Instruction *> &FusedInsts) {
2057 assert(Opts.matrix_default_layout == MatrixLayoutTy::ColumnMajor &&
2058 "Tiling only supported for column-major matrixes at the moment!");
2059 if (!isFusionProfitable(MatMul))
2065 const unsigned R = LShape.NumRows;
2066 const unsigned C = RShape.NumColumns;
2067 const unsigned M = LShape.NumColumns;
2070 auto [APtr, AAlloca] = getNonAliasingPointer(LoadOp0,
Store, MatMul);
2071 auto [BPtr, BAlloca] = getNonAliasingPointer(LoadOp1,
Store, MatMul);
2076 unsigned NumOps = getNumNativeVectorOps(EltType, R, M,
C);
2077 bool UseLoops = (
NumOps > Opts.fuse_matrix_loops_threshold) &&
2078 R % Opts.fuse_matrix_tile_size == 0 &&
2079 C % Opts.fuse_matrix_tile_size == 0;
2081 createTiledLoops(MatMul, APtr, LShape, BPtr, RShape,
Store);
2085 for (
unsigned J = 0; J <
C; J += Opts.fuse_matrix_tile_size)
2086 for (
unsigned I = 0;
I <
R;
I += Opts.fuse_matrix_tile_size) {
2087 const unsigned TileR = std::min(R -
I, Opts.fuse_matrix_tile_size);
2088 const unsigned TileC = std::min(
C - J, Opts.fuse_matrix_tile_size);
2089 MatrixTy Res = getZeroMatrix(EltType, TileR, TileC);
2091 for (
unsigned K = 0;
K <
M;
K += Opts.fuse_matrix_tile_size) {
2092 const unsigned TileM = std::min(M - K, Opts.fuse_matrix_tile_size);
2095 LShape, getIndex(APtr,
I), getIndex(APtr, K),
2096 {TileR, TileM}, EltType, Builder);
2099 RShape, getIndex(BPtr, K), getIndex(BPtr, J),
2100 {TileM, TileC}, EltType, Builder);
2101 emitMatrixMultiply(Res,
A,
B, Builder,
true,
false,
2104 storeMatrix(Res, CPtr,
Store->getAlign(),
Store->isVolatile(), {R, M},
2105 getIndex(CPtr,
I), getIndex(CPtr, J), EltType, Builder);
2120 FusedInsts.
insert(MatMul);
2121 eraseFromParentAndRemoveFromShapeMap(
Store);
2122 eraseFromParentAndRemoveFromShapeMap(MatMul);
2124 FusedInsts.
insert(LoadOp0);
2125 eraseFromParentAndRemoveFromShapeMap(LoadOp0);
2127 if (LoadOp1 != LoadOp0 && LoadOp1->
use_empty()) {
2128 FusedInsts.
insert(LoadOp1);
2129 eraseFromParentAndRemoveFromShapeMap(LoadOp1);
2138 LowerMatrixMultiplyFused(CallInst *MatMul,
2139 SmallPtrSetImpl<Instruction *> &FusedInsts,
2141 if (!Opts.fuse_matrix || !DT || Opts.fuse_matrix_tile_size == 0)
2144 assert(AA && LI &&
"Analyses should be available");
2151 if (Opts.matrix_default_layout == MatrixLayoutTy::ColumnMajor
2159 const unsigned R = LShape.NumRows;
2160 const unsigned M = LShape.NumColumns;
2161 const unsigned C = RShape.NumColumns;
2167 if (Opts.matrix_default_layout == MatrixLayoutTy::ColumnMajor) {
2168 MA = getMatrix(
A, ShapeInfo(R, M), Builder);
2169 MB = getMatrix(
T, ShapeInfo(
C, M), Builder);
2172 MA = getMatrix(
T, ShapeInfo(R, M), Builder);
2173 MB = getMatrix(
B, ShapeInfo(
C, M), Builder);
2178 MatrixTy
Result(isColumnMajor(), R,
C, EltType);
2180 emitMatrixMultiply(Result, MA, MB, Builder,
false,
true,
2183 FusedInsts.
insert(MatMul);
2189 Inst2ColumnMatrix[Transpose] = MatrixTy(isColumnMajor(), M,
C, EltType);
2191 finalizeLowering(MatMul, Result, Builder);
2196 Opts.matrix_default_layout != MatrixLayoutTy::ColumnMajor)
2204 if (LoadOp0 && LoadOp1 &&
Store) {
2207 SetVector<Value *> WorkList;
2209 SmallVector<Instruction *> ToHoist;
2210 for (
unsigned I = 0;
I != WorkList.
size(); ++
I) {
2211 Value *Current = WorkList[
I];
2217 if (DT->dominates(CurrI, MatMul))
2219 if (CurrI->mayHaveSideEffects() || CurrI->mayReadFromMemory())
2225 sort(ToHoist, [
this](Instruction *
A, Instruction *
B) {
2226 return DT->dominates(
A,
B);
2228 for (Instruction *
I : ToHoist)
2241 bool FusableOpsInSameBlock = LoadOp0->
getParent() == StoreParent &&
2243 for (
unsigned Idx = 0;
Idx != LifetimeEnds.
size();) {
2244 IntrinsicInst *
End = LifetimeEnds[
Idx];
2248 if (DT->dominates(
End, LoadOp0) && DT->dominates(
End, LoadOp1))
2254 if (FusableOpsInSameBlock &&
End->getParent() != StoreParent)
2262 if (AA->isNoAlias(Load0Loc, EndLoc) && AA->isNoAlias(Load1Loc, EndLoc))
2268 if (
End->getParent() == StoreParent) {
2274 ToRemove.push_back(
End);
2280 emitSIMDTiling(MatMul, LoadOp0, LoadOp1,
Store, FusedInsts);
2286 MatrixTy LowerMultiply(CallInst *MatMul,
IRBuilder<> &Builder) {
2291 const MatrixTy &Lhs = getMatrix(MatMul->
getArgOperand(0), LShape, Builder);
2292 const MatrixTy &Rhs = getMatrix(MatMul->
getArgOperand(1), RShape, Builder);
2293 assert(Lhs.getElementType() == Rhs.getElementType() &&
2294 "Matrix multiply argument element types do not match.");
2296 const unsigned R = LShape.NumRows;
2297 const unsigned C = RShape.NumColumns;
2298 assert(LShape.NumColumns == RShape.NumRows);
2301 MatrixTy
Result(isColumnMajor(), R,
C, EltType);
2302 assert(Lhs.getElementType() ==
Result.getElementType() &&
2303 "Matrix multiply result element type does not match arguments.");
2305 emitMatrixMultiply(Result, Lhs, Rhs, Builder,
false,
false,
2311 MatrixTy LowerTranspose(CallInst *Inst,
IRBuilder<> &Builder) {
2312 MatrixTy
Result(isColumnMajor());
2316 MatrixTy InputMatrix = getMatrix(InputVal, ArgShape, Builder);
2318 const unsigned NewNumVecs =
2319 InputMatrix.isColumnMajor() ? ArgShape.NumRows : ArgShape.NumColumns;
2320 const unsigned NewNumElts =
2321 InputMatrix.isColumnMajor() ? ArgShape.NumColumns : ArgShape.NumRows;
2323 for (
unsigned I = 0;
I < NewNumVecs; ++
I) {
2328 for (
auto J :
enumerate(InputMatrix.vectors())) {
2334 Result.addVector(ResultVector);
2340 return Result.addNumComputeOps(2 * ArgShape.NumRows * ArgShape.NumColumns)
2341 .addNumExposedTransposes(1);
2345 MatrixTy VisitLoad(LoadInst *Inst,
const ShapeInfo &SI,
Value *Ptr,
2351 MatrixTy VisitStore(StoreInst *Inst,
const ShapeInfo &SI,
Value *StoredVal,
2354 getIndex(Ptr, getStride(SI)), Inst->
isVolatile(), SI,
2358 MatrixTy VisitPHI(PHINode *Inst,
const ShapeInfo &SI,
IRBuilder<> &Builder) {
2359 auto BlockIP = Inst->
getParent()->getFirstInsertionPt();
2361 MatrixTy PhiM = getMatrix(Inst, SI, Builder);
2365 SmallDenseMap<BasicBlock *, MatrixTy> ReshapedIncoming;
2366 for (
auto [IncomingV, IncomingB] :
2373 if (
auto MaybeIP = IncomingInst->getInsertionPointAfterDef())
2378 It->second = getMatrix(IncomingV, SI, Builder);
2379 const MatrixTy &OpM = It->second;
2381 for (
unsigned VI = 0, VE = PhiM.getNumVectors(); VI != VE; ++VI) {
2383 NewPHI->
addIncoming(OpM.getVector(VI), IncomingB);
2394 MatrixTy VisitBinaryOperator(BinaryOperator *Inst,
const ShapeInfo &SI,
2399 MatrixTy
Result(isColumnMajor());
2400 MatrixTy
A = getMatrix(Lhs, SI, Builder);
2401 MatrixTy
B = getMatrix(Rhs, SI, Builder);
2402 assert(
A.isColumnMajor() ==
B.isColumnMajor() &&
2403 Result.isColumnMajor() ==
A.isColumnMajor() &&
2404 "operands must agree on matrix layout");
2411 return Result.addNumComputeOps(getNumOps(
Result.getVectorTy()) *
2416 MatrixTy VisitUnaryOperator(UnaryOperator *Inst,
const ShapeInfo &SI,
2420 MatrixTy
Result(isColumnMajor());
2421 MatrixTy
M = getMatrix(
Op, SI, Builder);
2426 auto BuildVectorOp = [&Builder, Inst](
Value *
Op) {
2428 case Instruction::FNeg:
2435 for (
auto *
Vector :
M.vectors())
2438 return Result.addNumComputeOps(getNumOps(
Result.getVectorTy()) *
2443 MatrixTy VisitCastInstruction(CastInst *Inst,
const ShapeInfo &Shape,
2447 MatrixTy
Result(isColumnMajor());
2448 MatrixTy
M = getMatrix(
Op, Shape, Builder);
2453 auto *NewVTy = VectorType::get(OrigVTy->getElementType(),
2456 for (
auto *
Vector :
M.vectors())
2459 return Result.addNumComputeOps(getNumOps(
Result.getVectorTy()) *
2464 MatrixTy VisitSelectInst(SelectInst *Inst,
const ShapeInfo &Shape,
2470 MatrixTy
Result(isColumnMajor());
2471 MatrixTy
A = getMatrix(OpA, Shape, Builder);
2472 MatrixTy
B = getMatrix(OpB, Shape, Builder);
2477 MatrixTy
C = getMatrix(
Cond, Shape, Builder);
2478 llvm::copy(
C.vectors(), std::back_inserter(CondV));
2480 CondV.
resize(
A.getNumVectors());
2487 "If we have a vector conditional, we should be propagating "
2488 "profile information.");
2492 return Result.addNumComputeOps(getNumOps(
Result.getVectorTy()) *
2499 struct ExprLinearizer {
2500 unsigned LengthToBreak = 100;
2502 raw_string_ostream Stream;
2503 unsigned LineLength = 0;
2504 const DataLayout &DL;
2508 const MapVector<Value *, MatrixTy> &Inst2Matrix;
2512 const DenseMap<Value *, SmallPtrSet<Value *, 2>> &Shared;
2515 const SmallSetVector<Value *, 32> &ExprsInSubprogram;
2522 SmallPtrSet<Value *, 8> ReusedExprs;
2524 ExprLinearizer(
const DataLayout &DL,
2525 const MapVector<Value *, MatrixTy> &Inst2Matrix,
2526 const DenseMap<
Value *, SmallPtrSet<Value *, 2>> &Shared,
2527 const SmallSetVector<Value *, 32> &ExprsInSubprogram,
2529 : Stream(Str), DL(DL), Inst2Matrix(Inst2Matrix), Shared(Shared),
2530 ExprsInSubprogram(ExprsInSubprogram), Leaf(Leaf) {}
2532 void indent(
unsigned N) {
2534 for (
unsigned i = 0; i <
N; i++)
2543 void maybeIndent(
unsigned Indent) {
2544 if (LineLength >= LengthToBreak)
2547 if (LineLength == 0)
2551 void write(StringRef S) {
2552 LineLength += S.
size();
2556 Value *getUnderlyingObjectThroughLoads(
Value *
V) {
2558 return getUnderlyingObjectThroughLoads(Ptr);
2559 else if (
V->getType()->isPointerTy())
2565 bool isMatrix(
Value *
V)
const {
return ExprsInSubprogram.count(
V); }
2569 void prettyPrintMatrixType(
Value *
V, raw_string_ostream &SS) {
2570 auto M = Inst2Matrix.find(
V);
2571 if (M == Inst2Matrix.end())
2574 SS <<
M->second.getNumRows();
2576 SS <<
M->second.getNumColumns();
2583 void writeFnName(CallInst *CI) {
2585 write(
"<no called fn>");
2588 if (!
Name.starts_with(
"llvm.matrix")) {
2597 raw_string_ostream
SS(Tmp);
2599 switch (
II->getIntrinsicID()) {
2600 case Intrinsic::matrix_multiply:
2601 prettyPrintMatrixType(
II->getOperand(0), SS);
2603 prettyPrintMatrixType(
II->getOperand(1), SS);
2604 SS <<
"." << *
II->getType()->getScalarType();
2606 case Intrinsic::matrix_transpose:
2607 prettyPrintMatrixType(
II->getOperand(0), SS);
2608 SS <<
"." << *
II->getType()->getScalarType();
2610 case Intrinsic::matrix_column_major_load:
2611 prettyPrintMatrixType(
II, SS);
2612 SS <<
"." << *
II->getType()->getScalarType();
2614 case Intrinsic::matrix_column_major_store:
2615 prettyPrintMatrixType(
II->getOperand(0), SS);
2616 SS <<
"." << *
II->getOperand(0)->getType()->getScalarType();
2625 unsigned getNumShapeArgs(CallInst *CI)
const {
2627 switch (
II->getIntrinsicID()) {
2628 case Intrinsic::matrix_multiply:
2630 case Intrinsic::matrix_transpose:
2632 case Intrinsic::matrix_column_major_load:
2633 case Intrinsic::matrix_column_major_store:
2646 V = getUnderlyingObjectThroughLoads(
V);
2647 if (
V->getType()->isPointerTy()) {
2649 Stream <<
"stack addr";
2650 LineLength += StringRef(
"stack addr").size();
2653 LineLength += StringRef(
"addr").size();
2655 if (!
V->getName().empty()) {
2656 Stream <<
" %" <<
V->getName() <<
"";
2657 LineLength +=
V->getName().size() + 2;
2663 raw_string_ostream TmpStream(Tmp);
2666 TmpStream << CI->getValue();
2668 TmpStream <<
"constant";
2671 TmpStream <<
"matrix";
2673 TmpStream <<
"scalar";
2675 Tmp = std::string(StringRef(Tmp).trim());
2676 LineLength += Tmp.size();
2683 void linearizeExpr(
Value *Expr,
unsigned Indent,
bool ParentReused,
2684 bool ParentShared) {
2686 maybeIndent(Indent);
2687 SmallVector<Value *, 8>
Ops;
2690 bool ExprShared =
false;
2693 if (!ParentShared) {
2694 auto SI = Shared.find(Expr);
2695 assert(SI != Shared.end() &&
SI->second.count(Leaf));
2701 write(
"shared with remark at line " + std::to_string(DL.getLine()) +
2702 " column " + std::to_string(DL.getCol()) +
" (");
2704 ExprShared =
SI->second.size() > 1;
2707 bool Reused = !ReusedExprs.insert(Expr).second;
2708 if (Reused && !ParentReused)
2721 Ops.append(
I->value_op_begin(),
I->value_op_end());
2722 write(
I->getOpcodeName());
2727 unsigned NumOpsToBreak = 1;
2732 if (
Ops.size() > NumOpsToBreak)
2735 maybeIndent(Indent + 1);
2737 linearizeExpr(
Op, Indent + 1, Reused, ExprShared);
2740 if (
Op !=
Ops.back())
2747 const std::string &getResult() {
2765 struct RemarkGenerator {
2766 const MapVector<Value *, MatrixTy> &Inst2Matrix;
2767 OptimizationRemarkEmitter &ORE;
2769 const DataLayout &DL;
2771 RemarkGenerator(
const MapVector<Value *, MatrixTy> &Inst2Matrix,
2772 OptimizationRemarkEmitter &ORE,
Function &Func)
2773 : Inst2Matrix(Inst2Matrix), ORE(ORE), Func(Func),
2774 DL(Func.getDataLayout()) {}
2779 SmallVector<Value *, 4>
2780 getExpressionLeaves(
const SmallSetVector<Value *, 32> &ExprsInSubprogram) {
2781 SmallVector<Value *, 4> Leaves;
2782 for (
auto *Expr : ExprsInSubprogram)
2784 !
any_of(Expr->
users(), [&ExprsInSubprogram](User *U) {
2785 return ExprsInSubprogram.count(U);
2795 const SmallSetVector<Value *, 32> &ExprsInSubprogram,
2796 DenseMap<
Value *, SmallPtrSet<Value *, 2>> &Shared) {
2798 if (!ExprsInSubprogram.
count(
V))
2804 collectSharedInfo(Leaf,
Op, ExprsInSubprogram, Shared);
2810 std::pair<OpInfoTy, OpInfoTy>
2811 sumOpInfos(
Value *Root, SmallPtrSetImpl<Value *> &ReusedExprs,
2812 const SmallSetVector<Value *, 32> &ExprsInSubprogram,
2813 DenseMap<
Value *, SmallPtrSet<Value *, 2>> &Shared)
const {
2814 if (!ExprsInSubprogram.
count(Root))
2818 if (!ReusedExprs.
insert(Root).second)
2821 OpInfoTy SharedCount;
2825 auto CM = Inst2Matrix.find(Root);
2826 if (
I->second.size() == 1)
2827 Count = CM->second.getOpInfo();
2829 SharedCount = CM->second.getOpInfo();
2832 auto C = sumOpInfos(
Op, ReusedExprs, ExprsInSubprogram, Shared);
2834 SharedCount +=
C.second;
2836 return {
Count, SharedCount};
2839 void emitRemarks() {
2846 MapVector<DISubprogram *, SmallVector<Value *, 8>> Subprog2Exprs;
2847 for (
const auto &KV : Inst2Matrix) {
2848 if (Func.getSubprogram()) {
2850 DILocation *
Context =
I->getDebugLoc();
2857 Subprog2Exprs[
nullptr].push_back(KV.first);
2860 for (
auto &KV : Subprog2Exprs) {
2861 SmallSetVector<Value *, 32> ExprsInSubprogram(KV.second.begin(),
2863 auto Leaves = getExpressionLeaves(ExprsInSubprogram);
2865 DenseMap<Value *, SmallPtrSet<Value *, 2>>
Shared;
2866 for (
Value *Leaf : Leaves)
2867 collectSharedInfo(Leaf, Leaf, ExprsInSubprogram, Shared);
2870 for (
auto *L : Leaves) {
2882 SmallPtrSet<Value *, 8> ReusedExprs;
2883 OpInfoTy Counts, SharedCounts;
2884 std::tie(Counts, SharedCounts) =
2885 sumOpInfos(L, ReusedExprs, ExprsInSubprogram, Shared);
2887 OptimizationRemark Rem(
DEBUG_TYPE,
"matrix-lowered", Loc,
2890 Rem <<
"Lowered with ";
2891 Rem <<
ore::NV(
"NumStores", Counts.NumStores) <<
" stores, "
2892 <<
ore::NV(
"NumLoads", Counts.NumLoads) <<
" loads, "
2893 <<
ore::NV(
"NumComputeOps", Counts.NumComputeOps)
2895 <<
ore::NV(
"NumExposedTransposes", Counts.NumExposedTransposes)
2896 <<
" exposed transposes";
2898 if (SharedCounts.NumStores > 0 || SharedCounts.NumLoads > 0 ||
2899 SharedCounts.NumComputeOps > 0) {
2900 Rem <<
",\nadditionally "
2901 <<
ore::NV(
"NumStores", SharedCounts.NumStores) <<
" stores, "
2902 <<
ore::NV(
"NumLoads", SharedCounts.NumLoads) <<
" loads, "
2903 <<
ore::NV(
"NumFPOps", SharedCounts.NumComputeOps)
2905 <<
" are shared with other expressions";
2908 Rem << (
"\n" + linearize(L, Shared, ExprsInSubprogram, DL));
2916 const DenseMap<
Value *, SmallPtrSet<Value *, 2>> &Shared,
2917 const SmallSetVector<Value *, 32> &ExprsInSubprogram,
2918 const DataLayout &DL) {
2919 ExprLinearizer Lin(DL, Inst2Matrix, Shared, ExprsInSubprogram, L);
2920 Lin.linearizeExpr(L, 0,
false,
false);
2921 return Lin.getResult();
2932 LowerMatrixIntrinsics LMT(Opts,
F,
TTI, Minimal ?
nullptr : &AM);
2946 static_cast<PassInfoMixin<LowerMatrixIntrinsicsPass> *
>(
this)->
printPipeline(
2947 OS, MapClassName2PassName);
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Register Bank Select
static const Function * getParent(const Value *V)
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
static Type * getIndexType(Value *In)
hexagon Hexagon specific predictive commoning for HVX vectors
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
static DISubprogram * getSubprogram(DIScope *Scope)
Helper function to either return Scope, if it is a subprogram or the attached subprogram for a local ...
static auto m_AnyAdd(const LTy &L, const RTy &R)
Match any add operation (fp or integer).
static bool isShapePreserving(Value *V)
static auto m_AnyMul(const LTy &L, const RTy &R)
Match any mul operation (fp or integer).
static bool isSplat(Value *V)
Return true if V is a splat of a value (which is used when multiplying a matrix with a scalar).
static std::optional< ShapeInfo > computeShapeInfoForInst(Instruction *I, const DenseMap< Value *, ShapeInfo > &ShapeMap)
Return the ShapeInfo for the result of I, it it can be determined.
static iterator_range< Use * > getShapedOperandsForInst(Instruction *I)
Return an iterator over the operands of I that should share shape information with I.
static Value * computeVectorAddr(Value *BasePtr, Value *VecIdx, Value *Stride, unsigned NumElements, Type *EltType, IRBuilder<> &Builder)
uint64_t IntrinsicInst * II
PowerPC Reduce CR logical Operation
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
This file contains the declarations for profiling metadata utility functions.
SmallVector< MachineOperand, 4 > Cond
static unsigned getFastMathFlags(const MachineInstr &I, const SPIRVSubtarget &ST)
static Value * extractVector(IRBuilderTy &IRB, Value *V, unsigned BeginIndex, unsigned EndIndex, const Twine &Name)
static Value * insertVector(IRBuilderTy &IRB, Value *Old, Value *V, unsigned BeginIndex, const Twine &Name)
This file defines the scope_exit class, which executes user-defined cleanup logic at scope exit.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static SymbolRef::Type getType(const Symbol *Sym)
static const int BlockSize
static SDValue LowerStore(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue LowerLoad(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
void setAlignment(Align Align)
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
iterator begin()
Instruction iterator methods.
const Function * getParent() const
Return the enclosing method, or null if none.
reverse_iterator rbegin()
InstListType::reverse_iterator reverse_iterator
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
BinaryOps getOpcode() const
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
MaybeAlign getParamAlign(unsigned ArgNo) const
Extract the alignment for a call or parameter (0=unknown).
Value * getArgOperand(unsigned i) const
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
static LLVM_ABI ConstantAggregateZero * get(Type *Ty)
static ConstantFP * getNegativeZero(Type *Ty)
LLVM_ABI DISubprogram * getSubprogram() const
Get the subprogram for this scope.
Base class for scope-like contexts.
Subprogram description. Uses SubclassData1.
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)
Analysis pass which computes a DominatorTree.
static constexpr ElementCount getFixed(ScalarTy MinVal)
void setAllowContract(bool B=true)
bool allowReassoc() const
Flag queries.
bool allowContract() const
unsigned getNumElements() const
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
bool isIntrinsic() const
isIntrinsic - Returns true if the function's name starts with "llvm.".
static GEPNoWrapFlags inBounds()
static GEPNoWrapFlags noUnsignedWrap()
Value * CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreatePtrAddWithUnsignedOffset(Value *Ptr, uint64_t Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Create a ptradd with an unsigned constant offset.
LLVM_ABI CallInst * CreateLifetimeStart(Value *Ptr)
Create a lifetime.start intrinsic.
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
LLVM_ABI CallInst * CreateLifetimeEnd(Value *Ptr)
Create a lifetime.end intrinsic.
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
CallInst * CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, uint64_t Size, bool isVolatile=false, const AAMDNodes &AAInfo=AAMDNodes())
Create and insert a memcpy between the specified pointers.
CondBrInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
Value * CreateFAdd(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
LLVM_ABI Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
LLVM_ABI Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="", MDNode *FPMathTag=nullptr, FMFSource FMFSource={})
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Value * CreateInBoundsGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="")
LLVM_ABI Value * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
void disableGEPCanonicalization()
Disable canonicalization of getelementptr instructions to ptradd form.
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
LLVM_ABI Value * CreateAddReduce(Value *Src)
Create a vector int add reduction intrinsic of the source vector.
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended from a 64-bit value.
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
LLVM_ABI Value * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > OverloadTypes, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="", ArrayRef< OperandBundleDef > OpBundles={}, function_ref< void(CallInst *)> SetFn=[](CallInst *) {})
Variant to create a possibly constant-folded intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
LLVM_ABI Value * CreateFAddReduce(Value *Acc, Value *Src)
Create a sequential vector fadd reduction intrinsic of the source vector.
Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateFNeg(Value *V, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
LLVM_ABI Value * CreateUnaryIntrinsic(Intrinsic::ID ID, Value *Op, FMFSource FMFSource={}, const Twine &Name="")
Create a call to intrinsic ID with 1 operand which is mangled on its type.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
static InstructionCost getInvalid(CostType Val=0)
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
iterator_range< user_iterator > users()
LLVM_ABI FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
user_iterator user_begin()
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
bool isVolatile() const
Return true if this is a load from a volatile memory location.
Align getAlign() const
Return the alignment of the access that is being performed.
TypeSize getValue() const
Analysis pass that exposes the LoopInfo for a function.
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
LLVM_ABI void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
CallInst * CreateMatrixTranspose(Value *Matrix, unsigned Rows, unsigned Columns, const Twine &Name="")
Create a llvm.matrix.transpose call, transposing Matrix with Rows rows and Columns columns.
CallInst * CreateMatrixMultiply(Value *LHS, Value *RHS, unsigned LHSRows, unsigned LHSColumns, unsigned RHSColumns, const Twine &Name="")
Create a llvm.matrix.multiply call, multiplying matrixes LHS and RHS.
static LLVM_ABI MemoryLocation get(const LoadInst *LI)
Return a location with information about the memory reference by the given instruction.
LocationSize Size
The maximum size of the location, in address-units, or UnknownSize if the size is not known.
const Value * Ptr
The address of the start of the location.
static LLVM_ABI MemoryLocation getForArgument(const CallBase *Call, unsigned ArgIdx, const TargetLibraryInfo *TLI)
Return a location representing a particular argument of a call.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
iterator_range< const_block_iterator > blocks() const
op_range incoming_values()
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
PreservedAnalyses & preserve()
Mark an analysis as preserved.
size_type size() const
Determine the number of elements in the SetVector.
void insert_range(Range &&R)
size_type count(const_arg_type key) const
Count the number of elements of a given key in the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool erase(PtrType Ptr)
Remove pointer from the set.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
bool isVolatile() const
Return true if this is a store to a volatile memory location.
Represent a constant reference to a string, i.e.
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
Get the string size.
Analysis pass providing the TargetTransformInfo.
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isVoidTy() const
Return true if this is 'void'.
UnaryOps getOpcode() const
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
iterator_range< use_iterator > uses()
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Type * getElementType() const
constexpr ScalarTy getFixedValue() const
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
self_iterator getIterator()
A range adaptor for a pair of iterators.
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char IsVolatile[]
Key for Kernel::Arg::Metadata::mIsVolatile.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ BasicBlock
Various leaf nodes.
LLVM_ABI StringRef getBaseName(ID id)
Return the LLVM name for an intrinsic, without encoded types for overloading, such as "llvm....
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
match_combine_or< Ty... > m_CombineOr(const Ty &...Ps)
Combine pattern matchers matching any of Ps patterns.
TwoOps_match< ValueOpTy, PointerOpTy, Instruction::Store > m_Store(const ValueOpTy &ValueOp, const PointerOpTy &PointerOp)
Matches StoreInst.
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::FMul > m_FMul(const LHS &L, const RHS &R)
bool match(Val *V, const Pattern &P)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
auto m_BinOp()
Match an arbitrary binary operation and ignore it.
auto m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::FAdd > m_FAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
OneOps_match< OpTy, Instruction::Load > m_Load(const OpTy &Op)
Matches LoadInst.
auto m_Intrinsic(const Ts &...Ops)
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
auto m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
ElementType
The element type of an SRV or UAV resource.
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< PhiNode * > Phi
friend class Instruction
Iterator for Instructions in a `BasicBlock.
unsigned getOpcode(const VPValue *V)
Return the instruction opcode for the recipe defining V or 0 for unsupported recipes and VPValues not...
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
void fill(R &&Range, T &&Value)
Provide wrappers to std::fill which take ranges instead of having to pass begin/end explicitly.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
detail::zippy< detail::zip_first, T, U, Args... > zip_equal(T &&t, U &&u, Args &&...args)
zip iterator that assumes that all iteratees have the same length.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
LLVM_ABI void setExplicitlyUnknownBranchWeightsIfProfiled(Instruction &I, StringRef PassName, const Function *F=nullptr)
Like setExplicitlyUnknownBranchWeights(...), but only sets unknown branch weights in the new instruct...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
auto successors(const MachineBasicBlock *BB)
scope_exit(Callable) -> scope_exit< Callable >
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
bool operator!=(uint64_t V1, const APInt &V2)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
LLVM_ATTRIBUTE_ALWAYS_INLINE DynamicAPInt & operator+=(DynamicAPInt &A, int64_t B)
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
LLVM_ABI Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
const ScalarOptions & getScalarOptions(const LLVMContext &C)
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
const Value * getPointerOperand(const Value *V)
A helper function that returns the pointer operand of a load, store or GEP instruction.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
LLVM_ABI void addStringMetadataToLoop(Loop *TheLoop, const char *MDString, unsigned V=0)
Set input string into loop metadata by keeping other values intact.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
void sort(IteratorTy Start, IteratorTy End)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
IRBuilder(LLVMContext &, FolderTy, InserterTy) -> IRBuilder< FolderTy, InserterTy >
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ABI const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=MaxLookupSearchDepth, bool MustPreserveProvenance=false)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Mul
Product of integers.
LLVM_ABI BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")
Split the specified block at the specified instruction.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
DWARFExpression::Operation Op
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
ArrayRef(const T &OneElt) -> ArrayRef< T >
OutputIt copy(R &&Range, OutputIt Out)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
@ End
Run the machine verifier once, at the end of the pipeline.
AAResults AliasAnalysis
Temporary typedef for legacy code that uses a generic AliasAnalysis pointer or reference.
LLVM_ABI Error write(DWPWriter &Out, ArrayRef< std::string > Inputs, OnCuIndexOverflow OverflowOptValue, Dwarf64StrOffsetsPromotion StrOffsetsOptValue, raw_pwrite_stream *OS=nullptr)
LLVM_ABI llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.