LLVM 24.0.0git
X86LowerAMXIntrinsics.cpp
Go to the documentation of this file.
1//===-- X86LowerAMXIntrinsics.cpp -X86 Scalarize AMX Intrinsics------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file Pass to transform amx intrinsics to scalar operations.
10/// This pass is always enabled and it skips when it is not -O0 and has no
11/// optnone attributes. With -O0 or optnone attribute, the def of shape to amx
12/// intrinsics is near the amx intrinsics code. We are not able to find a
13/// point which post-dominate all the shape and dominate all amx intrinsics.
14/// To decouple the dependency of the shape, we transform amx intrinsics
15/// to scalar operation, so that compiling doesn't fail. In long term, we
16/// should improve fast register allocation to allocate amx register.
17//===----------------------------------------------------------------------===//
18//
19#include "X86.h"
20#include "X86TargetMachine.h"
24#include "llvm/CodeGen/Passes.h"
27#include "llvm/IR/Analysis.h"
28#include "llvm/IR/DataLayout.h"
29#include "llvm/IR/Dominators.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/IRBuilder.h"
34#include "llvm/IR/IntrinsicsX86.h"
35#include "llvm/IR/MDBuilder.h"
36#include "llvm/IR/PassManager.h"
40#include "llvm/Pass.h"
45
46using namespace llvm;
47using namespace PatternMatch;
48
49#define DEBUG_TYPE "x86-lower-amx-intrinsics"
50
51#ifndef NDEBUG
52static bool isV256I32Ty(Type *Ty) {
53 if (auto *FVT = dyn_cast<FixedVectorType>(Ty))
54 return FVT->getNumElements() == 256 &&
55 FVT->getElementType()->isIntegerTy(32);
56 return false;
57}
58#endif
59
60namespace {
61class X86LowerAMXIntrinsics {
62 Function &Func;
63
64public:
65 X86LowerAMXIntrinsics(Function &F, DomTreeUpdater &DomTU, LoopInfo *LoopI)
66 : Func(F), DTU(DomTU), LI(LoopI) {}
67 bool visit();
68
69private:
70 DomTreeUpdater &DTU;
71 LoopInfo *LI;
72 BasicBlock *createLoop(BasicBlock *Preheader, BasicBlock *Exit, Value *Bound,
73 ConstantInt *Step, StringRef Name, IRBuilderBase &B,
74 Loop *L);
75 template <bool IsTileLoad>
76 Value *createTileLoadStoreLoops(BasicBlock *Start, BasicBlock *End,
77 IRBuilderBase &B, Value *Row, Value *Col,
78 Value *Ptr, Value *Stride, Value *Tile);
79 template <Intrinsic::ID IntrID>
80 std::enable_if_t<IntrID == Intrinsic::x86_tdpbssd_internal ||
81 IntrID == Intrinsic::x86_tdpbsud_internal ||
82 IntrID == Intrinsic::x86_tdpbusd_internal ||
83 IntrID == Intrinsic::x86_tdpbuud_internal ||
84 IntrID == Intrinsic::x86_tdpbf16ps_internal,
85 Value *>
86 createTileDPLoops(BasicBlock *Start, BasicBlock *End, IRBuilderBase &B,
87 Value *Row, Value *Col, Value *K, Value *Acc, Value *LHS,
88 Value *RHS);
89 template <bool IsTileLoad>
90 bool lowerTileLoadStore(Instruction *TileLoadStore);
91 template <Intrinsic::ID IntrID>
92 std::enable_if_t<IntrID == Intrinsic::x86_tdpbssd_internal ||
93 IntrID == Intrinsic::x86_tdpbsud_internal ||
94 IntrID == Intrinsic::x86_tdpbusd_internal ||
95 IntrID == Intrinsic::x86_tdpbuud_internal ||
96 IntrID == Intrinsic::x86_tdpbf16ps_internal,
97 bool>
98 lowerTileDP(Instruction *TileDP);
99 bool lowerTileZero(Instruction *TileZero);
100};
101} // anonymous namespace
102
103BasicBlock *X86LowerAMXIntrinsics::createLoop(BasicBlock *Preheader,
104 BasicBlock *Exit, Value *Bound,
105 ConstantInt *Step, StringRef Name,
106 IRBuilderBase &B, Loop *L) {
107 LLVMContext &Ctx = Preheader->getContext();
108 BasicBlock *Header =
109 BasicBlock::Create(Ctx, Name + ".header", Preheader->getParent(), Exit);
110 BasicBlock *Body =
111 BasicBlock::Create(Ctx, Name + ".body", Header->getParent(), Exit);
112 BasicBlock *Latch =
113 BasicBlock::Create(Ctx, Name + ".latch", Header->getParent(), Exit);
114
115 Type *I16Ty = Type::getInt16Ty(Ctx);
116 UncondBrInst::Create(Body, Header);
117 UncondBrInst::Create(Latch, Body);
118 PHINode *IV =
119 PHINode::Create(I16Ty, 2, Name + ".iv", Header->getTerminator()->getIterator());
120 IV->addIncoming(ConstantInt::get(I16Ty, 0), Preheader);
121
122 B.SetInsertPoint(Latch);
123 Value *Inc = B.CreateAdd(IV, Step, Name + ".step");
124 Value *Cond = B.CreateICmpNE(Inc, Bound, Name + ".cond");
125 auto *BR = CondBrInst::Create(Cond, Header, Exit, Latch);
126 if (auto *BoundInt = dyn_cast<ConstantInt>(Bound)) {
127 assert(Step->getZExtValue() != 0 &&
128 "Expected a non-zero step size. This is chosen by the pass and "
129 "should always be non-zero to imply a finite loop.");
130 MDBuilder MDB(Preheader->getContext());
132 *BR, {BoundInt->getZExtValue() / Step->getZExtValue(), 1}, false);
133 } else {
135 }
136 IV->addIncoming(Inc, Latch);
137
138 UncondBrInst *PreheaderBr = cast<UncondBrInst>(Preheader->getTerminator());
139 BasicBlock *Tmp = PreheaderBr->getSuccessor();
140 PreheaderBr->setSuccessor(Header);
142 {DominatorTree::Delete, Preheader, Tmp},
143 {DominatorTree::Insert, Header, Body},
144 {DominatorTree::Insert, Body, Latch},
145 {DominatorTree::Insert, Latch, Header},
146 {DominatorTree::Insert, Latch, Exit},
147 {DominatorTree::Insert, Preheader, Header},
148 });
149 if (LI) {
150 L->addBasicBlockToLoop(Header, *LI);
151 L->addBasicBlockToLoop(Body, *LI);
152 L->addBasicBlockToLoop(Latch, *LI);
153 }
154 return Body;
155}
156
157template <bool IsTileLoad>
158Value *X86LowerAMXIntrinsics::createTileLoadStoreLoops(
159 BasicBlock *Start, BasicBlock *End, IRBuilderBase &B, Value *Row,
160 Value *Col, Value *Ptr, Value *Stride, Value *Tile) {
161 std::string IntrinName = IsTileLoad ? "tileload" : "tilestore";
162 Loop *RowLoop = nullptr;
163 Loop *ColLoop = nullptr;
164 if (LI) {
165 RowLoop = LI->AllocateLoop();
166 ColLoop = LI->AllocateLoop();
167 RowLoop->addChildLoop(ColLoop);
168 if (Loop *ParentL = LI->getLoopFor(Start))
169 ParentL->addChildLoop(RowLoop);
170 else
171 LI->addTopLevelLoop(RowLoop);
172 }
173
174 BasicBlock *RowBody = createLoop(Start, End, Row, B.getInt16(1),
175 IntrinName + ".scalarize.rows", B, RowLoop);
176 BasicBlock *RowLatch = RowBody->getSingleSuccessor();
177
178 BasicBlock *ColBody = createLoop(RowBody, RowLatch, Col, B.getInt16(1),
179 IntrinName + ".scalarize.cols", B, ColLoop);
180
181 BasicBlock *ColLoopLatch = ColBody->getSingleSuccessor();
182 BasicBlock *ColLoopHeader = ColBody->getSinglePredecessor();
183 BasicBlock *RowLoopHeader = RowBody->getSinglePredecessor();
184 Value *CurrentRow = &*RowLoopHeader->begin();
185 Value *CurrentCol = &*ColLoopHeader->begin();
186 Type *EltTy = B.getInt32Ty();
187 FixedVectorType *V256I32Ty = FixedVectorType::get(EltTy, 256);
188
189 // Common part for tileload and tilestore
190 // *.scalarize.cols.body:
191 // Calculate %idxmem and %idxvec
192 B.SetInsertPoint(ColBody->getTerminator());
193 Value *CurrentRowZExt = B.CreateZExt(CurrentRow, Stride->getType());
194 Value *CurrentColZExt = B.CreateZExt(CurrentCol, Stride->getType());
195 Value *Offset =
196 B.CreateAdd(B.CreateMul(CurrentRowZExt, Stride), CurrentColZExt);
197 Value *EltPtr = B.CreateGEP(EltTy, Ptr, Offset);
198 Value *Idx = B.CreateAdd(B.CreateMul(CurrentRow, B.getInt16(16)), CurrentCol);
199 if (IsTileLoad) {
200 // tileload.scalarize.rows.header:
201 // %vec.phi.row = phi <256 x i32> [ zeroinitializer, %entry ], [ %ResVec,
202 // %tileload.scalarize.rows.latch ]
203 B.SetInsertPoint(RowLoopHeader->getTerminator());
204 Value *VecZero = Constant::getNullValue(V256I32Ty);
205 PHINode *VecCPhiRowLoop = B.CreatePHI(V256I32Ty, 2, "vec.phi.row");
206 VecCPhiRowLoop->addIncoming(VecZero, Start);
207
208 // tileload.scalarize.cols.header:
209 // %vec.phi = phi <256 x i32> [ %vec.phi.row, %tileload.scalarize.rows.body
210 // ], [ %ResVec, %tileload.scalarize.cols.latch ]
211 B.SetInsertPoint(ColLoopHeader->getTerminator());
212 PHINode *VecPhi = B.CreatePHI(V256I32Ty, 2, "vec.phi");
213 VecPhi->addIncoming(VecCPhiRowLoop, RowBody);
214
215 // tileload.scalarize.cols.body:
216 // Calculate %idxmem and %idxvec
217 // %eltptr = getelementptr i32, i32* %base, i64 %idxmem
218 // %elt = load i32, i32* %ptr
219 // %ResVec = insertelement <256 x i32> %vec.phi, i32 %elt, i16 %idxvec
220 B.SetInsertPoint(ColBody->getTerminator());
221 Value *Elt = B.CreateLoad(EltTy, EltPtr);
222 Value *ResVec = B.CreateInsertElement(VecPhi, Elt, Idx);
223 VecPhi->addIncoming(ResVec, ColLoopLatch);
224 VecCPhiRowLoop->addIncoming(ResVec, RowLatch);
225
226 return ResVec;
227 } else {
228 auto *BitCast = cast<BitCastInst>(Tile);
229 Value *Vec = BitCast->getOperand(0);
230 assert(isV256I32Ty(Vec->getType()) && "bitcast from non-v256i32 to x86amx");
231 // tilestore.scalarize.cols.body:
232 // %mul = mul i16 %row.iv, i16 16
233 // %idx = add i16 %mul, i16 %col.iv
234 // %vec = extractelement <16 x i32> %vec, i16 %idx
235 // store i32 %vec, i32* %ptr
236 B.SetInsertPoint(ColBody->getTerminator());
237 Value *Elt = B.CreateExtractElement(Vec, Idx);
238
239 B.CreateStore(Elt, EltPtr);
240 return nullptr;
241 }
242}
243
244template <Intrinsic::ID IntrID>
245std::enable_if_t<IntrID == Intrinsic::x86_tdpbssd_internal ||
246 IntrID == Intrinsic::x86_tdpbsud_internal ||
247 IntrID == Intrinsic::x86_tdpbusd_internal ||
248 IntrID == Intrinsic::x86_tdpbuud_internal ||
249 IntrID == Intrinsic::x86_tdpbf16ps_internal,
250 Value *>
251X86LowerAMXIntrinsics::createTileDPLoops(BasicBlock *Start, BasicBlock *End,
252 IRBuilderBase &B, Value *Row,
253 Value *Col, Value *K, Value *Acc,
254 Value *LHS, Value *RHS) {
255 std::string IntrinName;
256 switch (IntrID) {
257 case Intrinsic::x86_tdpbssd_internal:
258 IntrinName = "tiledpbssd";
259 break;
260 case Intrinsic::x86_tdpbsud_internal:
261 IntrinName = "tiledpbsud";
262 break;
263 case Intrinsic::x86_tdpbusd_internal:
264 IntrinName = "tiledpbusd";
265 break;
266 case Intrinsic::x86_tdpbuud_internal:
267 IntrinName = "tiledpbuud";
268 break;
269 case Intrinsic::x86_tdpbf16ps_internal:
270 IntrinName = "tiledpbf16ps";
271 break;
272 }
273 Loop *RowLoop = nullptr;
274 Loop *ColLoop = nullptr;
275 Loop *InnerLoop = nullptr;
276 if (LI) {
277 RowLoop = LI->AllocateLoop();
278 ColLoop = LI->AllocateLoop();
279 InnerLoop = LI->AllocateLoop();
280 ColLoop->addChildLoop(InnerLoop);
281 RowLoop->addChildLoop(ColLoop);
282 if (Loop *ParentL = LI->getLoopFor(Start))
283 ParentL->addChildLoop(RowLoop);
284 else
285 LI->addTopLevelLoop(RowLoop);
286 }
287
288 BasicBlock *RowBody = createLoop(Start, End, Row, B.getInt16(1),
289 IntrinName + ".scalarize.rows", B, RowLoop);
290 BasicBlock *RowLatch = RowBody->getSingleSuccessor();
291
292 BasicBlock *ColBody = createLoop(RowBody, RowLatch, Col, B.getInt16(1),
293 IntrinName + ".scalarize.cols", B, ColLoop);
294
295 BasicBlock *ColLoopLatch = ColBody->getSingleSuccessor();
296
297 B.SetInsertPoint(ColBody->getTerminator());
298 BasicBlock *InnerBody =
299 createLoop(ColBody, ColLoopLatch, K, B.getInt16(1),
300 IntrinName + ".scalarize.inner", B, InnerLoop);
301
302 BasicBlock *ColLoopHeader = ColBody->getSinglePredecessor();
303 BasicBlock *RowLoopHeader = RowBody->getSinglePredecessor();
304 BasicBlock *InnerLoopHeader = InnerBody->getSinglePredecessor();
305 BasicBlock *InnerLoopLatch = InnerBody->getSingleSuccessor();
306 Value *CurrentRow = &*RowLoopHeader->begin();
307 Value *CurrentCol = &*ColLoopHeader->begin();
308 Value *CurrentInner = &*InnerLoopHeader->begin();
309
310 FixedVectorType *V256I32Ty = FixedVectorType::get(B.getInt32Ty(), 256);
311 auto *BitCastAcc = cast<BitCastInst>(Acc);
312 Value *VecC = BitCastAcc->getOperand(0);
313 assert(isV256I32Ty(VecC->getType()) && "bitcast from non-v256i32 to x86amx");
314 // TODO else create BitCast from x86amx to v256i32.
315 // Store x86amx to memory, and reload from memory
316 // to vector. However with -O0, it doesn't happen.
317 auto *BitCastLHS = cast<BitCastInst>(LHS);
318 Value *VecA = BitCastLHS->getOperand(0);
319 assert(isV256I32Ty(VecA->getType()) && "bitcast from non-v256i32 to x86amx");
320 auto *BitCastRHS = cast<BitCastInst>(RHS);
321 Value *VecB = BitCastRHS->getOperand(0);
322 assert(isV256I32Ty(VecB->getType()) && "bitcast from non-v256i32 to x86amx");
323
324 // tiledpbssd.scalarize.rows.header:
325 // %vec.c.phi.row = phi <256 x i32> [ %VecC, %continue ], [ %NewVecC,
326 // %tiledpbssd.scalarize.rows.latch ]
327
328 // %vec.d.phi.row = phi <256 x i32> [ zeroinitializer, %continue ], [
329 // %NewVecD, %tiledpbssd.scalarize.rows.latch ]
330 B.SetInsertPoint(RowLoopHeader->getTerminator());
331 PHINode *VecCPhiRowLoop = B.CreatePHI(V256I32Ty, 2, "vec.c.phi.row");
332 VecCPhiRowLoop->addIncoming(VecC, Start);
333 Value *VecZero = Constant::getNullValue(V256I32Ty);
334 PHINode *VecDPhiRowLoop = B.CreatePHI(V256I32Ty, 2, "vec.d.phi.row");
335 VecDPhiRowLoop->addIncoming(VecZero, Start);
336
337 // tiledpbssd.scalarize.cols.header:
338 // %vec.c.phi.col = phi <256 x i32> [ %vec.c.phi.row,
339 // %tiledpbssd.scalarize.rows.body ], [ %NewVecC,
340 // %tiledpbssd.scalarize.cols.latch ]
341
342 // %vec.d.phi.col = phi <256 x i32> [
343 // %vec.d.phi.row, %tiledpbssd.scalarize.rows.body ], [ %NewVecD,
344 // %tiledpbssd.scalarize.cols.latch ]
345
346 // calculate idxc.
347 B.SetInsertPoint(ColLoopHeader->getTerminator());
348 PHINode *VecCPhiColLoop = B.CreatePHI(V256I32Ty, 2, "vec.c.phi.col");
349 VecCPhiColLoop->addIncoming(VecCPhiRowLoop, RowBody);
350 PHINode *VecDPhiColLoop = B.CreatePHI(V256I32Ty, 2, "vec.d.phi.col");
351 VecDPhiColLoop->addIncoming(VecDPhiRowLoop, RowBody);
352 Value *IdxC =
353 B.CreateAdd(B.CreateMul(CurrentRow, B.getInt16(16)), CurrentCol);
354
355 // tiledpbssd.scalarize.inner.header:
356 // %vec.c.inner.phi = phi <256 x i32> [ %vec.c.phi.col,
357 // %tiledpbssd.scalarize.cols.body ], [ %NewVecC,
358 // %tiledpbssd.scalarize.inner.latch ]
359
360 B.SetInsertPoint(InnerLoopHeader->getTerminator());
361 PHINode *VecCPhi = B.CreatePHI(V256I32Ty, 2, "vec.c.inner.phi");
362 VecCPhi->addIncoming(VecCPhiColLoop, ColBody);
363
364 B.SetInsertPoint(InnerBody->getTerminator());
365 Value *IdxA =
366 B.CreateAdd(B.CreateMul(CurrentRow, B.getInt16(16)), CurrentInner);
367 Value *IdxB =
368 B.CreateAdd(B.CreateMul(CurrentInner, B.getInt16(16)), CurrentCol);
369 Value *NewVecC = nullptr;
370
371 if (IntrID != Intrinsic::x86_tdpbf16ps_internal) {
372 // tiledpbssd.scalarize.inner.body:
373 // calculate idxa, idxb
374 // %eltc = extractelement <256 x i32> %vec.c.inner.phi, i16 %idxc
375 // %elta = extractelement <256 x i32> %veca, i16 %idxa
376 // %eltav4i8 = bitcast i32 %elta to <4 x i8>
377 // %eltb = extractelement <256 x i32> %vecb, i16 %idxb
378 // %eltbv4i8 = bitcast i32 %eltb to <4 x i8>
379 // %eltav4i32 = sext <4 x i8> %eltav4i8 to <4 x i32>
380 // %eltbv4i32 = sext <4 x i8> %eltbv4i8 to <4 x i32>
381 // %mulab = mul <4 x i32> %eltbv4i32, %eltav4i32
382 // %acc = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %131)
383 // %neweltc = add i32 %elt, %acc
384 // %NewVecC = insertelement <256 x i32> %vec.c.inner.phi, i32 %neweltc,
385 // i16 %idxc
386 FixedVectorType *V4I8Ty = FixedVectorType::get(B.getInt8Ty(), 4);
387 FixedVectorType *V4I32Ty = FixedVectorType::get(B.getInt32Ty(), 4);
388 Value *EltC = B.CreateExtractElement(VecCPhi, IdxC);
389 Value *EltA = B.CreateExtractElement(VecA, IdxA);
390 Value *SubVecA = B.CreateBitCast(EltA, V4I8Ty);
391 Value *EltB = B.CreateExtractElement(VecB, IdxB);
392 Value *SubVecB = B.CreateBitCast(EltB, V4I8Ty);
393 Value *SEXTSubVecB = nullptr;
394 Value *SEXTSubVecA = nullptr;
395 switch (IntrID) {
396 case Intrinsic::x86_tdpbssd_internal:
397 SEXTSubVecB = B.CreateSExt(SubVecB, V4I32Ty);
398 SEXTSubVecA = B.CreateSExt(SubVecA, V4I32Ty);
399 break;
400 case Intrinsic::x86_tdpbsud_internal:
401 SEXTSubVecB = B.CreateZExt(SubVecB, V4I32Ty);
402 SEXTSubVecA = B.CreateSExt(SubVecA, V4I32Ty);
403 break;
404 case Intrinsic::x86_tdpbusd_internal:
405 SEXTSubVecB = B.CreateSExt(SubVecB, V4I32Ty);
406 SEXTSubVecA = B.CreateZExt(SubVecA, V4I32Ty);
407 break;
408 case Intrinsic::x86_tdpbuud_internal:
409 SEXTSubVecB = B.CreateZExt(SubVecB, V4I32Ty);
410 SEXTSubVecA = B.CreateZExt(SubVecA, V4I32Ty);
411 break;
412 default:
413 llvm_unreachable("Invalid intrinsic ID!");
414 }
415 Value *SubVecR = B.CreateAddReduce(B.CreateMul(SEXTSubVecA, SEXTSubVecB));
416 Value *ResElt = B.CreateAdd(EltC, SubVecR);
417 NewVecC = B.CreateInsertElement(VecCPhi, ResElt, IdxC);
418 } else {
419 // tiledpbf16ps.scalarize.inner.body:
420 // calculate idxa, idxb, idxc
421 // %eltc = extractelement <256 x i32> %vec.c.inner.phi, i16 %idxc
422 // %eltcf32 = bitcast i32 %eltc to float
423 // %elta = extractelement <256 x i32> %veca, i16 %idxa
424 // %eltav2i16 = bitcast i32 %elta to <2 x i16>
425 // %eltb = extractelement <256 x i32> %vecb, i16 %idxb
426 // %eltbv2i16 = bitcast i32 %eltb to <2 x i16>
427 // %shufflea = shufflevector <2 x i16> %elta, <2 x i16> zeroinitializer, <4
428 // x i32> <i32 2, i32 0, i32 3, i32 1>
429 // %eltav2f32 = bitcast <4 x i16> %shufflea to <2 x float>
430 // %shuffleb = shufflevector <2 x i16> %eltb, <2 xi16> zeroinitializer, <4 x
431 // i32> <i32 2, i32 0, i32 3, i32 1>
432 // %eltbv2f32 = bitcast <4 x i16> %shuffleb to <2 x float>
433 // %mulab = fmul <2 x float> %eltav2f32, %eltbv2f32
434 // %acc = call float
435 // @llvm.vector.reduce.fadd.v2f32(float %eltcf32, <2 x float> %mulab)
436 // %neweltc = bitcast float %acc to i32
437 // %NewVecC = insertelement <256 x i32> %vec.c.inner.phi, i32 %neweltc,
438 // i16 %idxc
439 // %NewVecD = insertelement <256 x i32> %vec.d.inner.phi, i32 %neweltc,
440 // i16 %idxc
441 FixedVectorType *V2I16Ty = FixedVectorType::get(B.getInt16Ty(), 2);
442 FixedVectorType *V2F32Ty = FixedVectorType::get(B.getFloatTy(), 2);
443 Value *EltC = B.CreateExtractElement(VecCPhi, IdxC);
444 Value *EltCF32 = B.CreateBitCast(EltC, B.getFloatTy());
445 Value *EltA = B.CreateExtractElement(VecA, IdxA);
446 Value *SubVecA = B.CreateBitCast(EltA, V2I16Ty);
447 Value *EltB = B.CreateExtractElement(VecB, IdxB);
448 Value *SubVecB = B.CreateBitCast(EltB, V2I16Ty);
449 Value *ZeroV2I16 = Constant::getNullValue(V2I16Ty);
450 int ShuffleMask[4] = {2, 0, 3, 1};
451 auto ShuffleArray = ArrayRef(ShuffleMask);
452 Value *AV2F32 = B.CreateBitCast(
453 B.CreateShuffleVector(SubVecA, ZeroV2I16, ShuffleArray), V2F32Ty);
454 Value *BV2F32 = B.CreateBitCast(
455 B.CreateShuffleVector(SubVecB, ZeroV2I16, ShuffleArray), V2F32Ty);
456 Value *SubVecR = B.CreateFAddReduce(EltCF32, B.CreateFMul(AV2F32, BV2F32));
457 Value *ResElt = B.CreateBitCast(SubVecR, B.getInt32Ty());
458 NewVecC = B.CreateInsertElement(VecCPhi, ResElt, IdxC);
459 }
460
461 // tiledpbssd.scalarize.cols.latch:
462 // %NewEltC = extractelement <256 x i32> %vec.c.phi.col, i16 %idxc
463 // %NewVecD = insertelement <256 x i32> %vec.d.phi.col, i32 %NewEltC,
464 // i16 %idxc
465 B.SetInsertPoint(ColLoopLatch->getTerminator());
466 Value *NewEltC = B.CreateExtractElement(NewVecC, IdxC);
467 Value *NewVecD = B.CreateInsertElement(VecDPhiColLoop, NewEltC, IdxC);
468
469 VecCPhi->addIncoming(NewVecC, InnerLoopLatch);
470 VecCPhiRowLoop->addIncoming(NewVecC, RowLatch);
471 VecCPhiColLoop->addIncoming(NewVecC, ColLoopLatch);
472 VecDPhiRowLoop->addIncoming(NewVecD, RowLatch);
473 VecDPhiColLoop->addIncoming(NewVecD, ColLoopLatch);
474
475 return NewVecD;
476}
477
478template <Intrinsic::ID IntrID>
479std::enable_if_t<IntrID == Intrinsic::x86_tdpbssd_internal ||
480 IntrID == Intrinsic::x86_tdpbsud_internal ||
481 IntrID == Intrinsic::x86_tdpbusd_internal ||
482 IntrID == Intrinsic::x86_tdpbuud_internal ||
483 IntrID == Intrinsic::x86_tdpbf16ps_internal,
484 bool>
485X86LowerAMXIntrinsics::lowerTileDP(Instruction *TileDP) {
486 Value *M, *N, *K, *C, *A, *B;
488 m_Value(C), m_Value(A), m_Value(B)));
489 Instruction *InsertI = TileDP;
490 IRBuilder<> PreBuilder(TileDP);
491 PreBuilder.SetInsertPoint(TileDP);
492 // We visit the loop with (m, n/4, k/4):
493 // %n_dword = lshr i16 %n, 2
494 // %k_dword = lshr i16 %k, 2
495 Value *NDWord = PreBuilder.CreateLShr(N, PreBuilder.getInt16(2));
496 Value *KDWord = PreBuilder.CreateLShr(K, PreBuilder.getInt16(2));
497 BasicBlock *Start = InsertI->getParent();
498 BasicBlock *End =
499 SplitBlock(InsertI->getParent(), InsertI, &DTU, LI, nullptr, "continue");
500 IRBuilder<> Builder(TileDP);
501 Value *ResVec = createTileDPLoops<IntrID>(Start, End, Builder, M, NDWord,
502 KDWord, C, A, B);
503 // we cannot assume there always be bitcast after tiledpbssd. So we need to
504 // insert one bitcast as required
505 Builder.SetInsertPoint(End->getFirstNonPHIIt());
506 Value *ResAMX =
507 Builder.CreateBitCast(ResVec, Type::getX86_AMXTy(Builder.getContext()));
508 // Delete TileDP intrinsic and do some clean-up.
509 for (Use &U : llvm::make_early_inc_range(TileDP->uses())) {
510 Instruction *I = cast<Instruction>(U.getUser());
511 Value *Vec;
512 if (match(I, m_BitCast(m_Value(Vec)))) {
513 I->replaceAllUsesWith(ResVec);
514 I->eraseFromParent();
515 }
516 }
517 TileDP->replaceAllUsesWith(ResAMX);
518 TileDP->eraseFromParent();
519 return true;
520}
521
522template <bool IsTileLoad>
523bool X86LowerAMXIntrinsics::lowerTileLoadStore(Instruction *TileLoadStore) {
524 Value *M, *N, *Ptr, *Stride, *Tile;
525 if (IsTileLoad)
526 match(TileLoadStore,
528 m_Value(M), m_Value(N), m_Value(Ptr), m_Value(Stride)));
529 else
531 m_Value(M), m_Value(N), m_Value(Ptr),
532 m_Value(Stride), m_Value(Tile)));
533
534 Instruction *InsertI = TileLoadStore;
535 IRBuilder<> PreBuilder(TileLoadStore);
536 PreBuilder.SetInsertPoint(TileLoadStore);
537 Value *NDWord = PreBuilder.CreateLShr(N, PreBuilder.getInt16(2));
538 Value *StrideDWord = PreBuilder.CreateLShr(Stride, PreBuilder.getInt64(2));
539 BasicBlock *Start = InsertI->getParent();
540 BasicBlock *End =
541 SplitBlock(InsertI->getParent(), InsertI, &DTU, LI, nullptr, "continue");
542 IRBuilder<> Builder(TileLoadStore);
543 Value *ResVec = createTileLoadStoreLoops<IsTileLoad>(
544 Start, End, Builder, M, NDWord, Ptr, StrideDWord,
545 IsTileLoad ? nullptr : Tile);
546 if (IsTileLoad) {
547 // we cannot assume there always be bitcast after tileload. So we need to
548 // insert one bitcast as required
549 Builder.SetInsertPoint(End->getFirstNonPHIIt());
550 Value *ResAMX =
551 Builder.CreateBitCast(ResVec, Type::getX86_AMXTy(Builder.getContext()));
552 // Delete tileloadd6 intrinsic and do some clean-up
553 for (Use &U : llvm::make_early_inc_range(TileLoadStore->uses())) {
554 Instruction *I = cast<Instruction>(U.getUser());
555 Value *Vec;
556 if (match(I, m_BitCast(m_Value(Vec)))) {
557 I->replaceAllUsesWith(ResVec);
558 I->eraseFromParent();
559 }
560 }
561 TileLoadStore->replaceAllUsesWith(ResAMX);
562 }
563 TileLoadStore->eraseFromParent();
564 return true;
565}
566
567bool X86LowerAMXIntrinsics::lowerTileZero(Instruction *TileZero) {
568 IRBuilder<> Builder(TileZero);
569 FixedVectorType *V256I32Ty = FixedVectorType::get(Builder.getInt32Ty(), 256);
570 Value *VecZero = Constant::getNullValue(V256I32Ty);
571 for (Use &U : llvm::make_early_inc_range(TileZero->uses())) {
572 Instruction *I = cast<Instruction>(U.getUser());
573 Value *Vec;
574 if (match(I, m_BitCast(m_Value(Vec)))) {
575 I->replaceAllUsesWith(VecZero);
576 I->eraseFromParent();
577 }
578 }
579 TileZero->eraseFromParent();
580 return true;
581}
582
583bool X86LowerAMXIntrinsics::visit() {
584 bool C = false;
586 for (BasicBlock *BB : depth_first(&Func)) {
587 for (BasicBlock::iterator II = BB->begin(), IE = BB->end(); II != IE;) {
588 if (auto *Inst = dyn_cast<IntrinsicInst>(&*II++)) {
589 switch (Inst->getIntrinsicID()) {
590 case Intrinsic::x86_tdpbssd_internal:
591 case Intrinsic::x86_tdpbsud_internal:
592 case Intrinsic::x86_tdpbusd_internal:
593 case Intrinsic::x86_tdpbuud_internal:
594 case Intrinsic::x86_tileloadd64_internal:
595 case Intrinsic::x86_tilestored64_internal:
596 case Intrinsic::x86_tilezero_internal:
597 case Intrinsic::x86_tdpbf16ps_internal:
598 WorkList.push_back(Inst);
599 break;
600 default:
601 break;
602 }
603 }
604 }
605 }
606
607 for (auto *Inst : WorkList) {
608 switch (Inst->getIntrinsicID()) {
609 case Intrinsic::x86_tdpbssd_internal:
610 C = lowerTileDP<Intrinsic::x86_tdpbssd_internal>(Inst) || C;
611 break;
612 case Intrinsic::x86_tdpbsud_internal:
613 C = lowerTileDP<Intrinsic::x86_tdpbsud_internal>(Inst) || C;
614 break;
615 case Intrinsic::x86_tdpbusd_internal:
616 C = lowerTileDP<Intrinsic::x86_tdpbusd_internal>(Inst) || C;
617 break;
618 case Intrinsic::x86_tdpbuud_internal:
619 C = lowerTileDP<Intrinsic::x86_tdpbuud_internal>(Inst) || C;
620 break;
621 case Intrinsic::x86_tdpbf16ps_internal:
622 C = lowerTileDP<Intrinsic::x86_tdpbf16ps_internal>(Inst) || C;
623 break;
624 case Intrinsic::x86_tileloadd64_internal:
625 C = lowerTileLoadStore<true>(Inst) || C;
626 break;
627 case Intrinsic::x86_tilestored64_internal:
628 C = lowerTileLoadStore<false>(Inst) || C;
629 break;
630 case Intrinsic::x86_tilezero_internal:
631 C = lowerTileZero(Inst) || C;
632 break;
633 default:
634 llvm_unreachable("invalid amx intrinsics!");
635 }
636 }
637
638 return C;
639}
640
641namespace {
642bool shouldRunLowerAMXIntrinsics(const Function &F, const TargetMachine *TM) {
643 const X86Options &CLOpts =
644 static_cast<const X86TargetMachine *>(TM)->getCLOpts();
645 return CLOpts.enable_x86_scalar_amx &&
646 (F.hasFnAttribute(Attribute::OptimizeNone) ||
647 TM->getOptLevel() == CodeGenOptLevel::None);
648}
649
650bool runLowerAMXIntrinsics(Function &F, DominatorTree *DT, LoopInfo *LI) {
651 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
652
653 X86LowerAMXIntrinsics LAT(F, DTU, LI);
654 return LAT.visit();
655}
656} // namespace
657
660 if (!shouldRunLowerAMXIntrinsics(F, TM))
661 return PreservedAnalyses::all();
662
663 DominatorTree &DT = FAM.getResult<DominatorTreeAnalysis>(F);
664 LoopInfo &LI = FAM.getResult<LoopAnalysis>(F);
665 bool Changed = runLowerAMXIntrinsics(F, &DT, &LI);
666 if (!Changed)
667 return PreservedAnalyses::all();
668
672 return PA;
673}
674
675namespace {
676class X86LowerAMXIntrinsicsLegacyPass : public FunctionPass {
677public:
678 static char ID;
679
680 X86LowerAMXIntrinsicsLegacyPass() : FunctionPass(ID) {}
681
682 bool runOnFunction(Function &F) override {
683 TargetMachine *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
684 if (!shouldRunLowerAMXIntrinsics(F, TM))
685 return false;
686
687 auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
688 auto *DT = DTWP ? &DTWP->getDomTree() : nullptr;
689 auto *LIWP = getAnalysisIfAvailable<LoopInfoWrapperPass>();
690 auto *LI = LIWP ? &LIWP->getLoopInfo() : nullptr;
691 return runLowerAMXIntrinsics(F, DT, LI);
692 }
693 StringRef getPassName() const override { return "Lower AMX intrinsics"; }
694
695 void getAnalysisUsage(AnalysisUsage &AU) const override {
696 AU.addPreserved<DominatorTreeWrapperPass>();
697 AU.addPreserved<LoopInfoWrapperPass>();
698 AU.addRequired<TargetPassConfig>();
699 }
700};
701} // namespace
702
703static const char PassName[] = "Lower AMX intrinsics";
704char X86LowerAMXIntrinsicsLegacyPass::ID = 0;
705INITIALIZE_PASS_BEGIN(X86LowerAMXIntrinsicsLegacyPass, DEBUG_TYPE, PassName,
706 false, false)
708INITIALIZE_PASS_END(X86LowerAMXIntrinsicsLegacyPass, DEBUG_TYPE, PassName,
710
712 return new X86LowerAMXIntrinsicsLegacyPass();
713}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static bool runOnFunction(Function &F, bool PostInlining)
#define DEBUG_TYPE
This header defines various interfaces for pass management in LLVM.
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
uint64_t IntrinsicInst * II
FunctionAnalysisManager FAM
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
This file contains the declarations for profiling metadata utility functions.
SmallVector< MachineOperand, 4 > Cond
static void visit(BasicBlock &Start, std::function< bool(BasicBlock *)> op)
Target-Independent Code Generator Pass Configuration Options pass.
This pass exposes codegen information to IR-level passes.
static bool isV256I32Ty(Type *Ty)
static const char PassName[]
Value * RHS
Value * LHS
static const uint32_t IV[8]
Definition blake3_impl.h:83
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
LLVM Basic Block Representation.
Definition BasicBlock.h:62
iterator begin()
Instruction iterator methods.
Definition BasicBlock.h:440
const Function * getParent() const
Return the enclosing method, or null if none.
Definition BasicBlock.h:213
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition BasicBlock.h:206
LLVM_ABI const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
LLVM_ABI const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
InstListType::iterator iterator
Instruction iterators...
Definition BasicBlock.h:170
LLVM_ABI LLVMContext & getContext() const
Get the context in which this basic block lives.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Definition BasicBlock.h:237
static CondBrInst * Create(Value *Cond, BasicBlock *IfTrue, BasicBlock *IfFalse, InsertPosition InsertBefore=nullptr)
This is the shared class of boolean and integer constants.
Definition Constants.h:87
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition Constants.h:168
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Analysis pass which computes a DominatorTree.
Definition Dominators.h:241
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:122
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition Type.cpp:838
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:312
void applyUpdatesPermissive(ArrayRef< UpdateT > Updates)
Submit updates to all available trees.
Common base class shared among various IRBuilders.
Definition IRBuilder.h:111
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Analysis pass that exposes the LoopInfo for a function.
Definition LoopInfo.h:598
void addChildLoop(LoopT *NewChild)
Add the specified loop to be a child of this loop.
void addTopLevelLoop(LoopT *New)
This adds the specified loop to the collection of top-level loops.
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Represents a single loop in the control flow graph.
Definition LoopInfo.h:40
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition Analysis.h:115
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
PreservedAnalyses & preserve()
Mark an analysis as preserved.
Definition Analysis.h:133
void push_back(const T &Elt)
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
Primary interface to the complete machine description for the target machine.
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
Target-Independent Code Generator Pass Configuration Options.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
void setSuccessor(BasicBlock *NewSucc)
static UncondBrInst * Create(BasicBlock *Target, InsertPosition InsertBefore=nullptr)
BasicBlock * getSuccessor(unsigned i=0) const
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:548
iterator_range< use_iterator > uses()
Definition Value.h:382
PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM)
const ParentTy * getParent() const
Definition ilist_node.h:34
Pass manager infrastructure for declaring and invalidating analyses.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const AArch64MCOptions & getCLOpts(const MCTargetOptions &Options)
@ BR
Control flow instructions. These all have token chains.
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:83
bool match(Val *V, const Pattern &P)
auto m_Value()
Match an arbitrary value and ignore it.
CastOperator_match< OpTy, Instruction::BitCast > m_BitCast(const OpTy &Op)
Matches BitCast.
auto m_Intrinsic(const Ts &...Ops)
Match intrinsic calls like this: m_Intrinsic<Intrinsic::fabs>(m_Value(X))
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
This is an optimization pass for GlobalISel generic memory operations.
@ Offset
Definition DWP.cpp:577
FunctionPass * createX86LowerAMXIntrinsicsLegacyPass()
LLVM_ABI void setExplicitlyUnknownBranchWeightsIfProfiled(Instruction &I, StringRef PassName, const Function *F=nullptr)
Like setExplicitlyUnknownBranchWeights(...), but only sets unknown branch weights in the new instruct...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
IRBuilder(LLVMContext &, FolderTy, InserterTy) -> IRBuilder< FolderTy, InserterTy >
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
LLVM_ABI BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")
Split the specified block at the specified instruction.
ArrayRef(const T &OneElt) -> ArrayRef< T >
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
iterator_range< df_iterator< T > > depth_first(const T &G)
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
LLVM_ABI void setFittedBranchWeights(Instruction &I, ArrayRef< uint64_t > Weights, bool IsExpected, bool ElideAllZero=false)
Variant of setBranchWeights where the Weights will be fit first to uint32_t by shifting right.
#define N