LLVM 20.0.0git
X86FastPreTileConfig.cpp
Go to the documentation of this file.
1//===-- X86FastPreTileConfig.cpp - Fast Tile Register Configure------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file Pass to preconfig the shape of physical tile registers
10/// It inserts ldtilecfg ahead of each group of tile registers. The algorithm
11/// walk each instruction of basic block in reverse order. All the tile
12/// registers that live out the basic block would be spilled and reloaded
13/// before its user. It also check the depenedency of the shape to ensure
14/// the shape is defined before ldtilecfg.
15//
16//===----------------------------------------------------------------------===//
17
18#include "X86.h"
19#include "X86InstrBuilder.h"
21#include "X86RegisterInfo.h"
22#include "X86Subtarget.h"
24#include "llvm/ADT/Statistic.h"
29#include "llvm/CodeGen/Passes.h"
32#include "llvm/Support/Debug.h"
33
34using namespace llvm;
35
36#define DEBUG_TYPE "fastpretileconfig"
37
38STATISTIC(NumStores, "Number of stores added");
39STATISTIC(NumLoads, "Number of loads added");
40
41namespace {
42
43class X86FastPreTileConfig : public MachineFunctionPass {
44 MachineFunction *MF = nullptr;
45 const X86Subtarget *ST = nullptr;
46 const TargetInstrInfo *TII = nullptr;
47 MachineRegisterInfo *MRI = nullptr;
48 X86MachineFunctionInfo *X86FI = nullptr;
49 MachineFrameInfo *MFI = nullptr;
50 const TargetRegisterInfo *TRI = nullptr;
51 MachineBasicBlock *MBB = nullptr;
52 int CfgSS = -1;
53 struct PHIInfo {
54 Register Row;
55 Register Col;
56 Register StackAddr;
57 };
59
60 /// Maps virtual regs to the frame index where these values are spilled.
61 IndexedMap<int, VirtReg2IndexFunctor> StackSlotForVirtReg;
62
63 /// Has a bit set for tile virtual register for which it was determined
64 /// that it is alive across blocks.
65 BitVector MayLiveAcrossBlocks;
66
67 int getStackSpaceFor(Register VirtReg);
68 void InitializeTileConfigStackSpace();
69 bool mayLiveOut(Register VirtReg, MachineInstr *CfgMI);
70 void spill(MachineBasicBlock::iterator Before, Register VirtReg, bool Kill);
71 void reload(MachineBasicBlock::iterator UseMI, Register VirtReg,
72 MachineOperand *RowMO, MachineOperand *ColMO);
73 void canonicalizePHIs(MachineBasicBlock &MBB);
74 void convertPHI(MachineBasicBlock *MBB, MachineInstr &PHI);
75 void convertPHIs(MachineBasicBlock &MBB);
76 bool configBasicBlock(MachineBasicBlock &MBB);
77
78public:
79 X86FastPreTileConfig() : MachineFunctionPass(ID), StackSlotForVirtReg(-1) {}
80
81 /// Return the pass name.
82 StringRef getPassName() const override {
83 return "Fast Tile Register Preconfigure";
84 }
85
86 /// Perform tile register configure.
87 bool runOnMachineFunction(MachineFunction &MFunc) override;
88
89 static char ID;
90};
91
92} // end anonymous namespace
93
94char X86FastPreTileConfig::ID = 0;
95
96INITIALIZE_PASS_BEGIN(X86FastPreTileConfig, DEBUG_TYPE,
97 "Fast Tile Register Preconfigure", false, false)
98INITIALIZE_PASS_END(X86FastPreTileConfig, DEBUG_TYPE,
100
104 auto MBBEnd = MBB.end();
105 if (B == MBBEnd)
106 return true;
107
109 for (; &*I != A && &*I != B; ++I)
110 ;
111
112 return &*I == A;
113}
114
115/// This allocates space for the specified virtual register to be held on the
116/// stack.
117int X86FastPreTileConfig::getStackSpaceFor(Register VirtReg) {
118 // Find the location Reg would belong...
119 int SS = StackSlotForVirtReg[VirtReg];
120 // Already has space allocated?
121 if (SS != -1)
122 return SS;
123
124 // Allocate a new stack object for this spill location...
125 const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);
126 unsigned Size = TRI->getSpillSize(RC);
127 Align Alignment = TRI->getSpillAlign(RC);
128 int FrameIdx = MFI->CreateSpillStackObject(Size, Alignment);
129
130 // Assign the slot.
131 StackSlotForVirtReg[VirtReg] = FrameIdx;
132 return FrameIdx;
133}
134
135/// Returns false if \p VirtReg is known to not live out of the current config.
136/// If \p VirtReg live out of the current MBB, it must live out of the current
137/// config
138bool X86FastPreTileConfig::mayLiveOut(Register VirtReg, MachineInstr *CfgMI) {
139 if (MayLiveAcrossBlocks.test(Register::virtReg2Index(VirtReg)))
140 return true;
141
142 for (const MachineInstr &UseInst : MRI->use_nodbg_instructions(VirtReg)) {
143 if (UseInst.getParent() != MBB) {
144 MayLiveAcrossBlocks.set(Register::virtReg2Index(VirtReg));
145 return true;
146 }
147
148 // The use and def are in the same MBB. If the tile register is
149 // reconfigured, it is crobbered and we need to spill and reload
150 // tile register.
151 if (CfgMI) {
152 if (dominates(*MBB, *CfgMI, UseInst)) {
153 MayLiveAcrossBlocks.set(Register::virtReg2Index(VirtReg));
154 return true;
155 }
156 }
157 }
158
159 return false;
160}
161
162void X86FastPreTileConfig::InitializeTileConfigStackSpace() {
163 MachineBasicBlock &MBB = MF->front();
165 DebugLoc DL;
166 if (ST->hasAVX512()) {
167 Register Zmm = MRI->createVirtualRegister(&X86::VR512RegClass);
168 BuildMI(MBB, MI, DL, TII->get(X86::AVX512_512_SET0), Zmm);
169 addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZmr)), CfgSS)
170 .addReg(Zmm);
171 } else if (ST->hasAVX2()) {
172 Register Ymm = MRI->createVirtualRegister(&X86::VR256RegClass);
173 BuildMI(MBB, MI, DL, TII->get(X86::AVX_SET0), Ymm);
174 addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSYmr)), CfgSS)
175 .addReg(Ymm);
176 addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSYmr)), CfgSS,
177 32)
178 .addReg(Ymm);
179 } else {
180 assert(ST->hasSSE2() && "AMX should assume SSE2 enabled");
181 unsigned StoreOpc = ST->hasAVX() ? X86::VMOVUPSmr : X86::MOVUPSmr;
182 Register Xmm = MRI->createVirtualRegister(&X86::VR128RegClass);
183 BuildMI(MBB, MI, DL, TII->get(X86::V_SET0), Xmm);
184 addFrameReference(BuildMI(MBB, MI, DL, TII->get(StoreOpc)), CfgSS)
185 .addReg(Xmm);
186 addFrameReference(BuildMI(MBB, MI, DL, TII->get(StoreOpc)), CfgSS, 16)
187 .addReg(Xmm);
188 addFrameReference(BuildMI(MBB, MI, DL, TII->get(StoreOpc)), CfgSS, 32)
189 .addReg(Xmm);
190 addFrameReference(BuildMI(MBB, MI, DL, TII->get(StoreOpc)), CfgSS, 48)
191 .addReg(Xmm);
192 }
193 // Fill in the palette first.
194 addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::MOV8mi)), CfgSS)
195 .addImm(1);
196}
197
198/// Insert spill instruction for \p AssignedReg before \p Before.
199/// TODO: Update DBG_VALUEs with \p VirtReg operands with the stack slot.
200void X86FastPreTileConfig::spill(MachineBasicBlock::iterator Before,
201 Register VirtReg, bool Kill) {
202 LLVM_DEBUG(dbgs() << "Spilling " << printReg(VirtReg, TRI) << " \n");
203 int FI = getStackSpaceFor(VirtReg);
204 LLVM_DEBUG(dbgs() << " to stack slot #" << FI << '\n');
205
206 const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);
207 // Don't need shape information for tile store, becasue it is adjacent to
208 // the tile def instruction.
209 TII->storeRegToStackSlot(*MBB, Before, VirtReg, Kill, FI, &RC, TRI,
210 Register());
211 ++NumStores;
212
213 // TODO: update DBG_VALUEs
214}
215
216/// Insert reload instruction for \p PhysReg before \p Before.
217void X86FastPreTileConfig::reload(MachineBasicBlock::iterator UseMI,
218 Register OrigReg, MachineOperand *RowMO,
219 MachineOperand *ColMO) {
220 int FI = getStackSpaceFor(OrigReg);
221 const TargetRegisterClass &RC = *MRI->getRegClass(OrigReg);
222 Register TileReg;
223 // Fold copy to tileload
224 // BB1:
225 // spill src to s
226 //
227 // BB2:
228 // t = copy src
229 // -->
230 // t = tileload (s)
231 if (UseMI->isCopy())
232 TileReg = UseMI->getOperand(0).getReg();
233 else
234 TileReg = MRI->createVirtualRegister(&RC);
235 // Can't use TII->loadRegFromStackSlot(), because we need the shape
236 // information for reload.
237 // tileloadd (%sp, %idx), %tmm
238 unsigned Opc = X86::PTILELOADDV;
239 Register StrideReg = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);
240 // FIXME: MBB is not the parent of UseMI.
242 TII->get(X86::MOV64ri), StrideReg)
243 .addImm(64);
244 NewMI = addFrameReference(
245 BuildMI(*UseMI->getParent(), UseMI, DebugLoc(), TII->get(Opc), TileReg)
246 .addReg(RowMO->getReg())
247 .addReg(ColMO->getReg()),
248 FI);
249 MachineOperand &MO = NewMI->getOperand(5);
250 MO.setReg(StrideReg);
251 MO.setIsKill(true);
252 RowMO->setIsKill(false);
253 ColMO->setIsKill(false);
254 // Erase copy instruction after it is folded.
255 if (UseMI->isCopy()) {
257 } else {
258 // Replace the register in the user MI.
259 for (auto &MO : UseMI->operands()) {
260 if (MO.isReg() && MO.getReg() == OrigReg)
261 MO.setReg(TileReg);
262 }
263 }
264
265 ++NumLoads;
266 LLVM_DEBUG(dbgs() << "Reloading " << printReg(OrigReg, TRI) << " into "
267 << printReg(TileReg, TRI) << '\n');
268}
269
271 if (Reg.isVirtual()) {
272 unsigned RegClassID = MRI->getRegClass(Reg)->getID();
273 if (RegClassID == X86::TILERegClassID)
274 return 1;
275 if (RegClassID == X86::TILEPAIRRegClassID)
276 return 2;
277 } else {
278 if (Reg >= X86::TMM0 && Reg <= X86::TMM7)
279 return 1;
280 if (Reg >= X86::TMM0_TMM1 && Reg <= X86::TMM6_TMM7)
281 return 2;
282 }
283 return 0;
284}
285
287 return getTileDefNum(MRI, VirtReg) > 0;
288}
289
291 // The instruction must have 3 operands: tile def, row, col.
292 if (MI.isDebugInstr() || MI.getNumOperands() < 3 || !MI.isPseudo())
293 return false;
294 MachineOperand &MO = MI.getOperand(0);
295
296 if (!MO.isReg())
297 return false;
298
299 return getTileDefNum(MRI, MO.getReg()) > 0;
300}
301
303 MachineInstr *MI = MRI->getVRegDef(TileReg);
304 if (isTileDef(MRI, *MI)) {
305 MachineOperand *RowMO = &MI->getOperand(1);
306 MachineOperand *ColMO = &MI->getOperand(2);
307 return ShapeT(RowMO, ColMO, MRI);
308 } else if (MI->isCopy()) {
309 TileReg = MI->getOperand(1).getReg();
310 return getShape(MRI, TileReg);
311 }
312
313 // The def should not be PHI node, because we walk the MBB in reverse post
314 // order.
315 assert(MI->isPHI() && "Unexpected PHI when get shape.");
316 llvm_unreachable("Unexpected MI when get shape.");
317}
318
319// BB0:
320// spill t0 to s0
321// BB1:
322// spill t1 to s1
323//
324// BB2:
325// t = phi [t0, bb0] [t1, bb1]
326// -->
327// row = phi [r0, bb0] [r1, bb1]
328// col = phi [c0, bb0] [c1, bb1]
329// s = phi [s0, bb0] [s1, bb1]
330// t = tileload row, col, s
331// The new instruction is inserted at the end of the phi node. The order
332// of the original phi node is not ensured.
333void X86FastPreTileConfig::convertPHI(MachineBasicBlock *MBB,
334 MachineInstr &PHI) {
335 // 1. Create instruction to get stack slot address of each incoming block.
336 // 2. Create PHI node for the stack address.
337 // 3. Create PHI node for shape. If one of the incoming shape is immediate
338 // use the immediate and delete the PHI node.
339 // 4. Create tileload instruction from the stack address.
340 Register StackAddrReg = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);
341 MachineInstrBuilder AddrPHI = BuildMI(*MBB, ++PHI.getIterator(), DebugLoc(),
342 TII->get(X86::PHI), StackAddrReg);
343 Register RowReg = MRI->createVirtualRegister(&X86::GR16RegClass);
344 MachineInstrBuilder RowPHI = BuildMI(*MBB, ++PHI.getIterator(), DebugLoc(),
345 TII->get(X86::PHI), RowReg);
346 Register ColReg = MRI->createVirtualRegister(&X86::GR16RegClass);
347 MachineInstrBuilder ColPHI = BuildMI(*MBB, ++PHI.getIterator(), DebugLoc(),
348 TII->get(X86::PHI), ColReg);
349 // Record the mapping of phi node and its row/column information.
350 VisitedPHIs[&PHI] = {RowReg, ColReg, StackAddrReg};
351
352 for (unsigned I = 1, E = PHI.getNumOperands(); I != E; I += 2) {
353 // Get the 2 incoming value of tile register and MBB.
354 Register InTileReg = PHI.getOperand(I).getReg();
355 // Mark it as liveout, so that it will be spilled when visit
356 // the incoming MBB. Otherwise since phi will be deleted, it
357 // would miss spill when visit incoming MBB.
358 MayLiveAcrossBlocks.set(Register::virtReg2Index(InTileReg));
359 MachineBasicBlock *InMBB = PHI.getOperand(I + 1).getMBB();
360
361 MachineInstr *TileDefMI = MRI->getVRegDef(InTileReg);
363 if (TileDefMI->isPHI()) {
364 InsertPos = TileDefMI->getParent()->getFirstNonPHI();
365 if (VisitedPHIs.count(TileDefMI)) { // circular phi reference
366 // def t1
367 // / \
368 // def t2 t3 = phi(t1, t4) <--
369 // \ / |
370 // t4 = phi(t2, t3)-------------
371 //
372 // For each (row, column and stack address) append phi incoming value.
373 // Create r3 = phi(r1, r4)
374 // Create r4 = phi(r2, r3)
375 Register InRowReg = VisitedPHIs[TileDefMI].Row;
376 Register InColReg = VisitedPHIs[TileDefMI].Col;
377 Register InStackAddrReg = VisitedPHIs[TileDefMI].StackAddr;
378 RowPHI.addReg(InRowReg).addMBB(InMBB);
379 ColPHI.addReg(InColReg).addMBB(InMBB);
380 AddrPHI.addReg(InStackAddrReg).addMBB(InMBB);
381 continue;
382 } else {
383 // Recursively convert PHI to tileload
384 convertPHI(TileDefMI->getParent(), *TileDefMI);
385 // The PHI node is coverted to tileload instruction. Get the stack
386 // address from tileload operands.
387 MachineInstr *TileLoad = MRI->getVRegDef(InTileReg);
388 assert(TileLoad && TileLoad->getOpcode() == X86::PTILELOADDV);
389 Register InRowReg = TileLoad->getOperand(1).getReg();
390 Register InColReg = TileLoad->getOperand(2).getReg();
391 Register InStackAddrReg = TileLoad->getOperand(3).getReg();
392 RowPHI.addReg(InRowReg).addMBB(InMBB);
393 ColPHI.addReg(InColReg).addMBB(InMBB);
394 AddrPHI.addReg(InStackAddrReg).addMBB(InMBB);
395 }
396 } else {
397 InsertPos = TileDefMI->getIterator();
398
399 // Fill the incoming operand of row/column phi instruction.
400 ShapeT Shape = getShape(MRI, InTileReg);
401 Shape.getRow()->setIsKill(false);
402 Shape.getCol()->setIsKill(false);
403 RowPHI.addReg(Shape.getRow()->getReg()).addMBB(InMBB);
404 ColPHI.addReg(Shape.getCol()->getReg()).addMBB(InMBB);
405
406 // The incoming tile register live out of its def BB, it would be spilled.
407 // Create MI to get the spill stack slot address for the tile register
408 int FI = getStackSpaceFor(InTileReg);
409 Register InStackAddrReg =
410 MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);
411 addOffset(BuildMI(*TileDefMI->getParent(), InsertPos, DebugLoc(),
412 TII->get(X86::LEA64r), InStackAddrReg)
413 .addFrameIndex(FI),
414 0);
415 AddrPHI.addReg(InStackAddrReg).addMBB(InMBB);
416 }
417 }
418
420 Register StrideReg = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);
421 BuildMI(*MBB, InsertPos, DebugLoc(), TII->get(X86::MOV64ri), StrideReg)
422 .addImm(64);
423 Register TileReg = PHI.getOperand(0).getReg();
424 MachineInstr *NewMI = addDirectMem(
425 BuildMI(*MBB, InsertPos, DebugLoc(), TII->get(X86::PTILELOADDV), TileReg)
426 .addReg(RowReg)
427 .addReg(ColReg),
428 StackAddrReg);
429 MachineOperand &MO = NewMI->getOperand(5);
430 MO.setReg(StrideReg);
431 MO.setIsKill(true);
432 PHI.eraseFromParent();
433 VisitedPHIs.erase(&PHI);
434}
435
437 MachineOperand &MO = MI.getOperand(0);
438 if (MO.isReg() && MO.getReg().isVirtual() && isTileRegister(MRI, MO.getReg()))
439 return true;
440 return false;
441}
442
443void X86FastPreTileConfig::canonicalizePHIs(MachineBasicBlock &MBB) {
445
446 for (MachineInstr &MI : MBB) {
447 if (!MI.isPHI())
448 break;
449 if (!isTileRegDef(MRI, MI))
450 continue;
451 PHIs.push_back(&MI);
452 }
453 // Canonicalize the phi node first. One tile phi may depeneds previous
454 // phi node. For below case, we need convert %t4.
455 //
456 // BB0:
457 // %t3 = phi (t1 BB1, t2 BB0)
458 // %t4 = phi (t5 BB1, t3 BB0)
459 // -->
460 // %t3 = phi (t1 BB1, t2 BB0)
461 // %t4 = phi (t5 BB1, t2 BB0)
462 //
463 while (!PHIs.empty()) {
464 MachineInstr *PHI = PHIs.pop_back_val();
465
466 // Find the operand that is incoming from the same MBB and the def
467 // is also phi node.
468 MachineOperand *InMO = nullptr;
469 MachineInstr *DefMI = nullptr;
470 for (unsigned I = 1, E = PHI->getNumOperands(); I != E; I += 2) {
471 Register InTileReg = PHI->getOperand(I).getReg();
472 MachineBasicBlock *InMBB = PHI->getOperand(I + 1).getMBB();
473 DefMI = MRI->getVRegDef(InTileReg);
474 if (InMBB != &MBB || !DefMI->isPHI())
475 continue;
476
477 InMO = &PHI->getOperand(I);
478 break;
479 }
480 // If can't find such operand, do nothing.
481 if (!InMO)
482 continue;
483
484 // Current phi node depends on previous phi node. Break the
485 // dependency.
486 Register DefTileReg;
487 for (unsigned I = 1, E = DefMI->getNumOperands(); I != E; I += 2) {
488 MachineBasicBlock *InMBB = PHI->getOperand(I + 1).getMBB();
489 if (InMBB != &MBB)
490 continue;
491 DefTileReg = DefMI->getOperand(I).getReg();
492 InMO->setReg(DefTileReg);
493 break;
494 }
495 }
496}
497
498void X86FastPreTileConfig::convertPHIs(MachineBasicBlock &MBB) {
500 for (MachineInstr &MI : MBB) {
501 if (!MI.isPHI())
502 break;
503 if (!isTileRegDef(MRI, MI))
504 continue;
505 PHIs.push_back(&MI);
506 }
507 while (!PHIs.empty()) {
508 MachineInstr *MI = PHIs.pop_back_val();
509 VisitedPHIs.clear();
510 convertPHI(&MBB, *MI);
511 }
512}
513
514// PreTileConfig should configure the tile registers based on basic
515// block.
516bool X86FastPreTileConfig::configBasicBlock(MachineBasicBlock &MBB) {
517 this->MBB = &MBB;
518 bool Change = false;
519 MachineInstr *LastShapeMI = nullptr;
520 MachineInstr *LastTileCfg = nullptr;
521 bool HasUnconfigTile = false;
522
523 auto Config = [&](MachineInstr &Before) {
524 if (CfgSS == -1)
525 CfgSS = MFI->CreateStackObject(ST->getTileConfigSize(),
526 ST->getTileConfigAlignment(), false);
527 LastTileCfg = addFrameReference(
528 BuildMI(MBB, Before, DebugLoc(), TII->get(X86::PLDTILECFGV)), CfgSS);
529 LastShapeMI = nullptr;
530 Change = true;
531 };
532 auto HasTileOperand = [](MachineRegisterInfo *MRI, MachineInstr &MI) {
533 for (const MachineOperand &MO : MI.operands()) {
534 if (!MO.isReg())
535 continue;
536 Register Reg = MO.getReg();
537 if (Reg.isVirtual() && isTileRegister(MRI, Reg))
538 return true;
539 }
540 return false;
541 };
542 for (MachineInstr &MI : reverse(MBB)) {
543 // We have transformed phi node before configuring BB.
544 if (MI.isPHI())
545 break;
546 // Don't collect the shape of used tile, the tile should be defined
547 // before the tile use. Spill and reload would happen if there is only
548 // tile use after ldtilecfg, so the shape can be collected from reload.
549 // Take below code for example. %t would be reloaded before tilestore
550 // call
551 // ....
552 // tilestore %r, %c, %t
553 // -->
554 // call
555 // ldtilecfg
556 // %t = tileload %r, %c
557 // tilestore %r, %c, %t
558 if (HasTileOperand(MRI, MI))
559 HasUnconfigTile = true;
560 // According to AMX ABI, all the tile registers including config register
561 // are volatile. Caller need to save/restore config register.
562 if (MI.isCall() && HasUnconfigTile) {
564 if (LastShapeMI && dominates(MBB, MI, LastShapeMI))
565 I = ++LastShapeMI->getIterator();
566 else
567 I = ++MI.getIterator();
568 Config(*I);
569 HasUnconfigTile = false;
570 continue;
571 }
572 if (!isTileDef(MRI, MI))
573 continue;
574 //
575 //---------------------------------------------------------------------
576 // Don't handle COPY instruction. If the src and dst of the COPY can be
577 // in the same config in below case, we just check the shape of t0.
578 // def row0
579 // def col0
580 // ldtilecfg
581 // t0 = tielzero(row0, col0)
582 // t1 = copy t0
583 // ...
584 // If the src and dst of the COPY can NOT be in the same config in below
585 // case. Reload would be generated befor the copy instruction.
586 // def row0
587 // def col0
588 // t0 = tielzero(row0, col0)
589 // spill t0
590 // ...
591 // def row1
592 // def col1
593 // ldtilecfg
594 // t1 = tilezero(row1, col1)
595 // reload t0
596 // t1 = copy t0
597 //---------------------------------------------------------------------
598 //
599 // If MI dominate the last shape def instruction, we need insert
600 // ldtilecfg after LastShapeMI now. The config doesn't include
601 // current MI.
602 // def row0
603 // def col0
604 // tilezero(row0, col0) <- MI
605 // def row1
606 // def col1
607 // ldtilecfg <- insert
608 // tilezero(row1, col1)
609 if (LastShapeMI && dominates(MBB, MI, LastShapeMI))
610 Config(*(++LastShapeMI->getIterator()));
611 MachineOperand *RowMO = &MI.getOperand(1);
612 MachineOperand *ColMO = &MI.getOperand(2);
613 MachineInstr *RowMI = MRI->getVRegDef(RowMO->getReg());
614 MachineInstr *ColMI = MRI->getVRegDef(ColMO->getReg());
615 // If the shape is defined in current MBB, check the domination.
616 // FIXME how about loop?
617 if (RowMI->getParent() == &MBB) {
618 if (!LastShapeMI)
619 LastShapeMI = RowMI;
620 else if (dominates(MBB, LastShapeMI, RowMI))
621 LastShapeMI = RowMI;
622 }
623 if (ColMI->getParent() == &MBB) {
624 if (!LastShapeMI)
625 LastShapeMI = ColMI;
626 else if (dominates(MBB, LastShapeMI, ColMI))
627 LastShapeMI = ColMI;
628 }
629 unsigned TileDefNum = getTileDefNum(MRI, MI.getOperand(0).getReg());
630 if (TileDefNum > 1) {
631 for (unsigned I = 1; I < TileDefNum; I++) {
632 MachineOperand *ColxMO = &MI.getOperand(2 + I);
633 MachineInstr *ColxMI = MRI->getVRegDef(ColxMO->getReg());
634 if (ColxMI->getParent() == &MBB) {
635 if (!LastShapeMI)
636 LastShapeMI = ColxMI;
637 else if (dominates(MBB, LastShapeMI, ColxMI))
638 LastShapeMI = ColxMI;
639 }
640 }
641 }
642 // If there is user live out of the tilecfg, spill it and reload in
643 // before the user.
644 Register TileReg = MI.getOperand(0).getReg();
645 if (mayLiveOut(TileReg, LastTileCfg))
646 spill(++MI.getIterator(), TileReg, false);
647 for (MachineInstr &UseMI : MRI->use_instructions(TileReg)) {
648 if (UseMI.getParent() == &MBB) {
649 // check user should not across ldtilecfg
650 if (!LastTileCfg || !dominates(MBB, LastTileCfg, UseMI))
651 continue;
652 // reload befor UseMI
653 reload(UseMI.getIterator(), TileReg, RowMO, ColMO);
654 } else {
655 // Don't reload for phi instruction, we handle phi reload separately.
656 // TODO: merge the reload for the same user MBB.
657 if (!UseMI.isPHI())
658 reload(UseMI.getIterator(), TileReg, RowMO, ColMO);
659 }
660 }
661 }
662
663 // Configure tile registers at the head of the MBB
664 if (HasUnconfigTile) {
666 if (LastShapeMI == nullptr || LastShapeMI->isPHI())
668 else
669 Before = &*(++LastShapeMI->getIterator());
670
671 Config(*Before);
672 }
673
674 return Change;
675}
676
677bool X86FastPreTileConfig::runOnMachineFunction(MachineFunction &MFunc) {
678 X86FI = MFunc.getInfo<X86MachineFunctionInfo>();
679 // Early exit in the common case of non-AMX code.
680 if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA)
681 return false;
682
683 MF = &MFunc;
684 MRI = &MFunc.getRegInfo();
685 ST = &MFunc.getSubtarget<X86Subtarget>();
686 TII = ST->getInstrInfo();
687 MFI = &MFunc.getFrameInfo();
688 TRI = ST->getRegisterInfo();
689 CfgSS = -1;
690
691 unsigned NumVirtRegs = MRI->getNumVirtRegs();
692
693 StackSlotForVirtReg.resize(NumVirtRegs);
694 MayLiveAcrossBlocks.clear();
695 // We will create register during config. *3 is to make sure
696 // the virtual register number doesn't exceed the size of
697 // the bit vector.
698 MayLiveAcrossBlocks.resize(NumVirtRegs * 3);
699 bool Change = false;
700 assert(MRI->isSSA());
701
702 // Canonicalize the phi node first.
703 for (MachineBasicBlock &MBB : MFunc)
704 canonicalizePHIs(MBB);
705
706 // Loop over all of the basic blocks in reverse post order and insert
707 // ldtilecfg for tile registers. The reserse post order is to facilitate
708 // PHI node convert.
710 for (MachineBasicBlock *MBB : RPOT) {
711 convertPHIs(*MBB);
712 Change |= configBasicBlock(*MBB);
713 }
714
715 if (Change)
716 InitializeTileConfigStackSpace();
717
718 StackSlotForVirtReg.clear();
719 return Change;
720}
721
723 return new X86FastPreTileConfig();
724}
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
Rewrite undef for PHI
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
#define LLVM_DEBUG(...)
Definition: Debug.h:106
uint64_t Size
RelaxConfig Config
Definition: ELF_riscv.cpp:506
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:57
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition: Statistic.h:166
static unsigned getTileDefNum(MachineRegisterInfo *MRI, Register Reg)
Fast Tile Register static false bool dominates(MachineBasicBlock &MBB, MachineBasicBlock::const_iterator A, MachineBasicBlock::const_iterator B)
static bool isTileRegDef(MachineRegisterInfo *MRI, MachineInstr &MI)
static bool isTileRegister(MachineRegisterInfo *MRI, Register VirtReg)
Fast Tile Register Preconfigure
static ShapeT getShape(MachineRegisterInfo *MRI, Register TileReg)
#define DEBUG_TYPE
static bool isTileDef(MachineRegisterInfo *MRI, MachineInstr &MI)
A debug info location.
Definition: DebugLoc.h:33
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:310
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Store the specified register of the given register class to the specified stack frame index.
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
Definition: MachineInstr.h:69
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:575
bool isCopy() const
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:347
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:578
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:691
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
bool isPHI() const
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:585
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
void setIsKill(bool Val=true)
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
static unsigned virtReg2Index(Register Reg)
Convert a virtual register number to a 0-based index.
Definition: Register.h:77
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:91
MachineOperand * getRow(unsigned I=0) const
Definition: TileShapeInfo.h:78
MachineOperand * getCol(unsigned I=0) const
Definition: TileShapeInfo.h:85
bool empty() const
Definition: SmallVector.h:81
void push_back(const T &Elt)
Definition: SmallVector.h:413
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1196
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:51
TargetInstrInfo - Interface to description of machine instruction set.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
X86MachineFunctionInfo - This class is derived from MachineFunction and contains private X86 target-s...
self_iterator getIterator()
Definition: ilist_node.h:132
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ SS
Definition: X86.h:212
Reg
All possible values of the reg field in the ModR/M byte.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
static const MachineInstrBuilder & addFrameReference(const MachineInstrBuilder &MIB, int FI, int Offset=0, bool mem=true)
addFrameReference - This function is used to add a reference to the base of an abstract object on the...
FunctionPass * createX86FastPreTileConfigPass()
Return a pass that preconfig the tile registers before fast reg allocation.
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:420
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
static const MachineInstrBuilder & addOffset(const MachineInstrBuilder &MIB, int Offset)
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
static const MachineInstrBuilder & addDirectMem(const MachineInstrBuilder &MIB, unsigned Reg)
addDirectMem - This function is used to add a direct memory reference to the current instruction – th...
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39