LLVM 24.0.0git
MachineSMEABIPass.cpp
Go to the documentation of this file.
1//===- MachineSMEABIPass.cpp ----------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass implements the SME ABI requirements for ZA state. This includes
10// implementing the lazy (and agnostic) ZA state save schemes around calls.
11//
12//===----------------------------------------------------------------------===//
13//
14// This pass works by collecting instructions that require ZA to be in a
15// specific state (e.g., "ACTIVE" or "SAVED") and inserting the necessary state
16// transitions to ensure ZA is in the required state before instructions. State
17// transitions represent actions such as setting up or restoring a lazy save.
18// Certain points within a function may also have predefined states independent
19// of any instructions, for example, a "shared_za" function is always entered
20// and exited in the "ACTIVE" state.
21//
22// To handle ZA state across control flow, we make use of edge bundling. This
23// assigns each block an "incoming" and "outgoing" edge bundle (representing
24// incoming and outgoing edges). Initially, these are unique to each block;
25// then, in the process of forming bundles, the outgoing bundle of a block is
26// joined with the incoming bundle of all successors. The result is that each
27// bundle can be assigned a single ZA state, which ensures the state required by
28// all a blocks' successors is the same, and that each basic block will always
29// be entered with the same ZA state. This eliminates the need for splitting
30// edges to insert state transitions or "phi" nodes for ZA states.
31//
32// See below for a simple example of edge bundling.
33//
34// The following shows a conditionally executed basic block (BB1):
35//
36// if (cond)
37// BB1
38// BB2
39//
40// Initial Bundles Joined Bundles
41//
42// ┌──0──┐ ┌──0──┐
43// │ BB0 │ │ BB0 │
44// └──1──┘ └──1──┘
45// ├───────┐ ├───────┐
46// ▼ │ ▼ │
47// ┌──2──┐ │ ─────► ┌──1──┐ │
48// │ BB1 │ ▼ │ BB1 │ ▼
49// └──3──┘ ┌──4──┐ └──1──┘ ┌──1──┐
50// └───►4 BB2 │ └───►1 BB2 │
51// └──5──┘ └──2──┘
52//
53// On the left are the initial per-block bundles, and on the right are the
54// joined bundles (which are the result of the EdgeBundles analysis).
55
56#include "AArch64InstrInfo.h"
58#include "AArch64Subtarget.h"
69
70using namespace llvm;
71
72#define DEBUG_TYPE "aarch64-machine-sme-abi"
73
74namespace {
75
76// Note: For agnostic ZA, we assume the function is always entered/exited in the
77// "ACTIVE" state -- this _may_ not be the case (since OFF is also a
78// possibility, but for the purpose of placing ZA saves/restores, that does not
79// matter).
80enum ZAState : uint8_t {
81 // Any/unknown state (not valid)
82 ANY = 0,
83
84 // ZA is in use and active (i.e. within the accumulator)
85 ACTIVE,
86
87 // ZA is active, but ZT0 has been saved.
88 // This handles the edge case of sharedZA && !sharesZT0.
89 ACTIVE_ZT0_SAVED,
90
91 // A ZA save has been set up or committed (i.e. ZA is dormant or off)
92 // If the function uses ZT0 it must also be saved.
93 LOCAL_SAVED,
94
95 // ZA has been committed to the lazy save buffer of the current function.
96 // If the function uses ZT0 it must also be saved.
97 // ZA is off.
98 LOCAL_COMMITTED,
99
100 // The ZA/ZT0 state on entry to the function.
101 ENTRY,
102
103 // ZA is off.
104 OFF,
105
106 // The number of ZA states (not a valid state)
107 NUM_ZA_STATE
108};
109
110/// A bitmask enum to record live physical registers that the "emit*" routines
111/// may need to preserve. Note: This only tracks registers we may clobber.
112enum LiveRegs : uint8_t {
113 None = 0,
114 NZCV = 1 << 0,
115 W0 = 1 << 1,
116 W0_HI = 1 << 2,
117 X0 = W0 | W0_HI,
118 LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue = */ W0_HI)
119};
120
121/// Holds the virtual registers live physical registers have been saved to.
122struct PhysRegSave {
123 LiveRegs PhysLiveRegs;
124 Register StatusFlags = Register();
125 Register X0Save = Register();
126};
127
128/// Contains the needed ZA state (and live registers) at an instruction. That is
129/// the state ZA must be in _before_ "InsertPt".
130struct InstInfo {
131 ZAState NeededState{ZAState::ANY};
133 LiveRegs PhysLiveRegs = LiveRegs::None;
134};
135
136/// Contains the needed ZA state for each instruction in a block. Instructions
137/// that do not require a ZA state are not recorded.
138struct BlockInfo {
140 ZAState FixedEntryState{ZAState::ANY};
141 ZAState DesiredIncomingState{ZAState::ANY};
142 ZAState DesiredOutgoingState{ZAState::ANY};
143 LiveRegs PhysLiveRegsAtEntry = LiveRegs::None;
144 LiveRegs PhysLiveRegsAtExit = LiveRegs::None;
145};
146
147/// Contains the needed ZA state information for all blocks within a function.
148struct FunctionInfo {
150 std::optional<MachineBasicBlock::iterator> AfterSMEProloguePt;
151 LiveRegs PhysLiveRegsAfterSMEPrologue = LiveRegs::None;
152};
153
154/// State/helpers that is only needed when emitting code to handle
155/// saving/restoring ZA.
156class EmitContext {
157public:
158 EmitContext() = default;
159
160 /// Get or create a TPIDR2 block in \p MF.
161 int getTPIDR2Block(MachineFunction &MF) {
162 if (TPIDR2BlockFI)
163 return *TPIDR2BlockFI;
164 MachineFrameInfo &MFI = MF.getFrameInfo();
165 TPIDR2BlockFI = MFI.CreateStackObject(16, Align(16), false);
166 return *TPIDR2BlockFI;
167 }
168
169 /// Get or create agnostic ZA buffer pointer in \p MF.
170 Register getAgnosticZABufferPtr(MachineFunction &MF) {
171 if (AgnosticZABufferPtr.isValid())
172 return AgnosticZABufferPtr;
173 Register BufferPtr =
174 MF.getInfo<AArch64FunctionInfo>()->getEarlyAllocSMESaveBuffer();
175 AgnosticZABufferPtr =
176 BufferPtr.isValid()
177 ? BufferPtr
178 : MF.getRegInfo().createVirtualRegister(&AArch64::GPR64RegClass);
179 return AgnosticZABufferPtr;
180 }
181
182 int getZT0SaveSlot(MachineFunction &MF) {
183 if (ZT0SaveFI)
184 return *ZT0SaveFI;
185 MachineFrameInfo &MFI = MF.getFrameInfo();
186 ZT0SaveFI = MFI.CreateSpillStackObject(64, Align(16));
187 return *ZT0SaveFI;
188 }
189
190 /// Returns true if the function must allocate a ZA save buffer on entry. This
191 /// will be the case if, at any point in the function, a ZA save was emitted.
192 bool needsSaveBuffer() const {
193 assert(!(TPIDR2BlockFI && AgnosticZABufferPtr) &&
194 "Cannot have both a TPIDR2 block and agnostic ZA buffer");
195 return TPIDR2BlockFI || AgnosticZABufferPtr.isValid();
196 }
197
198private:
199 std::optional<int> ZT0SaveFI;
200 std::optional<int> TPIDR2BlockFI;
201 Register AgnosticZABufferPtr = Register();
202};
203
204StringRef getZAStateString(ZAState State) {
205#define MAKE_CASE(V) \
206 case V: \
207 return #V;
208 switch (State) {
209 MAKE_CASE(ZAState::ANY)
210 MAKE_CASE(ZAState::ACTIVE)
211 MAKE_CASE(ZAState::ACTIVE_ZT0_SAVED)
212 MAKE_CASE(ZAState::LOCAL_SAVED)
213 MAKE_CASE(ZAState::LOCAL_COMMITTED)
214 MAKE_CASE(ZAState::ENTRY)
215 MAKE_CASE(ZAState::OFF)
216 default:
217 llvm_unreachable("Unexpected ZAState");
218 }
219#undef MAKE_CASE
220}
221
222static bool isZAorZTRegOp(const TargetRegisterInfo &TRI,
223 const MachineOperand &MO) {
224 if (!MO.isReg() || !MO.getReg().isPhysical())
225 return false;
226 return any_of(TRI.subregs_inclusive(MO.getReg()), [](const MCPhysReg &SR) {
227 return AArch64::MPR128RegClass.contains(SR) ||
228 AArch64::ZTRRegClass.contains(SR);
229 });
230}
231
232/// Returns the required ZA state needed before \p MI and an iterator pointing
233/// to where any code required to change the ZA state should be inserted.
234static std::pair<ZAState, MachineBasicBlock::iterator>
235getInstNeededZAState(const TargetRegisterInfo &TRI, MachineInstr &MI,
236 SMEAttrs SMEFnAttrs) {
238
239 // Note: InOutZAUsePseudo, RequiresZASavePseudo, and RequiresZT0SavePseudo are
240 // intended to mark the position immediately before a call. Due to
241 // SelectionDAG constraints, these markers occur after the ADJCALLSTACKDOWN,
242 // so we use std::prev(InsertPt) to get the position before the call.
243
244 if (MI.getOpcode() == AArch64::InOutZAUsePseudo)
245 return {ZAState::ACTIVE, std::prev(InsertPt)};
246
247 // Note: If we need to save both ZA and ZT0 we use RequiresZASavePseudo.
248 if (MI.getOpcode() == AArch64::RequiresZASavePseudo)
249 return {ZAState::LOCAL_SAVED, std::prev(InsertPt)};
250
251 // If we only need to save ZT0 there's two cases to consider:
252 // 1. The function has ZA state (that we don't need to save).
253 // - In this case we switch to the "ACTIVE_ZT0_SAVED" state.
254 // This only saves ZT0.
255 // 2. The function does not have ZA state
256 // - In this case we switch to "LOCAL_COMMITTED" state.
257 // This saves ZT0 and turns ZA off.
258 if (MI.getOpcode() == AArch64::RequiresZT0SavePseudo) {
259 return {SMEFnAttrs.hasZAState() ? ZAState::ACTIVE_ZT0_SAVED
260 : ZAState::LOCAL_COMMITTED,
261 std::prev(InsertPt)};
262 }
263
264 if (MI.isReturn()) {
265 bool ZAOffAtReturn = SMEFnAttrs.hasPrivateZAInterface();
266 return {ZAOffAtReturn ? ZAState::OFF : ZAState::ACTIVE, InsertPt};
267 }
268
269 for (auto &MO : MI.operands()) {
270 if (isZAorZTRegOp(TRI, MO))
271 return {ZAState::ACTIVE, InsertPt};
272 }
273
274 return {ZAState::ANY, InsertPt};
275}
276
277struct MachineSMEABI : public MachineFunctionPass {
278 inline static char ID = 0;
279
280 MachineSMEABI() : MachineFunctionPass(ID) {}
281
282 bool runOnMachineFunction(MachineFunction &MF) override;
283
284 StringRef getPassName() const override { return "Machine SME ABI pass"; }
285
286 void getAnalysisUsage(AnalysisUsage &AU) const override {
287 AU.setPreservesCFG();
294 }
295
296 /// Collects the needed ZA state (and live registers) before each instruction
297 /// within the machine function.
298 FunctionInfo collectNeededZAStates(SMEAttrs SMEFnAttrs);
299
300 /// Assigns each edge bundle a ZA state based on the desired states of
301 /// incoming and outgoing blocks in the bundle.
302 SmallVector<ZAState> assignBundleZAStates(const EdgeBundles &Bundles,
303 const FunctionInfo &FnInfo);
304
305 /// Inserts code to handle changes between ZA states within the function.
306 /// E.g., ACTIVE -> LOCAL_SAVED will insert code required to save ZA.
307 void insertStateChanges(EmitContext &, const FunctionInfo &FnInfo,
308 const EdgeBundles &Bundles,
309 ArrayRef<ZAState> BundleStates);
310
311 void addSMELibCall(MachineInstrBuilder &MIB, RTLIB::Libcall LC,
312 CallingConv::ID ExpectedCC);
313
314 void emitZT0SaveRestore(EmitContext &, MachineBasicBlock &MBB,
315 MachineBasicBlock::iterator MBBI, bool IsSave);
316
317 // Emission routines for private and shared ZA functions (using lazy saves).
318 void emitSMEPrologue(MachineBasicBlock &MBB,
320 void emitRestoreLazySave(EmitContext &, MachineBasicBlock &MBB,
322 LiveRegs PhysLiveRegs);
323 void emitSetupLazySave(EmitContext &, MachineBasicBlock &MBB,
325 void emitAllocateLazySaveBuffer(EmitContext &, MachineBasicBlock &MBB,
328 bool ClearTPIDR2, bool On);
329
330 // Emission routines for agnostic ZA functions.
331 void emitSetupFullZASave(MachineBasicBlock &MBB,
333 LiveRegs PhysLiveRegs);
334 // Emit a "full" ZA save or restore. It is "full" in the sense that this
335 // function will emit a call to __arm_sme_save or __arm_sme_restore, which
336 // handles saving and restoring both ZA and ZT0.
337 void emitFullZASaveRestore(EmitContext &, MachineBasicBlock &MBB,
339 LiveRegs PhysLiveRegs, bool IsSave);
340 void emitAllocateFullZASaveBuffer(EmitContext &, MachineBasicBlock &MBB,
342 LiveRegs PhysLiveRegs);
343
344 /// Attempts to find an insertion point before \p Inst where the status flags
345 /// are not live. If \p Inst is `Block.Insts.end()` a point before the end of
346 /// the block is found.
347 std::pair<MachineBasicBlock::iterator, LiveRegs>
348 findStateChangeInsertionPoint(MachineBasicBlock &MBB, const BlockInfo &Block,
350 void emitStateChange(EmitContext &, MachineBasicBlock &MBB,
351 MachineBasicBlock::iterator MBBI, ZAState From,
352 ZAState To, LiveRegs PhysLiveRegs);
353
354 // Helpers for switching between lazy/full ZA save/restore routines.
355 void emitZASave(EmitContext &Context, MachineBasicBlock &MBB,
357 if (AFI->getSMEFnAttrs().hasAgnosticZAInterface())
358 return emitFullZASaveRestore(Context, MBB, MBBI, PhysLiveRegs,
359 /*IsSave=*/true);
360 return emitSetupLazySave(Context, MBB, MBBI);
361 }
362 void emitZARestore(EmitContext &Context, MachineBasicBlock &MBB,
364 if (AFI->getSMEFnAttrs().hasAgnosticZAInterface())
365 return emitFullZASaveRestore(Context, MBB, MBBI, PhysLiveRegs,
366 /*IsSave=*/false);
367 return emitRestoreLazySave(Context, MBB, MBBI, PhysLiveRegs);
368 }
369 void emitAllocateZASaveBuffer(EmitContext &Context, MachineBasicBlock &MBB,
371 LiveRegs PhysLiveRegs) {
372 if (AFI->getSMEFnAttrs().hasAgnosticZAInterface())
373 return emitAllocateFullZASaveBuffer(Context, MBB, MBBI, PhysLiveRegs);
374 return emitAllocateLazySaveBuffer(Context, MBB, MBBI);
375 }
376
377 /// Collects the reachable calls from \p MBBI marked with \p Marker. This is
378 /// intended to be used to emit lazy save remarks. Note: This stops at the
379 /// first marked call along any path.
380 void collectReachableMarkedCalls(const MachineBasicBlock &MBB,
383 unsigned Marker) const;
384
385 void emitCallSaveRemarks(const MachineBasicBlock &MBB,
387 unsigned Marker, StringRef RemarkName,
388 StringRef SaveName) const;
389
390 void emitError(const Twine &Message) {
391 LLVMContext &Context = MF->getFunction().getContext();
392 Context.emitError(MF->getName() + ": " + Message);
393 }
394
395 /// Save live physical registers to virtual registers.
396 PhysRegSave createPhysRegSave(LiveRegs PhysLiveRegs, MachineBasicBlock &MBB,
398 /// Restore physical registers from a save of their previous values.
399 void restorePhyRegSave(const PhysRegSave &RegSave, MachineBasicBlock &MBB,
401
402private:
403 MachineFunction *MF = nullptr;
404 const AArch64Subtarget *Subtarget = nullptr;
405 const AArch64RegisterInfo *TRI = nullptr;
406 const AArch64FunctionInfo *AFI = nullptr;
407 const AArch64InstrInfo *TII = nullptr;
408 const LibcallLoweringInfo *LLI = nullptr;
409
411 MachineRegisterInfo *MRI = nullptr;
412};
413
414static LiveRegs getPhysLiveRegs(LiveRegUnits const &LiveUnits) {
415 LiveRegs PhysLiveRegs = LiveRegs::None;
416 if (!LiveUnits.available(AArch64::NZCV))
417 PhysLiveRegs |= LiveRegs::NZCV;
418 // We have to track W0 and X0 separately as otherwise things can get
419 // confused if we attempt to preserve X0 but only W0 was defined.
420 if (!LiveUnits.available(AArch64::W0))
421 PhysLiveRegs |= LiveRegs::W0;
422 if (!LiveUnits.available(AArch64::W0_HI))
423 PhysLiveRegs |= LiveRegs::W0_HI;
424 return PhysLiveRegs;
425}
426
427static void setPhysLiveRegs(LiveRegUnits &LiveUnits, LiveRegs PhysLiveRegs) {
428 if (PhysLiveRegs & LiveRegs::NZCV)
429 LiveUnits.addReg(AArch64::NZCV);
430 if (PhysLiveRegs & LiveRegs::W0)
431 LiveUnits.addReg(AArch64::W0);
432 if (PhysLiveRegs & LiveRegs::W0_HI)
433 LiveUnits.addReg(AArch64::W0_HI);
434}
435
436[[maybe_unused]] bool isCallStartOpcode(unsigned Opc) {
437 switch (Opc) {
438 case AArch64::BLR:
439 case AArch64::BLRA:
440 case AArch64::TLSDESC_CALLSEQ:
441 case AArch64::TLSDESC_AUTH_CALLSEQ:
442 case AArch64::ADJCALLSTACKDOWN:
443 return true;
444 default:
445 return false;
446 }
447}
448
449FunctionInfo MachineSMEABI::collectNeededZAStates(SMEAttrs SMEFnAttrs) {
450 assert((SMEFnAttrs.hasAgnosticZAInterface() || SMEFnAttrs.hasZT0State() ||
451 SMEFnAttrs.hasZAState()) &&
452 "Expected function to have ZA/ZT0 state!");
453
455 LiveRegs PhysLiveRegsAfterSMEPrologue = LiveRegs::None;
456 std::optional<MachineBasicBlock::iterator> AfterSMEProloguePt;
457
458 for (MachineBasicBlock &MBB : *MF) {
459 BlockInfo &Block = Blocks[MBB.getNumber()];
460
461 if (MBB.isEntryBlock()) {
462 // Entry block:
463 Block.FixedEntryState = ZAState::ENTRY;
464 } else if (MBB.isEHPad()) {
465 // EH entry block:
466 Block.FixedEntryState = ZAState::LOCAL_COMMITTED;
467 }
468
469 LiveRegUnits LiveUnits(*TRI);
470 LiveUnits.addLiveOuts(MBB);
471
472 Block.PhysLiveRegsAtExit = getPhysLiveRegs(LiveUnits);
473 auto FirstTerminatorInsertPt = MBB.getFirstTerminator();
474 auto FirstNonPhiInsertPt = MBB.getFirstNonPHI();
475 for (MachineInstr &MI : reverse(MBB)) {
476 if (MI.isDebugInstr())
477 continue;
478
480 LiveUnits.stepBackward(MI);
481 LiveRegs PhysLiveRegs = getPhysLiveRegs(LiveUnits);
482 // The SMEStateAllocPseudo marker is added to a function if the save
483 // buffer was allocated in SelectionDAG. It marks the end of the
484 // allocation -- which is a safe point for this pass to insert any TPIDR2
485 // block setup.
486 if (MI.getOpcode() == AArch64::SMEStateAllocPseudo) {
487 AfterSMEProloguePt = MBBI;
488 PhysLiveRegsAfterSMEPrologue = PhysLiveRegs;
489 }
490 // Note: We treat Agnostic ZA as inout_za with an alternate save/restore.
491 auto [NeededState, InsertPt] = getInstNeededZAState(*TRI, MI, SMEFnAttrs);
492 assert((InsertPt == MBBI || isCallStartOpcode(InsertPt->getOpcode())) &&
493 "Unexpected state change insertion point!");
494 if (MBBI == FirstTerminatorInsertPt)
495 Block.PhysLiveRegsAtExit = PhysLiveRegs;
496 if (MBBI == FirstNonPhiInsertPt)
497 Block.PhysLiveRegsAtEntry = PhysLiveRegs;
498 if (NeededState != ZAState::ANY)
499 Block.Insts.push_back({NeededState, InsertPt, PhysLiveRegs});
500 }
501
502 // Reverse vector (as we had to iterate backwards for liveness).
503 std::reverse(Block.Insts.begin(), Block.Insts.end());
504
505 // Record the desired states on entry/exit of this block. These are the
506 // states that would not incur a state transition.
507 if (!Block.Insts.empty()) {
508 Block.DesiredIncomingState = Block.Insts.front().NeededState;
509 Block.DesiredOutgoingState = Block.Insts.back().NeededState;
510 }
511 }
512
513 return FunctionInfo{std::move(Blocks), AfterSMEProloguePt,
514 PhysLiveRegsAfterSMEPrologue};
515}
516
517/// Assigns each edge bundle a ZA state based on the desired states of incoming
518/// and outgoing blocks in the bundle.
520MachineSMEABI::assignBundleZAStates(const EdgeBundles &Bundles,
521 const FunctionInfo &FnInfo) {
522 SmallVector<ZAState> BundleStates(Bundles.getNumBundles());
523 for (unsigned I = 0, E = Bundles.getNumBundles(); I != E; ++I) {
524 std::optional<ZAState> BundleState;
525 for (unsigned BlockID : Bundles.getBlocks(I)) {
526 const BlockInfo &Block = FnInfo.Blocks[BlockID];
527 // Check if the block is an incoming block in the bundle. Note: We skip
528 // Block.FixedEntryState != ANY to ignore EH pads (which are only
529 // reachable via exceptions).
530 if (Block.FixedEntryState != ZAState::ANY ||
531 Bundles.getBundle(BlockID, /*Out=*/false) != I)
532 continue;
533
534 // Pick a state that matches all incoming blocks. Fall back to "ACTIVE" if
535 // any incoming state doesn't match. This will hoist the state from
536 // incoming blocks to outgoing blocks.
537 if (!BundleState)
538 BundleState = Block.DesiredIncomingState;
539 else if (BundleState != Block.DesiredIncomingState)
540 BundleState = ZAState::ACTIVE;
541 }
542
543 if (!BundleState || BundleState == ZAState::ANY)
544 BundleState = ZAState::ACTIVE;
545
546 BundleStates[I] = *BundleState;
547 }
548
549 return BundleStates;
550}
551
552std::pair<MachineBasicBlock::iterator, LiveRegs>
553MachineSMEABI::findStateChangeInsertionPoint(
554 MachineBasicBlock &MBB, const BlockInfo &Block,
556 LiveRegs PhysLiveRegs;
558 if (Inst != Block.Insts.end()) {
559 InsertPt = Inst->InsertPt;
560 PhysLiveRegs = Inst->PhysLiveRegs;
561 } else {
562 InsertPt = MBB.getFirstTerminator();
563 PhysLiveRegs = Block.PhysLiveRegsAtExit;
564 }
565
566 if (PhysLiveRegs == LiveRegs::None)
567 return {InsertPt, PhysLiveRegs}; // Nothing to do (no live regs).
568
569 // Find the previous state change. We can not move before this point.
570 MachineBasicBlock::iterator PrevStateChangeI;
571 if (Inst == Block.Insts.begin()) {
572 PrevStateChangeI = MBB.begin();
573 } else {
574 // Note: `std::prev(Inst)` is the previous InstInfo. We only create an
575 // InstInfo object for instructions that require a specific ZA state, so the
576 // InstInfo is the site of the previous state change in the block (which can
577 // be several MIs earlier).
578 PrevStateChangeI = std::prev(Inst)->InsertPt;
579 }
580
581 // Note: LiveUnits will only accurately track X0 and NZCV.
582 LiveRegUnits LiveUnits(*TRI);
583 setPhysLiveRegs(LiveUnits, PhysLiveRegs);
584 auto BestCandidate = std::make_pair(InsertPt, PhysLiveRegs);
585 for (MachineBasicBlock::iterator I = InsertPt; I != PrevStateChangeI; --I) {
586 if (I->isDebugInstr())
587 continue;
588
589 // Don't move before/into a call (which may have a state change before it).
590 if (I->getOpcode() == TII->getCallFrameDestroyOpcode() || I->isCall())
591 break;
592 LiveUnits.stepBackward(*I);
593 LiveRegs CurrentPhysLiveRegs = getPhysLiveRegs(LiveUnits);
594 // Find places where NZCV is available, but keep looking for locations where
595 // both NZCV and X0 are available, which can avoid some copies.
596 if (!(CurrentPhysLiveRegs & LiveRegs::NZCV))
597 BestCandidate = {I, CurrentPhysLiveRegs};
598 if (CurrentPhysLiveRegs == LiveRegs::None)
599 break;
600 }
601 return BestCandidate;
602}
603
604void MachineSMEABI::insertStateChanges(EmitContext &Context,
605 const FunctionInfo &FnInfo,
606 const EdgeBundles &Bundles,
607 ArrayRef<ZAState> BundleStates) {
608 for (MachineBasicBlock &MBB : *MF) {
609 const BlockInfo &Block = FnInfo.Blocks[MBB.getNumber()];
610 ZAState InState = BundleStates[Bundles.getBundle(MBB.getNumber(),
611 /*Out=*/false)];
612
613 ZAState CurrentState = Block.FixedEntryState;
614 if (CurrentState == ZAState::ANY)
615 CurrentState = InState;
616
617 for (auto &Inst : Block.Insts) {
618 if (CurrentState != Inst.NeededState) {
619 auto [InsertPt, PhysLiveRegs] =
620 findStateChangeInsertionPoint(MBB, Block, &Inst);
621 emitStateChange(Context, MBB, InsertPt, CurrentState, Inst.NeededState,
622 PhysLiveRegs);
623 CurrentState = Inst.NeededState;
624 }
625 }
626
627 if (MBB.succ_empty())
628 continue;
629
630 ZAState OutState =
631 BundleStates[Bundles.getBundle(MBB.getNumber(), /*Out=*/true)];
632 if (CurrentState != OutState) {
633 auto [InsertPt, PhysLiveRegs] =
634 findStateChangeInsertionPoint(MBB, Block, Block.Insts.end());
635 emitStateChange(Context, MBB, InsertPt, CurrentState, OutState,
636 PhysLiveRegs);
637 }
638 }
639}
640
643 if (MBB.empty())
644 return DebugLoc();
645 return MBBI != MBB.end() ? MBBI->getDebugLoc() : MBB.back().getDebugLoc();
646}
647
648/// Finds the first call (as determined by MachineInstr::isCall()) starting from
649/// \p MBBI in \p MBB marked with \p Marker (which is a marker opcode such as
650/// RequiresZASavePseudo). If a marked call is found, it is pushed to \p Calls
651/// and the function returns true.
652static bool findMarkedCall(const MachineBasicBlock &MBB,
655 unsigned Marker, unsigned CallDestroyOpcode) {
656 auto IsMarker = [&](auto &MI) { return MI.getOpcode() == Marker; };
657 auto MarkerInst = std::find_if(MBBI, MBB.end(), IsMarker);
658 if (MarkerInst == MBB.end())
659 return false;
661 while (++I != MBB.end()) {
662 if (I->isCall() || I->getOpcode() == CallDestroyOpcode)
663 break;
664 }
665 if (I != MBB.end() && I->isCall())
666 Calls.push_back(&*I);
667 // Note: This function always returns true if a "Marker" was found.
668 return true;
669}
670
671void MachineSMEABI::collectReachableMarkedCalls(
672 const MachineBasicBlock &StartMBB,
674 SmallVectorImpl<const MachineInstr *> &Calls, unsigned Marker) const {
675 assert(Marker == AArch64::InOutZAUsePseudo ||
676 Marker == AArch64::RequiresZASavePseudo ||
677 Marker == AArch64::RequiresZT0SavePseudo);
678 unsigned CallDestroyOpcode = TII->getCallFrameDestroyOpcode();
679 if (findMarkedCall(StartMBB, StartInst, Calls, Marker, CallDestroyOpcode))
680 return;
681
684 StartMBB.succ_rend());
685 while (!Worklist.empty()) {
686 const MachineBasicBlock *MBB = Worklist.pop_back_val();
687 auto [_, Inserted] = Visited.insert(MBB);
688 if (!Inserted)
689 continue;
690
691 if (!findMarkedCall(*MBB, MBB->begin(), Calls, Marker, CallDestroyOpcode))
692 Worklist.append(MBB->succ_rbegin(), MBB->succ_rend());
693 }
694}
695
696static StringRef getCalleeName(const MachineInstr &CallInst) {
697 assert(CallInst.isCall() && "expected a call");
698 for (const MachineOperand &MO : CallInst.operands()) {
699 if (MO.isSymbol())
700 return MO.getSymbolName();
701 if (MO.isGlobal())
702 return MO.getGlobal()->getName();
703 }
704 return {};
705}
706
707void MachineSMEABI::emitCallSaveRemarks(const MachineBasicBlock &MBB,
709 DebugLoc DL, unsigned Marker,
710 StringRef RemarkName,
711 StringRef SaveName) const {
712 auto SaveRemark = [&](DebugLoc DL, const MachineBasicBlock &MBB) {
713 return MachineOptimizationRemarkAnalysis("sme", RemarkName, DL, &MBB);
714 };
715 StringRef StateName = Marker == AArch64::RequiresZT0SavePseudo ? "ZT0" : "ZA";
716 ORE->emit([&] {
717 return SaveRemark(DL, MBB) << SaveName << " of " << StateName
718 << " emitted in '" << MF->getName() << "'";
719 });
720 if (!ORE->allowExtraAnalysis("sme"))
721 return;
722 SmallVector<const MachineInstr *> CallsRequiringSaves;
723 collectReachableMarkedCalls(MBB, MBBI, CallsRequiringSaves, Marker);
724 for (const MachineInstr *CallInst : CallsRequiringSaves) {
725 auto R = SaveRemark(CallInst->getDebugLoc(), *CallInst->getParent());
726 R << "call";
727 if (StringRef CalleeName = getCalleeName(*CallInst); !CalleeName.empty())
728 R << " to '" << CalleeName << "'";
729 R << " requires " << StateName << " save";
730 ORE->emit(R);
731 }
732}
733
734void MachineSMEABI::emitSetupLazySave(EmitContext &Context,
738
739 emitCallSaveRemarks(MBB, MBBI, DL, AArch64::RequiresZASavePseudo,
740 "SMELazySaveZA", "lazy save");
741
742 // Get pointer to TPIDR2 block.
743 Register TPIDR2 = MRI->createVirtualRegister(&AArch64::GPR64spRegClass);
744 Register TPIDR2Ptr = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
745 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXri), TPIDR2)
746 .addFrameIndex(Context.getTPIDR2Block(*MF))
747 .addImm(0)
748 .addImm(0);
749 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), TPIDR2Ptr)
750 .addReg(TPIDR2);
751 // Set TPIDR2_EL0 to point to TPIDR2 block.
752 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MSR))
753 .addImm(AArch64SysReg::TPIDR2_EL0)
754 .addReg(TPIDR2Ptr);
755}
756
757PhysRegSave MachineSMEABI::createPhysRegSave(LiveRegs PhysLiveRegs,
760 DebugLoc DL) {
761 PhysRegSave RegSave{PhysLiveRegs};
762 if (PhysLiveRegs & LiveRegs::NZCV) {
763 RegSave.StatusFlags = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
764 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MRS), RegSave.StatusFlags)
765 .addImm(AArch64SysReg::NZCV)
766 .addReg(AArch64::NZCV, RegState::Implicit)
767 .setOperandDead(2); // implicit-def $nzcv
768 }
769 // Note: Preserving X0 is "free" as this is before register allocation, so
770 // the register allocator is still able to optimize these copies.
771 if (PhysLiveRegs & LiveRegs::W0) {
772 RegSave.X0Save = MRI->createVirtualRegister(PhysLiveRegs & LiveRegs::W0_HI
773 ? &AArch64::GPR64RegClass
774 : &AArch64::GPR32RegClass);
775 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), RegSave.X0Save)
776 .addReg(PhysLiveRegs & LiveRegs::W0_HI ? AArch64::X0 : AArch64::W0);
777 }
778 return RegSave;
779}
780
781void MachineSMEABI::restorePhyRegSave(const PhysRegSave &RegSave,
784 DebugLoc DL) {
785 if (RegSave.StatusFlags.isValid())
786 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MSR))
787 .addImm(AArch64SysReg::NZCV)
788 .addReg(RegSave.StatusFlags)
789 .addReg(AArch64::NZCV, RegState::ImplicitDefine);
790
791 if (RegSave.X0Save.isValid())
792 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY),
793 RegSave.PhysLiveRegs & LiveRegs::W0_HI ? AArch64::X0 : AArch64::W0)
794 .addReg(RegSave.X0Save);
795}
796
797void MachineSMEABI::addSMELibCall(MachineInstrBuilder &MIB, RTLIB::Libcall LC,
798 CallingConv::ID ExpectedCC) {
799 RTLIB::LibcallImpl LCImpl = LLI->getLibcallImpl(LC);
800 if (LCImpl == RTLIB::Unsupported)
801 emitError("cannot lower SME ABI (SME routines unsupported)");
804 if (CC != ExpectedCC)
805 emitError("invalid calling convention for SME routine: '" + ImplName + "'");
806 // FIXME: This assumes the ImplName StringRef is null-terminated.
807 MIB.addExternalSymbol(ImplName.data());
808 MIB.addRegMask(TRI->getCallPreservedMask(*MF, CC));
809}
810
811void MachineSMEABI::emitRestoreLazySave(EmitContext &Context,
814 LiveRegs PhysLiveRegs) {
816 Register TPIDR2EL0 = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
817 Register TPIDR2 = AArch64::X0;
818
819 // TODO: Emit these within the restore MBB to prevent unnecessary saves.
820 PhysRegSave RegSave = createPhysRegSave(PhysLiveRegs, MBB, MBBI, DL);
821
822 // Enable ZA.
823 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MSRpstatesvcrImm1))
824 .addImm(AArch64SVCR::SVCRZA)
825 .addImm(1);
826 // Get current TPIDR2_EL0.
827 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MRS), TPIDR2EL0)
828 .addImm(AArch64SysReg::TPIDR2_EL0)
829 .setOperandDead(2); // implicit-def $nzcv
830 // Get pointer to TPIDR2 block.
831 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXri), TPIDR2)
832 .addFrameIndex(Context.getTPIDR2Block(*MF))
833 .addImm(0)
834 .addImm(0);
835 // (Conditionally) restore ZA state.
836 auto RestoreZA = BuildMI(MBB, MBBI, DL, TII->get(AArch64::RestoreZAPseudo))
837 .addReg(TPIDR2EL0)
838 .addReg(TPIDR2);
839 addSMELibCall(
840 RestoreZA, RTLIB::SMEABI_TPIDR2_RESTORE,
842 // Zero TPIDR2_EL0.
843 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MSR))
844 .addImm(AArch64SysReg::TPIDR2_EL0)
845 .addReg(AArch64::XZR);
846
847 restorePhyRegSave(RegSave, MBB, MBBI, DL);
848}
849
850void MachineSMEABI::emitZAMode(MachineBasicBlock &MBB,
852 bool ClearTPIDR2, bool On) {
854
855 if (ClearTPIDR2)
856 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MSR))
857 .addImm(AArch64SysReg::TPIDR2_EL0)
858 .addReg(AArch64::XZR);
859
860 // Disable ZA.
861 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MSRpstatesvcrImm1))
862 .addImm(AArch64SVCR::SVCRZA)
863 .addImm(On ? 1 : 0);
864}
865
866void MachineSMEABI::emitAllocateLazySaveBuffer(
867 EmitContext &Context, MachineBasicBlock &MBB,
869 MachineFrameInfo &MFI = MF->getFrameInfo();
871 Register SP = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
872 Register SVL = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
873 Register Buffer = AFI->getEarlyAllocSMESaveBuffer();
874
875 // Calculate SVL.
876 BuildMI(MBB, MBBI, DL, TII->get(AArch64::RDSVLI_XI), SVL).addImm(1);
877
878 // 1. Allocate the lazy save buffer.
879 if (!Buffer.isValid()) {
880 // TODO: On Windows, we allocate the lazy save buffer in SelectionDAG (so
881 // Buffer is valid). This is done to reuse the existing
882 // expansions (which can insert stack checks). This works, but it means we
883 // will always allocate the lazy save buffer (even if the function contains
884 // no lazy saves). If we want to handle Windows here, we'll need to
885 // implement something similar to LowerWindowsDYNAMIC_STACKALLOC.
886 assert(!Subtarget->isTargetWindowsOrUEFI() &&
887 "Lazy ZA save is not yet supported on Windows or UEFI");
888 Buffer = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
889 // Get original stack pointer.
890 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), SP)
891 .addReg(AArch64::SP);
892 // Allocate a lazy-save buffer object of the size given, normally SVL * SVL
893 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MSUBXrrr), Buffer)
894 .addReg(SVL)
895 .addReg(SVL)
896 .addReg(SP);
897 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), AArch64::SP)
898 .addReg(Buffer);
899 // We have just allocated a variable sized object, tell this to PEI.
900 MFI.CreateVariableSizedObject(Align(16), nullptr);
901 }
902
903 // 2. Setup the TPIDR2 block.
904 {
905 // Note: This case just needs to do `SVL << 48`. It is not implemented as we
906 // generally don't support big-endian SVE/SME.
907 if (!Subtarget->isLittleEndian())
909 "TPIDR2 block initialization is not supported on big-endian targets");
910
911 // Store buffer pointer and num_za_save_slices.
912 // Bytes 10-15 are implicitly zeroed.
913 BuildMI(MBB, MBBI, DL, TII->get(AArch64::STPXi))
914 .addReg(Buffer)
915 .addReg(SVL)
916 .addFrameIndex(Context.getTPIDR2Block(*MF))
917 .addImm(0);
918 }
919}
920
921static constexpr unsigned ZERO_ALL_ZA_MASK = 0b11111111;
922
923void MachineSMEABI::emitSMEPrologue(MachineBasicBlock &MBB,
926
927 bool ZeroZA = AFI->getSMEFnAttrs().isNewZA();
928 bool ZeroZT0 = AFI->getSMEFnAttrs().isNewZT0();
930 // Get current TPIDR2_EL0.
931 Register TPIDR2EL0 = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
932 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MRS))
933 .addReg(TPIDR2EL0, RegState::Define)
934 .addImm(AArch64SysReg::TPIDR2_EL0)
935 .setOperandDead(2); // implicit-def $nzcv
936 // If TPIDR2_EL0 is non-zero, commit the lazy save.
937 // NOTE: Functions that only use ZT0 don't need to zero ZA.
938 auto CommitZASave =
939 BuildMI(MBB, MBBI, DL, TII->get(AArch64::CommitZASavePseudo))
940 .addReg(TPIDR2EL0)
941 .addImm(ZeroZA)
942 .addImm(ZeroZT0);
943 addSMELibCall(
944 CommitZASave, RTLIB::SMEABI_TPIDR2_SAVE,
946 if (ZeroZA)
947 CommitZASave.addDef(AArch64::ZAB0, RegState::ImplicitDefine);
948 if (ZeroZT0)
949 CommitZASave.addDef(AArch64::ZT0, RegState::ImplicitDefine);
950 // Enable ZA (as ZA could have previously been in the OFF state).
951 BuildMI(MBB, MBBI, DL, TII->get(AArch64::MSRpstatesvcrImm1))
952 .addImm(AArch64SVCR::SVCRZA)
953 .addImm(1);
954 } else if (AFI->getSMEFnAttrs().hasSharedZAInterface()) {
955 if (ZeroZA)
956 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ZERO_M))
958 .addDef(AArch64::ZAB0, RegState::ImplicitDefine);
959 if (ZeroZT0)
960 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ZERO_T)).addDef(AArch64::ZT0);
961 }
962}
963
964void MachineSMEABI::emitFullZASaveRestore(EmitContext &Context,
967 LiveRegs PhysLiveRegs, bool IsSave) {
969
970 if (IsSave)
971 emitCallSaveRemarks(MBB, MBBI, DL, AArch64::RequiresZASavePseudo,
972 "SMEFullZASave", "full save");
973
974 PhysRegSave RegSave = createPhysRegSave(PhysLiveRegs, MBB, MBBI, DL);
975
976 // Copy the buffer pointer into X0.
977 Register BufferPtr = AArch64::X0;
978 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), BufferPtr)
979 .addReg(Context.getAgnosticZABufferPtr(*MF));
980
981 // Call __arm_sme_save/__arm_sme_restore.
982 auto SaveRestoreZA = BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
983 .addReg(BufferPtr, RegState::Implicit);
984 addSMELibCall(
985 SaveRestoreZA,
986 IsSave ? RTLIB::SMEABI_SME_SAVE : RTLIB::SMEABI_SME_RESTORE,
988
989 restorePhyRegSave(RegSave, MBB, MBBI, DL);
990}
991
992void MachineSMEABI::emitZT0SaveRestore(EmitContext &Context,
995 bool IsSave) {
997
998 // Note: This will report calls that _only_ need ZT0 saved. Call that save
999 // both ZA and ZT0 will be under the SMELazySaveZA remark. This prevents
1000 // reporting the same calls twice.
1001 if (IsSave)
1002 emitCallSaveRemarks(MBB, MBBI, DL, AArch64::RequiresZT0SavePseudo,
1003 "SMEZT0Save", "spill");
1004
1005 Register ZT0Save = MRI->createVirtualRegister(&AArch64::GPR64spRegClass);
1006
1007 BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXri), ZT0Save)
1008 .addFrameIndex(Context.getZT0SaveSlot(*MF))
1009 .addImm(0)
1010 .addImm(0);
1011
1012 if (IsSave) {
1013 BuildMI(MBB, MBBI, DL, TII->get(AArch64::STR_TX))
1014 .addReg(AArch64::ZT0)
1015 .addReg(ZT0Save);
1016 } else {
1017 BuildMI(MBB, MBBI, DL, TII->get(AArch64::LDR_TX), AArch64::ZT0)
1018 .addReg(ZT0Save);
1019 }
1020}
1021
1022void MachineSMEABI::emitAllocateFullZASaveBuffer(
1023 EmitContext &Context, MachineBasicBlock &MBB,
1025 // Buffer already allocated in SelectionDAG.
1026 if (AFI->getEarlyAllocSMESaveBuffer())
1027 return;
1028
1030 Register BufferPtr = Context.getAgnosticZABufferPtr(*MF);
1031 Register BufferSize = MRI->createVirtualRegister(&AArch64::GPR64RegClass);
1032
1033 PhysRegSave RegSave = createPhysRegSave(PhysLiveRegs, MBB, MBBI, DL);
1034
1035 // Calculate the SME state size.
1036 {
1037 auto SMEStateSize = BuildMI(MBB, MBBI, DL, TII->get(AArch64::BL))
1038 .addReg(AArch64::X0, RegState::ImplicitDefine);
1039 addSMELibCall(
1040 SMEStateSize, RTLIB::SMEABI_SME_STATE_SIZE,
1042 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), BufferSize)
1043 .addReg(AArch64::X0);
1044 }
1045
1046 // Allocate a buffer object of the size given __arm_sme_state_size.
1047 {
1048 MachineFrameInfo &MFI = MF->getFrameInfo();
1049 BuildMI(MBB, MBBI, DL, TII->get(AArch64::SUBXrx64), AArch64::SP)
1050 .addReg(AArch64::SP)
1051 .addReg(BufferSize)
1053 BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::COPY), BufferPtr)
1054 .addReg(AArch64::SP);
1055
1056 // We have just allocated a variable sized object, tell this to PEI.
1057 MFI.CreateVariableSizedObject(Align(16), nullptr);
1058 }
1059
1060 restorePhyRegSave(RegSave, MBB, MBBI, DL);
1061}
1062
1063struct FromState {
1064 ZAState From;
1065
1066 constexpr uint8_t to(ZAState To) const {
1067 static_assert(NUM_ZA_STATE < 16, "expected ZAState to fit in 4-bits");
1068 return uint8_t(From) << 4 | uint8_t(To);
1069 }
1070};
1071
1072constexpr FromState transitionFrom(ZAState From) { return FromState{From}; }
1073
1074void MachineSMEABI::emitStateChange(EmitContext &Context,
1077 ZAState From, ZAState To,
1078 LiveRegs PhysLiveRegs) {
1079 // ZA not used.
1080 if (From == ZAState::ANY || To == ZAState::ANY)
1081 return;
1082
1083 // If we're exiting from the ENTRY state that means that the function has not
1084 // used ZA, so in the case of private ZA/ZT0 functions we can omit any set up.
1085 if (From == ZAState::ENTRY && To == ZAState::OFF)
1086 return;
1087
1088 // TODO: Avoid setting up the save buffer if there's no transition to
1089 // LOCAL_SAVED.
1090 if (From == ZAState::ENTRY) {
1091 assert(&MBB == &MBB.getParent()->front() &&
1092 "ENTRY state only valid in entry block");
1093 emitSMEPrologue(MBB, MBB.getFirstNonPHI());
1094 if (To == ZAState::ACTIVE)
1095 return; // Nothing more to do (ZA is active after the prologue).
1096
1097 // Note: "emitNewZAPrologue" zeros ZA, so we may need to setup a lazy save
1098 // if "To" is "ZAState::LOCAL_SAVED". It may be possible to improve this
1099 // case by changing the placement of the zero instruction.
1100 From = ZAState::ACTIVE;
1101 }
1102
1103 SMEAttrs SMEFnAttrs = AFI->getSMEFnAttrs();
1104 bool IsAgnosticZA = SMEFnAttrs.hasAgnosticZAInterface();
1105 bool HasZT0State = SMEFnAttrs.hasZT0State();
1106 bool HasZAState = IsAgnosticZA || SMEFnAttrs.hasZAState();
1107
1108 switch (transitionFrom(From).to(To)) {
1109 // This section handles: ACTIVE <-> ACTIVE_ZT0_SAVED
1110 case transitionFrom(ZAState::ACTIVE).to(ZAState::ACTIVE_ZT0_SAVED):
1111 emitZT0SaveRestore(Context, MBB, InsertPt, /*IsSave=*/true);
1112 break;
1113 case transitionFrom(ZAState::ACTIVE_ZT0_SAVED).to(ZAState::ACTIVE):
1114 emitZT0SaveRestore(Context, MBB, InsertPt, /*IsSave=*/false);
1115 break;
1116
1117 // This section handles: ACTIVE[_ZT0_SAVED] -> LOCAL_SAVED
1118 case transitionFrom(ZAState::ACTIVE).to(ZAState::LOCAL_SAVED):
1119 case transitionFrom(ZAState::ACTIVE_ZT0_SAVED).to(ZAState::LOCAL_SAVED):
1120 if (HasZT0State && From == ZAState::ACTIVE)
1121 emitZT0SaveRestore(Context, MBB, InsertPt, /*IsSave=*/true);
1122 if (HasZAState)
1123 emitZASave(Context, MBB, InsertPt, PhysLiveRegs);
1124 break;
1125
1126 // This section handles: ACTIVE -> LOCAL_COMMITTED
1127 case transitionFrom(ZAState::ACTIVE).to(ZAState::LOCAL_COMMITTED):
1128 // TODO: We could support ZA state here, but this transition is currently
1129 // only possible when we _don't_ have ZA state.
1130 assert(HasZT0State && !HasZAState && "Expect to only have ZT0 state.");
1131 emitZT0SaveRestore(Context, MBB, InsertPt, /*IsSave=*/true);
1132 emitZAMode(MBB, InsertPt, /*ClearTPIDR2=*/false, /*On=*/false);
1133 break;
1134
1135 // This section handles: LOCAL_COMMITTED -> (OFF|LOCAL_SAVED)
1136 case transitionFrom(ZAState::LOCAL_COMMITTED).to(ZAState::OFF):
1137 case transitionFrom(ZAState::LOCAL_COMMITTED).to(ZAState::LOCAL_SAVED):
1138 // These transitions are a no-op.
1139 break;
1140
1141 // This section handles: LOCAL_(SAVED|COMMITTED) -> ACTIVE[_ZT0_SAVED]
1142 case transitionFrom(ZAState::LOCAL_COMMITTED).to(ZAState::ACTIVE):
1143 case transitionFrom(ZAState::LOCAL_COMMITTED).to(ZAState::ACTIVE_ZT0_SAVED):
1144 case transitionFrom(ZAState::LOCAL_SAVED).to(ZAState::ACTIVE):
1145 case transitionFrom(ZAState::LOCAL_SAVED).to(ZAState::ACTIVE_ZT0_SAVED):
1146 if (HasZAState)
1147 emitZARestore(Context, MBB, InsertPt, PhysLiveRegs);
1148 else
1149 emitZAMode(MBB, InsertPt, /*ClearTPIDR2=*/false, /*On=*/true);
1150 if (HasZT0State && To == ZAState::ACTIVE)
1151 emitZT0SaveRestore(Context, MBB, InsertPt, /*IsSave=*/false);
1152 break;
1153
1154 // This section handles transitions to OFF (not previously covered)
1155 case transitionFrom(ZAState::ACTIVE).to(ZAState::OFF):
1156 case transitionFrom(ZAState::ACTIVE_ZT0_SAVED).to(ZAState::OFF):
1157 case transitionFrom(ZAState::LOCAL_SAVED).to(ZAState::OFF):
1158 assert(SMEFnAttrs.hasPrivateZAInterface() &&
1159 "Did not expect to turn ZA off in shared/agnostic ZA function");
1160 emitZAMode(MBB, InsertPt, /*ClearTPIDR2=*/From == ZAState::LOCAL_SAVED,
1161 /*On=*/false);
1162 break;
1163
1164 default:
1165 dbgs() << "Error: Transition from " << getZAStateString(From) << " to "
1166 << getZAStateString(To) << '\n';
1167 llvm_unreachable("Unimplemented state transition");
1168 }
1169}
1170
1171/// Returns true if private ZA setup can be elided. This occurs when there is
1172/// no instruction within the function that requires ZA to be active.
1173static bool canElidePrivateZASetup(const FunctionInfo &FnInfo) {
1174 for (const BlockInfo &BlockInfo : FnInfo.Blocks) {
1175 for (const InstInfo &InstInfo : BlockInfo.Insts) {
1176 if (InstInfo.NeededState == ZAState::ACTIVE ||
1177 InstInfo.NeededState == ZAState::ACTIVE_ZT0_SAVED)
1178 return false;
1179 }
1180 }
1181 return true;
1182}
1183
1184} // end anonymous namespace
1185
1186INITIALIZE_PASS(MachineSMEABI, "aarch64-machine-sme-abi", "Machine SME ABI",
1187 false, false)
1188
1189bool MachineSMEABI::runOnMachineFunction(MachineFunction &MF) {
1190 AFI = MF.getInfo<AArch64FunctionInfo>();
1191 SMEAttrs SMEFnAttrs = AFI->getSMEFnAttrs();
1192 if (!SMEFnAttrs.hasZAState() && !SMEFnAttrs.hasZT0State() &&
1193 !SMEFnAttrs.hasAgnosticZAInterface())
1194 return false;
1195
1196 Subtarget = &MF.getSubtarget<AArch64Subtarget>();
1197 if (!Subtarget->hasSME() && !SMEFnAttrs.hasAgnosticZAInterface())
1198 return false;
1199
1200 assert(MF.getRegInfo().isSSA() && "Expected to be run on SSA form!");
1201
1202 this->MF = &MF;
1203 ORE = &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE();
1204 LLI = &getAnalysis<LibcallLoweringInfoWrapper>().getLibcallLowering(
1205 *MF.getFunction().getParent(), *Subtarget);
1206 TII = Subtarget->getInstrInfo();
1207 TRI = Subtarget->getRegisterInfo();
1208 MRI = &MF.getRegInfo();
1209
1210 const EdgeBundles &Bundles =
1211 getAnalysis<EdgeBundlesWrapperLegacy>().getEdgeBundles();
1212
1213 FunctionInfo FnInfo = collectNeededZAStates(SMEFnAttrs);
1214
1215 if (SMEFnAttrs.hasPrivateZAInterface() && canElidePrivateZASetup(FnInfo))
1216 return false;
1217
1218 SmallVector<ZAState> BundleStates = assignBundleZAStates(Bundles, FnInfo);
1219
1220 EmitContext Context;
1221 insertStateChanges(Context, FnInfo, Bundles, BundleStates);
1222
1223 if (Context.needsSaveBuffer()) {
1224 if (FnInfo.AfterSMEProloguePt) {
1225 // Note: With inline stack probes the AfterSMEProloguePt may not be in the
1226 // entry block (due to the probing loop).
1227 MachineBasicBlock::iterator MBBI = *FnInfo.AfterSMEProloguePt;
1228 emitAllocateZASaveBuffer(Context, *MBBI->getParent(), MBBI,
1229 FnInfo.PhysLiveRegsAfterSMEPrologue);
1230 } else {
1231 MachineBasicBlock &EntryBlock = MF.front();
1232 emitAllocateZASaveBuffer(
1233 Context, EntryBlock, EntryBlock.getFirstNonPHI(),
1234 FnInfo.Blocks[EntryBlock.getNumber()].PhysLiveRegsAtEntry);
1235 }
1236 }
1237
1238 return true;
1239}
1240
1241FunctionPass *llvm::createMachineSMEABIPass() { return new MachineSMEABI(); }
static constexpr unsigned ZERO_ALL_ZA_MASK
const AArch64Subtarget & Subtarget
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
const HexagonInstrInfo * TII
#define _
IRTranslator LLVM IR MI
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
#define ENTRY(ASMNAME, ENUM)
#define I(x, y, z)
Definition MD5.cpp:57
===- MachineOptimizationRemarkEmitter.h - Opt Diagnostics -*- C++ -*-—===//
#define MAKE_CASE(V)
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
if(PassOpts->AAPipeline)
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition PassSupport.h:56
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
Func MI getDebugLoc()))
This file defines the SmallVector class.
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
Represent the analysis usage information of a pass.
AnalysisUsage & addPreservedID(const void *ID)
AnalysisUsage & addRequired()
LLVM_ABI void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition Pass.cpp:280
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
This class represents a function call, abstracting a target machine's calling convention.
A debug info location.
Definition DebugLoc.h:126
ArrayRef< unsigned > getBlocks(unsigned Bundle) const
getBlocks - Return an array of blocks that are connected to Bundle.
Definition EdgeBundles.h:53
unsigned getBundle(unsigned N, bool Out) const
getBundle - Return the ingoing (Out = false) or outgoing (Out = true) bundle number for basic block N
Definition EdgeBundles.h:47
unsigned getNumBundles() const
getNumBundles - Return the total number of bundles in the CFG.
Definition EdgeBundles.h:50
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:312
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:352
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
Tracks which library functions to use for a particular subtarget or function.
CallingConv::ID getLibcallImplCallingConv(RTLIB::LibcallImpl Call) const
Get the CallingConv that should be used for the specified libcall.
RTLIB::LibcallImpl getLibcallImpl(RTLIB::Libcall Call) const
Return the lowering's selection of implementation call for Call.
A set of register units used to track register liveness.
bool available(MCRegister Reg) const
Returns true if no part of physical register Reg is live.
void addReg(MCRegister Reg)
Adds register units covered by physical register Reg.
LLVM_ABI void stepBackward(const MachineInstr &MI)
Updates liveness when stepping backwards over the instruction MI.
LLVM_ABI void addLiveOuts(const MachineBasicBlock &MBB)
Adds registers living out of block MBB.
MachineInstrBundleIterator< const MachineInstr > const_iterator
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
LLVM_ABI iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
succ_reverse_iterator succ_rbegin()
MachineInstrBundleIterator< MachineInstr > iterator
succ_reverse_iterator succ_rend()
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
LLVM_ABI int CreateSpillStackObject(uint64_t Size, Align Alignment, TargetStackID::Value StackID=TargetStackID::Default)
Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...
LLVM_ABI int CreateVariableSizedObject(Align Alignment, const AllocaInst *Alloca)
Notify the MachineFrameInfo object that a variable sized object has been created.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
unsigned getNumBlockIDs() const
getNumBlockIDs - Return the number of MBB ID's allocated.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
const MachineInstrBuilder & setOperandDead(unsigned OpIdx) const
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
MachineOperand class - Representation of each machine instruction operand.
const GlobalValue * getGlobal() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool isSymbol() const
isSymbol - Tests if this is a MO_ExternalSymbol operand.
bool isGlobal() const
isGlobal - Tests if this is a MO_GlobalAddress operand.
const char * getSymbolName() const
Register getReg() const
getReg - Returns the register number.
Diagnostic information for optimization analysis remarks.
LLVM_ABI void emit(DiagnosticInfoOptimizationBase &OptDiag)
Emit an optimization remark.
bool allowExtraAnalysis(StringRef PassName) const
Whether we allow for extra compile-time budget to perform more analysis to be more informative.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isValid() const
Definition Register.h:112
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition Register.h:83
SMEAttrs is a utility class to parse the SME ACLE attributes on functions.
bool hasAgnosticZAInterface() const
bool hasPrivateZAInterface() const
bool hasSharedZAInterface() const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
typename SuperClass::const_iterator const_iterator
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
constexpr bool empty() const
Check if the string is empty.
Definition StringRef.h:141
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
Definition StringRef.h:138
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
op_range operands()
Definition User.h:267
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
const ParentTy * getParent() const
Definition ilist_node.h:34
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)
getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ AArch64_SME_ABI_Support_Routines_PreserveMost_From_X0
Preserve X0-X13, X19-X29, SP, Z0-Z31, P0-P15.
@ AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1
Preserve X1-X15, X19-X29, SP, Z0-Z31, P0-P15.
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
FunctionPass * createMachineSMEABIPass()
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
LLVM_ABI char & MachineDominatorsID
MachineDominators - This pass is a machine dominators analysis pass.
LLVM_ABI void reportFatalInternalError(Error Err)
Report a fatal error that indicates a bug in LLVM.
Definition Error.cpp:173
LLVM_ABI char & MachineLoopInfoID
MachineLoopInfo - This pass is a loop analysis pass.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition MCRegister.h:21
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
static StringRef getLibcallImplName(RTLIB::LibcallImpl CallImpl)
Get the libcall routine name for the specified libcall implementation.