LLVM 24.0.0git
SMEPeepholeOpt.cpp
Go to the documentation of this file.
1//===- SMEPeepholeOpt.cpp - SME peephole optimization pass-----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// This pass tries to remove back-to-back (smstart, smstop) and
9// (smstop, smstart) sequences. The pass is conservative when it cannot
10// determine that it is safe to remove these sequences.
11//===----------------------------------------------------------------------===//
12
13#include "AArch64InstrInfo.h"
15#include "AArch64Subtarget.h"
20
21using namespace llvm;
22
23#define DEBUG_TYPE "aarch64-sme-peephole-opt"
24
25namespace {
26
27struct SMEPeepholeOpt : public MachineFunctionPass {
28 static char ID;
29
30 SMEPeepholeOpt() : MachineFunctionPass(ID) {}
31
32 bool runOnMachineFunction(MachineFunction &MF) override;
33
34 StringRef getPassName() const override {
35 return "SME Peephole Optimization pass";
36 }
37
38 void getAnalysisUsage(AnalysisUsage &AU) const override {
39 AU.setPreservesCFG();
41 }
42
43 bool optimizeStartStopPairs(MachineBasicBlock &MBB,
44 bool &HasRemovedAllSMChanges) const;
45 bool visitRegSequence(MachineInstr &MI);
46};
47
48char SMEPeepholeOpt::ID = 0;
49
50} // end anonymous namespace
51
53 return MI->getOpcode() == AArch64::MSRpstatePseudo;
54}
55
57 const MachineInstr *MI2) {
58 // We only consider the same type of streaming mode change here, i.e.
59 // start/stop SM, or start/stop ZA pairs.
60 if (MI1->getOperand(0).getImm() != MI2->getOperand(0).getImm())
61 return false;
62
63 // One must be 'start', the other must be 'stop'
64 if (MI1->getOperand(1).getImm() == MI2->getOperand(1).getImm())
65 return false;
66
67 bool IsConditional = isConditionalStartStop(MI2);
68 if (isConditionalStartStop(MI1) != IsConditional)
69 return false;
70
71 if (!IsConditional)
72 return true;
73
74 // Check to make sure the conditional start/stop pairs are identical.
75 if (MI1->getOperand(2).getImm() != MI2->getOperand(2).getImm())
76 return false;
77
78 // Ensure reg masks are identical.
79 if (MI1->getOperand(4).getRegMask() != MI2->getOperand(4).getRegMask())
80 return false;
81
82 // Only consider conditional start/stop pairs which read the same register
83 // holding the original value of pstate.sm. This is somewhat over conservative
84 // as all conditional streaming mode changes only look at the state on entry
85 // to the function.
86 if (MI1->getOperand(3).isReg() && MI2->getOperand(3).isReg()) {
87 Register Reg1 = MI1->getOperand(3).getReg();
88 Register Reg2 = MI2->getOperand(3).getReg();
89 if (Reg1.isPhysical() || Reg2.isPhysical() || Reg1 != Reg2)
90 return false;
91 }
92
93 return true;
94}
95
97 assert((MI->getOpcode() == AArch64::MSRpstatesvcrImm1 ||
98 MI->getOpcode() == AArch64::MSRpstatePseudo) &&
99 "Expected MI to be a smstart/smstop instruction");
100 return MI->getOperand(0).getImm() == AArch64SVCR::SVCRSM ||
101 MI->getOperand(0).getImm() == AArch64SVCR::SVCRSMZA;
102}
103
106 const MachineOperand &MO) {
107 if (!MO.isReg())
108 return false;
109
110 Register R = MO.getReg();
111 if (R.isPhysical())
112 return llvm::any_of(TRI.subregs_inclusive(R), [](const MCPhysReg &SR) {
113 return AArch64::ZPRRegClass.contains(SR) ||
114 AArch64::PPRRegClass.contains(SR);
115 });
116
117 const TargetRegisterClass *RC = MRI.getRegClass(R);
118 return TRI.getCommonSubClass(&AArch64::ZPRRegClass, RC) ||
119 TRI.getCommonSubClass(&AArch64::PPRRegClass, RC);
120}
121
122bool SMEPeepholeOpt::optimizeStartStopPairs(
123 MachineBasicBlock &MBB, bool &HasRemovedAllSMChanges) const {
124 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
125 const TargetRegisterInfo &TRI =
127
128 bool Changed = false;
129 MachineInstr *Prev = nullptr;
130
131 // Walk through instructions in the block trying to find pairs of smstart
132 // and smstop nodes that cancel each other out. We only permit a limited
133 // set of instructions to appear between them, otherwise we reset our
134 // tracking.
135 unsigned NumSMChanges = 0;
136 unsigned NumSMChangesRemoved = 0;
137 for (MachineInstr &MI : make_early_inc_range(MBB)) {
138 switch (MI.getOpcode()) {
139 case AArch64::MSRpstatesvcrImm1:
140 case AArch64::MSRpstatePseudo: {
142 NumSMChanges++;
143
144 if (!Prev)
145 Prev = &MI;
146 else if (isMatchingStartStopPair(Prev, &MI)) {
147 // If they match, we can remove them, and possibly any instructions
148 // that we marked for deletion in between.
149 Prev->eraseFromParent();
150 MI.eraseFromParent();
151 Prev = nullptr;
152 Changed = true;
153 NumSMChangesRemoved += 2;
154 } else {
155 Prev = &MI;
156 }
157 continue;
158 }
159 default:
160 if (!Prev)
161 // Avoid doing expensive checks when Prev is nullptr.
162 continue;
163 break;
164 }
165
166 // Debug instructions are not executed and are agnostic of streaming mode.
167 if (MI.isDebugInstr())
168 continue;
169
170 // Test if the instructions in between the start/stop sequence are agnostic
171 // of streaming mode. If not, the algorithm should reset.
172 switch (MI.getOpcode()) {
173 default:
174 Prev = nullptr;
175 break;
176 case AArch64::COALESCER_BARRIER_FPR16:
177 case AArch64::COALESCER_BARRIER_FPR32:
178 case AArch64::COALESCER_BARRIER_FPR64:
179 case AArch64::COALESCER_BARRIER_FPR128:
180 case AArch64::COPY:
181 // These instructions should be safe when executed on their own, but
182 // the code remains conservative when SVE registers are used. There may
183 // exist subtle cases where executing a COPY in a different mode results
184 // in different behaviour, even if we can't yet come up with any
185 // concrete example/test-case.
186 if (isSVERegOp(TRI, MRI, MI.getOperand(0)) ||
187 isSVERegOp(TRI, MRI, MI.getOperand(1)))
188 Prev = nullptr;
189 break;
190 case AArch64::RestoreZAPseudo:
191 case AArch64::InOutZAUsePseudo:
192 case AArch64::CommitZASavePseudo:
193 case AArch64::SMEStateAllocPseudo:
194 case AArch64::RequiresZASavePseudo:
195 // These instructions only depend on the ZA state, not the streaming mode,
196 // so if the pair of smstart/stop is only changing the streaming mode, we
197 // can permit these instructions.
198 if (Prev->getOperand(0).getImm() != AArch64SVCR::SVCRSM)
199 Prev = nullptr;
200 break;
201 case AArch64::ADJCALLSTACKDOWN:
202 case AArch64::ADJCALLSTACKUP:
203 case AArch64::ANDXri:
204 case AArch64::ADDXri:
205 // We permit these as they don't generate SVE/NEON instructions.
206 break;
207 case AArch64::MSRpstatesvcrImm1:
208 case AArch64::MSRpstatePseudo:
209 llvm_unreachable("Should have been handled");
210 }
211 }
212
213 HasRemovedAllSMChanges =
214 NumSMChanges && (NumSMChanges == NumSMChangesRemoved);
215 return Changed;
216}
217
218// Using the FORM_TRANSPOSED_REG_TUPLE pseudo can improve register allocation
219// of multi-vector intrinsics. However, the pseudo should only be emitted if
220// the input registers of the REG_SEQUENCE are copy nodes where the source
221// register is in a StridedOrContiguous class. For example:
222//
223// %3:zpr2stridedorcontiguous = LD1B_2Z_IMM_PSEUDO ..
224// %4:zpr = COPY %3.zsub1:zpr2stridedorcontiguous
225// %5:zpr = COPY %3.zsub0:zpr2stridedorcontiguous
226// %6:zpr2stridedorcontiguous = LD1B_2Z_PSEUDO ..
227// %7:zpr = COPY %6.zsub1:zpr2stridedorcontiguous
228// %8:zpr = COPY %6.zsub0:zpr2stridedorcontiguous
229// %9:zpr2mul2 = REG_SEQUENCE %5:zpr, %subreg.zsub0, %8:zpr, %subreg.zsub1
230//
231// -> %9:zpr2mul2 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO %5:zpr, %8:zpr
232//
233bool SMEPeepholeOpt::visitRegSequence(MachineInstr &MI) {
234 assert(MI.getMF()->getRegInfo().isSSA() && "Expected to be run on SSA form!");
235
236 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
237 switch (MRI.getRegClass(MI.getOperand(0).getReg())->getID()) {
238 case AArch64::ZPR2RegClassID:
239 case AArch64::ZPR4RegClassID:
240 case AArch64::ZPR2Mul2RegClassID:
241 case AArch64::ZPR4Mul4RegClassID:
242 break;
243 default:
244 return false;
245 }
246
247 // The first operand is the register class created by the REG_SEQUENCE.
248 // Each operand pair after this consists of a vreg + subreg index, so
249 // for example a sequence of 2 registers will have a total of 5 operands.
250 if (MI.getNumOperands() != 5 && MI.getNumOperands() != 9)
251 return false;
252
253 MCRegister SubReg = MCRegister::NoRegister;
254 for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
255 MachineOperand &MO = MI.getOperand(I);
256
257 MachineOperand *Def = MRI.getOneDef(MO.getReg());
258 if (!Def || !Def->getParent()->isCopy())
259 return false;
260
261 const MachineOperand &CopySrc = Def->getParent()->getOperand(1);
262 unsigned OpSubReg = CopySrc.getSubReg();
263 if (SubReg == MCRegister::NoRegister)
264 SubReg = OpSubReg;
265
266 MachineOperand *CopySrcOp = MRI.getOneDef(CopySrc.getReg());
267 if (!CopySrcOp || !CopySrcOp->isReg() || OpSubReg != SubReg ||
268 CopySrcOp->getReg().isPhysical())
269 return false;
270
271 const TargetRegisterClass *CopySrcClass =
272 MRI.getRegClass(CopySrcOp->getReg());
273 if (CopySrcClass != &AArch64::ZPR2StridedOrContiguousRegClass &&
274 CopySrcClass != &AArch64::ZPR4StridedOrContiguousRegClass)
275 return false;
276 }
277
278 unsigned Opc = MI.getNumOperands() == 5
279 ? AArch64::FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO
280 : AArch64::FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO;
281
282 const TargetInstrInfo *TII =
283 MI.getMF()->getSubtarget<AArch64Subtarget>().getInstrInfo();
284 MachineInstrBuilder MIB = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
285 TII->get(Opc), MI.getOperand(0).getReg());
286 for (unsigned I = 1; I < MI.getNumOperands(); I += 2)
287 MIB.addReg(MI.getOperand(I).getReg());
288
289 MI.eraseFromParent();
290 return true;
291}
292
293INITIALIZE_PASS(SMEPeepholeOpt, "aarch64-sme-peephole-opt",
294 "SME Peephole Optimization", false, false)
295
296bool SMEPeepholeOpt::runOnMachineFunction(MachineFunction &MF) {
297 if (skipFunction(MF.getFunction()))
298 return false;
299
300 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
301 SMEAttrs SMEFnAttrs = AFI->getSMEFnAttrs();
302
303 if (!MF.getSubtarget<AArch64Subtarget>().hasSME() &&
304 !SMEFnAttrs.hasStreamingCompatibleInterface())
305 return false;
306
307 assert(MF.getRegInfo().isSSA() && "Expected to be run on SSA form!");
308
309 bool Changed = false;
310 bool FunctionHasAllSMChangesRemoved = false;
311
312 // Even if the block lives in a function with no SME attributes attached we
313 // still have to analyze all the blocks because we may call a streaming
314 // function that requires smstart/smstop pairs.
315 for (MachineBasicBlock &MBB : MF) {
316 bool BlockHasAllSMChangesRemoved;
317 Changed |= optimizeStartStopPairs(MBB, BlockHasAllSMChangesRemoved);
318 FunctionHasAllSMChangesRemoved |= BlockHasAllSMChangesRemoved;
319
320 if (MF.getSubtarget<AArch64Subtarget>().isStreaming()) {
322 if (MI.getOpcode() == AArch64::REG_SEQUENCE)
323 Changed |= visitRegSequence(MI);
324 }
325 }
326
327 if (FunctionHasAllSMChangesRemoved)
328 AFI->setHasStreamingModeChanges(false);
329
330 return Changed;
331}
332
333FunctionPass *llvm::createSMEPeepholeOptPass() { return new SMEPeepholeOpt(); }
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock & MBB
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define I(x, y, z)
Definition MD5.cpp:57
Register const TargetRegisterInfo * TRI
if(PassOpts->AAPipeline)
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition PassSupport.h:56
MachineRegisterInfo & MRI
static bool isSVERegOp(const TargetRegisterInfo &TRI, const MachineRegisterInfo &MRI, const MachineOperand &MO)
static bool isMatchingStartStopPair(const MachineInstr *MI1, const MachineInstr *MI2)
static bool isConditionalStartStop(const MachineInstr *MI)
static bool ChangesStreamingMode(const MachineInstr *MI)
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
bool isStreaming() const
Returns true if the function has a streaming body.
Represent the analysis usage information of a pass.
LLVM_ABI void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition Pass.cpp:280
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:312
unsigned getID() const
getID() - Return the register class ID number.
static constexpr unsigned NoRegister
Definition MCRegister.h:60
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
const MachineOperand & getOperand(unsigned i) const
LLVM_ABI MachineInstrBundleIterator< MachineInstr > eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
MachineOperand class - Representation of each machine instruction operand.
unsigned getSubReg() const
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Register getReg() const
getReg - Returns the register number.
const uint32_t * getRegMask() const
getRegMask - Returns a bit mask of registers preserved by this RegMask operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
MachineOperand * getOneDef(Register Reg) const
Returns the defining operand if there is exactly one operand defining the specified register,...
const MachineFunction & getMF() const
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition Register.h:83
SMEAttrs is a utility class to parse the SME ACLE attributes on functions.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
NodeAddr< DefNode * > Def
Definition RDFGraph.h:384
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
FunctionPass * createSMEPeepholeOptPass()
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition MCRegister.h:21
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58