LLVM 24.0.0git
AArch64CodeLayoutOpt.cpp
Go to the documentation of this file.
1//===-- AArch64CodeLayoutOpt.cpp - Code Layout Optimizations --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass runs after instruction scheduling and employs code layout
10// optimizations for certain patterns.
11//
12// Option -aarch64-code-layout-opt-enable selects instruction pairs to optimize:
13// cmp-csel: Enable CMP/CMN-CSEL code layout optimization
14// fcmp-fcsel: Enable FCMP-FCSEL code layout optimization
15//
16// The initial implementation induces function alignment when a supported
17// pattern is detected, and possibly instruction-alignment when a pair would
18// straddle cache-lines.
19//===----------------------------------------------------------------------===//
20
21#include "AArch64.h"
22#include "AArch64InstrInfo.h"
23#include "AArch64Subtarget.h"
26#include "llvm/ADT/Statistic.h"
30#include "llvm/Support/Debug.h"
33
34using namespace llvm;
35
36#define DEBUG_TYPE "aarch64-code-layout-opt"
37#define DBG(...) LLVM_DEBUG(dbgs() << DEBUG_TYPE ": " << __VA_ARGS__)
38#define AARCH64_CODE_LAYOUT_OPT_NAME "AArch64 Code Layout Optimization"
39
41 None = 0,
42 CmpCsel = 1 << 0, // Align CMP/CMN-CSEL pairs
43 FcmpFcsel = 1 << 1, // Align FCMP-FCSEL pairs
45};
46
48 "aarch64-code-layout-opt-enable", cl::Hidden, cl::CommaSeparated,
49 cl::desc("Enable code alignment optimization for instruction pairs"),
51 clEnumValN(None, "none", "Disable the code alignment pass"),
52 clEnumValN(CmpCsel, "cmp-csel", "CMP/CMN-CSEL pair alignment (32-bit)"),
53 clEnumValN(FcmpFcsel, "fcmp-fcsel", "FCMP-FCSEL pair alignment")));
54
55STATISTIC(NumFunctionsAligned,
56 "Number of functions with aligned (to 64-bytes by default)");
57STATISTIC(NumCmpCselPairsDetected,
58 "Number of CMP/CMN-CSEL pairs detected for alignment");
59STATISTIC(NumFcmpFcselPairsDetected,
60 "Number of FCMP-FCSEL pairs detected for alignment");
61
62namespace {
63
64class AArch64CodeLayoutOpt : public MachineFunctionPass {
65public:
66 static char ID;
67 AArch64CodeLayoutOpt() : MachineFunctionPass(ID) {}
68 void getAnalysisUsage(AnalysisUsage &AU) const override;
69 bool runOnMachineFunction(MachineFunction &MF) override;
70 StringRef getPassName() const override {
72 }
73
74private:
75 /// Align each fusible CMP/CMN-CSEL or FCMP-FCSEL pair in \p MBB by emitting
76 /// .p2align before the lead instruction (splitting the block if needed).
77 /// \returns true iff at least one pair was found and aligned.
78 bool alignLayoutSensitivePatterns(MachineBasicBlock *MBB, CodeLayoutOpt CLO);
79
80 /// Emit .p2align before MI. Splits the block if MI is not at its start.
81 void emitP2Align(MachineInstr &MI, Align DesiredAlign,
82 unsigned MaxSkipBytes = 4);
83
84 bool optimizeForCodeLayout(MachineFunction &MF, CodeLayoutOpt CLO);
85};
86
87} // end anonymous namespace
88
89char AArch64CodeLayoutOpt::ID = 0;
90
91INITIALIZE_PASS(AArch64CodeLayoutOpt, "aarch64-code-layout-opt",
92 AARCH64_CODE_LAYOUT_OPT_NAME, false, false)
93
94void AArch64CodeLayoutOpt::getAnalysisUsage(AnalysisUsage &AU) const {
95 AU.setPreservesAll();
97}
98
100 return new AArch64CodeLayoutOpt();
101}
102
103/// \returns true iff Opc is a floating-point comparison (FCMP/FCMPE).
104static bool isFloatingPointCompare(unsigned Opc) {
105 switch (Opc) {
106 case AArch64::FCMPSrr:
107 case AArch64::FCMPDrr:
108 case AArch64::FCMPESrr:
109 case AArch64::FCMPEDrr:
110 case AArch64::FCMPHrr:
111 case AArch64::FCMPEHrr:
112 return true;
113 default:
114 return false;
115 }
116}
117
118/// \returns true iff Opc is a floating-point conditional select (FCSEL).
120 switch (Opc) {
121 case AArch64::FCSELSrrr:
122 case AArch64::FCSELDrrr:
123 case AArch64::FCSELHrrr:
124 return true;
125 default:
126 return false;
127 }
128}
129
130/// \returns true if MI is a qualifying 32-bit CMP or CMN instruction.
131/// CMP is encoded as SUBS with WZR destination, CMN as ADDS with WZR.
132/// Only simple variants (no shifted/extended reg) qualify, and immediate
133/// variants require no LSL shift and small immediates (<=15).
135 switch (MI.getOpcode()) {
136 case AArch64::SUBSWrr:
137 case AArch64::ADDSWrr:
138 return MI.definesRegister(AArch64::WZR, /*TRI=*/nullptr);
139 case AArch64::SUBSWri:
140 case AArch64::ADDSWri:
141 return MI.definesRegister(AArch64::WZR, /*TRI=*/nullptr) &&
142 MI.getOperand(3).getImm() == 0 && MI.getOperand(2).getImm() <= 15;
143 case AArch64::SUBSWrs:
144 case AArch64::ADDSWrs:
145 return MI.definesRegister(AArch64::WZR, /*TRI=*/nullptr) &&
146 !AArch64InstrInfo::hasShiftedReg(MI);
147 case AArch64::SUBSWrx:
148 return MI.definesRegister(AArch64::WZR, /*TRI=*/nullptr) &&
149 !AArch64InstrInfo::hasExtendedReg(MI);
150 default:
151 return false;
152 }
153}
154
155bool AArch64CodeLayoutOpt::runOnMachineFunction(MachineFunction &MF) {
156 const Function &F = MF.getFunction();
157 // hasOptSize() returns true for both -Os and -Oz.
158 if (F.hasOptSize())
159 return false;
160
161 const auto *Subtarget = &MF.getSubtarget<AArch64Subtarget>();
162
163 // Aligning basic blocks currently isn't compatible with Windows unwind info.
165 return false;
166
167 CodeLayoutOpt CLO = None;
168 if (EnableCodeAlignment.getNumOccurrences()) {
173 } else {
174 // Default: enable when the subtarget opts in via FeatureAlignCmpCSelPairs.
175 if (Subtarget->hasAlignCmpCSelPairs()) {
176 if (Subtarget->hasFuseCmpCSel())
178 if (Subtarget->hasFuseFCmpFCSel())
180 }
181 }
182
183 if (CLO == None)
184 return false;
185
186 return optimizeForCodeLayout(MF, CLO);
187}
188
189void AArch64CodeLayoutOpt::emitP2Align(MachineInstr &MI, Align DesiredAlign,
190 unsigned MaxSkipBytes) {
191 MachineBasicBlock *MBB = MI.getParent();
192
193 auto FirstReal =
195 if (&*FirstReal != &MI) {
196 auto PrevIt = prev_nodbg(MI.getIterator(), MBB->instr_begin());
197 MBB = MBB->splitAt(*PrevIt, /*UpdateLiveIns=*/true);
198 }
199
200 MBB->setAlignment(DesiredAlign);
201 MBB->setMaxBytesForAlignment(MaxSkipBytes);
202}
203
204// Align each fusible CMP/CMN-CSEL or FCMP-FCSEL pair in MBB by emitting
205// .p2align before the lead instruction (splitting the block if needed).
206// A pair is: a qualifying lead instruction immediately followed by its
207// consumer (CMP/CMN→CSEL or FCMP→FCSEL), with no intervening instructions.
208// Returns true iff at least one pair was found and aligned.
209bool AArch64CodeLayoutOpt::alignLayoutSensitivePatterns(MachineBasicBlock *MBB,
210 CodeLayoutOpt CLO) {
211 auto End = MBB->instr_end();
213
214 for (auto &MI : instructionsWithoutDebug(MBB->begin(), MBB->end())) {
215 auto NextIt =
216 skipDebugInstructionsForward(std::next(MI.getIterator()), End);
217 if (NextIt == End)
218 break;
219
220 // --- CMP/CMN-CSEL detection ---
222 NextIt->getOpcode() == AArch64::CSELWr) {
223 Pairs.push_back({&MI, true});
224 continue;
225 }
226
227 // --- FCMP-FCSEL detection ---
228 if ((CLO & CodeLayoutOpt::FcmpFcsel) &&
229 isFloatingPointCompare(MI.getOpcode()) &&
230 isFloatingPointConditionalSelect(NextIt->getOpcode())) {
231 Pairs.push_back({&MI, false});
232 continue;
233 }
234 }
235
236 for (auto &[MI, IsCmpCsel] : Pairs) {
237 emitP2Align(*MI, Align(64));
238 DBG(".p2align 6, , 4 before " << *MI);
239 ++(IsCmpCsel ? NumCmpCselPairsDetected : NumFcmpFcselPairsDetected);
240 }
241
242 return !Pairs.empty();
243}
244
245bool AArch64CodeLayoutOpt::optimizeForCodeLayout(MachineFunction &MF,
246 CodeLayoutOpt CLO) {
247 DBG("optimizeForCodeLayout: " << MF.getName() << "\n");
248
249 bool Changed = false;
250 for (auto &MBB : MF)
251 Changed |= alignLayoutSensitivePatterns(&MBB, CLO);
252
253 if (!Changed)
254 return false;
255
256 unsigned FunctionAlignBytes = MF.getSubtarget<AArch64Subtarget>()
257 .getCLOpts()
258 .code_layout_opt_align_functions;
259 if (!isPowerOf2_32(FunctionAlignBytes))
261 "aarch64-code-layout-opt-align-functions must be a power of 2");
262 if (MF.getAlignment() < Align(FunctionAlignBytes)) {
263 MF.setAlignment(Align(FunctionAlignBytes));
264 ++NumFunctionsAligned;
265 DBG("Set " << FunctionAlignBytes << "-byte alignment for function "
266 << MF.getName() << "\n");
267 } else {
268 DBG("Function " << MF.getName() << " already has sufficient alignment\n");
269 }
270 return true;
271}
static bool isFloatingPointConditionalSelect(unsigned Opc)
#define AARCH64_CODE_LAYOUT_OPT_NAME
static cl::list< CodeLayoutOpt > EnableCodeAlignment("aarch64-code-layout-opt-enable", cl::Hidden, cl::CommaSeparated, cl::desc("Enable code alignment optimization for instruction pairs"), cl::values(clEnumValN(None, "none", "Disable the code alignment pass"), clEnumValN(CmpCsel, "cmp-csel", "CMP/CMN-CSEL pair alignment (32-bit)"), clEnumValN(FcmpFcsel, "fcmp-fcsel", "FCMP-FCSEL pair alignment")))
static bool isFloatingPointCompare(unsigned Opc)
#define DBG(...)
static bool isQualifyingIntCompare(const MachineInstr &MI)
const AArch64Subtarget & Subtarget
MachineBasicBlock & MBB
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition MD5.cpp:54
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition PassSupport.h:56
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
bool isTargetWindowsOrUEFI() const
UEFI follows the Windows ARM64 ABI and produces PE/COFF images, so most of the Windows specific ABI h...
Represent the analysis usage information of a pass.
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:312
void setMaxBytesForAlignment(unsigned MaxBytes)
Set the maximum amount of padding allowed for aligning the basic block.
void setAlignment(Align A)
Set alignment of the basic block.
LLVM_ABI MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Function & getFunction()
Return the LLVM function that this machine code represents.
Representation of each machine instruction.
void push_back(const T &Elt)
const AArch64MCOptions & getCLOpts(const MCTargetOptions &Options)
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
This is an optimization pass for GlobalISel generic memory operations.
IterT skipDebugInstructionsForward(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It until it points to a non-debug instruction or to End and return the resulting iterator.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
FunctionPass * createAArch64CodeLayoutOptPass()
IterT prev_nodbg(IterT It, IterT Begin, bool SkipPseudoOp=true)
Decrement It, then continue decrementing it while it points to a debug instruction.
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177