LLVM 22.0.0git
StaticDataSplitter.cpp
Go to the documentation of this file.
1//===- StaticDataSplitter.cpp ---------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The pass uses branch profile data to assign hotness based section qualifiers
10// for the following types of static data:
11// - Jump tables
12// - Module-internal global variables
13// - Constant pools
14//
15// For the original RFC of this pass please see
16// https://discourse.llvm.org/t/rfc-profile-guided-static-data-partitioning/83744
17
18#include "llvm/ADT/Statistic.h"
29#include "llvm/CodeGen/Passes.h"
32#include "llvm/Pass.h"
34
35using namespace llvm;
36
37#define DEBUG_TYPE "static-data-splitter"
38
39STATISTIC(NumHotJumpTables, "Number of hot jump tables seen.");
40STATISTIC(NumColdJumpTables, "Number of cold jump tables seen.");
41STATISTIC(NumUnknownJumpTables,
42 "Number of jump tables with unknown hotness. They are from functions "
43 "without profile information.");
44
46 const MachineBranchProbabilityInfo *MBPI = nullptr;
47 const MachineBlockFrequencyInfo *MBFI = nullptr;
48 const ProfileSummaryInfo *PSI = nullptr;
49 StaticDataProfileInfo *SDPI = nullptr;
50
51 // If the global value is a local linkage global variable, return it.
52 // Otherwise, return nullptr.
53 const GlobalVariable *getLocalLinkageGlobalVariable(const GlobalValue *GV);
54
55 // Returns true if the global variable is in one of {.rodata, .bss, .data,
56 // .data.rel.ro} sections.
57 bool inStaticDataSection(const GlobalVariable &GV, const TargetMachine &TM);
58
59 // Returns the constant if the operand refers to a global variable or constant
60 // that gets lowered to static data sections. Otherwise, return nullptr.
61 const Constant *getConstant(const MachineOperand &Op, const TargetMachine &TM,
62 const MachineConstantPool *MCP);
63
64 // Use profiles to partition static data.
65 bool partitionStaticDataWithProfiles(MachineFunction &MF);
66
67 // Update LLVM statistics for a machine function with profiles.
68 void updateStatsWithProfiles(const MachineFunction &MF);
69
70 // Update LLVM statistics for a machine function without profiles.
71 void updateStatsWithoutProfiles(const MachineFunction &MF);
72
73 void annotateStaticDataWithoutProfiles(const MachineFunction &MF);
74
75public:
76 static char ID;
77
81
82 StringRef getPassName() const override { return "Static Data Splitter"; }
83
84 void getAnalysisUsage(AnalysisUsage &AU) const override {
90 // This pass does not modify any required analysis results except
91 // StaticDataProfileInfoWrapperPass, but StaticDataProfileInfoWrapperPass
92 // is made an immutable pass that it won't be re-scheduled by pass manager
93 // anyway. So mark setPreservesAll() here for faster compile time.
94 AU.setPreservesAll();
95 }
96
97 bool runOnMachineFunction(MachineFunction &MF) override;
98};
99
104
106 .getStaticDataProfileInfo();
107
108 const bool ProfileAvailable = PSI && PSI->hasProfileSummary() && MBFI &&
110
111 if (!ProfileAvailable) {
112 annotateStaticDataWithoutProfiles(MF);
113 updateStatsWithoutProfiles(MF);
114 return false;
115 }
116
117 bool Changed = partitionStaticDataWithProfiles(MF);
118
119 updateStatsWithProfiles(MF);
120 return Changed;
121}
122
123const Constant *
124StaticDataSplitter::getConstant(const MachineOperand &Op,
125 const TargetMachine &TM,
126 const MachineConstantPool *MCP) {
127 if (!Op.isGlobal() && !Op.isCPI())
128 return nullptr;
129
130 if (Op.isGlobal()) {
131 // Find global variables with local linkage.
132 const GlobalVariable *GV = getLocalLinkageGlobalVariable(Op.getGlobal());
133 // Skip 'llvm.'-prefixed global variables conservatively because they are
134 // often handled specially, and skip those not in static data
135 // sections.
136 if (!GV || GV->getName().starts_with("llvm.") ||
137 !inStaticDataSection(*GV, TM))
138 return nullptr;
139 return GV;
140 }
141 assert(Op.isCPI() && "Op must be constant pool index in this branch");
142 int CPI = Op.getIndex();
143 if (CPI == -1)
144 return nullptr;
145
146 assert(MCP != nullptr && "Constant pool info is not available.");
147 const MachineConstantPoolEntry &CPE = MCP->getConstants()[CPI];
148
150 return nullptr;
151
152 return CPE.Val.ConstVal;
153}
154
155bool StaticDataSplitter::partitionStaticDataWithProfiles(MachineFunction &MF) {
156 // If any of the static data (jump tables, global variables, constant pools)
157 // are captured by the analysis, set `Changed` to true. Note this pass won't
158 // invalidate any analysis pass (see `getAnalysisUsage` above), so the main
159 // purpose of tracking and conveying the change (to pass manager) is
160 // informative as opposed to invalidating any analysis results. As an example
161 // of where this information is useful, `PMDataManager::dumpPassInfo` will
162 // only dump pass info if a local change happens, otherwise a pass appears as
163 // "skipped".
164 bool Changed = false;
165
166 MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
167
168 // Jump table could be used by either terminating instructions or
169 // non-terminating ones, so we walk all instructions and use
170 // `MachineOperand::isJTI()` to identify jump table operands.
171 // Similarly, `MachineOperand::isCPI()` is used to identify constant pool
172 // usages in the same loop.
173 for (const auto &MBB : MF) {
174 std::optional<uint64_t> Count = MBFI->getBlockProfileCount(&MBB);
175 for (const MachineInstr &I : MBB) {
176 for (const MachineOperand &Op : I.operands()) {
177 if (!Op.isJTI() && !Op.isGlobal() && !Op.isCPI())
178 continue;
179
180 if (Op.isJTI()) {
181 assert(MJTI != nullptr && "Jump table info is not available.");
182 const int JTI = Op.getIndex();
183 // This is not a source block of jump table.
184 if (JTI == -1)
185 continue;
186
187 auto Hotness = MachineFunctionDataHotness::Hot;
188
189 // Hotness is based on source basic block hotness.
190 // TODO: PSI APIs are about instruction hotness. Introduce API for
191 // data access hotness.
192 if (Count && PSI->isColdCount(*Count))
193 Hotness = MachineFunctionDataHotness::Cold;
194
195 Changed |= MJTI->updateJumpTableEntryHotness(JTI, Hotness);
196 } else if (const Constant *C =
197 getConstant(Op, MF.getTarget(), MF.getConstantPool())) {
198 SDPI->addConstantProfileCount(C, Count);
199 Changed = true;
200 }
201 }
202 }
203 }
204 return Changed;
205}
206
207const GlobalVariable *
208StaticDataSplitter::getLocalLinkageGlobalVariable(const GlobalValue *GV) {
209 // LLVM IR Verifier requires that a declaration must have valid declaration
210 // linkage, and local linkages are not among the valid ones. So there is no
211 // need to check GV is not a declaration here.
212 return (GV && GV->hasLocalLinkage()) ? dyn_cast<GlobalVariable>(GV) : nullptr;
213}
214
215bool StaticDataSplitter::inStaticDataSection(const GlobalVariable &GV,
216 const TargetMachine &TM) {
217
218 SectionKind Kind = TargetLoweringObjectFile::getKindForGlobal(&GV, TM);
219 return Kind.isData() || Kind.isReadOnly() || Kind.isReadOnlyWithRel() ||
220 Kind.isBSS();
221}
222
223void StaticDataSplitter::updateStatsWithProfiles(const MachineFunction &MF) {
225 return;
226
227 if (const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo()) {
228 for (const auto &JumpTable : MJTI->getJumpTables()) {
229 if (JumpTable.Hotness == MachineFunctionDataHotness::Hot) {
230 ++NumHotJumpTables;
231 } else {
232 assert(JumpTable.Hotness == MachineFunctionDataHotness::Cold &&
233 "A jump table is either hot or cold when profile information is "
234 "available.");
235 ++NumColdJumpTables;
236 }
237 }
238 }
239}
240
241void StaticDataSplitter::annotateStaticDataWithoutProfiles(
242 const MachineFunction &MF) {
243 for (const auto &MBB : MF)
244 for (const MachineInstr &I : MBB)
245 for (const MachineOperand &Op : I.operands())
246 if (const Constant *C =
247 getConstant(Op, MF.getTarget(), MF.getConstantPool()))
248 SDPI->addConstantProfileCount(C, std::nullopt);
249}
250
251void StaticDataSplitter::updateStatsWithoutProfiles(const MachineFunction &MF) {
253 return;
254
255 if (const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo()) {
256 NumUnknownJumpTables += MJTI->getJumpTables().size();
257 }
258}
259
261
263 false, false)
269 false)
270
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock & MBB
#define DEBUG_TYPE
#define I(x, y, z)
Definition MD5.cpp:58
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:167
StringRef getPassName() const override
getPassName - Return a nice clean name for a pass.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesAll()
Set by analyses that do not transform their input at all.
This is an important base class in LLVM.
Definition Constant.h:43
bool hasProfileData(bool IncludeSynthetic=false) const
Return true if the function is annotated with profile data.
Definition Function.h:334
bool hasLocalLinkage() const
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
union llvm::MachineConstantPoolEntry::@004270020304201266316354007027341142157160323045 Val
The constant itself.
bool isMachineConstantPoolEntry() const
isMachineConstantPoolEntry - Return true if the MachineConstantPoolEntry is indeed a target specific ...
The MachineConstantPool class keeps track of constants referenced by a function which must be spilled...
const std::vector< MachineConstantPoolEntry > & getConstants() const
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
LLVM_ABI bool updateJumpTableEntryHotness(size_t JTI, MachineFunctionDataHotness Hotness)
const std::vector< MachineJumpTableEntry > & getJumpTables() const
MachineOperand class - Representation of each machine instruction operand.
static LLVM_ABI PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
AnalysisType & getAnalysis() const
getAnalysis<AnalysisType>() - This function is used by subclasses to get to the analysis information ...
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
Analysis providing profile information.
This wraps the StaticDataProfileInfo object as an immutable pass, for a backend pass to operate on.
A class that holds the constants that represent static data and their profile information and provide...
StringRef - Represent a constant reference to a string, i.e.
Definition StringRef.h:55
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:269
static SectionKind getKindForGlobal(const GlobalObject *GO, const TargetMachine &TM)
Classify the specified global variable into a set of target independent categories embodied in Sectio...
Primary interface to the complete machine description for the target machine.
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:322
Changed
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
This is an optimization pass for GlobalISel generic memory operations.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:649
LLVM_ABI void initializeStaticDataSplitterPass(PassRegistry &)
LLVM_ABI MachineFunctionPass * createStaticDataSplitterPass()
createStaticDataSplitterPass - This is a machine-function pass that categorizes static data hotness u...
FunctionAddr VTableAddr Count
Definition InstrProf.h:139
LLVM_ABI bool AreStatisticsEnabled()
Check if statistics are enabled.
DWARFExpression::Operation Op