LLVM 24.0.0git
ProfileVerify.cpp
Go to the documentation of this file.
1//===- ProfileVerify.cpp - Verify profile info for testing ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
11#include "llvm/ADT/STLExtras.h"
13#include "llvm/IR/Analysis.h"
14#include "llvm/IR/Constants.h"
15#include "llvm/IR/Dominators.h"
16#include "llvm/IR/Function.h"
17#include "llvm/IR/GlobalValue.h"
21#include "llvm/IR/LLVMContext.h"
22#include "llvm/IR/MDBuilder.h"
23#include "llvm/IR/Module.h"
24#include "llvm/IR/PassManager.h"
30
31using namespace llvm;
33 DefaultFunctionEntryCount("profcheck-default-function-entry-count",
34 cl::init(1000));
35static cl::opt<bool>
36 AnnotateSelect("profcheck-annotate-select", cl::init(true),
37 cl::desc("Also inject (if missing) and verify MD_prof for "
38 "`select` instructions"));
39static cl::opt<bool>
40 WeightsForTest("profcheck-weights-for-test", cl::init(false),
41 cl::desc("Generate weights with small values for tests."));
42
44 "profcheck-default-select-true-weight", cl::init(2U),
45 cl::desc("When annotating `select` instructions, this value will be used "
46 "for the first ('true') case."));
48 "profcheck-default-select-false-weight", cl::init(3U),
49 cl::desc("When annotating `select` instructions, this value will be used "
50 "for the second ('false') case."));
52 "profcheck-annotate-indirect-calls", cl::init(false),
53 cl::desc("Also inject (if missing) and verify MD_prof for indirect calls"));
55 "profcheck-annotate-mem-ops", cl::init(false),
57 "Also inject (if missing) and verify MD_prof for memory operations"));
58namespace {
59class ProfileInjector {
60 Function &F;
62
63public:
64 static const Instruction *
65 getTerminatorBenefitingFromMDProf(const BasicBlock &BB) {
66 if (succ_size(&BB) < 2)
67 return nullptr;
68 auto *Term = BB.getTerminator();
69 return (isa<CondBrInst>(Term) || isa<SwitchInst>(Term) ||
71 ? Term
72 : nullptr;
73 }
74
75 static Instruction *getTerminatorBenefitingFromMDProf(BasicBlock &BB) {
76 return const_cast<Instruction *>(
77 getTerminatorBenefitingFromMDProf(const_cast<const BasicBlock &>(BB)));
78 }
79
80 ProfileInjector(Function &F, FunctionAnalysisManager &FAM) : F(F), FAM(FAM) {}
81 bool inject();
82};
83
84bool isAsmOnly(const Function &F) {
85 if (!F.hasFnAttribute(Attribute::AttrKind::Naked))
86 return false;
87 for (const auto &BB : F)
88 for (const auto &I : drop_end(BB)) {
89 const auto *CB = dyn_cast<CallBase>(&I);
90 if (!CB || !CB->isInlineAsm())
91 return false;
92 }
93 return true;
94}
95
96void emitProfileError(StringRef Msg, Function &F) {
97 F.getContext().emitError("Profile verification failed for function '" +
98 F.getName() + "': " + Msg);
99}
100
101} // namespace
102
103// FIXME: currently this injects only for terminators. Select isn't yet
104// supported.
105bool ProfileInjector::inject() {
106 // skip purely asm functions
107 if (isAsmOnly(F))
108 return false;
109 // Get whatever branch probability info can be derived from the given IR -
110 // whether it has or not metadata. The main intention for this pass is to
111 // ensure that other passes don't drop or "forget" to update MD_prof. We do
112 // this as a mode in which lit tests would run. We want to avoid changing the
113 // behavior of those tests. A pass may use BPI (or BFI, which is computed from
114 // BPI). If no metadata is present, BPI is guesstimated by
115 // BranchProbabilityAnalysis. The injector (this pass) only persists whatever
116 // information the analysis provides, in other words, the pass being tested
117 // will get the same BPI it does if the injector wasn't running.
118 auto &BPI = FAM.getResult<BranchProbabilityAnalysis>(F);
119
120 // Inject a function count if there's none. It's reasonable for a pass to
121 // want to clear the MD_prof of a function with zero entry count. If the
122 // original profile (iFDO or AFDO) is empty for a function, it's simpler to
123 // require assigning it the 0-entry count explicitly than to mark every branch
124 // as cold (we do want some explicit information in the spirit of what this
125 // verifier wants to achieve - make dropping / corrupting MD_prof
126 // unit-testable)
127 if (!F.getEntryCount())
128 F.setEntryCount(DefaultFunctionEntryCount);
129 // If there is an entry count that's 0, then don't bother injecting. We won't
130 // verify these either.
131 if (*F.getEntryCount() == 0)
132 return false;
133 bool Changed = false;
134 // Cycle through the weights list. If we didn't, tests with more than (say)
135 // one conditional branch would have the same !prof metadata on all of them,
136 // and numerically that may make for a poor unit test.
137 uint32_t WeightsForTestOffset = 0;
138 for (auto &BB : F) {
139 for (Instruction &I : BB) {
140 if (I.getMetadata(LLVMContext::MD_prof))
141 continue;
142 if (AnnotateSelect) {
143 if (auto *SI = dyn_cast<SelectInst>(&I)) {
144 if (SI->getCondition()->getType()->isVectorTy())
145 continue;
147 /*IsExpected=*/false);
148 Changed = true;
149 }
150 }
152 if (auto *MI = dyn_cast<MemIntrinsic>(&I)) {
153 if (isa<ConstantInt>(MI->getLength()))
154 continue;
155 annotateValueSite(*F.getParent(), *MI, {{19797, 2}, {18770, 3}}, 10,
156 InstrProfValueKind::IPVK_MemOPSize, 30);
157 Changed = true;
158 }
159 }
161 if (auto *CB = dyn_cast<CallBase>(&I)) {
162 if (!CB->isIndirectCall())
163 continue;
164 annotateValueSite(*F.getParent(), *CB, {{19797, 2}, {18770, 3}}, 10,
165 InstrProfValueKind::IPVK_IndirectCallTarget, 30);
166 Changed = true;
167 }
168 }
169 }
170 auto *Term = getTerminatorBenefitingFromMDProf(BB);
171 if (!Term || Term->getMetadata(LLVMContext::MD_prof))
172 continue;
174
175 SmallVector<uint32_t> Weights;
176 Weights.reserve(Term->getNumSuccessors());
177 if (WeightsForTest) {
178 static const std::array Primes{3, 5, 7, 11, 13, 17, 19, 23, 29, 31,
179 37, 41, 43, 47, 53, 59, 61, 67, 71};
180 for (uint32_t I = 0, E = Term->getNumSuccessors(); I < E; ++I)
181 Weights.emplace_back(
182 Primes[(WeightsForTestOffset + I) % Primes.size()]);
183 ++WeightsForTestOffset;
184 } else {
185 Probs.reserve(Term->getNumSuccessors());
186 for (auto I = 0U, E = Term->getNumSuccessors(); I < E; ++I)
187 Probs.emplace_back(BPI.getEdgeProbability(&BB, Term->getSuccessor(I)));
188
189 assert(llvm::find_if(Probs,
190 [](const BranchProbability &P) {
191 return P.isUnknown();
192 }) == Probs.end() &&
193 "All branch probabilities should be valid");
194 const auto *FirstZeroDenominator =
195 find_if(Probs, [](const BranchProbability &P) {
196 return P.getDenominator() == 0;
197 });
198 (void)FirstZeroDenominator;
199 assert(FirstZeroDenominator == Probs.end());
200 const auto *FirstNonZeroNumerator = find_if(
201 Probs, [](const BranchProbability &P) { return !P.isZero(); });
202 assert(FirstNonZeroNumerator != Probs.end());
203 DynamicAPInt LCM(Probs[0].getDenominator());
204 DynamicAPInt GCD(FirstNonZeroNumerator->getNumerator());
205 for (const auto &Prob : drop_begin(Probs)) {
206 if (!Prob.getNumerator())
207 continue;
208 LCM = llvm::lcm(LCM, DynamicAPInt(Prob.getDenominator()));
209 GCD = llvm::gcd(GCD, DynamicAPInt(Prob.getNumerator()));
210 }
211 for (const auto &Prob : Probs) {
212 DynamicAPInt W =
213 (Prob.getNumerator() * LCM / GCD) / Prob.getDenominator();
214 Weights.emplace_back(static_cast<uint32_t>((int64_t)W));
215 }
216 }
217 setBranchWeights(*Term, Weights, /*IsExpected=*/false);
218 Changed = true;
219 }
220 return Changed;
221}
222
225 ProfileInjector PI(F, FAM);
226 if (!PI.inject())
227 return PreservedAnalyses::all();
228
230}
231
232PreservedAnalyses ProfileVerifierPass::run(Module &M,
234 auto PopulateIgnoreList = [&](StringRef GVName) {
235 if (const auto *CT = M.getGlobalVariable(GVName))
236 if (CT->hasInitializer())
237 if (const auto *CA =
238 dyn_cast_if_present<ConstantArray>(CT->getInitializer()))
239 for (const auto &Elt : CA->operands())
240 if (const auto *CS = dyn_cast<ConstantStruct>(Elt))
241 if (CS->getNumOperands() >= 2 && CS->getOperand(1))
242 if (const auto *F = dyn_cast<Function>(
243 CS->getOperand(1)->stripPointerCasts()))
244 IgnoreList.insert(F);
245 };
246 PopulateIgnoreList("llvm.global_ctors");
247 PopulateIgnoreList("llvm.global_dtors");
248
249 // expose the function-level run as public through a wrapper, so we can use
250 // pass manager mechanisms dealing with declarations and with composing the
251 // returned PreservedAnalyses values.
252 struct Wrapper : OptionalPassInfoMixin<Wrapper> {
255 return PVP.run(F, FAM);
256 }
257 explicit Wrapper(ProfileVerifierPass &PVP) : PVP(PVP) {}
258 };
259
261}
262
263PreservedAnalyses ProfileVerifierPass::run(Function &F,
265 // skip purely asm functions
266 if (isAsmOnly(F))
267 return PreservedAnalyses::all();
268 if (IgnoreList.contains(&F))
269 return PreservedAnalyses::all();
270
271 const auto EntryCount = F.getEntryCount();
272 if (!EntryCount) {
273 auto *MD = F.getMetadata(LLVMContext::MD_prof);
274 if (!MD || !isExplicitlyUnknownProfileMetadata(*MD)) {
275 emitProfileError("function entry count missing (set to 0 if cold)", F);
276 return PreservedAnalyses::all();
277 }
278 } else if (*EntryCount == 0) {
279 return PreservedAnalyses::all();
280 }
281 for (const auto &BB : F) {
282 for (const auto &I : BB) {
283 if (I.getMetadata(LLVMContext::MD_prof))
284 continue;
285 if (AnnotateSelect) {
286 if (auto *SI = dyn_cast<SelectInst>(&I)) {
287 if (SI->getCondition()->getType()->isVectorTy())
288 continue;
289 emitProfileError("select annotation missing", F);
290 }
291 }
293 if (auto *MI = dyn_cast<MemIntrinsic>(&I)) {
294 if (isa<ConstantInt>(MI->getLength()))
295 continue;
296 emitProfileError("memory operation annotation missing", F);
297 }
298 }
300 if (auto *CB = dyn_cast<CallBase>(&I)) {
301 if (!CB->isIndirectCall())
302 continue;
303 emitProfileError("indirect call annotation missing", F);
304 }
305 }
306 }
307 if (const auto *Term =
308 ProfileInjector::getTerminatorBenefitingFromMDProf(BB))
309 if (!Term->getMetadata(LLVMContext::MD_prof))
310 emitProfileError("branch annotation missing", F);
311 }
312 return PreservedAnalyses::all();
313}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
IRTranslator LLVM IR MI
Module.h This file contains the declarations for the Module class.
This header defines various interfaces for pass management in LLVM.
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
#define P(N)
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This file contains the declarations for profiling metadata utility functions.
static cl::opt< bool > WeightsForTest("profcheck-weights-for-test", cl::init(false), cl::desc("Generate weights with small values for tests."))
static cl::opt< uint32_t > SelectFalseWeight("profcheck-default-select-false-weight", cl::init(3U), cl::desc("When annotating `select` instructions, this value will be used " "for the second ('false') case."))
static cl::opt< int64_t > DefaultFunctionEntryCount("profcheck-default-function-entry-count", cl::init(1000))
static cl::opt< bool > AnnotateSelect("profcheck-annotate-select", cl::init(true), cl::desc("Also inject (if missing) and verify MD_prof for " "`select` instructions"))
static cl::opt< uint32_t > SelectTrueWeight("profcheck-default-select-true-weight", cl::init(2U), cl::desc("When annotating `select` instructions, this value will be used " "for the first ('true') case."))
static cl::opt< bool > AnnotateIndirectCalls("profcheck-annotate-indirect-calls", cl::init(false), cl::desc("Also inject (if missing) and verify MD_prof for indirect calls"))
static cl::opt< bool > AnnotateMemoryOperations("profcheck-annotate-mem-ops", cl::init(false), cl::desc("Also inject (if missing) and verify MD_prof for memory operations"))
const char * Msg
This file contains some templates that are useful if you are working with the STL at all.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
LLVM Basic Block Representation.
Definition BasicBlock.h:62
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
Definition BasicBlock.h:237
LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
Runs the function pass across every function in the module.
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition Analysis.h:115
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM)
Checks that MD_prof is present on every instruction that supports it.
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
Pass manager infrastructure for declaring and invalidating analyses.
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:83
initializer< Ty > init(const Ty &Val)
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
LLVM_ATTRIBUTE_ALWAYS_INLINE DynamicAPInt gcd(const DynamicAPInt &A, const DynamicAPInt &B)
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
ModuleToFunctionPassAdaptor createModuleToFunctionPassAdaptor(FunctionPassT &&Pass, bool EagerlyInvalidate=false)
A function to deduce a function pass type and wrap it in the templated adaptor.
LLVM_ABI bool isExplicitlyUnknownProfileMetadata(const MDNode &MD)
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected, bool ElideAllZero=false)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
LLVM_ABI void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
auto succ_size(const MachineBasicBlock *BB)
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
Definition STLExtras.h:323
LLVM_ATTRIBUTE_ALWAYS_INLINE DynamicAPInt lcm(const DynamicAPInt &A, const DynamicAPInt &B)
Returns the least common multiple of A and B.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
AnalysisManager< Module > ModuleAnalysisManager
Convenience typedef for the Module analysis manager.
Definition MIRParser.h:39
A CRTP mix-in for passes that can be skipped.