32#include <unordered_set>
36#define DEBUG_TYPE "pseudo-probe"
39 "Number of probes that have an artificial debug line");
43 cl::desc(
"Do pseudo probe verification"));
47 cl::desc(
"The option to specify the name of the functions to verify."));
51 cl::desc(
"Update pseudo probe distribution factor"));
55 const DILocation *InlinedAt = DIL ? DIL->getInlinedAt() :
nullptr;
57 Hash ^=
MD5Hash(std::to_string(InlinedAt->getLine()));
58 Hash ^=
MD5Hash(std::to_string(InlinedAt->getColumn()));
59 auto Name = InlinedAt->getSubprogramLinkageName();
61 InlinedAt = InlinedAt->getInlinedAt();
70bool PseudoProbeVerifier::shouldVerifyFunction(
const Function *
F) {
72 if (
F->isDeclaration())
76 if (
F->hasAvailableExternallyLinkage())
79 static std::unordered_set<std::string> VerifyFuncNames(
81 return VerifyFuncNames.empty() || VerifyFuncNames.count(
F->getName().str());
96 "\n*** Pseudo Probe Verification After " + PassID.
str() +
" ***\n";
98 if (
const auto **M = any_cast<const Module *>(&
IR))
100 else if (
const auto **
F = any_cast<const Function *>(&
IR))
102 else if (
const auto **
C = any_cast<const LazyCallGraph::SCC *>(&
IR))
104 else if (
const auto **L = any_cast<const Loop *>(&
IR))
121 if (!shouldVerifyFunction(
F))
124 for (
const auto &BB : *
F)
125 collectProbeFactors(&BB, ProbeFactors);
126 verifyProbeFactors(
F, ProbeFactors);
130 const Function *
F =
L->getHeader()->getParent();
136 for (
const auto &
I : *
Block) {
139 ProbeFactors[{Probe->Id, Hash}] += Probe->Factor;
144void PseudoProbeVerifier::verifyProbeFactors(
146 bool BannerPrinted =
false;
147 auto &PrevProbeFactors = FunctionProbeFactors[
F->getName()];
148 for (
const auto &
I : ProbeFactors) {
149 float CurProbeFactor =
I.second;
150 if (PrevProbeFactors.count(
I.first)) {
151 float PrevProbeFactor = PrevProbeFactors[
I.first];
152 if (std::abs(CurProbeFactor - PrevProbeFactor) >
153 DistributionFactorVariance) {
154 if (!BannerPrinted) {
155 dbgs() <<
"Function " <<
F->getName() <<
":\n";
156 BannerPrinted =
true;
158 dbgs() <<
"Probe " <<
I.first.first <<
"\tprevious factor "
159 <<
format(
"%0.2f", PrevProbeFactor) <<
"\tcurrent factor "
160 <<
format(
"%0.2f", CurProbeFactor) <<
"\n";
165 PrevProbeFactors[
I.first] =
I.second;
171 for (
const auto *Operand :
FuncInfo->operands()) {
172 const auto *MD = cast<MDNode>(Operand);
174 mdconst::dyn_extract<ConstantInt>(MD->getOperand(0))->getZExtValue();
176 mdconst::dyn_extract<ConstantInt>(MD->getOperand(1))->getZExtValue();
183PseudoProbeManager::getDesc(
const Function &
F)
const {
184 auto I = GUIDToProbeDescMap.find(
186 return I == GUIDToProbeDescMap.end() ? nullptr : &
I->second;
195 const auto *Desc = getDesc(
F);
197 LLVM_DEBUG(
dbgs() <<
"Probe descriptor missing for Function " <<
F.getName()
211 const std::string &CurModuleUniqueId)
212 :
F(&Func), CurModuleUniqueId(CurModuleUniqueId) {
213 BlockProbeIds.clear();
214 CallProbeIds.clear();
216 computeProbeIdForBlocks();
217 computeProbeIdForCallsites();
225void SampleProfileProber::computeCFGHash() {
226 std::vector<uint8_t> Indexes;
228 for (
auto &BB : *F) {
229 auto *TI = BB.getTerminator();
230 for (
unsigned I = 0,
E = TI->getNumSuccessors();
I !=
E; ++
I) {
231 auto *Succ = TI->getSuccessor(
I);
232 auto Index = getBlockId(Succ);
233 for (
int J = 0; J < 4; J++)
234 Indexes.push_back((uint8_t)(
Index >> (J * 8)));
240 FunctionHash = (
uint64_t)CallProbeIds.size() << 48 |
243 FunctionHash &= 0x0FFFFFFFFFFFFFFF;
244 assert(FunctionHash &&
"Function checksum should not be zero");
245 LLVM_DEBUG(
dbgs() <<
"\nFunction Hash Computation for " <<
F->getName()
247 <<
" CRC = " << JC.
getCRC() <<
", Edges = "
248 << Indexes.size() <<
", ICSites = " << CallProbeIds.size()
249 <<
", Hash = " << FunctionHash <<
"\n");
252void SampleProfileProber::computeProbeIdForBlocks() {
257 for (
auto &BB : *F) {
260 BlockProbeIds[&BB] = LastProbeId;
264void SampleProfileProber::computeProbeIdForCallsites() {
265 for (
auto &BB : *F) {
267 if (!isa<CallBase>(
I))
269 if (isa<IntrinsicInst>(&
I))
271 CallProbeIds[&
I] = ++LastProbeId;
277 auto I = BlockProbeIds.find(
const_cast<BasicBlock *
>(BB));
278 return I == BlockProbeIds.end() ? 0 :
I->second;
282 auto Iter = CallProbeIds.find(
const_cast<Instruction *
>(Call));
283 return Iter == CallProbeIds.end() ? 0 : Iter->second;
293 if (
auto *SP =
F.getSubprogram()) {
294 FName = SP->getLinkageName();
296 FName = SP->getName();
306 assert((isa<PseudoProbeInst>(
I) || isa<CallBase>(
I)) &&
307 "Expecting pseudo probe or call instructions");
308 if (!
I->getDebugLoc()) {
309 if (
auto *SP =
F.getSubprogram()) {
314 dbgs() <<
"\nIn Function " <<
F.getName()
315 <<
" Probe gets an artificial debug line\n";
323 for (
auto &
I : BlockProbeIds) {
333 return !isa<PHINode>(J) && !isa<DbgInfoIntrinsic>(J) &&
334 !J->isLifetimeStartOrEnd() && J->getDebugLoc();
344 "Cannot get the probing point");
350 auto *Probe =
Builder.CreateCall(ProbeFn, Args);
351 AssignDebugLoc(Probe);
357 for (
auto &
I : CallProbeIds) {
358 auto *Call =
I.first;
360 uint32_t Type = cast<CallBase>(Call)->getCalledFunction()
363 AssignDebugLoc(Call);
369 if (
auto DIL = Call->getDebugLoc()) {
370 DIL = DIL->cloneWithDiscriminator(V);
371 Call->setDebugLoc(DIL);
380 auto Hash = getFunctionHash();
383 assert(NMD &&
"llvm.pseudo_probe_desc should be pre-created");
396 if (!
F.isDeclarationForLinker()) {
398 auto Triple =
TM->getTargetTriple();
414 if (
F.isDeclaration())
423void PseudoProbeUpdatePass::runOnFunction(
Function &
F,
426 auto BBProfileCount = [&BFI](
BasicBlock *BB) {
427 return BFI.getBlockProfileCount(BB).value_or(0);
436 ProbeFactors[{Probe->Id, Hash}] += BBProfileCount(&
Block);
446 float Sum = ProbeFactors[{Probe->Id, Hash}];
458 if (
F.isDeclaration())
462 runOnFunction(
F,
FAM);
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Statically lint checks LLVM IR
typename CallsiteContextGraph< DerivedCCG, FuncTy, CallTy >::FuncInfo FuncInfo
FunctionAnalysisManager FAM
const char LLVMTargetMachineRef TM
PassInstrumentationCallbacks PIC
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static cl::opt< bool > UpdatePseudoProbe("update-pseudo-probe", cl::init(true), cl::Hidden, cl::desc("Update pseudo probe distribution factor"))
static cl::opt< bool > VerifyPseudoProbe("verify-pseudo-probe", cl::init(false), cl::Hidden, cl::desc("Do pseudo probe verification"))
static cl::list< std::string > VerifyPseudoProbeFuncList("verify-pseudo-probe-funcs", cl::Hidden, cl::desc("The option to specify the name of the functions to verify."))
static uint64_t computeCallStackHash(const Instruction &Inst)
static uint64_t getCallStackHash(const DILocation *DIL)
This file provides the interface for the pseudo probe implementation for AutoFDO.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Implements a dense probed hash-table based set.
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
void update(ArrayRef< uint8_t > Data)
A node in the call graph.
An SCC of the call graph.
Represents a single loop in the control flow graph.
MDNode * createPseudoProbeDesc(uint64_t GUID, uint64_t Hash, StringRef FName)
Return metadata containing the pseudo probe descriptor for a function.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
A Module instance is used to store all the information related to an LLVM module.
This class manages callbacks registration, as well as provides a way for PassInstrumentation to pass ...
void registerAfterPassCallback(CallableT C, bool ToFront=false)
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
bool profileIsValid(const Function &F, const FunctionSamples &Samples) const
PseudoProbeManager(const Module &M)
bool moduleIsProbed(const Module &M) const
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
void registerCallbacks(PassInstrumentationCallbacks &PIC)
void runAfterPass(StringRef PassID, Any IR)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
Sample profile pseudo prober.
SampleProfileProber(Function &F, const std::string &CurModuleUniqueId)
void instrumentOneFunc(Function &F, TargetMachine *TM)
StringRef - Represent a constant reference to a string, i.e.
std::string str() const
str - Get the contents as an std::string.
constexpr bool empty() const
empty - Check if the string is empty.
Primary interface to the complete machine description for the target machine.
Triple - Helper class for working with autoconf configuration names.
bool supportsCOMDAT() const
Tests whether the target supports comdat.
The instances of the Type class are immutable: once they are created, they are never changed.
LLVM Value Representation.
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Representation of the samples collected for a function.
uint64_t getFunctionHash() const
static StringRef getCanonicalFnName(const Function &F)
Return the canonical name for a function, taking into account suffix elision policy attributes.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
static constexpr uint64_t PseudoProbeFullDistributionFactor
void setProbeDistributionFactor(Instruction &Inst, float Factor)
std::string getUniqueModuleId(Module *M)
Produce a unique identifier for this module by taking the MD5 sum of the names of the module's strong...
std::optional< PseudoProbe > extractProbe(const Instruction &Inst)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
static void computeEHOnlyBlocks(FunctionT &F, DenseSet< BlockT * > &EHBlocks)
Compute a list of blocks that are only reachable via EH paths.
Comdat * getOrCreateFunctionComdat(Function &F, Triple &T)
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
std::unordered_map< std::pair< uint64_t, uint64_t >, float, pair_hash< uint64_t, uint64_t > > ProbeFactorMap
uint64_t MD5Hash(StringRef Str)
Helper to compute and return lower 64 bits of the given string's MD5 hash.
constexpr const char * PseudoProbeDescMetadataName
static constexpr uint8_t FullDistributionFactor
static uint32_t packProbeData(uint32_t Index, uint32_t Type, uint32_t Flags, uint32_t Factor)