33#include <unordered_set>
37#define DEBUG_TYPE "pseudo-probe"
40 "Number of probes that have an artificial debug line");
44 cl::desc(
"Do pseudo probe verification"));
48 cl::desc(
"The option to specify the name of the functions to verify."));
52 cl::desc(
"Update pseudo probe distribution factor"));
56 const DILocation *InlinedAt = DIL ? DIL->getInlinedAt() :
nullptr;
58 Hash ^=
MD5Hash(std::to_string(InlinedAt->getLine()));
59 Hash ^=
MD5Hash(std::to_string(InlinedAt->getColumn()));
60 auto Name = InlinedAt->getSubprogramLinkageName();
62 InlinedAt = InlinedAt->getInlinedAt();
71bool PseudoProbeVerifier::shouldVerifyFunction(
const Function *
F) {
73 if (
F->isDeclaration())
77 if (
F->hasAvailableExternallyLinkage())
80 static std::unordered_set<std::string> VerifyFuncNames(
82 return VerifyFuncNames.empty() || VerifyFuncNames.count(
F->getName().str());
97 "\n*** Pseudo Probe Verification After " + PassID.
str() +
" ***\n";
99 if (
const auto **M = llvm::any_cast<const Module *>(&
IR))
101 else if (
const auto **
F = llvm::any_cast<const Function *>(&
IR))
103 else if (
const auto **
C = llvm::any_cast<const LazyCallGraph::SCC *>(&
IR))
105 else if (
const auto **L = llvm::any_cast<const Loop *>(&
IR))
122 if (!shouldVerifyFunction(
F))
125 for (
const auto &BB : *
F)
126 collectProbeFactors(&BB, ProbeFactors);
127 verifyProbeFactors(
F, ProbeFactors);
131 const Function *
F =
L->getHeader()->getParent();
137 for (
const auto &
I : *
Block) {
140 ProbeFactors[{Probe->Id, Hash}] += Probe->Factor;
145void PseudoProbeVerifier::verifyProbeFactors(
147 bool BannerPrinted =
false;
148 auto &PrevProbeFactors = FunctionProbeFactors[
F->getName()];
149 for (
const auto &
I : ProbeFactors) {
150 float CurProbeFactor =
I.second;
151 if (PrevProbeFactors.count(
I.first)) {
152 float PrevProbeFactor = PrevProbeFactors[
I.first];
153 if (std::abs(CurProbeFactor - PrevProbeFactor) >
154 DistributionFactorVariance) {
155 if (!BannerPrinted) {
156 dbgs() <<
"Function " <<
F->getName() <<
":\n";
157 BannerPrinted =
true;
159 dbgs() <<
"Probe " <<
I.first.first <<
"\tprevious factor "
160 <<
format(
"%0.2f", PrevProbeFactor) <<
"\tcurrent factor "
161 <<
format(
"%0.2f", CurProbeFactor) <<
"\n";
166 PrevProbeFactors[
I.first] =
I.second;
171 const std::string &CurModuleUniqueId)
172 :
F(&Func), CurModuleUniqueId(CurModuleUniqueId) {
173 BlockProbeIds.clear();
174 CallProbeIds.clear();
176 computeProbeIdForBlocks();
177 computeProbeIdForCallsites();
185void SampleProfileProber::computeCFGHash() {
186 std::vector<uint8_t> Indexes;
188 for (
auto &BB : *F) {
190 auto Index = getBlockId(Succ);
191 for (
int J = 0; J < 4; J++)
192 Indexes.push_back((uint8_t)(
Index >> (J * 8)));
198 FunctionHash = (
uint64_t)CallProbeIds.size() << 48 |
201 FunctionHash &= 0x0FFFFFFFFFFFFFFF;
202 assert(FunctionHash &&
"Function checksum should not be zero");
203 LLVM_DEBUG(
dbgs() <<
"\nFunction Hash Computation for " <<
F->getName()
205 <<
" CRC = " << JC.
getCRC() <<
", Edges = "
206 << Indexes.size() <<
", ICSites = " << CallProbeIds.size()
207 <<
", Hash = " << FunctionHash <<
"\n");
210void SampleProfileProber::computeProbeIdForBlocks() {
215 for (
auto &BB : *F) {
218 BlockProbeIds[&BB] = LastProbeId;
222void SampleProfileProber::computeProbeIdForCallsites() {
226 for (
auto &BB : *F) {
228 if (!isa<CallBase>(
I))
230 if (isa<IntrinsicInst>(&
I))
235 if (LastProbeId >= 0xFFFF) {
236 std::string
Msg =
"Pseudo instrumentation incomplete for " +
237 std::string(
F->getName()) +
" because it's too large";
243 CallProbeIds[&
I] = ++LastProbeId;
249 auto I = BlockProbeIds.find(
const_cast<BasicBlock *
>(BB));
250 return I == BlockProbeIds.end() ? 0 :
I->second;
254 auto Iter = CallProbeIds.find(
const_cast<Instruction *
>(Call));
255 return Iter == CallProbeIds.end() ? 0 : Iter->second;
265 if (
auto *SP =
F.getSubprogram()) {
266 FName = SP->getLinkageName();
268 FName = SP->getName();
278 assert((isa<PseudoProbeInst>(
I) || isa<CallBase>(
I)) &&
279 "Expecting pseudo probe or call instructions");
280 if (!
I->getDebugLoc()) {
281 if (
auto *SP =
F.getSubprogram()) {
286 dbgs() <<
"\nIn Function " <<
F.getName()
287 <<
" Probe gets an artificial debug line\n";
295 for (
auto &
I : BlockProbeIds) {
305 return !isa<PHINode>(J) && !isa<DbgInfoIntrinsic>(J) &&
306 !J->isLifetimeStartOrEnd() && J->getDebugLoc();
316 "Cannot get the probing point");
322 auto *Probe = Builder.
CreateCall(ProbeFn, Args);
323 AssignDebugLoc(Probe);
326 if (
auto DIL = Probe->getDebugLoc()) {
327 if (DIL->getDiscriminator()) {
328 DIL = DIL->cloneWithDiscriminator(0);
329 Probe->setDebugLoc(DIL);
337 for (
auto &
I : CallProbeIds) {
338 auto *Call =
I.first;
340 uint32_t Type = cast<CallBase>(Call)->getCalledFunction()
343 AssignDebugLoc(Call);
344 if (
auto DIL = Call->getDebugLoc()) {
351 DIL = DIL->cloneWithDiscriminator(V);
352 Call->setDebugLoc(DIL);
361 auto Hash = getFunctionHash();
364 assert(NMD &&
"llvm.pseudo_probe_desc should be pre-created");
377 if (
F.isDeclaration())
386void PseudoProbeUpdatePass::runOnFunction(
Function &
F,
389 auto BBProfileCount = [&BFI](
BasicBlock *BB) {
390 return BFI.getBlockProfileCount(BB).value_or(0);
399 ProbeFactors[{Probe->Id, Hash}] += BBProfileCount(&
Block);
409 float Sum = ProbeFactors[{Probe->Id, Hash}];
421 if (
F.isDeclaration())
425 runOnFunction(
F,
FAM);
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Legalize the Machine IR a function s Machine IR
FunctionAnalysisManager FAM
const char LLVMTargetMachineRef TM
PassInstrumentationCallbacks PIC
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static cl::opt< bool > UpdatePseudoProbe("update-pseudo-probe", cl::init(true), cl::Hidden, cl::desc("Update pseudo probe distribution factor"))
static cl::opt< bool > VerifyPseudoProbe("verify-pseudo-probe", cl::init(false), cl::Hidden, cl::desc("Do pseudo probe verification"))
static cl::list< std::string > VerifyPseudoProbeFuncList("verify-pseudo-probe-funcs", cl::Hidden, cl::desc("The option to specify the name of the functions to verify."))
static uint64_t computeCallStackHash(const Instruction &Inst)
static uint64_t getCallStackHash(const DILocation *DIL)
This file provides the interface for the pseudo probe implementation for AutoFDO.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Implements a dense probed hash-table based set.
Diagnostic information for the sample profiler.
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
BasicBlock::iterator GetInsertPoint() const
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
void update(ArrayRef< uint8_t > Data)
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
A node in the call graph.
An SCC of the call graph.
Represents a single loop in the control flow graph.
MDNode * createPseudoProbeDesc(uint64_t GUID, uint64_t Hash, StringRef FName)
Return metadata containing the pseudo probe descriptor for a function.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
A Module instance is used to store all the information related to an LLVM module.
This class manages callbacks registration, as well as provides a way for PassInstrumentation to pass ...
void registerAfterPassCallback(CallableT C, bool ToFront=false)
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
void registerCallbacks(PassInstrumentationCallbacks &PIC)
void runAfterPass(StringRef PassID, Any IR)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
Sample profile pseudo prober.
SampleProfileProber(Function &F, const std::string &CurModuleUniqueId)
void instrumentOneFunc(Function &F, TargetMachine *TM)
StringRef - Represent a constant reference to a string, i.e.
std::string str() const
str - Get the contents as an std::string.
constexpr bool empty() const
empty - Check if the string is empty.
Primary interface to the complete machine description for the target machine.
The instances of the Type class are immutable: once they are created, they are never changed.
LLVM Value Representation.
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const CustomOperand< const MCSubtargetInfo & > Msg[]
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
initializer< Ty > init(const Ty &Val)
uint64_t MD5Hash(const FunctionId &Obj)
This is an optimization pass for GlobalISel generic memory operations.
static constexpr uint64_t PseudoProbeFullDistributionFactor
auto successors(const MachineBasicBlock *BB)
void setProbeDistributionFactor(Instruction &Inst, float Factor)
std::string getUniqueModuleId(Module *M)
Produce a unique identifier for this module by taking the MD5 sum of the names of the module's strong...
std::optional< PseudoProbe > extractProbe(const Instruction &Inst)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
static void computeEHOnlyBlocks(FunctionT &F, DenseSet< BlockT * > &EHBlocks)
Compute a list of blocks that are only reachable via EH paths.
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
std::unordered_map< std::pair< uint64_t, uint64_t >, float, pair_hash< uint64_t, uint64_t > > ProbeFactorMap
constexpr const char * PseudoProbeDescMetadataName
static constexpr uint8_t FullDistributionFactor
static uint32_t packProbeData(uint32_t Index, uint32_t Type, uint32_t Flags, uint32_t Factor)