LLVM 22.0.0git
MIRSampleProfile.cpp
Go to the documentation of this file.
1//===-------- MIRSampleProfile.cpp: MIRSampleFDO (For FSAFDO) -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides the implementation of the MIRSampleProfile loader, mainly
10// for flow sensitive SampleFDO.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/DenseMap.h"
16#include "llvm/ADT/DenseSet.h"
26#include "llvm/CodeGen/Passes.h"
27#include "llvm/IR/Function.h"
28#include "llvm/IR/PseudoProbe.h"
31#include "llvm/Support/Debug.h"
36#include <optional>
37
38using namespace llvm;
39using namespace sampleprof;
40using namespace llvm::sampleprofutil;
42
43#define DEBUG_TYPE "fs-profile-loader"
44
46 "show-fs-branchprob", cl::Hidden, cl::init(false),
47 cl::desc("Print setting flow sensitive branch probabilities"));
49 "fs-profile-debug-prob-diff-threshold", cl::init(10),
51 "Only show debug message if the branch probability is greater than "
52 "this value (in percentage)."));
53
55 "fs-profile-debug-bw-threshold", cl::init(10000),
56 cl::desc("Only show debug message if the source branch weight is greater "
57 " than this value."));
58
59static cl::opt<bool> ViewBFIBefore("fs-viewbfi-before", cl::Hidden,
60 cl::init(false),
61 cl::desc("View BFI before MIR loader"));
62static cl::opt<bool> ViewBFIAfter("fs-viewbfi-after", cl::Hidden,
63 cl::init(false),
64 cl::desc("View BFI after MIR loader"));
65
67
69 "Load MIR Sample Profile",
70 /* cfg = */ false, /* is_analysis = */ false)
77 /* cfg = */ false, /* is_analysis = */ false)
78
80
82llvm::createMIRProfileLoaderPass(std::string File, std::string RemappingFile,
84 IntrusiveRefCntPtr<vfs::FileSystem> FS) {
85 return new MIRProfileLoaderPass(File, RemappingFile, P, std::move(FS));
86}
87
88namespace llvm {
89
90// Internal option used to control BFI display only after MBP pass.
91// Defined in CodeGen/MachineBlockFrequencyInfo.cpp:
92// -view-block-layout-with-bfi={none | fraction | integer | count}
94
95// Command line option to specify the name of the function for CFG dump
96// Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name=
98
99std::optional<PseudoProbe> extractProbe(const MachineInstr &MI) {
100 if (MI.isPseudoProbe()) {
101 PseudoProbe Probe;
102 Probe.Id = MI.getOperand(1).getImm();
103 Probe.Type = MI.getOperand(2).getImm();
104 Probe.Attr = MI.getOperand(3).getImm();
105 Probe.Factor = 1;
106 DILocation *DebugLoc = MI.getDebugLoc();
107 Probe.Discriminator = DebugLoc ? DebugLoc->getDiscriminator() : 0;
108 return Probe;
109 }
110
111 // Ignore callsite probes since they do not have FS discriminators.
112 return std::nullopt;
113}
114
115namespace afdo_detail {
143} // namespace afdo_detail
144
146 : public SampleProfileLoaderBaseImpl<MachineFunction> {
147public:
151 DT = MDT;
152 PDT = MPDT;
153 LI = MLI;
154 BFI = MBFI;
155 ORE = MORE;
156 }
158 P = Pass;
161 assert(LowBit < HighBit && "HighBit needs to be greater than Lowbit");
162 }
163
166 : SampleProfileLoaderBaseImpl(std::string(Name), std::string(RemapName),
167 std::move(FS)) {}
168
171 bool doInitialization(Module &M);
172 bool isValid() const { return ProfileIsValid; }
173
174protected:
176
177 /// Hold the information of the basic block frequency.
179
180 /// PassNum is the sequence number this pass is called, start from 1.
182
183 // LowBit in the FS discriminator used by this instance. Note the number is
184 // 0-based. Base discrimnator use bit 0 to bit 11.
185 unsigned LowBit;
186 // HighwBit in the FS discriminator used by this instance. Note the number
187 // is 0-based.
188 unsigned HighBit;
189
190 bool ProfileIsValid = true;
193 return getProbeWeight(MI);
194 if (ImprovedFSDiscriminator && MI.isMetaInstruction())
195 return std::error_code();
196 return getInstWeightImpl(MI);
197 }
198};
199
200template <>
203
205 LLVM_DEBUG(dbgs() << "\nPropagation complete. Setting branch probs\n");
206 for (auto &BI : F) {
207 MachineBasicBlock *BB = &BI;
208 if (BB->succ_size() < 2)
209 continue;
210 const MachineBasicBlock *EC = EquivalenceClass[BB];
211 uint64_t BBWeight = BlockWeights[EC];
212 uint64_t SumEdgeWeight = 0;
213 for (MachineBasicBlock *Succ : BB->successors()) {
214 Edge E = std::make_pair(BB, Succ);
215 SumEdgeWeight += EdgeWeights[E];
216 }
217
218 if (BBWeight != SumEdgeWeight) {
219 LLVM_DEBUG(dbgs() << "BBweight is not equal to SumEdgeWeight: BBWWeight="
220 << BBWeight << " SumEdgeWeight= " << SumEdgeWeight
221 << "\n");
222 BBWeight = SumEdgeWeight;
223 }
224 if (BBWeight == 0) {
225 LLVM_DEBUG(dbgs() << "SKIPPED. All branch weights are zero.\n");
226 continue;
227 }
228
229#ifndef NDEBUG
230 uint64_t BBWeightOrig = BBWeight;
231#endif
232 uint32_t MaxWeight = std::numeric_limits<uint32_t>::max();
233 uint32_t Factor = 1;
234 if (BBWeight > MaxWeight) {
235 Factor = BBWeight / MaxWeight + 1;
236 BBWeight /= Factor;
237 LLVM_DEBUG(dbgs() << "Scaling weights by " << Factor << "\n");
238 }
239
241 SE = BB->succ_end();
242 SI != SE; ++SI) {
243 MachineBasicBlock *Succ = *SI;
244 Edge E = std::make_pair(BB, Succ);
245 uint64_t EdgeWeight = EdgeWeights[E];
246 EdgeWeight /= Factor;
247
248 assert(BBWeight >= EdgeWeight &&
249 "BBweight is larger than EdgeWeight -- should not happen.\n");
250
251 BranchProbability OldProb = BFI->getMBPI()->getEdgeProbability(BB, SI);
252 BranchProbability NewProb(EdgeWeight, BBWeight);
253 if (OldProb == NewProb)
254 continue;
255 BB->setSuccProbability(SI, NewProb);
256#ifndef NDEBUG
257 if (!ShowFSBranchProb)
258 continue;
259 bool Show = false;
261 if (OldProb > NewProb)
262 Diff = OldProb - NewProb;
263 else
264 Diff = NewProb - OldProb;
266 Show &= (BBWeightOrig >= FSProfileDebugBWThreshold);
267
268 auto DIL = BB->findBranchDebugLoc();
269 auto SuccDIL = Succ->findBranchDebugLoc();
270 if (Show) {
271 dbgs() << "Set branch fs prob: MBB (" << BB->getNumber() << " -> "
272 << Succ->getNumber() << "): ";
273 if (DIL)
274 dbgs() << DIL->getFilename() << ":" << DIL->getLine() << ":"
275 << DIL->getColumn();
276 if (SuccDIL)
277 dbgs() << "-->" << SuccDIL->getFilename() << ":" << SuccDIL->getLine()
278 << ":" << SuccDIL->getColumn();
279 dbgs() << " W=" << BBWeightOrig << " " << OldProb << " --> " << NewProb
280 << "\n";
281 }
282#endif
283 }
284 }
285}
286
288 auto &Ctx = M.getContext();
289
291 Filename, Ctx, *FS, P, RemappingFilename);
292 if (std::error_code EC = ReaderOrErr.getError()) {
293 std::string Msg = "Could not open profile: " + EC.message();
294 Ctx.diagnose(DiagnosticInfoSampleProfile(Filename, Msg));
295 return false;
296 }
297
298 Reader = std::move(ReaderOrErr.get());
299 Reader->setModule(&M);
301
302 // Load pseudo probe descriptors for probe-based function samples.
303 if (Reader->profileIsProbeBased()) {
304 ProbeManager = std::make_unique<PseudoProbeManager>(M);
305 if (!ProbeManager->moduleIsProbed(M)) {
306 return false;
307 }
308 }
309
310 return true;
311}
312
314 // Do not load non-FS profiles. A line or probe can get a zero-valued
315 // discriminator at certain pass which could result in accidentally loading
316 // the corresponding base counter in the non-FS profile, while a non-zero
317 // discriminator would end up getting zero samples. This could in turn undo
318 // the sample distribution effort done by previous BFI maintenance and the
319 // probe distribution factor work for pseudo probes.
320 if (!Reader->profileIsFS())
321 return false;
322
323 Function &Func = MF.getFunction();
324 clearFunctionData(false);
325 Samples = Reader->getSamplesFor(Func);
326 if (!Samples || Samples->empty())
327 return false;
328
330 if (!ProbeManager->profileIsValid(MF.getFunction(), *Samples))
331 return false;
332 } else {
333 if (getFunctionLoc(MF) == 0)
334 return false;
335 }
336
337 DenseSet<GlobalValue::GUID> InlinedGUIDs;
338 bool Changed = computeAndPropagateWeights(MF, InlinedGUIDs);
339
340 // Set the new BPI, BFI.
341 setBranchProbs(MF);
342
343 return Changed;
344}
345
346} // namespace llvm
347
349 std::string FileName, std::string RemappingFileName, FSDiscriminatorPass P,
351 : MachineFunctionPass(ID), ProfileFileName(FileName), P(P) {
352 LowBit = getFSPassBitBegin(P);
353 HighBit = getFSPassBitEnd(P);
354
355 auto VFS = FS ? std::move(FS) : vfs::getRealFileSystem();
356 MIRSampleLoader = std::make_unique<MIRProfileLoader>(
357 FileName, RemappingFileName, std::move(VFS));
358 assert(LowBit < HighBit && "HighBit needs to be greater than Lowbit");
359}
360
361bool MIRProfileLoaderPass::runOnMachineFunction(MachineFunction &MF) {
362 if (!MIRSampleLoader->isValid())
363 return false;
364
365 LLVM_DEBUG(dbgs() << "MIRProfileLoader pass working on Func: "
366 << MF.getFunction().getName() << "\n");
368 auto *MDT = &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
369 auto *MPDT =
371
372 MF.RenumberBlocks();
373 MDT->updateBlockNumbers();
374 MPDT->updateBlockNumbers();
375
376 MIRSampleLoader->setInitVals(
377 MDT, MPDT, &getAnalysis<MachineLoopInfoWrapperPass>().getLI(), MBFI,
379
381 (ViewBlockFreqFuncName.empty() ||
383 MBFI->view("MIR_Prof_loader_b." + MF.getName(), false);
384 }
385
386 bool Changed = MIRSampleLoader->runOnFunction(MF);
387 if (Changed)
388 MBFI->calculate(MF, *MBFI->getMBPI(),
390
392 (ViewBlockFreqFuncName.empty() ||
394 MBFI->view("MIR_prof_loader_a." + MF.getName(), false);
395 }
396
397 return Changed;
398}
399
400bool MIRProfileLoaderPass::doInitialization(Module &M) {
401 LLVM_DEBUG(dbgs() << "MIRProfileLoader pass working on Module " << M.getName()
402 << "\n");
403
404 MIRSampleLoader->setFSPass(P);
405 return MIRSampleLoader->doInitialization(M);
406}
407
408void MIRProfileLoaderPass::getAnalysisUsage(AnalysisUsage &AU) const {
409 AU.setPreservesAll();
410 AU.addRequired<MachineBlockFrequencyInfoWrapperPass>();
411 AU.addRequired<MachineDominatorTreeWrapperPass>();
412 AU.addRequired<MachinePostDominatorTreeWrapperPass>();
413 AU.addRequiredTransitive<MachineLoopInfoWrapperPass>();
414 AU.addRequired<MachineOptimizationRemarkEmitterPass>();
416}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
#define DEBUG_TYPE
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition MD5.cpp:55
static cl::opt< bool > ShowFSBranchProb("show-fs-branchprob", cl::Hidden, cl::init(false), cl::desc("Print setting flow sensitive branch probabilities"))
static cl::opt< bool > ViewBFIAfter("fs-viewbfi-after", cl::Hidden, cl::init(false), cl::desc("View BFI after MIR loader"))
static cl::opt< unsigned > FSProfileDebugBWThreshold("fs-profile-debug-bw-threshold", cl::init(10000), cl::desc("Only show debug message if the source branch weight is greater " " than this value."))
static cl::opt< unsigned > FSProfileDebugProbDiffThreshold("fs-profile-debug-prob-diff-threshold", cl::init(10), cl::desc("Only show debug message if the branch probability is greater than " "this value (in percentage)."))
static cl::opt< bool > ViewBFIBefore("fs-viewbfi-before", cl::Hidden, cl::init(false), cl::desc("View BFI before MIR loader"))
===- MachineOptimizationRemarkEmitter.h - Opt Diagnostics -*- C++ -*-—===//
#define P(N)
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
This file provides the interface for the sampled PGO profile loader base implementation.
This file provides the utility functions for the sampled PGO loader base implementation.
#define LLVM_DEBUG(...)
Definition Debug.h:114
Defines the virtual file system interface vfs::FileSystem.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesAll()
Set by analyses that do not transform their input at all.
AnalysisUsage & addRequiredTransitive()
A debug info location.
Definition DebugLoc.h:124
Implements a dense probed hash-table based set.
Definition DenseSet.h:279
Diagnostic information for the sample profiler.
Represents either an error or a value T.
Definition ErrorOr.h:56
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:314
Class to represent profile counts.
Definition Function.h:297
A smart pointer to a reference-counted object that inherits from RefCountedBase or ThreadSafeRefCount...
MIRProfileLoaderPass(std::string FileName="", std::string RemappingFileName="", FSDiscriminatorPass P=FSDiscriminatorPass::Pass1, IntrusiveRefCntPtr< vfs::FileSystem > FS=nullptr)
FS bits will only use the '1' bits in the Mask.
MIRProfileLoader(StringRef Name, StringRef RemapName, IntrusiveRefCntPtr< vfs::FileSystem > FS)
void setBranchProbs(MachineFunction &F)
ErrorOr< uint64_t > getInstWeight(const MachineInstr &MI) override
bool runOnFunction(MachineFunction &F)
MachineBlockFrequencyInfo * BFI
Hold the information of the basic block frequency.
FSDiscriminatorPass P
PassNum is the sequence number this pass is called, start from 1.
bool doInitialization(Module &M)
void setInitVals(MachineDominatorTree *MDT, MachinePostDominatorTree *MPDT, MachineLoopInfo *MLI, MachineBlockFrequencyInfo *MBFI, MachineOptimizationRemarkEmitter *MORE)
void setFSPass(FSDiscriminatorPass Pass)
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
LLVM_ABI void setSuccProbability(succ_iterator I, BranchProbability Prob)
Set successor probability of a given iterator.
SmallVectorImpl< MachineBasicBlock * >::iterator succ_iterator
LLVM_ABI DebugLoc findBranchDebugLoc()
Find and return the merged DebugLoc of the branch instructions of the block.
iterator_range< succ_iterator > successors()
iterator_range< pred_iterator > predecessors()
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
LLVM_ABI void view(const Twine &Name, bool isSimple=true) const
Pop up a ghostview window with the current block frequency propagation rendered using dot.
LLVM_ABI const MachineBranchProbabilityInfo * getMBPI() const
LLVM_ABI void calculate(const MachineFunction &F, const MachineBranchProbabilityInfo &MBPI, const MachineLoopInfo &MLI)
calculate - compute block frequency info for the given function.
Analysis pass which computes a MachineDominatorTree.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Function & getFunction()
Return the LLVM function that this machine code represents.
void RenumberBlocks(MachineBasicBlock *MBBFrom=nullptr)
RenumberBlocks - This discards all of the MachineBasicBlock numbers and recomputes them.
Representation of each machine instruction.
Diagnostic information for optimization analysis remarks.
MachinePostDominatorTree - an analysis pass wrapper for DominatorTree used to compute the post-domina...
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:67
Pass interface - Implemented by all 'passes'.
Definition Pass.h:99
AnalysisType & getAnalysis() const
getAnalysis<AnalysisType>() - This function is used by subclasses to get to the analysis information ...
bool computeAndPropagateWeights(FunctionT &F, const DenseSet< GlobalValue::GUID > &InlinedGUIDs)
void computeDominanceAndLoopInfo(FunctionT &F)
ErrorOr< uint64_t > getInstWeightImpl(const InstructionT &Inst)
SampleProfileLoaderBaseImpl(std::string Name, std::string RemapName, IntrusiveRefCntPtr< vfs::FileSystem > FS)
virtual ErrorOr< uint64_t > getProbeWeight(const InstructionT &Inst)
std::pair< const BasicBlockT *, const BasicBlockT * > Edge
StringRef - Represent a constant reference to a string, i.e.
Definition StringRef.h:55
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:322
A range adaptor for a pair of iterators.
static LLVM_ABI bool ProfileIsProbeBased
static LLVM_ABI ErrorOr< std::unique_ptr< SampleProfileReader > > create(StringRef Filename, LLVMContext &C, vfs::FileSystem &FS, FSDiscriminatorPass P=FSDiscriminatorPass::Base, StringRef RemapFilename="")
Create a sample profile reader appropriate to the file format.
Changed
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
initializer< Ty > init(const Ty &Val)
LLVM_ABI IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
static unsigned getFSPassBitBegin(sampleprof::FSDiscriminatorPass P)
cl::opt< bool > ImprovedFSDiscriminator
LLVM_ABI char & MIRProfileLoaderPassID
This pass reads flow sensitive profile.
static unsigned getFSPassBitEnd(sampleprof::FSDiscriminatorPass P)
cl::opt< std::string > ViewBlockFreqFuncName("view-bfi-func-name", cl::Hidden, cl::desc("The option to specify " "the name of the function " "whose CFG will be displayed."))
LLVM_ABI std::optional< PseudoProbe > extractProbe(const Instruction &Inst)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:207
cl::opt< GVDAGType > ViewBlockLayoutWithBFI("view-block-layout-with-bfi", cl::Hidden, cl::desc("Pop up a window to show a dag displaying MBP layout and associated " "block frequencies of the CFG."), cl::values(clEnumValN(GVDT_None, "none", "do not display graphs."), clEnumValN(GVDT_Fraction, "fraction", "display a graph using the " "fractional block frequency representation."), clEnumValN(GVDT_Integer, "integer", "display a graph using the raw " "integer fractional block frequency representation."), clEnumValN(GVDT_Count, "count", "display a graph using the real " "profile count if available.")))
Function::ProfileCount ProfileCount
LLVM_ABI FunctionPass * createMIRProfileLoaderPass(std::string File, std::string RemappingFile, sampleprof::FSDiscriminatorPass P, IntrusiveRefCntPtr< vfs::FileSystem > FS)
Read Flow Sensitive Profile.
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1867
Implement std::hash so that hash_code can be used in STL containers.
Definition BitVector.h:867
#define MORE()
Definition regcomp.c:246
uint32_t Discriminator
MachineOptimizationRemarkEmitter OptRemarkEmitterT
static PredRangeT getPredecessors(MachineBasicBlock *BB)
MachineOptimizationRemarkAnalysis OptRemarkAnalysisT
iterator_range< SmallVectorImpl< MachineBasicBlock * >::iterator > SuccRangeT
static SuccRangeT getSuccessors(MachineBasicBlock *BB)
static const MachineBasicBlock * getEntryBB(const MachineFunction *F)
iterator_range< SmallVectorImpl< MachineBasicBlock * >::iterator > PredRangeT
static Function & getFunction(MachineFunction &F)