51 using namespace sampleprof;
53 #define DEBUG_TYPE "sample-profile"
61 "sample-profile-max-propagate-iterations",
cl::init(100),
62 cl::desc(
"Maximum number of iterations to go through when propagating "
63 "sample block/edge weights through the CFG."));
68 typedef std::pair<BasicBlock *, BasicBlock *> Edge;
83 :
FunctionPass(
ID), DT(nullptr), PDT(nullptr), LI(nullptr), Ctx(nullptr),
84 Reader(), Samples(nullptr), Filename(
Name), ProfileIsValid(
false) {
88 bool doInitialization(
Module &M)
override;
90 void dump() { Reader->dump(); }
92 const char *getPassName()
const override {
return "Sample profile pass"; }
112 void findEquivalenceClasses(
Function &
F);
117 unsigned visitEdge(Edge E,
unsigned *NumUnknownEdges, Edge *UnknownEdge);
123 unsigned HeaderLineno;
129 BlockWeightMap BlockWeights;
135 EdgeWeightMap EdgeWeights;
149 EquivalenceClassMap EquivalenceClass;
157 BlockEdgeMap Predecessors;
160 BlockEdgeMap Successors;
166 std::unique_ptr<SampleProfileReader> Reader;
183 void SampleProfileLoader::printEdgeWeight(
raw_ostream &OS, Edge E) {
184 OS <<
"weight[" << E.first->getName() <<
"->" << E.second->getName()
185 <<
"]: " << EdgeWeights[E] <<
"\n";
192 void SampleProfileLoader::printBlockEquivalence(
raw_ostream &OS,
195 OS <<
"equivalence[" << BB->
getName()
196 <<
"]: " << ((Equiv) ? EquivalenceClass[BB]->
getName() :
"NONE") <<
"\n";
204 OS <<
"weight[" << BB->
getName() <<
"]: " << BlockWeights[BB] <<
"\n";
218 unsigned SampleProfileLoader::getInstWeight(
Instruction &Inst) {
223 unsigned Lineno = DLoc.
getLine();
224 if (Lineno < HeaderLineno)
228 int LOffset = Lineno - HeaderLineno;
229 unsigned Discriminator = DIL->getDiscriminator();
230 unsigned Weight = Samples->samplesAt(LOffset, Discriminator);
231 DEBUG(
dbgs() <<
" " << Lineno <<
"." << Discriminator <<
":" << Inst
232 <<
" (line offset: " << LOffset <<
"." << Discriminator
233 <<
" - weight: " << Weight <<
")\n");
246 unsigned SampleProfileLoader::getBlockWeight(
BasicBlock *BB) {
248 std::pair<BlockWeightMap::iterator, bool> Entry =
249 BlockWeights.insert(std::make_pair(BB, 0));
251 return Entry.first->second;
256 unsigned InstWeight = getInstWeight(
I);
257 if (InstWeight > Weight)
260 Entry.first->second = Weight;
270 bool SampleProfileLoader::computeBlockWeights(
Function &
F) {
271 bool Changed =
false;
274 unsigned Weight = getBlockWeight(&BB);
275 Changed |= (Weight > 0);
305 void SampleProfileLoader::findEquivalencesFor(
308 for (
auto *BB2 : Descendants) {
309 bool IsDomParent = DomTree->
dominates(BB2, BB1);
310 bool IsInSameLoop = LI->getLoopFor(BB1) == LI->getLoopFor(BB2);
311 if (BB1 != BB2 && VisitedBlocks.insert(BB2).second && IsDomParent &&
313 EquivalenceClass[BB2] = BB1;
323 unsigned &BB1Weight = BlockWeights[BB1];
324 unsigned &BB2Weight = BlockWeights[BB2];
325 BB1Weight = std::max(BB1Weight, BB2Weight);
339 void SampleProfileLoader::findEquivalenceClasses(
Function &F) {
341 DEBUG(
dbgs() <<
"\nBlock equivalence classes\n");
347 if (EquivalenceClass.count(BB1)) {
348 DEBUG(printBlockEquivalence(
dbgs(), BB1));
353 EquivalenceClass[BB1] = BB1;
365 DominatedBBs.
clear();
366 DT->getDescendants(BB1, DominatedBBs);
367 findEquivalencesFor(BB1, DominatedBBs, PDT->DT);
377 DominatedBBs.
clear();
378 PDT->getDescendants(BB1, DominatedBBs);
379 findEquivalencesFor(BB1, DominatedBBs, DT);
381 DEBUG(printBlockEquivalence(
dbgs(), BB1));
390 DEBUG(
dbgs() <<
"\nAssign the same weight to all blocks in the same class\n");
395 BlockWeights[BB] = BlockWeights[EquivBB];
410 unsigned SampleProfileLoader::visitEdge(Edge E,
unsigned *NumUnknownEdges,
412 if (!VisitedEdges.count(E)) {
413 (*NumUnknownEdges)++;
418 return EdgeWeights[E];
432 bool SampleProfileLoader::propagateThroughEdges(
Function &F) {
433 bool Changed =
false;
434 DEBUG(
dbgs() <<
"\nPropagation through edges\n");
443 for (
unsigned i = 0; i < 2; i++) {
444 unsigned TotalWeight = 0;
445 unsigned NumUnknownEdges = 0;
446 Edge UnknownEdge, SelfReferentialEdge;
450 for (
auto *Pred : Predecessors[BB]) {
451 Edge E = std::make_pair(Pred, BB);
452 TotalWeight += visitEdge(E, &NumUnknownEdges, &UnknownEdge);
453 if (E.first == E.second)
454 SelfReferentialEdge = E;
458 for (
auto *Succ : Successors[BB]) {
459 Edge E = std::make_pair(BB, Succ);
460 TotalWeight += visitEdge(E, &NumUnknownEdges, &UnknownEdge);
487 if (NumUnknownEdges <= 1) {
488 unsigned &BBWeight = BlockWeights[BB];
489 if (NumUnknownEdges == 0) {
493 if (TotalWeight > BBWeight) {
494 BBWeight = TotalWeight;
496 DEBUG(
dbgs() <<
"All edge weights for " << BB->getName()
497 <<
" known. Set weight for block: ";
498 printBlockWeight(
dbgs(), BB););
500 if (VisitedBlocks.insert(BB).second)
502 }
else if (NumUnknownEdges == 1 && VisitedBlocks.count(BB)) {
505 if (BBWeight >= TotalWeight)
506 EdgeWeights[UnknownEdge] = BBWeight - TotalWeight;
508 EdgeWeights[UnknownEdge] = 0;
509 VisitedEdges.insert(UnknownEdge);
512 printEdgeWeight(
dbgs(), UnknownEdge));
514 }
else if (SelfReferentialEdge.first && VisitedBlocks.count(BB)) {
515 unsigned &BBWeight = BlockWeights[BB];
517 if (BBWeight >= TotalWeight)
518 EdgeWeights[SelfReferentialEdge] = BBWeight - TotalWeight;
520 EdgeWeights[SelfReferentialEdge] = 0;
521 VisitedEdges.insert(SelfReferentialEdge);
523 DEBUG(
dbgs() <<
"Set self-referential edge weight to: ";
524 printEdgeWeight(
dbgs(), SelfReferentialEdge));
536 void SampleProfileLoader::buildEdges(
Function &F) {
542 if (!Predecessors[B1].empty())
546 if (Visited.
insert(B2).second)
547 Predecessors[B1].push_back(B2);
552 if (!Successors[B1].empty())
556 if (Visited.
insert(B2).second)
557 Successors[B1].push_back(B2);
579 void SampleProfileLoader::propagateWeights(
Function &F) {
596 Changed = propagateThroughEdges(F);
601 DEBUG(
dbgs() <<
"\nPropagation complete. Setting branch weights\n");
608 if (!isa<BranchInst>(TI) && !isa<SwitchInst>(TI))
611 DEBUG(
dbgs() <<
"\nGetting weights for branch at line "
614 bool AllWeightsZero =
true;
617 Edge E = std::make_pair(BB, Succ);
618 unsigned Weight = EdgeWeights[E];
620 Weights.push_back(Weight);
622 AllWeightsZero =
false;
627 if (!AllWeightsZero) {
628 DEBUG(
dbgs() <<
"SUCCESS. Found non-zero weights.\n");
630 MDB.createBranchWeights(Weights));
632 DEBUG(
dbgs() <<
"SKIPPED. All branch weights are zero.\n");
648 unsigned SampleProfileLoader::getFunctionLoc(
Function &F) {
655 "No debug information found in function " + F.
getName() +
656 ": Function profile not used",
710 bool SampleProfileLoader::emitAnnotations(
Function &F) {
711 bool Changed =
false;
714 HeaderLineno = getFunctionLoc(F);
715 if (HeaderLineno == 0)
719 <<
": " << HeaderLineno <<
"\n");
722 Changed |= computeBlockWeights(F);
726 findEquivalenceClasses(F);
737 "Sample Profile loader",
false,
false)
745 bool SampleProfileLoader::doInitialization(
Module &M) {
747 if (std::error_code EC = ReaderOrErr.getError()) {
748 std::string Msg =
"Could not open profile: " + EC.message();
752 Reader = std::move(ReaderOrErr.get());
762 return new SampleProfileLoader(Name);
765 bool SampleProfileLoader::runOnFunction(
Function &F) {
769 DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
770 PDT = &getAnalysis<PostDominatorTree>();
771 LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
773 Samples = Reader->getSamplesFor(F);
774 if (!Samples->empty())
775 return emitAnnotations(F);
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
A Module instance is used to store all the information related to an LLVM module. ...
INITIALIZE_PASS_BEGIN(SampleProfileLoader,"sample-profile","Sample Profile loader", false, false) INITIALIZE_PASS_END(SampleProfileLoader
StringRef getName() const
Return a constant reference to the value's name.
Representation of the samples collected for a function.
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
static StringRef getName(Value *V)
DISubprogram * getDISubprogram(const MDNode *Scope)
Find subprogram that is enclosing this scope.
Interval::succ_iterator succ_begin(Interval *I)
succ_begin/succ_end - define methods so that Intervals may be used just like BasicBlocks can with the...
static cl::opt< std::string > SampleProfileFile("sample-profile-file", cl::init(""), cl::value_desc("filename"), cl::desc("Profile file loaded by -sample-profile"), cl::Hidden)
static ErrorOr< std::unique_ptr< SampleProfileReader > > create(StringRef Filename, LLVMContext &C)
Create a sample profile reader appropriate to the file format.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Interval::succ_iterator succ_end(Interval *I)
unsigned getNumSuccessors() const
Return the number of successors that this terminator has.
initializer< Ty > init(const Ty &Val)
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
Subclasses of this class are all able to terminate a basic block.
LLVM Basic Block Representation.
bool dominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
dominates - Returns true iff A dominates B.
BasicBlock * getSuccessor(unsigned idx) const
Return the specified successor.
This is an important class for using LLVM in a threaded context.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
const DebugLoc & getDebugLoc() const
getDebugLoc - Return the debug location for this node as a DebugLoc.
Represent the analysis usage information of a pass.
const InstListType & getInstList() const
Return the underlying instruction list container.
FunctionPass class - This class is used to implement most global optimizations.
Interval::pred_iterator pred_end(Interval *I)
void setMetadata(unsigned KindID, MDNode *Node)
setMetadata - Set the metadata of the specified kind to the specified node.
void setEntryCount(uint64_t Count)
Set the entry count for this function.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
static cl::opt< unsigned > SampleProfileMaxPropagateIterations("sample-profile-max-propagate-iterations", cl::init(100), cl::desc("Maximum number of iterations to go through when propagating ""sample block/edge weights through the CFG."))
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
PostDominatorTree Class - Concrete subclass of DominatorTree that is used to compute the post-dominat...
FunctionPass * createSampleProfileLoaderPass()
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
sample Sample Profile false
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Module * getParent()
Get the module that this global value is contained inside of...
sample Sample Profile loader
This class implements an extremely fast bulk output stream that can only output to a stream...
The legacy pass manager's analysis pass to compute loop information.
StringRef - Represent a constant reference to a string, i.e.
Legacy analysis pass which computes a DominatorTree.
Diagnostic information for the sample profiler.
void initializeSampleProfileLoaderPass(PassRegistry &)
LLVMContext & getContext() const
Get the global data context.