28 #define DEBUG_TYPE "branch-prob"
31 "Branch Probability Analysis",
false,
true)
36 char BranchProbabilityInfoWrapperPass::
ID = 0;
118 if (isa<UnreachableInst>(TI) ||
123 BB->getTerminatingDeoptimizeCall())
124 PostDominatedByUnreachable.insert(BB);
132 if (PostDominatedByUnreachable.count(*
I))
133 UnreachableEdges.
push_back(
I.getSuccessorIndex());
135 ReachableEdges.
push_back(
I.getSuccessorIndex());
141 PostDominatedByUnreachable.insert(BB);
150 if (
auto *II = dyn_cast<InvokeInst>(TI))
151 if (PostDominatedByUnreachable.count(II->getNormalDest())) {
152 PostDominatedByUnreachable.insert(BB);
158 if (ReachableEdges.
empty()) {
160 for (
unsigned SuccIdx : UnreachableEdges)
161 setEdgeProbability(BB, SuccIdx, Prob);
166 UR_TAKEN_WEIGHT, (UR_TAKEN_WEIGHT + UR_NONTAKEN_WEIGHT) *
167 uint64_t(UnreachableEdges.
size()));
170 (UR_TAKEN_WEIGHT + UR_NONTAKEN_WEIGHT) * uint64_t(ReachableEdges.
size()));
172 for (
unsigned SuccIdx : UnreachableEdges)
173 setEdgeProbability(BB, SuccIdx, UnreachableProb);
174 for (
unsigned SuccIdx : ReachableEdges)
175 setEdgeProbability(BB, SuccIdx, ReachableProb);
182 bool BranchProbabilityInfo::calcMetadataWeights(
const BasicBlock *BB) {
186 if (!isa<BranchInst>(TI) && !isa<SwitchInst>(TI))
204 uint64_t WeightSum = 0;
209 mdconst::dyn_extract<ConstantInt>(WeightsNode->
getOperand(
i));
213 "Too many bits for uint32_t");
215 WeightSum += Weights.
back();
221 uint64_t ScalingFactor =
222 (WeightSum > UINT32_MAX) ? WeightSum / UINT32_MAX + 1 : 1;
226 Weights[
i] /= ScalingFactor;
227 WeightSum += Weights[
i];
230 if (WeightSum == 0) {
238 assert(WeightSum <= UINT32_MAX &&
239 "Expected weights to scale down to 32 bits");
252 bool BranchProbabilityInfo::calcColdCallHeuristics(
const BasicBlock *BB) {
261 if (PostDominatedByColdCall.count(*
I))
269 PostDominatedByColdCall.insert(BB);
273 assert(!PostDominatedByColdCall.count(BB));
275 if (
const CallInst *CI = dyn_cast<CallInst>(
I))
277 PostDominatedByColdCall.insert(BB);
282 if (
auto *II = dyn_cast<InvokeInst>(TI)) {
285 if (PostDominatedByColdCall.count(II->getNormalDest()))
286 PostDominatedByColdCall.insert(BB);
296 if (NormalEdges.
empty()) {
298 for (
unsigned SuccIdx : ColdEdges)
310 for (
unsigned SuccIdx : ColdEdges)
312 for (
unsigned SuccIdx : NormalEdges)
320 bool BranchProbabilityInfo::calcPointerHeuristics(
const BasicBlock *BB) {
341 unsigned TakenIdx = 0, NonTakenIdx = 1;
355 bool BranchProbabilityInfo::calcLoopBranchHeuristics(
const BasicBlock *BB,
367 ExitingEdges.
push_back(
I.getSuccessorIndex());
374 if (BackEdges.
empty() && ExitingEdges.
empty())
385 if (!BackEdges.
empty())
387 if (!InEdges.
empty())
389 if (!ExitingEdges.
empty())
393 auto Prob = Probs[0] / numBackEdges;
394 for (
unsigned SuccIdx : BackEdges)
399 auto Prob = Probs[1] / numInEdges;
400 for (
unsigned SuccIdx : InEdges)
405 auto Prob = Probs[2] / numExitingEdges;
406 for (
unsigned SuccIdx : ExitingEdges)
413 bool BranchProbabilityInfo::calcZeroHeuristics(
const BasicBlock *BB) {
432 if (
ConstantInt *AndRHS = dyn_cast<ConstantInt>(LHS->getOperand(1)))
433 if (AndRHS->getUniqueInteger().isPowerOf2())
484 unsigned TakenIdx = 0, NonTakenIdx = 1;
496 bool BranchProbabilityInfo::calcFloatingPointHeuristics(
const BasicBlock *BB) {
521 unsigned TakenIdx = 0, NonTakenIdx = 1;
533 bool BranchProbabilityInfo::calcInvokeHeuristics(
const BasicBlock *BB) {
550 OS <<
"---- Branch Probabilities ----\n";
553 assert(LastF &&
"Cannot print prior to running over a function");
554 for (
const auto &BI : *LastF) {
577 if (Prob > MaxProb) {
596 unsigned IndexInSuccessors)
const {
597 auto I = Probs.find(std::make_pair(Src, IndexInSuccessors));
599 if (
I != Probs.end())
618 bool FoundProb =
false;
621 auto MapI = Probs.find(std::make_pair(Src,
I.getSuccessorIndex()));
622 if (MapI != Probs.end()) {
624 Prob += MapI->second;
634 unsigned IndexInSuccessors,
636 Probs[std::make_pair(Src, IndexInSuccessors)] = Prob;
637 Handles.insert(BasicBlockCallbackVH(Src,
this));
638 DEBUG(
dbgs() <<
"set edge " << Src->
getName() <<
" -> " << IndexInSuccessors
639 <<
" successor probability to " << Prob <<
"\n");
649 <<
" probability is " << Prob
650 << (
isEdgeHot(Src, Dst) ?
" [HOT edge]\n" :
"\n");
656 for (
auto I = Probs.begin(),
E = Probs.end();
I !=
E; ++
I) {
667 assert(PostDominatedByUnreachable.empty());
668 assert(PostDominatedByColdCall.empty());
674 if (calcUnreachableHeuristics(BB))
676 if (calcMetadataWeights(BB))
678 if (calcColdCallHeuristics(BB))
680 if (calcLoopBranchHeuristics(BB, LI))
682 if (calcPointerHeuristics(BB))
684 if (calcZeroHeuristics(BB))
686 if (calcFloatingPointHeuristics(BB))
688 calcInvokeHeuristics(BB);
691 PostDominatedByUnreachable.clear();
692 PostDominatedByColdCall.clear();
702 const LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
703 BPI.calculate(F, LI);
724 OS <<
"Printing analysis results of BPI for function "
void push_back(const T &Elt)
static bool isEquality(Predicate Pred)
raw_ostream & printEdgeProbability(raw_ostream &OS, const BasicBlock *Src, const BasicBlock *Dst) const
Print an edge's probability.
const BasicBlock * getHotSucc(const BasicBlock *BB) const
Retrieve the hot successor of a block if one exists.
A Module instance is used to store all the information related to an LLVM module. ...
unsigned getNumOperands() const
Return number of MDNode operands.
This class represents a function call, abstracting a target machine's calling convention.
static const uint32_t FPH_TAKEN_WEIGHT
static bool isEquality(Predicate P)
Return true if this predicate is either EQ or NE.
bool runOnFunction(Function &F) override
runOnFunction - Virtual method overriden by subclasses to do the per-function processing of the pass...
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
void reserve(size_type N)
BlockT * getHeader() const
StringRef getName() const
Return a constant reference to the value's name.
iterator begin()
Instruction iterator methods.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
branch Branch Probability false
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
const APInt & getValue() const
Return the constant as an APInt value reference.
Analysis pass that exposes the LoopInfo for a function.
LLVM_NODISCARD bool empty() const
Interval::succ_iterator succ_begin(Interval *I)
succ_begin/succ_end - define methods so that Intervals may be used just like BasicBlocks can with the...
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Analysis pass which computes BranchProbabilityInfo.
This instruction compares its operands according to the predicate given to the constructor.
static const uint32_t UR_TAKEN_WEIGHT
Unreachable-terminating branch taken weight.
unsigned getActiveBits() const
Compute the number of active bits in the value.
Legacy analysis pass which computes BranchProbabilityInfo.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
Interval::succ_iterator succ_end(Interval *I)
unsigned getNumSuccessors() const
Return the number of successors that this terminator has.
Subclasses of this class are all able to terminate a basic block.
A set of analyses that are preserved following a run of a transformation pass.
static const uint32_t IH_NONTAKEN_WEIGHT
Invoke-terminating normal branch not-taken weight.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs...ExtraArgs)
Get the result of an analysis pass for a given IR unit.
LLVM Basic Block Representation.
Conditional or Unconditional Branch instruction.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static const uint32_t CC_TAKEN_WEIGHT
Weight for a branch taken going into a cold block.
void eraseBlock(const BasicBlock *BB)
Forget analysis results for the given basic block.
Represent the analysis usage information of a pass.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
This instruction compares its operands according to the predicate given to the constructor.
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
Value * getOperand(unsigned i) const
0 1 1 1 True if ordered (no nans)
iterator_range< po_iterator< T > > post_order(const T &G)
Predicate getPredicate() const
Return the predicate for this instruction.
static const uint32_t ZH_NONTAKEN_WEIGHT
BranchProbabilityInfo run(Function &F, FunctionAnalysisManager &AM)
Run the analysis pass over a function and produce BPI.
bool isPointerTy() const
True if this is an instance of PointerType.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
bool isConditional() const
void calculate(const Function &F, const LoopInfo &LI)
const MDOperand & getOperand(unsigned I) const
Iterator for intrusive lists based on ilist_node.
This is the shared class of boolean and integer constants.
Type * getType() const
All values are typed, get the type of this value.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
static BranchProbability getBranchProbability(uint64_t Numerator, uint64_t Denominator)
void setEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors, BranchProbability Prob)
Set the raw edge probability for the given edge.
bool isTrueWhenEqual() const
This is just a convenience.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
const BasicBlock & getEntryBlock() const
bool isEdgeHot(const BasicBlock *Src, const BasicBlock *Dst) const
Test if an edge is hot relative to other out-edges of the Src.
branch Branch Probability Analysis
void print(raw_ostream &OS) const
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isAllOnesValue() const
Return true if this is the value that would be returned by getAllOnesValue.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static const uint32_t FPH_NONTAKEN_WEIGHT
void setPreservesAll()
Set by analyses that do not transform their input at all.
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
static const uint32_t IH_TAKEN_WEIGHT
Invoke-terminating normal branch taken weight.
Value * getCondition() const
Analysis providing branch probability information.
static const uint32_t PH_NONTAKEN_WEIGHT
Represents a single loop in the control flow graph.
static const uint32_t LBH_NONTAKEN_WEIGHT
static const uint32_t LBH_TAKEN_WEIGHT
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
void releaseMemory() override
releaseMemory() - This member can be implemented by a pass if it wants to be able to release its memo...
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
INITIALIZE_PASS_BEGIN(BranchProbabilityInfoWrapperPass,"branch-prob","Branch Probability Analysis", false, true) INITIALIZE_PASS_END(BranchProbabilityInfoWrapperPass
static const uint32_t CC_NONTAKEN_WEIGHT
Weight for a branch not-taken into a cold block.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVM Value Representation.
static const uint32_t ZH_TAKEN_WEIGHT
unsigned getSuccessorIndex() const
This is used to interface between code that wants to operate on terminator instructions directly...
static const uint32_t PH_TAKEN_WEIGHT
This class implements an extremely fast bulk output stream that can only output to a stream...
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
The legacy pass manager's analysis pass to compute loop information.
A container for analyses that lazily runs them and caches their results.
static BranchProbability getZero()
void print(raw_ostream &OS, const Module *M=nullptr) const override
print - Print out the internal state of the pass.
A special type used by analysis passes to provide an address that identifies that particular analysis...
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
static const uint32_t UR_NONTAKEN_WEIGHT
Unreachable-terminating branch not-taken weight.