15 #ifndef LLVM_ANALYSIS_BLOCKFREQUENCYINFOIMPL_H
16 #define LLVM_ANALYSIS_BLOCKFREQUENCYINFOIMPL_H
36 #define DEBUG_TYPE "block-freq"
41 class BranchProbabilityInfo;
45 class MachineBasicBlock;
46 class MachineBranchProbabilityInfo;
47 class MachineFunction;
49 class MachineLoopInfo;
51 namespace bfi_detail {
83 bool isFull()
const {
return Mass == UINT64_MAX; }
92 uint64_t Sum = Mass + X.Mass;
93 Mass = Sum < Mass ? UINT64_MAX : Sum;
102 uint64_t Diff = Mass - X.Mass;
103 Mass = Diff > Mass ? 0 : Diff;
108 Mass = P.
scale(Mass);
216 template <
class It1,
class It2>
228 return Node ==
Nodes[0];
234 assert(
isHeader(B) &&
"this is only valid on loop header blocks");
269 return Loop->Parent->Parent;
287 return L ?
L->getHeader() :
Node;
293 while (
L->Parent &&
L->Parent->IsPackaged)
308 return Loop->Parent->Mass;
341 : Type(Type), TargetNode(TargetNode), Amount(Amount) {}
427 std::list<LoopData>::iterator Insert);
474 virtual std::string
getBlockName(
const BlockNode &Node)
const;
484 const BlockNode &Node)
const;
486 uint64_t Freq)
const;
488 void setBlockFreq(
const BlockNode &Node, uint64_t Freq);
496 return Freqs[0].Integer;
505 namespace bfi_detail {
530 assert(BB &&
"Unexpected nullptr");
531 auto MachineName =
"BB" +
Twine(BB->getNumber());
532 if (BB->getBasicBlock())
533 return (MachineName +
"[" + BB->getName() +
"]").str();
534 return MachineName.
str();
538 assert(BB &&
"Unexpected nullptr");
568 typedef std::deque<const IrrNode *>::const_iterator
iterator;
588 template <
class BlockEdgesAdder>
595 template <
class BlockEdgesAdder>
596 void initialize(
const BFIBase::LoopData *OuterLoop,
601 Nodes.emplace_back(Node);
605 template <
class BlockEdgesAdder>
609 const BFIBase::LoopData *OuterLoop);
611 template <
class BlockEdgesAdder>
616 for (
auto N : OuterLoop->
Nodes)
621 addEdges(Index, OuterLoop, addBlockEdges);
625 template <
class BlockEdgesAdder>
635 if (Working.isAPackage())
636 for (
const auto &
I : Working.Loop->Exits)
639 addBlockEdges(*
this, Irr, OuterLoop);
800 BranchProbabilityInfoT;
810 const BranchProbabilityInfoT *BPI;
815 std::vector<const BlockT *> RPOT;
818 typedef typename std::vector<const BlockT *>::const_iterator rpot_iterator;
820 rpot_iterator rpot_begin()
const {
return RPOT.begin(); }
821 rpot_iterator rpot_end()
const {
return RPOT.end(); }
823 size_t getIndex(
const rpot_iterator &
I)
const {
return I - rpot_begin(); }
825 BlockNode getNode(
const rpot_iterator &
I)
const {
826 return BlockNode(getIndex(I));
828 BlockNode getNode(
const BlockT *BB)
const {
return Nodes.
lookup(BB); }
830 const BlockT *getBlock(
const BlockNode &Node)
const {
831 assert(Node.Index < RPOT.size());
832 return RPOT[Node.Index];
838 void initializeRPOT();
847 void initializeLoops();
855 bool propagateMassToSuccessors(LoopData *OuterLoop,
const BlockNode &Node);
865 bool computeMassInLoop(LoopData &Loop);
875 bool tryToComputeMassInFunction();
889 void computeIrreducibleMass(LoopData *OuterLoop,
890 std::list<LoopData>::iterator Insert);
901 void computeMassInLoops();
909 void computeMassInFunction();
911 std::string
getBlockName(
const BlockNode &Node)
const override {
918 void calculate(
const FunctionT &
F,
const BranchProbabilityInfoT &BPI,
919 const LoopInfoT &LI);
927 const BlockT *BB)
const {
931 uint64_t Freq)
const {
939 const BranchProbabilityInfoT &
getBPI()
const {
return *BPI; }
963 const BranchProbabilityInfoT &BPI,
964 const LoopInfoT &LI) {
976 DEBUG(
dbgs() <<
"\nblock-frequency: " << F.getName() <<
"\n================="
977 << std::string(F.getName().size(),
'=') <<
"\n");
983 computeMassInLoops();
984 computeMassInFunction();
999 Freqs.emplace_back();
1005 const BlockT *Entry = &
F->front();
1006 RPOT.reserve(
F->size());
1007 std::copy(
po_begin(Entry),
po_end(Entry), std::back_inserter(RPOT));
1010 assert(RPOT.size() - 1 <= BlockNode::getMaxIndex() &&
1011 "More nodes in function than Block Frequency Info supports");
1013 DEBUG(
dbgs() <<
"reverse-post-order-traversal\n");
1014 for (rpot_iterator I = rpot_begin(),
E = rpot_end(); I !=
E; ++
I) {
1015 BlockNode Node = getNode(I);
1020 Working.reserve(RPOT.size());
1021 for (
size_t Index = 0; Index < RPOT.size(); ++Index)
1022 Working.emplace_back(Index);
1023 Freqs.resize(RPOT.size());
1026 template <
class BT>
void BlockFrequencyInfoImpl<BT>::initializeLoops() {
1032 std::deque<std::pair<const LoopT *, LoopData *>> Q;
1033 for (
const LoopT *
L : *LI)
1034 Q.emplace_back(
L,
nullptr);
1035 while (!Q.empty()) {
1036 const LoopT *Loop = Q.front().first;
1037 LoopData *Parent = Q.front().second;
1040 BlockNode Header = getNode(Loop->getHeader());
1041 assert(Header.isValid());
1043 Loops.emplace_back(Parent, Header);
1044 Working[Header.Index].Loop = &
Loops.back();
1047 for (
const LoopT *
L : *Loop)
1048 Q.emplace_back(
L, &
Loops.back());
1053 for (
size_t Index = 0; Index < RPOT.size(); ++Index) {
1055 if (Working[Index].isLoopHeader()) {
1056 LoopData *ContainingLoop = Working[Index].getContainingLoop();
1058 ContainingLoop->Nodes.push_back(Index);
1062 const LoopT *Loop = LI->getLoopFor(RPOT[Index]);
1067 BlockNode Header = getNode(Loop->getHeader());
1068 assert(Header.isValid());
1069 const auto &HeaderData = Working[Header.Index];
1070 assert(HeaderData.isLoopHeader());
1072 Working[Index].Loop = HeaderData.Loop;
1073 HeaderData.Loop->Nodes.push_back(Index);
1079 template <
class BT>
void BlockFrequencyInfoImpl<BT>::computeMassInLoops() {
1082 if (computeMassInLoop(*
L))
1084 auto Next = std::next(
L);
1085 computeIrreducibleMass(&*
L,
L.base());
1086 L = std::prev(Next);
1087 if (computeMassInLoop(*L))
1094 bool BlockFrequencyInfoImpl<BT>::computeMassInLoop(LoopData &Loop) {
1096 DEBUG(
dbgs() <<
"compute-mass-in-loop: " << getLoopName(Loop) <<
"\n");
1098 if (Loop.isIrreducible()) {
1099 BlockMass Remaining = BlockMass::getFull();
1101 auto &Mass = Working[Loop.Nodes[
H].Index].getMass();
1102 Mass = Remaining * BranchProbability(1, Loop.NumHeaders -
H);
1105 for (
const BlockNode &M : Loop.Nodes)
1106 if (!propagateMassToSuccessors(&Loop, M))
1109 adjustLoopHeaderMass(Loop);
1111 Working[Loop.getHeader().Index].getMass() = BlockMass::getFull();
1112 if (!propagateMassToSuccessors(&Loop, Loop.getHeader()))
1114 for (
const BlockNode &M : Loop.members())
1115 if (!propagateMassToSuccessors(&Loop, M))
1120 computeLoopScale(Loop);
1126 bool BlockFrequencyInfoImpl<BT>::tryToComputeMassInFunction() {
1128 DEBUG(
dbgs() <<
"compute-mass-in-function\n");
1129 assert(!Working.empty() &&
"no blocks in function");
1130 assert(!Working[0].isLoopHeader() &&
"entry block is a loop header");
1132 Working[0].getMass() = BlockMass::getFull();
1133 for (rpot_iterator I = rpot_begin(),
IE = rpot_end(); I !=
IE; ++
I) {
1135 BlockNode Node = getNode(I);
1136 if (Working[Node.Index].isPackaged())
1139 if (!propagateMassToSuccessors(
nullptr, Node))
1145 template <
class BT>
void BlockFrequencyInfoImpl<BT>::computeMassInFunction() {
1146 if (tryToComputeMassInFunction())
1148 computeIrreducibleMass(
nullptr,
Loops.begin());
1149 if (tryToComputeMassInFunction())
1155 namespace bfi_detail {
1156 template <
class BT>
struct BlockEdgesAdder {
1167 for (
auto I = Successor::child_begin(BB),
E = Successor::child_end(BB);
1174 void BlockFrequencyInfoImpl<BT>::computeIrreducibleMass(
1175 LoopData *OuterLoop, std::list<LoopData>::iterator Insert) {
1176 DEBUG(
dbgs() <<
"analyze-irreducible-in-";
1177 if (OuterLoop)
dbgs() <<
"loop: " << getLoopName(*OuterLoop) <<
"\n";
1178 else dbgs() <<
"function\n");
1180 using namespace bfi_detail;
1183 BlockEdgesAdder<BT> addBlockEdges(*
this);
1184 IrreducibleGraph
G(*
this, OuterLoop, addBlockEdges);
1186 for (
auto &L : analyzeIrreducible(
G, OuterLoop, Insert))
1187 computeMassInLoop(L);
1191 updateLoopWithIrreducible(*OuterLoop);
1201 BlockFrequencyInfoImpl<BT>::propagateMassToSuccessors(LoopData *OuterLoop,
1202 const BlockNode &Node) {
1206 if (
auto *Loop = Working[Node.Index].getPackagedLoop()) {
1207 assert(Loop != OuterLoop &&
"Cannot propagate mass in a packaged loop");
1208 if (!addLoopSuccessorsToDist(OuterLoop, *Loop, Dist))
1212 const BlockT *BB = getBlock(Node);
1213 for (
auto SI = Successor::child_begin(BB), SE = Successor::child_end(BB);
1215 if (!addToDist(Dist, OuterLoop, Node, getNode(*SI),
1223 distributeMass(Node, OuterLoop, Dist);
1231 OS <<
"block-frequency-info: " <<
F->getName() <<
"\n";
1232 for (
const BlockT &BB : *
F) {
1234 getFloatingBlockFreq(&BB).print(OS, 5)
1235 <<
", int = " << getBlockFreq(&BB).getFrequency() <<
"\n";
1248 template <
class BlockFrequencyInfoT,
class BranchProbabilityInfoT>
1260 return G->getFunction()->getName();
1264 unsigned HotPercentThreshold = 0) {
1266 if (!HotPercentThreshold)
1271 for (
NodeIter I = GTraits::nodes_begin(Graph),
1272 E = GTraits::nodes_end(Graph);
1276 std::max(
MaxFrequency, Graph->getBlockFreq(N).getFrequency());
1288 OS <<
"color=\"red\"";
1298 OS << Node->getName().
str() <<
" : ";
1301 Graph->printBlockFreq(OS, Node);
1304 OS << Graph->getBlockFreq(Node).getFrequency();
1307 auto Count = Graph->getBlockProfileCount(Node);
1309 OS << Count.getValue();
1316 "never reach this point.");
1322 const BlockFrequencyInfoT *
BFI,
1323 const BranchProbabilityInfoT *BPI,
1324 unsigned HotPercentThreshold = 0) {
1334 OS <<
format(
"label=\"%.1f%%\"", Percent);
1336 if (HotPercentThreshold) {
1341 if (EFreq >= HotFreq) {
1342 OS <<
",color=\"red\"";
BlockMass operator+(BlockMass L, BlockMass R)
void setBlockFreq(const BlockT *BB, uint64_t Freq)
void addNodesInLoop(const BFIBase::LoopData &OuterLoop)
GraphTraits< BlockFrequencyInfoT * > GTraits
bool isDoubleLoopHeader() const
bool IsPackaged
Whether this has been packaged.
LoopData * Loop
The loop this block is inside.
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Optional< uint64_t > getBlockProfileCount(const Function &F, const BlockT *BB) const
bfi_detail::BlockMass BlockMass
LoopData(LoopData *Parent, It1 FirstHeader, It1 LastHeader, It2 FirstOther, It2 LastOther)
SmallVector< BlockMass, 1 > HeaderMassList
This provides a very simple, boring adaptor for a begin and end iterator into a range type...
Index of loop information.
MachineBranchProbabilityInfo BranchProbabilityInfoT
Stats about a block itself.
const BranchProbabilityInfoT & getBPI() const
static std::string getGraphName(const BlockFrequencyInfoT *G)
uint32_t getNumerator() const
SmallVector< std::pair< BlockNode, BlockMass >, 4 > ExitMap
void addLocal(const BlockNode &Node, uint64_t Amount)
bool isADoublePackage() const
Has Loop been packaged up twice?
SmallDenseMap< uint32_t, IrrNode *, 4 > Lookup
LoopData * Parent
The parent loop.
bool operator==(BlockMass X) const
uint32_t NumHeaders
Number of headers.
ScaledNumber< uint64_t > Scaled64
IrreducibleGraph(BFIBase &BFI, const BFIBase::LoopData *OuterLoop, BlockEdgesAdder addBlockEdges)
Construct an explicit graph containing irreducible control flow.
const BlockFrequencyInfoImpl< BT > & BFI
void setBlockFreq(const BlockNode &Node, uint64_t Freq)
WorkingData(const BlockNode &Node)
BFIDOTGraphTraitsBase(bool isSimple=false)
raw_ostream & print(raw_ostream &OS) const
StringRef getName() const
Return a constant reference to the value's name.
void operator()(IrreducibleGraph &G, IrreducibleGraph::IrrNode &Irr, const LoopData *OuterLoop)
Weight(DistType Type, BlockNode TargetNode, uint64_t Amount)
uint32_t getWeightFromBranchProb(const BranchProbability Prob)
std::string str() const
Return the twine contents as a std::string.
virtual ~BlockFrequencyInfoImplBase()
Virtual destructor.
BlockMass & getMass()
Get the appropriate mass for a node.
Optional< uint64_t > getProfileCountFromFreq(const Function &F, uint64_t Freq) const
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
ScaledNumber< uint64_t > toScaled() const
Convert to scaled number.
BlockFrequency getBlockFreq(const BlockNode &Node) const
bool isPackaged() const
Has ContainingLoop been packaged up?
static GCRegistry::Add< StatepointGC > D("statepoint-example","an example strategy for statepoint")
void addEdge(IrrNode &Irr, const BlockNode &Succ, const BFIBase::LoopData *OuterLoop)
iterator pred_begin() const
BlockMass & operator*=(BranchProbability P)
void adjustLoopHeaderMass(LoopData &Loop)
Adjust the mass of all headers in an irreducible loop.
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Graph of irreducible control flow.
Function Alias Analysis false
IrrNode(const BlockNode &Node)
std::string getBlockName(const BasicBlock *BB)
Get the name of a BasicBlock.
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
void calculate(const FunctionT &F, const BranchProbabilityInfoT &BPI, const LoopInfoT &LI)
std::string getNodeLabel(NodeRef Node, const BlockFrequencyInfoT *Graph, GVDAGType GType)
bool operator>(BlockMass X) const
raw_ostream & print(raw_ostream &OS) const override
Print the frequencies for the current function.
ptrdiff_t difference_type
NodeList::const_iterator members_begin() const
void computeLoopScale(LoopData &Loop)
Compute the loop scale for a loop.
void addBackedge(const BlockNode &Node, uint64_t Amount)
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
void initialize(const BFIBase::LoopData *OuterLoop, BlockEdgesAdder addBlockEdges)
BlockNode getResolvedNode() const
Resolve a node to its representative.
format_object< Ts...> format(const char *Fmt, const Ts &...Vals)
These are helper functions used to produce formatted output.
Optional< uint64_t > getBlockProfileCount(const Function &F, const BlockNode &Node) const
bool addToDist(Distribution &Dist, const LoopData *OuterLoop, const BlockNode &Pred, const BlockNode &Succ, uint64_t Weight)
Add an edge to the distribution.
std::vector< FrequencyData > Freqs
Data about each block. This is used downstream.
GraphType::UnknownGraphTypeError NodeRef
iterator_range< std::list< LoopData >::iterator > analyzeIrreducible(const bfi_detail::IrreducibleGraph &G, LoopData *OuterLoop, std::list< LoopData >::iterator Insert)
Analyze irreducible SCCs.
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
void normalize()
Normalize the distribution.
BlockMass & operator-=(BlockMass X)
Subtract another mass.
BlockMass & operator+=(BlockMass X)
Add another mass.
bool operator>=(BlockMass X) const
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin()
std::deque< const IrrNode * >::const_iterator iterator
std::string getLoopName(const LoopData &Loop) const
SmallVector< Weight, 4 > WeightList
Distribution of unscaled probability weight.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang","erlang-compatible garbage collector")
void addNode(const BlockNode &Node)
bool DidOverflow
Whether Total did overflow.
po_iterator< T > po_end(const T &G)
Scaled64 getFloatingBlockFreq(const BlockNode &Node) const
raw_ostream & printBlockFreq(raw_ostream &OS, const BlockNode &Node) const
BlockMass Mass
Mass distribution from the entry block.
iterator_range< NodeList::const_iterator > members() const
bool isHeader(const BlockNode &Node) const
void clear()
Clear all memory.
LLVM_NODISCARD std::string str() const
str - Get the contents as an std::string.
BlockEdgesAdder(const BlockFrequencyInfoImpl< BT > &BFI)
void addNodesInFunction()
BlockMass operator*(BlockMass L, BranchProbability R)
bool operator<=(BlockMass X) const
std::string getEdgeAttributes(NodeRef Node, EdgeIter EI, const BlockFrequencyInfoT *BFI, const BranchProbabilityInfoT *BPI, unsigned HotPercentThreshold=0)
Scaled64 getFloatingBlockFreq(const BlockT *BB) const
bool operator<(BlockMass X) const
raw_ostream & operator<<(raw_ostream &OS, BlockMass X)
std::string & str()
Flushes the stream contents to the target string and returns the string's reference.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void addExit(const BlockNode &Node, uint64_t Amount)
LoopData * getContainingLoop() const
BlockFrequencyInfoImplBase BFIBase
BlockNode(IndexType Index)
iterator succ_begin() const
BlockFrequencyInfoImplBase::LoopData LoopData
std::string getNodeAttributes(NodeRef Node, const BlockFrequencyInfoT *Graph, unsigned HotPercentThreshold=0)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
isPodLike - This is a type trait that is used to determine whether a given type can be copied around ...
static uint32_t getDenominator()
bool isLoopHeader() const
ExitMap Exits
Successor edges (and weights).
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
SmallVector< BlockNode, 4 > NodeList
bool operator==(const BlockNode &X) const
GTraits::ChildIteratorType EdgeIter
static BranchProbability getBranchProbability(uint64_t Numerator, uint64_t Denominator)
BlockMass operator-(BlockMass L, BlockMass R)
virtual std::string getBlockName(const BlockNode &Node) const
uint64_t Total
Sum of all weights.
static BlockMass getEmpty()
BFIBase::BlockNode BlockNode
std::list< LoopData > Loops
Indexed information about loops.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
A range adaptor for a pair of iterators.
std::string getBlockName(const BlockT *BB)
Get the name of a MachineBasicBlock.
void updateLoopWithIrreducible(LoopData &OuterLoop)
Update a loop after packaging irreducible SCCs inside of it.
const FunctionT * getFunction() const
bool operator!=(BlockMass X) const
BlockFrequency getBlockFreq(const BlockT *BB) const
LoopData(LoopData *Parent, const BlockNode &Header)
iterator insert(iterator I, T &&Elt)
Unscaled probability weight.
bool operator<=(const BlockNode &X) const
NodeList::const_iterator members_end() const
LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end()
Analysis providing branch probability information.
iterator pred_end() const
uint64_t scale(uint64_t Num) const
Scale a large integer.
void addEdges(const BlockNode &Node, const BFIBase::LoopData *OuterLoop, BlockEdgesAdder addBlockEdges)
Represents a single loop in the control flow graph.
void packageLoop(LoopData &Loop)
Package up a loop.
bool isAPackage() const
Has Loop been packaged up?
uint64_t getEntryFreq() const
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
bool operator>=(const BlockNode &X) const
virtual raw_ostream & print(raw_ostream &OS) const
Optional< uint64_t > getProfileCountFromFreq(const Function &F, uint64_t Freq) const
static BlockMass getFull()
MachineLoopInfo LoopInfoT
BranchProbabilityInfo BranchProbabilityInfoT
bool operator<(const BlockNode &X) const
GTraits::nodes_iterator NodeIter
bool isIrreducible() const
raw_ostream & printBlockFreq(raw_ostream &OS, const BlockT *BB) const
void distributeMass(const BlockNode &Source, LoopData *OuterLoop, Distribution &Dist)
Distribute mass according to a distribution.
NodeList Nodes
Header and the members of the loop.
bool operator!=(const BlockNode &X) const
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
A raw_ostream that writes to an std::string.
LoopData * getPackagedLoop() const
std::vector< WorkingData > Working
Loop data: see initializeLoops().
std::vector< IrrNode > Nodes
HeaderMassList BackedgeMass
Mass returned to each loop header.
This class implements an extremely fast bulk output stream that can only output to a stream...
DefaultDOTGraphTraits - This class provides the default implementations of all of the DOTGraphTraits ...
Shared implementation for block frequency analysis.
static size_t getMaxIndex()
std::deque< const IrrNode * > Edges
Base class for BlockFrequencyInfoImpl.
bool addLoopSuccessorsToDist(const LoopData *OuterLoop, LoopData &Loop, Distribution &Dist)
Add all edges out of a packaged loop to the distribution.
GraphTraits< const BlockT * > Successor
iterator succ_end() const
void unwrapLoops()
Unwrap loops.
HeaderMassList::difference_type getHeaderIndex(const BlockNode &B)
LoopData & getLoopPackage(const BlockNode &Head)
BlockNode getHeader() const
Representative of a block.
po_iterator< T > po_begin(const T &G)
void finalizeMetrics()
Finalize frequency metrics.
MachineFunction FunctionT
bool operator>(const BlockNode &X) const
WeightList Weights
Individual successor weights.