15 #ifndef LLVM_ANALYSIS_BLOCKFREQUENCYINFOIMPL_H
16 #define LLVM_ANALYSIS_BLOCKFREQUENCYINFOIMPL_H
32 #define DEBUG_TYPE "block-freq"
37 class BranchProbabilityInfo;
41 class MachineBasicBlock;
42 class MachineBranchProbabilityInfo;
43 class MachineFunction;
45 class MachineLoopInfo;
47 namespace bfi_detail {
79 bool isFull()
const {
return Mass == UINT64_MAX; }
88 uint64_t Sum = Mass + X.Mass;
89 Mass = Sum < Mass ? UINT64_MAX : Sum;
98 uint64_t Diff = Mass - X.Mass;
99 Mass = Diff > Mass ? 0 : Diff;
104 Mass = P.
scale(Mass);
212 template <
class It1,
class It2>
224 return Node ==
Nodes[0];
230 assert(
isHeader(B) &&
"this is only valid on loop header blocks");
265 return Loop->Parent->Parent;
283 return L ? L->getHeader() :
Node;
289 while (L->Parent && L->Parent->IsPackaged)
304 return Loop->Parent->Mass;
337 : Type(Type), TargetNode(TargetNode), Amount(Amount) {}
423 std::list<LoopData>::iterator Insert);
470 virtual std::string
getBlockName(
const BlockNode &Node)
const;
485 assert(!
Freqs.empty());
486 return Freqs[0].Integer;
495 namespace bfi_detail {
520 assert(BB &&
"Unexpected nullptr");
521 auto MachineName =
"BB" +
Twine(BB->getNumber());
522 if (BB->getBasicBlock())
523 return (MachineName +
"[" + BB->getName() +
"]").str();
524 return MachineName.
str();
528 assert(BB &&
"Unexpected nullptr");
558 typedef std::deque<const IrrNode *>::const_iterator
iterator;
578 template <
class BlockEdgesAdder>
585 template <
class BlockEdgesAdder>
586 void initialize(
const BFIBase::LoopData *OuterLoop,
591 Nodes.emplace_back(Node);
595 template <
class BlockEdgesAdder>
599 const BFIBase::LoopData *OuterLoop);
601 template <
class BlockEdgesAdder>
606 for (
auto N : OuterLoop->
Nodes)
610 for (uint32_t Index = 0; Index <
BFI.
Working.size(); ++Index)
611 addEdges(Index, OuterLoop, addBlockEdges);
615 template <
class BlockEdgesAdder>
625 if (Working.isAPackage())
626 for (
const auto &
I : Working.Loop->Exits)
629 addBlockEdges(*
this, Irr, OuterLoop);
790 BranchProbabilityInfoT;
800 const BranchProbabilityInfoT *BPI;
805 std::vector<const BlockT *> RPOT;
808 typedef typename std::vector<const BlockT *>::const_iterator rpot_iterator;
810 rpot_iterator rpot_begin()
const {
return RPOT.begin(); }
811 rpot_iterator rpot_end()
const {
return RPOT.end(); }
813 size_t getIndex(
const rpot_iterator &
I)
const {
return I - rpot_begin(); }
815 BlockNode getNode(
const rpot_iterator &
I)
const {
816 return BlockNode(getIndex(I));
818 BlockNode getNode(
const BlockT *BB)
const {
return Nodes.
lookup(BB); }
820 const BlockT *getBlock(
const BlockNode &Node)
const {
821 assert(Node.Index < RPOT.size());
822 return RPOT[Node.Index];
828 void initializeRPOT();
837 void initializeLoops();
845 bool propagateMassToSuccessors(LoopData *OuterLoop,
const BlockNode &Node);
855 bool computeMassInLoop(LoopData &Loop);
865 bool tryToComputeMassInFunction();
879 void computeIrreducibleMass(LoopData *OuterLoop,
880 std::list<LoopData>::iterator Insert);
891 void computeMassInLoops();
899 void computeMassInFunction();
901 std::string
getBlockName(
const BlockNode &Node)
const override {
908 void doFunction(
const FunctionT *
F,
const BranchProbabilityInfoT *BPI,
909 const LoopInfoT *LI);
942 const BranchProbabilityInfoT *BPI,
943 const LoopInfoT *LI) {
955 DEBUG(
dbgs() <<
"\nblock-frequency: " << F->getName() <<
"\n================="
956 << std::string(F->getName().size(),
'=') <<
"\n");
962 computeMassInLoops();
963 computeMassInFunction();
969 const BlockT *Entry =
F->begin();
970 RPOT.reserve(
F->size());
971 std::copy(
po_begin(Entry),
po_end(Entry), std::back_inserter(RPOT));
972 std::reverse(RPOT.begin(), RPOT.end());
974 assert(RPOT.size() - 1 <= BlockNode::getMaxIndex() &&
975 "More nodes in function than Block Frequency Info supports");
977 DEBUG(
dbgs() <<
"reverse-post-order-traversal\n");
978 for (rpot_iterator I = rpot_begin(), E = rpot_end(); I != E; ++
I) {
979 BlockNode Node = getNode(I);
984 Working.reserve(RPOT.size());
985 for (
size_t Index = 0; Index < RPOT.size(); ++Index)
986 Working.emplace_back(Index);
987 Freqs.resize(RPOT.size());
990 template <
class BT>
void BlockFrequencyInfoImpl<BT>::initializeLoops() {
996 std::deque<std::pair<const LoopT *, LoopData *>> Q;
997 for (
const LoopT *L : *LI)
998 Q.emplace_back(L,
nullptr);
1000 const LoopT *Loop = Q.front().first;
1001 LoopData *Parent = Q.front().second;
1004 BlockNode Header = getNode(Loop->getHeader());
1005 assert(Header.isValid());
1007 Loops.emplace_back(Parent, Header);
1008 Working[Header.Index].Loop = &
Loops.back();
1011 for (
const LoopT *L : *Loop)
1012 Q.emplace_back(L, &
Loops.back());
1017 for (
size_t Index = 0; Index < RPOT.size(); ++Index) {
1019 if (Working[Index].isLoopHeader()) {
1020 LoopData *ContainingLoop = Working[Index].getContainingLoop();
1022 ContainingLoop->Nodes.push_back(Index);
1026 const LoopT *Loop = LI->getLoopFor(RPOT[Index]);
1031 BlockNode Header = getNode(Loop->getHeader());
1032 assert(Header.isValid());
1033 const auto &HeaderData = Working[Header.Index];
1034 assert(HeaderData.isLoopHeader());
1036 Working[Index].Loop = HeaderData.Loop;
1037 HeaderData.Loop->Nodes.push_back(Index);
1043 template <
class BT>
void BlockFrequencyInfoImpl<BT>::computeMassInLoops() {
1045 for (
auto L =
Loops.rbegin(), E =
Loops.rend(); L != E; ++L) {
1046 if (computeMassInLoop(*L))
1048 auto Next = std::next(L);
1049 computeIrreducibleMass(&*L, L.base());
1050 L = std::prev(Next);
1051 if (computeMassInLoop(*L))
1058 bool BlockFrequencyInfoImpl<BT>::computeMassInLoop(LoopData &Loop) {
1060 DEBUG(
dbgs() <<
"compute-mass-in-loop: " << getLoopName(Loop) <<
"\n");
1062 if (Loop.isIrreducible()) {
1063 BlockMass Remaining = BlockMass::getFull();
1064 for (uint32_t
H = 0;
H < Loop.NumHeaders; ++
H) {
1065 auto &Mass = Working[Loop.Nodes[
H].Index].getMass();
1066 Mass = Remaining * BranchProbability(1, Loop.NumHeaders -
H);
1069 for (
const BlockNode &M : Loop.Nodes)
1070 if (!propagateMassToSuccessors(&Loop, M))
1073 adjustLoopHeaderMass(Loop);
1075 Working[Loop.getHeader().Index].getMass() = BlockMass::getFull();
1076 if (!propagateMassToSuccessors(&Loop, Loop.getHeader()))
1078 for (
const BlockNode &M : Loop.members())
1079 if (!propagateMassToSuccessors(&Loop, M))
1084 computeLoopScale(Loop);
1090 bool BlockFrequencyInfoImpl<BT>::tryToComputeMassInFunction() {
1092 DEBUG(
dbgs() <<
"compute-mass-in-function\n");
1093 assert(!Working.empty() &&
"no blocks in function");
1094 assert(!Working[0].isLoopHeader() &&
"entry block is a loop header");
1096 Working[0].getMass() = BlockMass::getFull();
1097 for (rpot_iterator I = rpot_begin(),
IE = rpot_end(); I !=
IE; ++
I) {
1099 BlockNode Node = getNode(I);
1100 if (Working[Node.Index].isPackaged())
1103 if (!propagateMassToSuccessors(
nullptr, Node))
1109 template <
class BT>
void BlockFrequencyInfoImpl<BT>::computeMassInFunction() {
1110 if (tryToComputeMassInFunction())
1112 computeIrreducibleMass(
nullptr,
Loops.begin());
1113 if (tryToComputeMassInFunction())
1119 namespace bfi_detail {
1120 template <
class BT>
struct BlockEdgesAdder {
1131 for (
auto I = Successor::child_begin(BB), E = Successor::child_end(BB);
1138 void BlockFrequencyInfoImpl<BT>::computeIrreducibleMass(
1139 LoopData *OuterLoop, std::list<LoopData>::iterator Insert) {
1140 DEBUG(
dbgs() <<
"analyze-irreducible-in-";
1141 if (OuterLoop)
dbgs() <<
"loop: " << getLoopName(*OuterLoop) <<
"\n";
1142 else dbgs() <<
"function\n");
1144 using namespace bfi_detail;
1147 BlockEdgesAdder<BT> addBlockEdges(*
this);
1148 IrreducibleGraph
G(*
this, OuterLoop, addBlockEdges);
1150 for (
auto &L : analyzeIrreducible(
G, OuterLoop, Insert))
1151 computeMassInLoop(L);
1155 updateLoopWithIrreducible(*OuterLoop);
1160 BlockFrequencyInfoImpl<BT>::propagateMassToSuccessors(LoopData *OuterLoop,
1161 const BlockNode &Node) {
1165 if (
auto *Loop = Working[Node.Index].getPackagedLoop()) {
1166 assert(Loop != OuterLoop &&
"Cannot propagate mass in a packaged loop");
1167 if (!addLoopSuccessorsToDist(OuterLoop, *Loop, Dist))
1171 const BlockT *BB = getBlock(Node);
1172 for (
auto SI = Successor::child_begin(BB), SE = Successor::child_end(BB);
1176 if (!addToDist(Dist, OuterLoop, Node, getNode(*SI),
1177 BPI->getEdgeWeight(BB, SI)))
1184 distributeMass(Node, OuterLoop, Dist);
1192 OS <<
"block-frequency-info: " <<
F->getName() <<
"\n";
1193 for (
const BlockT &BB : *
F)
1195 <<
": float = " << getFloatingBlockFreq(&BB)
1196 <<
", int = " << getBlockFreq(&BB).getFrequency() <<
"\n";
void addNodesInLoop(const BFIBase::LoopData &OuterLoop)
bool isDoubleLoopHeader() const
bool IsPackaged
Whether this has been packaged.
bool operator!=(const BlockMass &X) const
BlockMass & operator-=(const BlockMass &X)
Subtract another mass.
LoopData * Loop
The loop this block is inside.
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
bfi_detail::BlockMass BlockMass
LoopData(LoopData *Parent, It1 FirstHeader, It1 LastHeader, It2 FirstOther, It2 LastOther)
SmallVector< BlockMass, 1 > HeaderMassList
BlockMass operator+(const BlockMass &L, const BlockMass &R)
This provides a very simple, boring adaptor for a begin and end iterator into a range type...
Index of loop information.
MachineBranchProbabilityInfo BranchProbabilityInfoT
Stats about a block itself.
void doFunction(const FunctionT *F, const BranchProbabilityInfoT *BPI, const LoopInfoT *LI)
bool operator>=(const BlockMass &X) const
std::string str() const
str - Get the contents as an std::string.
SmallVector< std::pair< BlockNode, BlockMass >, 4 > ExitMap
void addLocal(const BlockNode &Node, uint64_t Amount)
bool isADoublePackage() const
Has Loop been packaged up twice?
SmallDenseMap< uint32_t, IrrNode *, 4 > Lookup
LoopData * Parent
The parent loop.
uint32_t NumHeaders
Number of headers.
ScaledNumber< uint64_t > Scaled64
IrreducibleGraph(BFIBase &BFI, const BFIBase::LoopData *OuterLoop, BlockEdgesAdder addBlockEdges)
Construct an explicit graph containing irreducible control flow.
const BlockFrequencyInfoImpl< BT > & BFI
WorkingData(const BlockNode &Node)
raw_ostream & print(raw_ostream &OS) const
StringRef getName() const
Return a constant reference to the value's name.
void operator()(IrreducibleGraph &G, IrreducibleGraph::IrrNode &Irr, const LoopData *OuterLoop)
Weight(DistType Type, BlockNode TargetNode, uint64_t Amount)
std::string str() const
Return the twine contents as a std::string.
bool operator>(const BlockMass &X) const
virtual ~BlockFrequencyInfoImplBase()
Virtual destructor.
BlockMass & getMass()
Get the appropriate mass for a node.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
ScaledNumber< uint64_t > toScaled() const
Convert to scaled number.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
BlockFrequency getBlockFreq(const BlockNode &Node) const
BlockMass & operator+=(const BlockMass &X)
Add another mass.
bool isPackaged() const
Has ContainingLoop been packaged up?
void addEdge(IrrNode &Irr, const BlockNode &Succ, const BFIBase::LoopData *OuterLoop)
iterator pred_begin() const
void adjustLoopHeaderMass(LoopData &Loop)
Adjust the mass of all headers in an irreducible loop.
raw_ostream & operator<<(raw_ostream &OS, const BlockMass &X)
Graph of irreducible control flow.
IrrNode(const BlockNode &Node)
raw_ostream & print(raw_ostream &OS) const override
Print the frequencies for the current function.
ptrdiff_t difference_type
NodeList::const_iterator members_begin() const
void computeLoopScale(LoopData &Loop)
Compute the loop scale for a loop.
void addBackedge(const BlockNode &Node, uint64_t Amount)
void initialize(const BFIBase::LoopData *OuterLoop, BlockEdgesAdder addBlockEdges)
BlockNode getResolvedNode() const
Resolve a node to its representative.
bool addToDist(Distribution &Dist, const LoopData *OuterLoop, const BlockNode &Pred, const BlockNode &Succ, uint64_t Weight)
Add an edge to the distribution.
std::vector< FrequencyData > Freqs
Data about each block. This is used downstream.
iterator_range< std::list< LoopData >::iterator > analyzeIrreducible(const bfi_detail::IrreducibleGraph &G, LoopData *OuterLoop, std::list< LoopData >::iterator Insert)
Analyze irreducible SCCs.
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
void normalize()
Normalize the distribution.
std::deque< const IrrNode * >::const_iterator iterator
std::string getLoopName(const LoopData &Loop) const
SmallVector< Weight, 4 > WeightList
BlockMass & operator*=(const BranchProbability &P)
Distribution of unscaled probability weight.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang","erlang-compatible garbage collector")
void addNode(const BlockNode &Node)
bool DidOverflow
Whether Total did overflow.
po_iterator< T > po_end(const T &G)
Scaled64 getFloatingBlockFreq(const BlockNode &Node) const
raw_ostream & printBlockFreq(raw_ostream &OS, const BlockNode &Node) const
BlockMass Mass
Mass distribution from the entry block.
iterator_range< NodeList::const_iterator > members() const
bool isHeader(const BlockNode &Node) const
void clear()
Clear all memory.
BlockEdgesAdder(const BlockFrequencyInfoImpl< BT > &BFI)
void addNodesInFunction()
BlockMass operator-(const BlockMass &L, const BlockMass &R)
Scaled64 getFloatingBlockFreq(const BlockT *BB) const
bool operator==(const BlockMass &X) const
void addExit(const BlockNode &Node, uint64_t Amount)
LoopData * getContainingLoop() const
BlockFrequencyInfoImplBase BFIBase
BlockNode(IndexType Index)
iterator succ_begin() const
BlockFrequencyInfoImplBase::LoopData LoopData
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
isPodLike - This is a type trait that is used to determine whether a given type can be copied around ...
BlockMass operator*(const BlockMass &L, const BranchProbability &R)
bool isLoopHeader() const
ExitMap Exits
Successor edges (and weights).
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
SmallVector< BlockNode, 4 > NodeList
bool operator==(const BlockNode &X) const
block Branch Probability Basic Block static false std::string getBlockName(MachineBasicBlock *BB)
Helper to print the name of a MBB.
virtual std::string getBlockName(const BlockNode &Node) const
uint64_t Total
Sum of all weights.
static BlockMass getEmpty()
BFIBase::BlockNode BlockNode
std::list< LoopData > Loops
Indexed information about loops.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
A range adaptor for a pair of iterators.
std::string getBlockName(const BlockT *BB)
Get the name of a MachineBasicBlock.
void updateLoopWithIrreducible(LoopData &OuterLoop)
Update a loop after packaging irreducible SCCs inside of it.
const FunctionT * getFunction() const
BlockFrequency getBlockFreq(const BlockT *BB) const
LoopData(LoopData *Parent, const BlockNode &Header)
iterator insert(iterator I, T &&Elt)
Unscaled probability weight.
bool operator<=(const BlockNode &X) const
NodeList::const_iterator members_end() const
Analysis pass providing branch probability information.
iterator pred_end() const
uint64_t scale(uint64_t Num) const
Scale a large integer.
bool operator<=(const BlockMass &X) const
void addEdges(const BlockNode &Node, const BFIBase::LoopData *OuterLoop, BlockEdgesAdder addBlockEdges)
void packageLoop(LoopData &Loop)
Package up a loop.
bool isAPackage() const
Has Loop been packaged up?
uint64_t getEntryFreq() const
bool operator>=(const BlockNode &X) const
virtual raw_ostream & print(raw_ostream &OS) const
static BlockMass getFull()
MachineLoopInfo LoopInfoT
BranchProbabilityInfo BranchProbabilityInfoT
bool operator<(const BlockNode &X) const
bool isIrreducible() const
raw_ostream & printBlockFreq(raw_ostream &OS, const BlockT *BB) const
void distributeMass(const BlockNode &Source, LoopData *OuterLoop, Distribution &Dist)
Distribute mass according to a distribution.
NodeList Nodes
Header and the members of the loop.
bool operator!=(const BlockNode &X) const
LoopData * getPackagedLoop() const
std::vector< WorkingData > Working
Loop data: see initializeLoops().
std::vector< IrrNode > Nodes
HeaderMassList BackedgeMass
Mass returned to each loop header.
This class implements an extremely fast bulk output stream that can only output to a stream...
bool operator<(const BlockMass &X) const
Shared implementation for block frequency analysis.
static size_t getMaxIndex()
std::deque< const IrrNode * > Edges
Base class for BlockFrequencyInfoImpl.
bool addLoopSuccessorsToDist(const LoopData *OuterLoop, LoopData &Loop, Distribution &Dist)
Add all edges out of a packaged loop to the distribution.
GraphTraits< const BlockT * > Successor
iterator succ_end() const
void unwrapLoops()
Unwrap loops.
HeaderMassList::difference_type getHeaderIndex(const BlockNode &B)
LoopData & getLoopPackage(const BlockNode &Head)
BlockNode getHeader() const
Representative of a block.
po_iterator< T > po_begin(const T &G)
void finalizeMetrics()
Finalize frequency metrics.
MachineFunction FunctionT
bool operator>(const BlockNode &X) const
WeightList Weights
Individual successor weights.