20 using namespace llvm::bfi_detail;
22 #define DEBUG_TYPE "block-freq"
39 for (
int Digits = 0; Digits < 16; ++Digits)
72 struct DitheringDistributer {
76 DitheringDistributer(Distribution &Dist,
const BlockMass &Mass);
83 DitheringDistributer::DitheringDistributer(Distribution &Dist,
86 RemWeight = Dist.Total;
90 BlockMass DitheringDistributer::takeMass(uint32_t Weight) {
91 assert(Weight &&
"invalid weight");
92 assert(Weight <= RemWeight);
102 Weight::DistType
Type) {
103 assert(Amount &&
"invalid weight of 0");
104 uint64_t NewTotal = Total + Amount;
107 bool IsOverflow = NewTotal < Total;
108 assert(!(DidOverflow && IsOverflow) &&
"unexpected repeated overflow");
109 DidOverflow |= IsOverflow;
115 Weights.push_back(Weight(Type, Node, Amount));
119 assert(OtherW.TargetNode.isValid());
124 assert(W.Type == OtherW.Type);
125 assert(W.TargetNode == OtherW.TargetNode);
126 assert(OtherW.Amount &&
"Expected non-zero weight");
127 if (W.Amount > W.Amount + OtherW.Amount)
129 W.Amount = UINT64_MAX;
131 W.Amount += OtherW.Amount;
135 std::sort(Weights.begin(), Weights.end(),
137 const Weight &R) {
return L.TargetNode < R.TargetNode; });
140 WeightList::iterator O = Weights.begin();
141 for (WeightList::const_iterator
I = O, L = O, E = Weights.end();
I != E;
146 for (++L; L != E && I->TargetNode == L->TargetNode; ++L)
151 Weights.erase(O, Weights.end());
158 for (
const Weight &W : Weights)
162 if (Weights.size() == Combined.size())
167 Weights.reserve(Combined.size());
168 for (
const auto &
I : Combined)
169 Weights.push_back(
I.second);
173 if (Weights.size() > 128) {
185 return (N >> Shift) + (UINT64_C(1) & N >> (Shift - 1));
187 void Distribution::normalize() {
193 if (Weights.size() > 1)
197 if (Weights.size() == 1) {
199 Weights.front().Amount = 1;
210 else if (Total > UINT32_MAX)
217 assert(Total == std::accumulate(Weights.begin(), Weights.end(), UINT64_C(0),
218 [](uint64_t Sum,
const Weight &W) {
219 return Sum + W.Amount;
221 "Expected total to be correct");
230 for (
Weight &W : Weights) {
233 assert(W.TargetNode.isValid());
235 assert(W.Amount <= UINT32_MAX);
240 assert(Total <= UINT32_MAX);
246 std::vector<FrequencyData>().
swap(Freqs);
247 std::vector<WorkingData>().
swap(Working);
255 std::vector<FrequencyData> SavedFreqs(std::move(BFI.
Freqs));
257 BFI.
Freqs = std::move(SavedFreqs);
268 auto isLoopHeader = [&OuterLoop](
const BlockNode &Node) {
269 return OuterLoop && OuterLoop->
isHeader(Node);
275 auto debugSuccessor = [&](
const char *
Type) {
277 <<
" [" << Type <<
"] weight = " << Weight;
278 if (!isLoopHeader(Resolved))
280 if (Resolved != Succ)
284 (void)debugSuccessor;
287 if (isLoopHeader(Resolved)) {
288 DEBUG(debugSuccessor(
"backedge"));
293 if (Working[Resolved.
Index].getContainingLoop() != OuterLoop) {
294 DEBUG(debugSuccessor(
" exit "));
295 Dist.
addExit(Resolved, Weight);
299 if (Resolved < Pred) {
300 if (!isLoopHeader(Pred)) {
303 "unhandled irreducible control flow");
306 DEBUG(debugSuccessor(
"abort!!!"));
313 assert(OuterLoop && OuterLoop->
isIrreducible() && !isLoopHeader(Resolved) &&
314 "unhandled irreducible control flow");
317 DEBUG(debugSuccessor(
" local "));
325 for (
const auto &
I : Loop.
Exits)
326 if (!addToDist(Dist, OuterLoop, Loop.
getHeader(),
I.first,
337 DEBUG(
dbgs() <<
"compute-loop-scale: " << getLoopName(Loop) <<
"\n");
348 const Scaled64 InifiniteLoopScale(1, 12);
354 TotalBackedgeMass += Mass;
364 <<
" - " << TotalBackedgeMass <<
")\n"
365 <<
" - scale = " << Loop.
Scale <<
"\n");
370 DEBUG(
dbgs() <<
"packaging-loop: " << getLoopName(Loop) <<
"\n");
374 if (
auto *Loop = Working[M.
Index].getPackagedLoop())
383 const DitheringDistributer &D,
const BlockNode &
T,
385 dbgs() <<
" => assign " << M <<
" (" << D.RemMass <<
")";
387 dbgs() <<
" [" << Desc <<
"]";
398 DEBUG(
dbgs() <<
" => mass: " << Mass <<
"\n");
401 DitheringDistributer D(Dist, Mass);
413 assert(OuterLoop &&
"backedge or exit outside of loop");
416 if (W.
Type == Weight::Backedge) {
423 assert(W.
Type == Weight::Exit);
430 const Scaled64 &Min,
const Scaled64 &Max) {
437 const unsigned MaxBits = 64;
438 const unsigned SpreadBits = (Max / Min).lg();
439 Scaled64 ScalingFactor;
440 if (SpreadBits <= MaxBits - 3) {
443 ScalingFactor = Min.inverse();
449 ScalingFactor = Scaled64(1, MaxBits) / Max;
453 DEBUG(
dbgs() <<
"float-to-int: min = " << Min <<
", max = " << Max
454 <<
", factor = " << ScalingFactor <<
"\n");
455 for (
size_t Index = 0; Index < BFI.
Freqs.size(); ++Index) {
456 Scaled64 Scaled = BFI.
Freqs[Index].Scaled * ScalingFactor;
457 BFI.
Freqs[Index].Integer = std::max(UINT64_C(1), Scaled.toInt<uint64_t>());
459 << BFI.
Freqs[Index].Scaled <<
", scaled = " << Scaled
460 <<
", int = " << BFI.
Freqs[Index].Integer <<
"\n");
470 <<
": mass = " << Loop.Mass <<
", scale = " << Loop.Scale
472 Loop.Scale *= Loop.Mass.toScaled();
473 Loop.IsPackaged =
false;
474 DEBUG(
dbgs() <<
" => combined-scale = " << Loop.Scale <<
"\n");
479 for (
const BlockNode &
N : Loop.Nodes) {
480 const auto &Working = BFI.
Working[N.Index];
481 Scaled64 &
F = Working.isAPackage() ? Working.getPackagedLoop()->Scale
482 : BFI.
Freqs[N.Index].Scaled;
483 Scaled64 New = Loop.Scale *
F;
492 for (
size_t Index = 0; Index < Working.size(); ++Index)
493 Freqs[Index].Scaled = Working[Index].Mass.
toScaled();
502 auto Min = Scaled64::getLargest();
503 auto Max = Scaled64::getZero();
504 for (
size_t Index = 0; Index < Working.size(); ++Index) {
506 Min =
std::min(Min, Freqs[Index].Scaled);
507 Max = std::max(
Max, Freqs[Index].Scaled);
524 return Freqs[Node.
Index].Integer;
529 return Scaled64::getZero();
530 return Freqs[Node.
Index].Scaled;
535 return std::string();
545 return OS << getFloatingBlockFreq(Node);
554 return OS <<
Block / Entry;
560 for (
auto N : OuterLoop.
Nodes)
566 for (uint32_t Index = 0; Index <
BFI.
Working.size(); ++Index)
577 if (OuterLoop && OuterLoop->
isHeader(Succ))
583 Irr.
Edges.push_back(&SuccIrr);
584 SuccIrr.
Edges.push_front(&Irr);
610 const std::vector<const IrreducibleGraph::IrrNode *> &SCC,
611 LoopData::NodeList &Headers, LoopData::NodeList &Others) {
616 for (
const auto *
I : SCC)
619 for (
auto I = InSCC.
begin(), E = InSCC.
end();
I != E; ++
I) {
620 auto &Irr = *
I->first;
621 for (
const auto *
P :
make_range(Irr.pred_begin(), Irr.pred_end())) {
627 Headers.push_back(Irr.Node);
632 assert(Headers.size() >= 2 &&
633 "Expected irreducible CFG; -loop-info is likely invalid");
634 if (Headers.size() == InSCC.
size()) {
636 std::sort(Headers.begin(), Headers.end());
641 for (
const auto &
I : InSCC) {
646 auto &Irr = *
I.first;
647 for (
const auto *
P :
make_range(Irr.pred_begin(), Irr.pred_end())) {
649 if (
P->Node < Irr.Node)
658 Headers.push_back(Irr.Node);
662 if (Headers.back() == Irr.Node)
667 Others.push_back(Irr.Node);
670 std::sort(Headers.begin(), Headers.end());
671 std::sort(Others.begin(), Others.end());
676 LoopData *OuterLoop, std::list<LoopData>::iterator Insert,
677 const std::vector<const IrreducibleGraph::IrrNode *> &SCC) {
681 LoopData::NodeList Headers;
682 LoopData::NodeList Others;
685 auto Loop = BFI.
Loops.emplace(Insert, OuterLoop, Headers.begin(),
686 Headers.end(), Others.begin(), Others.end());
689 for (
const auto &
N :
Loop->Nodes)
690 if (BFI.
Working[
N.Index].isLoopHeader())
699 std::list<LoopData>::iterator Insert) {
700 assert((OuterLoop ==
nullptr) == (Insert ==
Loops.begin()));
701 auto Prev = OuterLoop ? std::prev(Insert) :
Loops.end();
722 for (
auto I = O, E = OuterLoop.
Nodes.
end();
I != E; ++
I)
723 if (!Working[
I->Index].isPackaged())
729 assert(Loop.
isIrreducible() &&
"this only makes sense on irreducible loops");
740 DEBUG(
dbgs() <<
"adjust-loop-header-mass:\n");
742 auto &HeaderNode = Loop.
Nodes[
H];
744 DEBUG(
dbgs() <<
" - Add back edge mass for node "
745 <<
getBlockName(HeaderNode) <<
": " << BackedgeMass <<
"\n");
746 Dist.
addLocal(HeaderNode, BackedgeMass.getMass());
749 DitheringDistributer D(Dist, LoopMass);
751 DEBUG(
dbgs() <<
" Distribute loop mass " << LoopMass
752 <<
" to headers using above weights\n");
void push_back(const T &Elt)
static void combineWeights(WeightList &Weights)
void addNodesInLoop(const BFIBase::LoopData &OuterLoop)
This builds on the llvm/ADT/GraphTraits.h file to find the strongly connected components (SCCs) of a ...
static uint64_t shiftRightAndRound(uint64_t N, int Shift)
static void combineWeightsBySorting(WeightList &Weights)
bool IsPackaged
Whether this has been packaged.
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
static char getHexDigit(int N)
Stats about a block itself.
void addLocal(const BlockNode &Node, uint64_t Amount)
SmallDenseMap< uint32_t, IrrNode *, 4 > Lookup
static void cleanup(BlockFrequencyInfoImplBase &BFI)
Clear all memory not needed downstream.
static void combineWeight(Weight &W, const Weight &OtherW)
uint64_t getFrequency() const
Returns the frequency as a fixpoint number scaled by the entry frequency.
uint32_t NumHeaders
Number of headers.
static const NodeType * getEntryNode(const GraphT &G)
std::size_t countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the most significant bit to the least stopping at the first 1...
GraphT::IrrNode::iterator ChildIteratorType
raw_ostream & print(raw_ostream &OS) const
ScaledNumber< uint64_t > toScaled() const
Convert to scaled number.
BlockFrequency getBlockFreq(const BlockNode &Node) const
Number of individual test Apply this number of consecutive mutations to each input exit after the first new interesting input is found the minimized corpus is saved into the first input directory Number of jobs to run If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
void addEdge(IrrNode &Irr, const BlockNode &Succ, const BFIBase::LoopData *OuterLoop)
void adjustLoopHeaderMass(LoopData &Loop)
Adjust the mass of all headers in an irreducible loop.
static void debugAssign(const BlockFrequencyInfoImplBase &BFI, const DitheringDistributer &D, const BlockNode &T, const BlockMass &M, const char *Desc)
scc_iterator< T > scc_begin(const T &G)
Construct the begin iterator for a deduced graph type T.
Graph of irreducible control flow.
static bool add(uint64_t *dest, const uint64_t *x, const uint64_t *y, unsigned len)
This function adds the integer array x to the integer array Y and places the result in dest...
ScaledNumber inverse() const
void computeLoopScale(LoopData &Loop)
Compute the loop scale for a loop.
void addBackedge(const BlockNode &Node, uint64_t Amount)
bool addToDist(Distribution &Dist, const LoopData *OuterLoop, const BlockNode &Pred, const BlockNode &Succ, uint64_t Weight)
Add an edge to the distribution.
std::vector< FrequencyData > Freqs
Data about each block. This is used downstream.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
static void convertFloatingToInteger(BlockFrequencyInfoImplBase &BFI, const Scaled64 &Min, const Scaled64 &Max)
iterator_range< std::list< LoopData >::iterator > analyzeIrreducible(const bfi_detail::IrreducibleGraph &G, LoopData *OuterLoop, std::list< LoopData >::iterator Insert)
Analyze irreducible SCCs.
The instances of the Type class are immutable: once they are created, they are never changed...
std::string getLoopName(const LoopData &Loop) const
Distribution of unscaled probability weight.
void addNode(const BlockNode &Node)
Scaled64 getFloatingBlockFreq(const BlockNode &Node) const
raw_ostream & printBlockFreq(raw_ostream &OS, const BlockNode &Node) const
static void unwrapLoop(BlockFrequencyInfoImplBase &BFI, LoopData &Loop)
Unwrap a loop package.
bool isHeader(const BlockNode &Node) const
void clear()
Clear all memory.
void addNodesInFunction()
uint64_t NextPowerOf2(uint64_t A)
NextPowerOf2 - Returns the next power of two (in 64-bits) that is strictly greater than A...
iterator erase(iterator I)
void addExit(const BlockNode &Node, uint64_t Amount)
iterator succ_begin() const
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
ExitMap Exits
Successor edges (and weights).
virtual std::string getBlockName(const BlockNode &Node) const
static void findIrreducibleHeaders(const BlockFrequencyInfoImplBase &BFI, const IrreducibleGraph &G, const std::vector< const IrreducibleGraph::IrrNode * > &SCC, LoopData::NodeList &Headers, LoopData::NodeList &Others)
Find extra irreducible headers.
static BlockMass getEmpty()
const GraphT::IrrNode NodeType
std::list< LoopData > Loops
Indexed information about loops.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static void createIrreducibleLoop(BlockFrequencyInfoImplBase &BFI, const IrreducibleGraph &G, LoopData *OuterLoop, std::list< LoopData >::iterator Insert, const std::vector< const IrreducibleGraph::IrrNode * > &SCC)
A range adaptor for a pair of iterators.
std::string getBlockName(const BlockT *BB)
Get the name of a MachineBasicBlock.
void updateLoopWithIrreducible(LoopData &OuterLoop)
Update a loop after packaging irreducible SCCs inside of it.
bfi_detail::IrreducibleGraph GraphT
Unscaled probability weight.
static void combineWeightsByHashing(WeightList &Weights)
static ChildIteratorType child_end(NodeType *N)
void packageLoop(LoopData &Loop)
Package up a loop.
static BlockMass getFull()
bool isIrreducible() const
void distributeMass(const BlockNode &Source, LoopData *OuterLoop, Distribution &Dist)
Distribute mass according to a distribution.
NodeList Nodes
Header and the members of the loop.
std::vector< WorkingData > Working
Loop data: see initializeLoops().
std::vector< IrrNode > Nodes
static ChildIteratorType child_begin(NodeType *N)
HeaderMassList BackedgeMass
Mass returned to each loop header.
This class implements an extremely fast bulk output stream that can only output to a stream...
std::deque< const IrrNode * > Edges
Base class for BlockFrequencyInfoImpl.
bool addLoopSuccessorsToDist(const LoopData *OuterLoop, LoopData &Loop, Distribution &Dist)
Add all edges out of a packaged loop to the distribution.
iterator succ_end() const
void unwrapLoops()
Unwrap loops.
HeaderMassList::difference_type getHeaderIndex(const BlockNode &B)
BlockNode getHeader() const
Representative of a block.
void finalizeMetrics()
Finalize frequency metrics.
WeightList Weights
Individual successor weights.