Bug Summary

File:build/source/llvm/tools/llvm-profgen/CSPreInliner.cpp
Warning:line 162, column 16
Value stored to 'SampleThreshold' during its initialization is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name CSPreInliner.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-17/lib/clang/17 -D _DEBUG -D _GLIBCXX_ASSERTIONS -D _GNU_SOURCE -D _LIBCPP_ENABLE_ASSERTIONS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I tools/llvm-profgen -I /build/source/llvm/tools/llvm-profgen -I include -I /build/source/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-17/lib/clang/17/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/source/= -fcoverage-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/source/= -source-date-epoch 1683717183 -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2023-05-10-133810-16478-1 -x c++ /build/source/llvm/tools/llvm-profgen/CSPreInliner.cpp
1//===-- CSPreInliner.cpp - Profile guided preinliner -------------- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "CSPreInliner.h"
10#include "ProfiledBinary.h"
11#include "llvm/ADT/SCCIterator.h"
12#include "llvm/ADT/Statistic.h"
13#include "llvm/DebugInfo/Symbolize/SymbolizableModule.h"
14#include <cstdint>
15#include <queue>
16
17#define DEBUG_TYPE"cs-preinliner" "cs-preinliner"
18
19using namespace llvm;
20using namespace sampleprof;
21
22STATISTIC(PreInlNumCSInlined,static llvm::Statistic PreInlNumCSInlined = {"cs-preinliner",
"PreInlNumCSInlined", "Number of functions inlined with context sensitive profile"
}
23 "Number of functions inlined with context sensitive profile")static llvm::Statistic PreInlNumCSInlined = {"cs-preinliner",
"PreInlNumCSInlined", "Number of functions inlined with context sensitive profile"
};
24STATISTIC(PreInlNumCSNotInlined,static llvm::Statistic PreInlNumCSNotInlined = {"cs-preinliner"
, "PreInlNumCSNotInlined", "Number of functions not inlined with context sensitive profile"
}
25 "Number of functions not inlined with context sensitive profile")static llvm::Statistic PreInlNumCSNotInlined = {"cs-preinliner"
, "PreInlNumCSNotInlined", "Number of functions not inlined with context sensitive profile"
};
26STATISTIC(PreInlNumCSInlinedHitMinLimit,static llvm::Statistic PreInlNumCSInlinedHitMinLimit = {"cs-preinliner"
, "PreInlNumCSInlinedHitMinLimit", "Number of functions with FDO inline stopped due to min size limit"
}
27 "Number of functions with FDO inline stopped due to min size limit")static llvm::Statistic PreInlNumCSInlinedHitMinLimit = {"cs-preinliner"
, "PreInlNumCSInlinedHitMinLimit", "Number of functions with FDO inline stopped due to min size limit"
};
28STATISTIC(PreInlNumCSInlinedHitMaxLimit,static llvm::Statistic PreInlNumCSInlinedHitMaxLimit = {"cs-preinliner"
, "PreInlNumCSInlinedHitMaxLimit", "Number of functions with FDO inline stopped due to max size limit"
}
29 "Number of functions with FDO inline stopped due to max size limit")static llvm::Statistic PreInlNumCSInlinedHitMaxLimit = {"cs-preinliner"
, "PreInlNumCSInlinedHitMaxLimit", "Number of functions with FDO inline stopped due to max size limit"
};
30STATISTIC(static llvm::Statistic PreInlNumCSInlinedHitGrowthLimit = {"cs-preinliner"
, "PreInlNumCSInlinedHitGrowthLimit", "Number of functions with FDO inline stopped due to growth size limit"
}
31 PreInlNumCSInlinedHitGrowthLimit,static llvm::Statistic PreInlNumCSInlinedHitGrowthLimit = {"cs-preinliner"
, "PreInlNumCSInlinedHitGrowthLimit", "Number of functions with FDO inline stopped due to growth size limit"
}
32 "Number of functions with FDO inline stopped due to growth size limit")static llvm::Statistic PreInlNumCSInlinedHitGrowthLimit = {"cs-preinliner"
, "PreInlNumCSInlinedHitGrowthLimit", "Number of functions with FDO inline stopped due to growth size limit"
};
33
34// The switches specify inline thresholds used in SampleProfileLoader inlining.
35// TODO: the actual threshold to be tuned here because the size here is based
36// on machine code not LLVM IR.
37namespace llvm {
38extern cl::opt<int> SampleHotCallSiteThreshold;
39extern cl::opt<int> SampleColdCallSiteThreshold;
40extern cl::opt<int> ProfileInlineGrowthLimit;
41extern cl::opt<int> ProfileInlineLimitMin;
42extern cl::opt<int> ProfileInlineLimitMax;
43extern cl::opt<bool> SortProfiledSCC;
44
45cl::opt<bool> EnableCSPreInliner(
46 "csspgo-preinliner", cl::Hidden, cl::init(true),
47 cl::desc("Run a global pre-inliner to merge context profile based on "
48 "estimated global top-down inline decisions"));
49
50cl::opt<bool> UseContextCostForPreInliner(
51 "use-context-cost-for-preinliner", cl::Hidden, cl::init(true),
52 cl::desc("Use context-sensitive byte size cost for preinliner decisions"));
53} // namespace llvm
54
55static cl::opt<bool> SamplePreInlineReplay(
56 "csspgo-replay-preinline", cl::Hidden, cl::init(false),
57 cl::desc(
58 "Replay previous inlining and adjust context profile accordingly"));
59
60CSPreInliner::CSPreInliner(SampleContextTracker &Tracker,
61 ProfiledBinary &Binary, ProfileSummary *Summary)
62 : UseContextCost(UseContextCostForPreInliner),
63 // TODO: Pass in a guid-to-name map in order for
64 // ContextTracker.getFuncNameFor to work, if `Profiles` can have md5 codes
65 // as their profile context.
66 ContextTracker(Tracker), Binary(Binary), Summary(Summary) {
67 // Set default preinliner hot/cold call site threshold tuned with CSSPGO.
68 // for good performance with reasonable profile size.
69 if (!SampleHotCallSiteThreshold.getNumOccurrences())
70 SampleHotCallSiteThreshold = 1500;
71 if (!SampleColdCallSiteThreshold.getNumOccurrences())
72 SampleColdCallSiteThreshold = 0;
73 if (!ProfileInlineLimitMax.getNumOccurrences())
74 ProfileInlineLimitMax = 50000;
75}
76
77std::vector<StringRef> CSPreInliner::buildTopDownOrder() {
78 std::vector<StringRef> Order;
79 // Trim cold edges to get a more stable call graph. This allows for a more
80 // stable top-down order which in turns helps the stablity of the generated
81 // profile from run to run.
82 uint64_t ColdCountThreshold = ProfileSummaryBuilder::getColdCountThreshold(
83 (Summary->getDetailedSummary()));
84 ProfiledCallGraph ProfiledCG(ContextTracker, ColdCountThreshold);
85
86 // Now that we have a profiled call graph, construct top-down order
87 // by building up SCC and reversing SCC order.
88 scc_iterator<ProfiledCallGraph *> I = scc_begin(&ProfiledCG);
89 while (!I.isAtEnd()) {
90 auto Range = *I;
91 if (SortProfiledSCC) {
92 // Sort nodes in one SCC based on callsite hotness.
93 scc_member_iterator<ProfiledCallGraph *> SI(*I);
94 Range = *SI;
95 }
96 for (auto *Node : Range) {
97 if (Node != ProfiledCG.getEntryNode())
98 Order.push_back(Node->Name);
99 }
100 ++I;
101 }
102 std::reverse(Order.begin(), Order.end());
103
104 return Order;
105}
106
107bool CSPreInliner::getInlineCandidates(ProfiledCandidateQueue &CQueue,
108 const FunctionSamples *CallerSamples) {
109 assert(CallerSamples && "Expect non-null caller samples")(static_cast <bool> (CallerSamples && "Expect non-null caller samples"
) ? void (0) : __assert_fail ("CallerSamples && \"Expect non-null caller samples\""
, "llvm/tools/llvm-profgen/CSPreInliner.cpp", 109, __extension__
__PRETTY_FUNCTION__));
110
111 // Ideally we want to consider everything a function calls, but as far as
112 // context profile is concerned, only those frames that are children of
113 // current one in the trie is relavent. So we walk the trie instead of call
114 // targets from function profile.
115 ContextTrieNode *CallerNode =
116 ContextTracker.getContextNodeForProfile(CallerSamples);
117
118 bool HasNewCandidate = false;
119 for (auto &Child : CallerNode->getAllChildContext()) {
120 ContextTrieNode *CalleeNode = &Child.second;
121 FunctionSamples *CalleeSamples = CalleeNode->getFunctionSamples();
122 if (!CalleeSamples)
123 continue;
124
125 // Call site count is more reliable, so we look up the corresponding call
126 // target profile in caller's context profile to retrieve call site count.
127 uint64_t CalleeEntryCount = CalleeSamples->getHeadSamplesEstimate();
128 uint64_t CallsiteCount = 0;
129 LineLocation Callsite = CalleeNode->getCallSiteLoc();
130 if (auto CallTargets = CallerSamples->findCallTargetMapAt(Callsite)) {
131 SampleRecord::CallTargetMap &TargetCounts = CallTargets.get();
132 auto It = TargetCounts.find(CalleeSamples->getName());
133 if (It != TargetCounts.end())
134 CallsiteCount = It->second;
135 }
136
137 // TODO: call site and callee entry count should be mostly consistent, add
138 // check for that.
139 HasNewCandidate = true;
140 uint32_t CalleeSize = getFuncSize(CalleeNode);
141 CQueue.emplace(CalleeSamples, std::max(CallsiteCount, CalleeEntryCount),
142 CalleeSize);
143 }
144
145 return HasNewCandidate;
146}
147
148uint32_t CSPreInliner::getFuncSize(const ContextTrieNode *ContextNode) {
149 if (UseContextCost)
150 return Binary.getFuncSizeForContext(ContextNode);
151
152 return ContextNode->getFunctionSamples()->getBodySamples().size();
153}
154
155bool CSPreInliner::shouldInline(ProfiledInlineCandidate &Candidate) {
156 // If replay inline is requested, simply follow the inline decision of the
157 // profiled binary.
158 if (SamplePreInlineReplay)
159 return Candidate.CalleeSamples->getContext().hasAttribute(
160 ContextWasInlined);
161
162 unsigned int SampleThreshold = SampleColdCallSiteThreshold;
Value stored to 'SampleThreshold' during its initialization is never read
163 uint64_t ColdCountThreshold = ProfileSummaryBuilder::getColdCountThreshold(
164 (Summary->getDetailedSummary()));
165
166 if (Candidate.CallsiteCount <= ColdCountThreshold)
167 SampleThreshold = SampleColdCallSiteThreshold;
168 else {
169 // Linearly adjust threshold based on normalized hotness, i.e, a value in
170 // [0,1]. Use 10% cutoff instead of the max count as the normalization
171 // upperbound for stability.
172 double NormalizationUpperBound =
173 ProfileSummaryBuilder::getEntryForPercentile(
174 Summary->getDetailedSummary(), 100000 /* 10% */)
175 .MinCount;
176 double NormalizationLowerBound = ColdCountThreshold;
177 double NormalizedHotness =
178 (Candidate.CallsiteCount - NormalizationLowerBound) /
179 (NormalizationUpperBound - NormalizationLowerBound);
180 if (NormalizedHotness > 1.0)
181 NormalizedHotness = 1.0;
182 // Add 1 to to ensure hot callsites get a non-zero threshold, which could
183 // happen when SampleColdCallSiteThreshold is 0. This is when we do not
184 // want any inlining for cold callsites.
185 SampleThreshold = SampleHotCallSiteThreshold * NormalizedHotness * 100 +
186 SampleColdCallSiteThreshold + 1;
187 }
188
189 return (Candidate.SizeCost < SampleThreshold);
190}
191
192void CSPreInliner::processFunction(const StringRef Name) {
193 FunctionSamples *FSamples = ContextTracker.getBaseSamplesFor(Name);
194 if (!FSamples)
195 return;
196
197 unsigned FuncSize =
198 getFuncSize(ContextTracker.getContextNodeForProfile(FSamples));
199 unsigned FuncFinalSize = FuncSize;
200 unsigned SizeLimit = FuncSize * ProfileInlineGrowthLimit;
201 SizeLimit = std::min(SizeLimit, (unsigned)ProfileInlineLimitMax);
202 SizeLimit = std::max(SizeLimit, (unsigned)ProfileInlineLimitMin);
203
204 LLVM_DEBUG(dbgs() << "Process " << Namedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { dbgs() << "Process " << Name
<< " for context-sensitive pre-inlining (pre-inline size: "
<< FuncSize << ", size limit: " << SizeLimit
<< ")\n"; } } while (false)
205 << " for context-sensitive pre-inlining (pre-inline size: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { dbgs() << "Process " << Name
<< " for context-sensitive pre-inlining (pre-inline size: "
<< FuncSize << ", size limit: " << SizeLimit
<< ")\n"; } } while (false)
206 << FuncSize << ", size limit: " << SizeLimit << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { dbgs() << "Process " << Name
<< " for context-sensitive pre-inlining (pre-inline size: "
<< FuncSize << ", size limit: " << SizeLimit
<< ")\n"; } } while (false);
207
208 ProfiledCandidateQueue CQueue;
209 getInlineCandidates(CQueue, FSamples);
210
211 while (!CQueue.empty() && FuncFinalSize < SizeLimit) {
212 ProfiledInlineCandidate Candidate = CQueue.top();
213 CQueue.pop();
214 bool ShouldInline = false;
215 if ((ShouldInline = shouldInline(Candidate))) {
216 // We mark context as inlined as the corresponding context profile
217 // won't be merged into that function's base profile.
218 ++PreInlNumCSInlined;
219 ContextTracker.markContextSamplesInlined(Candidate.CalleeSamples);
220 Candidate.CalleeSamples->getContext().setAttribute(
221 ContextShouldBeInlined);
222 FuncFinalSize += Candidate.SizeCost;
223 getInlineCandidates(CQueue, Candidate.CalleeSamples);
224 } else {
225 ++PreInlNumCSNotInlined;
226 }
227 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { dbgs() << (ShouldInline ? " Inlined"
: " Outlined") << " context profile for: " << ContextTracker
.getContextString(*Candidate.CalleeSamples) << " (callee size: "
<< Candidate.SizeCost << ", call count:" <<
Candidate.CallsiteCount << ")\n"; } } while (false)
228 dbgs() << (ShouldInline ? " Inlined" : " Outlined")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { dbgs() << (ShouldInline ? " Inlined"
: " Outlined") << " context profile for: " << ContextTracker
.getContextString(*Candidate.CalleeSamples) << " (callee size: "
<< Candidate.SizeCost << ", call count:" <<
Candidate.CallsiteCount << ")\n"; } } while (false)
229 << " context profile for: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { dbgs() << (ShouldInline ? " Inlined"
: " Outlined") << " context profile for: " << ContextTracker
.getContextString(*Candidate.CalleeSamples) << " (callee size: "
<< Candidate.SizeCost << ", call count:" <<
Candidate.CallsiteCount << ")\n"; } } while (false)
230 << ContextTracker.getContextString(*Candidate.CalleeSamples)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { dbgs() << (ShouldInline ? " Inlined"
: " Outlined") << " context profile for: " << ContextTracker
.getContextString(*Candidate.CalleeSamples) << " (callee size: "
<< Candidate.SizeCost << ", call count:" <<
Candidate.CallsiteCount << ")\n"; } } while (false)
231 << " (callee size: " << Candidate.SizeCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { dbgs() << (ShouldInline ? " Inlined"
: " Outlined") << " context profile for: " << ContextTracker
.getContextString(*Candidate.CalleeSamples) << " (callee size: "
<< Candidate.SizeCost << ", call count:" <<
Candidate.CallsiteCount << ")\n"; } } while (false)
232 << ", call count:" << Candidate.CallsiteCount << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { dbgs() << (ShouldInline ? " Inlined"
: " Outlined") << " context profile for: " << ContextTracker
.getContextString(*Candidate.CalleeSamples) << " (callee size: "
<< Candidate.SizeCost << ", call count:" <<
Candidate.CallsiteCount << ")\n"; } } while (false);
233 }
234
235 if (!CQueue.empty()) {
236 if (SizeLimit == (unsigned)ProfileInlineLimitMax)
237 ++PreInlNumCSInlinedHitMaxLimit;
238 else if (SizeLimit == (unsigned)ProfileInlineLimitMin)
239 ++PreInlNumCSInlinedHitMinLimit;
240 else
241 ++PreInlNumCSInlinedHitGrowthLimit;
242 }
243
244 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { { if (!CQueue.empty()) dbgs() << " Inline candidates ignored due to size limit (inliner "
"original size: " << FuncSize << ", inliner final size: "
<< FuncFinalSize << ", size limit: " << SizeLimit
<< ")\n"; while (!CQueue.empty()) { ProfiledInlineCandidate
Candidate = CQueue.top(); CQueue.pop(); bool WasInlined = Candidate
.CalleeSamples->getContext().hasAttribute(ContextWasInlined
); dbgs() << " " << ContextTracker.getContextString
(*Candidate.CalleeSamples) << " (candidate size:" <<
Candidate.SizeCost << ", call count: " << Candidate
.CallsiteCount << ", previously " << (WasInlined ?
"inlined)\n" : "not inlined)\n"); } }; } } while (false)
245 if (!CQueue.empty())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { { if (!CQueue.empty()) dbgs() << " Inline candidates ignored due to size limit (inliner "
"original size: " << FuncSize << ", inliner final size: "
<< FuncFinalSize << ", size limit: " << SizeLimit
<< ")\n"; while (!CQueue.empty()) { ProfiledInlineCandidate
Candidate = CQueue.top(); CQueue.pop(); bool WasInlined = Candidate
.CalleeSamples->getContext().hasAttribute(ContextWasInlined
); dbgs() << " " << ContextTracker.getContextString
(*Candidate.CalleeSamples) << " (candidate size:" <<
Candidate.SizeCost << ", call count: " << Candidate
.CallsiteCount << ", previously " << (WasInlined ?
"inlined)\n" : "not inlined)\n"); } }; } } while (false)
246 dbgs() << " Inline candidates ignored due to size limit (inliner "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { { if (!CQueue.empty()) dbgs() << " Inline candidates ignored due to size limit (inliner "
"original size: " << FuncSize << ", inliner final size: "
<< FuncFinalSize << ", size limit: " << SizeLimit
<< ")\n"; while (!CQueue.empty()) { ProfiledInlineCandidate
Candidate = CQueue.top(); CQueue.pop(); bool WasInlined = Candidate
.CalleeSamples->getContext().hasAttribute(ContextWasInlined
); dbgs() << " " << ContextTracker.getContextString
(*Candidate.CalleeSamples) << " (candidate size:" <<
Candidate.SizeCost << ", call count: " << Candidate
.CallsiteCount << ", previously " << (WasInlined ?
"inlined)\n" : "not inlined)\n"); } }; } } while (false)
247 "original size: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { { if (!CQueue.empty()) dbgs() << " Inline candidates ignored due to size limit (inliner "
"original size: " << FuncSize << ", inliner final size: "
<< FuncFinalSize << ", size limit: " << SizeLimit
<< ")\n"; while (!CQueue.empty()) { ProfiledInlineCandidate
Candidate = CQueue.top(); CQueue.pop(); bool WasInlined = Candidate
.CalleeSamples->getContext().hasAttribute(ContextWasInlined
); dbgs() << " " << ContextTracker.getContextString
(*Candidate.CalleeSamples) << " (candidate size:" <<
Candidate.SizeCost << ", call count: " << Candidate
.CallsiteCount << ", previously " << (WasInlined ?
"inlined)\n" : "not inlined)\n"); } }; } } while (false)
248 << FuncSize << ", inliner final size: " << FuncFinalSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { { if (!CQueue.empty()) dbgs() << " Inline candidates ignored due to size limit (inliner "
"original size: " << FuncSize << ", inliner final size: "
<< FuncFinalSize << ", size limit: " << SizeLimit
<< ")\n"; while (!CQueue.empty()) { ProfiledInlineCandidate
Candidate = CQueue.top(); CQueue.pop(); bool WasInlined = Candidate
.CalleeSamples->getContext().hasAttribute(ContextWasInlined
); dbgs() << " " << ContextTracker.getContextString
(*Candidate.CalleeSamples) << " (candidate size:" <<
Candidate.SizeCost << ", call count: " << Candidate
.CallsiteCount << ", previously " << (WasInlined ?
"inlined)\n" : "not inlined)\n"); } }; } } while (false)
249 << ", size limit: " << SizeLimit << ")\n";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { { if (!CQueue.empty()) dbgs() << " Inline candidates ignored due to size limit (inliner "
"original size: " << FuncSize << ", inliner final size: "
<< FuncFinalSize << ", size limit: " << SizeLimit
<< ")\n"; while (!CQueue.empty()) { ProfiledInlineCandidate
Candidate = CQueue.top(); CQueue.pop(); bool WasInlined = Candidate
.CalleeSamples->getContext().hasAttribute(ContextWasInlined
); dbgs() << " " << ContextTracker.getContextString
(*Candidate.CalleeSamples) << " (candidate size:" <<
Candidate.SizeCost << ", call count: " << Candidate
.CallsiteCount << ", previously " << (WasInlined ?
"inlined)\n" : "not inlined)\n"); } }; } } while (false)
250
251 while (!CQueue.empty()) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { { if (!CQueue.empty()) dbgs() << " Inline candidates ignored due to size limit (inliner "
"original size: " << FuncSize << ", inliner final size: "
<< FuncFinalSize << ", size limit: " << SizeLimit
<< ")\n"; while (!CQueue.empty()) { ProfiledInlineCandidate
Candidate = CQueue.top(); CQueue.pop(); bool WasInlined = Candidate
.CalleeSamples->getContext().hasAttribute(ContextWasInlined
); dbgs() << " " << ContextTracker.getContextString
(*Candidate.CalleeSamples) << " (candidate size:" <<
Candidate.SizeCost << ", call count: " << Candidate
.CallsiteCount << ", previously " << (WasInlined ?
"inlined)\n" : "not inlined)\n"); } }; } } while (false)
252 ProfiledInlineCandidate Candidate = CQueue.top();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { { if (!CQueue.empty()) dbgs() << " Inline candidates ignored due to size limit (inliner "
"original size: " << FuncSize << ", inliner final size: "
<< FuncFinalSize << ", size limit: " << SizeLimit
<< ")\n"; while (!CQueue.empty()) { ProfiledInlineCandidate
Candidate = CQueue.top(); CQueue.pop(); bool WasInlined = Candidate
.CalleeSamples->getContext().hasAttribute(ContextWasInlined
); dbgs() << " " << ContextTracker.getContextString
(*Candidate.CalleeSamples) << " (candidate size:" <<
Candidate.SizeCost << ", call count: " << Candidate
.CallsiteCount << ", previously " << (WasInlined ?
"inlined)\n" : "not inlined)\n"); } }; } } while (false)
253 CQueue.pop();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { { if (!CQueue.empty()) dbgs() << " Inline candidates ignored due to size limit (inliner "
"original size: " << FuncSize << ", inliner final size: "
<< FuncFinalSize << ", size limit: " << SizeLimit
<< ")\n"; while (!CQueue.empty()) { ProfiledInlineCandidate
Candidate = CQueue.top(); CQueue.pop(); bool WasInlined = Candidate
.CalleeSamples->getContext().hasAttribute(ContextWasInlined
); dbgs() << " " << ContextTracker.getContextString
(*Candidate.CalleeSamples) << " (candidate size:" <<
Candidate.SizeCost << ", call count: " << Candidate
.CallsiteCount << ", previously " << (WasInlined ?
"inlined)\n" : "not inlined)\n"); } }; } } while (false)
254 bool WasInlined =do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { { if (!CQueue.empty()) dbgs() << " Inline candidates ignored due to size limit (inliner "
"original size: " << FuncSize << ", inliner final size: "
<< FuncFinalSize << ", size limit: " << SizeLimit
<< ")\n"; while (!CQueue.empty()) { ProfiledInlineCandidate
Candidate = CQueue.top(); CQueue.pop(); bool WasInlined = Candidate
.CalleeSamples->getContext().hasAttribute(ContextWasInlined
); dbgs() << " " << ContextTracker.getContextString
(*Candidate.CalleeSamples) << " (candidate size:" <<
Candidate.SizeCost << ", call count: " << Candidate
.CallsiteCount << ", previously " << (WasInlined ?
"inlined)\n" : "not inlined)\n"); } }; } } while (false)
255 Candidate.CalleeSamples->getContext().hasAttribute(ContextWasInlined);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { { if (!CQueue.empty()) dbgs() << " Inline candidates ignored due to size limit (inliner "
"original size: " << FuncSize << ", inliner final size: "
<< FuncFinalSize << ", size limit: " << SizeLimit
<< ")\n"; while (!CQueue.empty()) { ProfiledInlineCandidate
Candidate = CQueue.top(); CQueue.pop(); bool WasInlined = Candidate
.CalleeSamples->getContext().hasAttribute(ContextWasInlined
); dbgs() << " " << ContextTracker.getContextString
(*Candidate.CalleeSamples) << " (candidate size:" <<
Candidate.SizeCost << ", call count: " << Candidate
.CallsiteCount << ", previously " << (WasInlined ?
"inlined)\n" : "not inlined)\n"); } }; } } while (false)
256 dbgs() << " "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { { if (!CQueue.empty()) dbgs() << " Inline candidates ignored due to size limit (inliner "
"original size: " << FuncSize << ", inliner final size: "
<< FuncFinalSize << ", size limit: " << SizeLimit
<< ")\n"; while (!CQueue.empty()) { ProfiledInlineCandidate
Candidate = CQueue.top(); CQueue.pop(); bool WasInlined = Candidate
.CalleeSamples->getContext().hasAttribute(ContextWasInlined
); dbgs() << " " << ContextTracker.getContextString
(*Candidate.CalleeSamples) << " (candidate size:" <<
Candidate.SizeCost << ", call count: " << Candidate
.CallsiteCount << ", previously " << (WasInlined ?
"inlined)\n" : "not inlined)\n"); } }; } } while (false)
257 << ContextTracker.getContextString(*Candidate.CalleeSamples)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { { if (!CQueue.empty()) dbgs() << " Inline candidates ignored due to size limit (inliner "
"original size: " << FuncSize << ", inliner final size: "
<< FuncFinalSize << ", size limit: " << SizeLimit
<< ")\n"; while (!CQueue.empty()) { ProfiledInlineCandidate
Candidate = CQueue.top(); CQueue.pop(); bool WasInlined = Candidate
.CalleeSamples->getContext().hasAttribute(ContextWasInlined
); dbgs() << " " << ContextTracker.getContextString
(*Candidate.CalleeSamples) << " (candidate size:" <<
Candidate.SizeCost << ", call count: " << Candidate
.CallsiteCount << ", previously " << (WasInlined ?
"inlined)\n" : "not inlined)\n"); } }; } } while (false)
258 << " (candidate size:" << Candidate.SizeCostdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { { if (!CQueue.empty()) dbgs() << " Inline candidates ignored due to size limit (inliner "
"original size: " << FuncSize << ", inliner final size: "
<< FuncFinalSize << ", size limit: " << SizeLimit
<< ")\n"; while (!CQueue.empty()) { ProfiledInlineCandidate
Candidate = CQueue.top(); CQueue.pop(); bool WasInlined = Candidate
.CalleeSamples->getContext().hasAttribute(ContextWasInlined
); dbgs() << " " << ContextTracker.getContextString
(*Candidate.CalleeSamples) << " (candidate size:" <<
Candidate.SizeCost << ", call count: " << Candidate
.CallsiteCount << ", previously " << (WasInlined ?
"inlined)\n" : "not inlined)\n"); } }; } } while (false)
259 << ", call count: " << Candidate.CallsiteCount << ", previously "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { { if (!CQueue.empty()) dbgs() << " Inline candidates ignored due to size limit (inliner "
"original size: " << FuncSize << ", inliner final size: "
<< FuncFinalSize << ", size limit: " << SizeLimit
<< ")\n"; while (!CQueue.empty()) { ProfiledInlineCandidate
Candidate = CQueue.top(); CQueue.pop(); bool WasInlined = Candidate
.CalleeSamples->getContext().hasAttribute(ContextWasInlined
); dbgs() << " " << ContextTracker.getContextString
(*Candidate.CalleeSamples) << " (candidate size:" <<
Candidate.SizeCost << ", call count: " << Candidate
.CallsiteCount << ", previously " << (WasInlined ?
"inlined)\n" : "not inlined)\n"); } }; } } while (false)
260 << (WasInlined ? "inlined)\n" : "not inlined)\n");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { { if (!CQueue.empty()) dbgs() << " Inline candidates ignored due to size limit (inliner "
"original size: " << FuncSize << ", inliner final size: "
<< FuncFinalSize << ", size limit: " << SizeLimit
<< ")\n"; while (!CQueue.empty()) { ProfiledInlineCandidate
Candidate = CQueue.top(); CQueue.pop(); bool WasInlined = Candidate
.CalleeSamples->getContext().hasAttribute(ContextWasInlined
); dbgs() << " " << ContextTracker.getContextString
(*Candidate.CalleeSamples) << " (candidate size:" <<
Candidate.SizeCost << ", call count: " << Candidate
.CallsiteCount << ", previously " << (WasInlined ?
"inlined)\n" : "not inlined)\n"); } }; } } while (false)
261 }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { { if (!CQueue.empty()) dbgs() << " Inline candidates ignored due to size limit (inliner "
"original size: " << FuncSize << ", inliner final size: "
<< FuncFinalSize << ", size limit: " << SizeLimit
<< ")\n"; while (!CQueue.empty()) { ProfiledInlineCandidate
Candidate = CQueue.top(); CQueue.pop(); bool WasInlined = Candidate
.CalleeSamples->getContext().hasAttribute(ContextWasInlined
); dbgs() << " " << ContextTracker.getContextString
(*Candidate.CalleeSamples) << " (candidate size:" <<
Candidate.SizeCost << ", call count: " << Candidate
.CallsiteCount << ", previously " << (WasInlined ?
"inlined)\n" : "not inlined)\n"); } }; } } while (false)
262 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { { if (!CQueue.empty()) dbgs() << " Inline candidates ignored due to size limit (inliner "
"original size: " << FuncSize << ", inliner final size: "
<< FuncFinalSize << ", size limit: " << SizeLimit
<< ")\n"; while (!CQueue.empty()) { ProfiledInlineCandidate
Candidate = CQueue.top(); CQueue.pop(); bool WasInlined = Candidate
.CalleeSamples->getContext().hasAttribute(ContextWasInlined
); dbgs() << " " << ContextTracker.getContextString
(*Candidate.CalleeSamples) << " (candidate size:" <<
Candidate.SizeCost << ", call count: " << Candidate
.CallsiteCount << ", previously " << (WasInlined ?
"inlined)\n" : "not inlined)\n"); } }; } } while (false);
263}
264
265void CSPreInliner::run() {
266#ifndef NDEBUG
267 auto printProfileNames = [](SampleContextTracker &ContextTracker,
268 bool IsInput) {
269 uint32_t Size = 0;
270 for (auto *Node : ContextTracker) {
271 FunctionSamples *FSamples = Node->getFunctionSamples();
272 if (FSamples) {
273 Size++;
274 dbgs() << " [" << ContextTracker.getContextString(Node) << "] "
275 << FSamples->getTotalSamples() << ":"
276 << FSamples->getHeadSamples() << "\n";
277 }
278 }
279 dbgs() << (IsInput ? "Input" : "Output") << " context-sensitive profiles ("
280 << Size << " total):\n";
281 };
282#endif
283
284 LLVM_DEBUG(printProfileNames(ContextTracker, true))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { printProfileNames(ContextTracker, true);
} } while (false);
285
286 // Execute global pre-inliner to estimate a global top-down inline
287 // decision and merge profiles accordingly. This helps with profile
288 // merge for ThinLTO otherwise we won't be able to merge profiles back
289 // to base profile across module/thin-backend boundaries.
290 // It also helps better compress context profile to control profile
291 // size, as we now only need context profile for functions going to
292 // be inlined.
293 for (StringRef FuncName : buildTopDownOrder()) {
294 processFunction(FuncName);
295 }
296
297 // Not inlined context profiles are merged into its base, so we can
298 // trim out such profiles from the output.
299 for (auto *Node : ContextTracker) {
300 FunctionSamples *FProfile = Node->getFunctionSamples();
301 if (FProfile &&
302 (Node->getParentContext() != &ContextTracker.getRootContext() &&
303 !FProfile->getContext().hasState(InlinedContext))) {
304 Node->setFunctionSamples(nullptr);
305 }
306 }
307 FunctionSamples::ProfileIsPreInlined = true;
308
309 LLVM_DEBUG(printProfileNames(ContextTracker, false))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("cs-preinliner")) { printProfileNames(ContextTracker, false)
; } } while (false);
310}