File: | build/source/llvm/lib/ProfileData/SampleProfReader.cpp |
Warning: | line 388, column 21 The left expression of the compound assignment is an uninitialized value. The computed value will also be garbage |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- SampleProfReader.cpp - Read LLVM sample profile data ---------------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | // This file implements the class that reads LLVM sample profiles. It | |||
10 | // supports three file formats: text, binary and gcov. | |||
11 | // | |||
12 | // The textual representation is useful for debugging and testing purposes. The | |||
13 | // binary representation is more compact, resulting in smaller file sizes. | |||
14 | // | |||
15 | // The gcov encoding is the one generated by GCC's AutoFDO profile creation | |||
16 | // tool (https://github.com/google/autofdo) | |||
17 | // | |||
18 | // All three encodings can be used interchangeably as an input sample profile. | |||
19 | // | |||
20 | //===----------------------------------------------------------------------===// | |||
21 | ||||
22 | #include "llvm/ProfileData/SampleProfReader.h" | |||
23 | #include "llvm/ADT/DenseMap.h" | |||
24 | #include "llvm/ADT/STLExtras.h" | |||
25 | #include "llvm/ADT/StringRef.h" | |||
26 | #include "llvm/IR/Module.h" | |||
27 | #include "llvm/IR/ProfileSummary.h" | |||
28 | #include "llvm/ProfileData/ProfileCommon.h" | |||
29 | #include "llvm/ProfileData/SampleProf.h" | |||
30 | #include "llvm/Support/CommandLine.h" | |||
31 | #include "llvm/Support/Compression.h" | |||
32 | #include "llvm/Support/ErrorOr.h" | |||
33 | #include "llvm/Support/JSON.h" | |||
34 | #include "llvm/Support/LEB128.h" | |||
35 | #include "llvm/Support/LineIterator.h" | |||
36 | #include "llvm/Support/MD5.h" | |||
37 | #include "llvm/Support/MemoryBuffer.h" | |||
38 | #include "llvm/Support/VirtualFileSystem.h" | |||
39 | #include "llvm/Support/raw_ostream.h" | |||
40 | #include <algorithm> | |||
41 | #include <cstddef> | |||
42 | #include <cstdint> | |||
43 | #include <limits> | |||
44 | #include <memory> | |||
45 | #include <system_error> | |||
46 | #include <vector> | |||
47 | ||||
48 | using namespace llvm; | |||
49 | using namespace sampleprof; | |||
50 | ||||
51 | #define DEBUG_TYPE"samplepgo-reader" "samplepgo-reader" | |||
52 | ||||
53 | // This internal option specifies if the profile uses FS discriminators. | |||
54 | // It only applies to text, binary and compact binary format profiles. | |||
55 | // For ext-binary format profiles, the flag is set in the summary. | |||
56 | static cl::opt<bool> ProfileIsFSDisciminator( | |||
57 | "profile-isfs", cl::Hidden, cl::init(false), | |||
58 | cl::desc("Profile uses flow sensitive discriminators")); | |||
59 | ||||
60 | /// Dump the function profile for \p FName. | |||
61 | /// | |||
62 | /// \param FContext Name + context of the function to print. | |||
63 | /// \param OS Stream to emit the output to. | |||
64 | void SampleProfileReader::dumpFunctionProfile(SampleContext FContext, | |||
65 | raw_ostream &OS) { | |||
66 | OS << "Function: " << FContext.toString() << ": " << Profiles[FContext]; | |||
67 | } | |||
68 | ||||
69 | /// Dump all the function profiles found on stream \p OS. | |||
70 | void SampleProfileReader::dump(raw_ostream &OS) { | |||
71 | std::vector<NameFunctionSamples> V; | |||
72 | sortFuncProfiles(Profiles, V); | |||
73 | for (const auto &I : V) | |||
74 | dumpFunctionProfile(I.first, OS); | |||
75 | } | |||
76 | ||||
77 | static void dumpFunctionProfileJson(const FunctionSamples &S, | |||
78 | json::OStream &JOS, bool TopLevel = false) { | |||
79 | auto DumpBody = [&](const BodySampleMap &BodySamples) { | |||
80 | for (const auto &I : BodySamples) { | |||
81 | const LineLocation &Loc = I.first; | |||
82 | const SampleRecord &Sample = I.second; | |||
83 | JOS.object([&] { | |||
84 | JOS.attribute("line", Loc.LineOffset); | |||
85 | if (Loc.Discriminator) | |||
86 | JOS.attribute("discriminator", Loc.Discriminator); | |||
87 | JOS.attribute("samples", Sample.getSamples()); | |||
88 | ||||
89 | auto CallTargets = Sample.getSortedCallTargets(); | |||
90 | if (!CallTargets.empty()) { | |||
91 | JOS.attributeArray("calls", [&] { | |||
92 | for (const auto &J : CallTargets) { | |||
93 | JOS.object([&] { | |||
94 | JOS.attribute("function", J.first); | |||
95 | JOS.attribute("samples", J.second); | |||
96 | }); | |||
97 | } | |||
98 | }); | |||
99 | } | |||
100 | }); | |||
101 | } | |||
102 | }; | |||
103 | ||||
104 | auto DumpCallsiteSamples = [&](const CallsiteSampleMap &CallsiteSamples) { | |||
105 | for (const auto &I : CallsiteSamples) | |||
106 | for (const auto &FS : I.second) { | |||
107 | const LineLocation &Loc = I.first; | |||
108 | const FunctionSamples &CalleeSamples = FS.second; | |||
109 | JOS.object([&] { | |||
110 | JOS.attribute("line", Loc.LineOffset); | |||
111 | if (Loc.Discriminator) | |||
112 | JOS.attribute("discriminator", Loc.Discriminator); | |||
113 | JOS.attributeArray( | |||
114 | "samples", [&] { dumpFunctionProfileJson(CalleeSamples, JOS); }); | |||
115 | }); | |||
116 | } | |||
117 | }; | |||
118 | ||||
119 | JOS.object([&] { | |||
120 | JOS.attribute("name", S.getName()); | |||
121 | JOS.attribute("total", S.getTotalSamples()); | |||
122 | if (TopLevel) | |||
123 | JOS.attribute("head", S.getHeadSamples()); | |||
124 | ||||
125 | const auto &BodySamples = S.getBodySamples(); | |||
126 | if (!BodySamples.empty()) | |||
127 | JOS.attributeArray("body", [&] { DumpBody(BodySamples); }); | |||
128 | ||||
129 | const auto &CallsiteSamples = S.getCallsiteSamples(); | |||
130 | if (!CallsiteSamples.empty()) | |||
131 | JOS.attributeArray("callsites", | |||
132 | [&] { DumpCallsiteSamples(CallsiteSamples); }); | |||
133 | }); | |||
134 | } | |||
135 | ||||
136 | /// Dump all the function profiles found on stream \p OS in the JSON format. | |||
137 | void SampleProfileReader::dumpJson(raw_ostream &OS) { | |||
138 | std::vector<NameFunctionSamples> V; | |||
139 | sortFuncProfiles(Profiles, V); | |||
140 | json::OStream JOS(OS, 2); | |||
141 | JOS.arrayBegin(); | |||
142 | for (const auto &F : V) | |||
143 | dumpFunctionProfileJson(*F.second, JOS, true); | |||
144 | JOS.arrayEnd(); | |||
145 | ||||
146 | // Emit a newline character at the end as json::OStream doesn't emit one. | |||
147 | OS << "\n"; | |||
148 | } | |||
149 | ||||
150 | /// Parse \p Input as function head. | |||
151 | /// | |||
152 | /// Parse one line of \p Input, and update function name in \p FName, | |||
153 | /// function's total sample count in \p NumSamples, function's entry | |||
154 | /// count in \p NumHeadSamples. | |||
155 | /// | |||
156 | /// \returns true if parsing is successful. | |||
157 | static bool ParseHead(const StringRef &Input, StringRef &FName, | |||
158 | uint64_t &NumSamples, uint64_t &NumHeadSamples) { | |||
159 | if (Input[0] == ' ') | |||
160 | return false; | |||
161 | size_t n2 = Input.rfind(':'); | |||
162 | size_t n1 = Input.rfind(':', n2 - 1); | |||
163 | FName = Input.substr(0, n1); | |||
164 | if (Input.substr(n1 + 1, n2 - n1 - 1).getAsInteger(10, NumSamples)) | |||
165 | return false; | |||
166 | if (Input.substr(n2 + 1).getAsInteger(10, NumHeadSamples)) | |||
167 | return false; | |||
168 | return true; | |||
169 | } | |||
170 | ||||
171 | /// Returns true if line offset \p L is legal (only has 16 bits). | |||
172 | static bool isOffsetLegal(unsigned L) { return (L & 0xffff) == L; } | |||
173 | ||||
174 | /// Parse \p Input that contains metadata. | |||
175 | /// Possible metadata: | |||
176 | /// - CFG Checksum information: | |||
177 | /// !CFGChecksum: 12345 | |||
178 | /// - CFG Checksum information: | |||
179 | /// !Attributes: 1 | |||
180 | /// Stores the FunctionHash (a.k.a. CFG Checksum) into \p FunctionHash. | |||
181 | static bool parseMetadata(const StringRef &Input, uint64_t &FunctionHash, | |||
182 | uint32_t &Attributes) { | |||
183 | if (Input.startswith("!CFGChecksum:")) { | |||
184 | StringRef CFGInfo = Input.substr(strlen("!CFGChecksum:")).trim(); | |||
185 | return !CFGInfo.getAsInteger(10, FunctionHash); | |||
186 | } | |||
187 | ||||
188 | if (Input.startswith("!Attributes:")) { | |||
189 | StringRef Attrib = Input.substr(strlen("!Attributes:")).trim(); | |||
190 | return !Attrib.getAsInteger(10, Attributes); | |||
191 | } | |||
192 | ||||
193 | return false; | |||
194 | } | |||
195 | ||||
196 | enum class LineType { | |||
197 | CallSiteProfile, | |||
198 | BodyProfile, | |||
199 | Metadata, | |||
200 | }; | |||
201 | ||||
202 | /// Parse \p Input as line sample. | |||
203 | /// | |||
204 | /// \param Input input line. | |||
205 | /// \param LineTy Type of this line. | |||
206 | /// \param Depth the depth of the inline stack. | |||
207 | /// \param NumSamples total samples of the line/inlined callsite. | |||
208 | /// \param LineOffset line offset to the start of the function. | |||
209 | /// \param Discriminator discriminator of the line. | |||
210 | /// \param TargetCountMap map from indirect call target to count. | |||
211 | /// \param FunctionHash the function's CFG hash, used by pseudo probe. | |||
212 | /// | |||
213 | /// returns true if parsing is successful. | |||
214 | static bool ParseLine(const StringRef &Input, LineType &LineTy, uint32_t &Depth, | |||
215 | uint64_t &NumSamples, uint32_t &LineOffset, | |||
216 | uint32_t &Discriminator, StringRef &CalleeName, | |||
217 | DenseMap<StringRef, uint64_t> &TargetCountMap, | |||
218 | uint64_t &FunctionHash, uint32_t &Attributes) { | |||
219 | for (Depth = 0; Input[Depth] == ' '; Depth++) | |||
220 | ; | |||
221 | if (Depth
| |||
222 | return false; | |||
223 | ||||
224 | if (Input[Depth] == '!') { | |||
225 | LineTy = LineType::Metadata; | |||
226 | return parseMetadata(Input.substr(Depth), FunctionHash, Attributes); | |||
227 | } | |||
228 | ||||
229 | size_t n1 = Input.find(':'); | |||
230 | StringRef Loc = Input.substr(Depth, n1 - Depth); | |||
231 | size_t n2 = Loc.find('.'); | |||
232 | if (n2 == StringRef::npos) { | |||
233 | if (Loc.getAsInteger(10, LineOffset) || !isOffsetLegal(LineOffset)) | |||
234 | return false; | |||
235 | Discriminator = 0; | |||
236 | } else { | |||
237 | if (Loc.substr(0, n2).getAsInteger(10, LineOffset)) | |||
238 | return false; | |||
239 | if (Loc.substr(n2 + 1).getAsInteger(10, Discriminator)) | |||
240 | return false; | |||
241 | } | |||
242 | ||||
243 | StringRef Rest = Input.substr(n1 + 2); | |||
244 | if (isDigit(Rest[0])) { | |||
245 | LineTy = LineType::BodyProfile; | |||
246 | size_t n3 = Rest.find(' '); | |||
247 | if (n3 == StringRef::npos) { | |||
248 | if (Rest.getAsInteger(10, NumSamples)) | |||
249 | return false; | |||
250 | } else { | |||
251 | if (Rest.substr(0, n3).getAsInteger(10, NumSamples)) | |||
252 | return false; | |||
253 | } | |||
254 | // Find call targets and their sample counts. | |||
255 | // Note: In some cases, there are symbols in the profile which are not | |||
256 | // mangled. To accommodate such cases, use colon + integer pairs as the | |||
257 | // anchor points. | |||
258 | // An example: | |||
259 | // _M_construct<char *>:1000 string_view<std::allocator<char> >:437 | |||
260 | // ":1000" and ":437" are used as anchor points so the string above will | |||
261 | // be interpreted as | |||
262 | // target: _M_construct<char *> | |||
263 | // count: 1000 | |||
264 | // target: string_view<std::allocator<char> > | |||
265 | // count: 437 | |||
266 | while (n3 != StringRef::npos) { | |||
267 | n3 += Rest.substr(n3).find_first_not_of(' '); | |||
268 | Rest = Rest.substr(n3); | |||
269 | n3 = Rest.find_first_of(':'); | |||
270 | if (n3 == StringRef::npos || n3 == 0) | |||
271 | return false; | |||
272 | ||||
273 | StringRef Target; | |||
274 | uint64_t count, n4; | |||
275 | while (true) { | |||
276 | // Get the segment after the current colon. | |||
277 | StringRef AfterColon = Rest.substr(n3 + 1); | |||
278 | // Get the target symbol before the current colon. | |||
279 | Target = Rest.substr(0, n3); | |||
280 | // Check if the word after the current colon is an integer. | |||
281 | n4 = AfterColon.find_first_of(' '); | |||
282 | n4 = (n4 != StringRef::npos) ? n3 + n4 + 1 : Rest.size(); | |||
283 | StringRef WordAfterColon = Rest.substr(n3 + 1, n4 - n3 - 1); | |||
284 | if (!WordAfterColon.getAsInteger(10, count)) | |||
285 | break; | |||
286 | ||||
287 | // Try to find the next colon. | |||
288 | uint64_t n5 = AfterColon.find_first_of(':'); | |||
289 | if (n5 == StringRef::npos) | |||
290 | return false; | |||
291 | n3 += n5 + 1; | |||
292 | } | |||
293 | ||||
294 | // An anchor point is found. Save the {target, count} pair | |||
295 | TargetCountMap[Target] = count; | |||
296 | if (n4 == Rest.size()) | |||
297 | break; | |||
298 | // Change n3 to the next blank space after colon + integer pair. | |||
299 | n3 = n4; | |||
300 | } | |||
301 | } else { | |||
302 | LineTy = LineType::CallSiteProfile; | |||
303 | size_t n3 = Rest.find_last_of(':'); | |||
304 | CalleeName = Rest.substr(0, n3); | |||
305 | if (Rest.substr(n3 + 1).getAsInteger(10, NumSamples)) | |||
306 | return false; | |||
307 | } | |||
308 | return true; | |||
309 | } | |||
310 | ||||
311 | /// Load samples from a text file. | |||
312 | /// | |||
313 | /// See the documentation at the top of the file for an explanation of | |||
314 | /// the expected format. | |||
315 | /// | |||
316 | /// \returns true if the file was loaded successfully, false otherwise. | |||
317 | std::error_code SampleProfileReaderText::readImpl() { | |||
318 | line_iterator LineIt(*Buffer, /*SkipBlanks=*/true, '#'); | |||
319 | sampleprof_error Result = sampleprof_error::success; | |||
320 | ||||
321 | InlineCallStack InlineStack; | |||
322 | uint32_t TopLevelProbeProfileCount = 0; | |||
323 | ||||
324 | // DepthMetadata tracks whether we have processed metadata for the current | |||
325 | // top-level or nested function profile. | |||
326 | uint32_t DepthMetadata = 0; | |||
327 | ||||
328 | ProfileIsFS = ProfileIsFSDisciminator; | |||
329 | FunctionSamples::ProfileIsFS = ProfileIsFS; | |||
330 | for (; !LineIt.is_at_eof(); ++LineIt) { | |||
| ||||
331 | if ((*LineIt)[(*LineIt).find_first_not_of(' ')] == '#') | |||
332 | continue; | |||
333 | // Read the header of each function. | |||
334 | // | |||
335 | // Note that for function identifiers we are actually expecting | |||
336 | // mangled names, but we may not always get them. This happens when | |||
337 | // the compiler decides not to emit the function (e.g., it was inlined | |||
338 | // and removed). In this case, the binary will not have the linkage | |||
339 | // name for the function, so the profiler will emit the function's | |||
340 | // unmangled name, which may contain characters like ':' and '>' in its | |||
341 | // name (member functions, templates, etc). | |||
342 | // | |||
343 | // The only requirement we place on the identifier, then, is that it | |||
344 | // should not begin with a number. | |||
345 | if ((*LineIt)[0] != ' ') { | |||
346 | uint64_t NumSamples, NumHeadSamples; | |||
347 | StringRef FName; | |||
348 | if (!ParseHead(*LineIt, FName, NumSamples, NumHeadSamples)) { | |||
349 | reportError(LineIt.line_number(), | |||
350 | "Expected 'mangled_name:NUM:NUM', found " + *LineIt); | |||
351 | return sampleprof_error::malformed; | |||
352 | } | |||
353 | DepthMetadata = 0; | |||
354 | SampleContext FContext(FName, CSNameTable); | |||
355 | if (FContext.hasContext()) | |||
356 | ++CSProfileCount; | |||
357 | Profiles[FContext] = FunctionSamples(); | |||
358 | FunctionSamples &FProfile = Profiles[FContext]; | |||
359 | FProfile.setContext(FContext); | |||
360 | MergeResult(Result, FProfile.addTotalSamples(NumSamples)); | |||
361 | MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples)); | |||
362 | InlineStack.clear(); | |||
363 | InlineStack.push_back(&FProfile); | |||
364 | } else { | |||
365 | uint64_t NumSamples; | |||
366 | StringRef FName; | |||
367 | DenseMap<StringRef, uint64_t> TargetCountMap; | |||
368 | uint32_t Depth, LineOffset, Discriminator; | |||
369 | LineType LineTy; | |||
370 | uint64_t FunctionHash = 0; | |||
371 | uint32_t Attributes = 0; | |||
372 | if (!ParseLine(*LineIt, LineTy, Depth, NumSamples, LineOffset, | |||
373 | Discriminator, FName, TargetCountMap, FunctionHash, | |||
374 | Attributes)) { | |||
375 | reportError(LineIt.line_number(), | |||
376 | "Expected 'NUM[.NUM]: NUM[ mangled_name:NUM]*', found " + | |||
377 | *LineIt); | |||
378 | return sampleprof_error::malformed; | |||
379 | } | |||
380 | if (LineTy
| |||
381 | // Metadata must be put at the end of a function profile. | |||
382 | reportError(LineIt.line_number(), | |||
383 | "Found non-metadata after metadata: " + *LineIt); | |||
384 | return sampleprof_error::malformed; | |||
385 | } | |||
386 | ||||
387 | // Here we handle FS discriminators. | |||
388 | Discriminator &= getDiscriminatorMask(); | |||
| ||||
389 | ||||
390 | while (InlineStack.size() > Depth) { | |||
391 | InlineStack.pop_back(); | |||
392 | } | |||
393 | switch (LineTy) { | |||
394 | case LineType::CallSiteProfile: { | |||
395 | FunctionSamples &FSamples = InlineStack.back()->functionSamplesAt( | |||
396 | LineLocation(LineOffset, Discriminator))[std::string(FName)]; | |||
397 | FSamples.setName(FName); | |||
398 | MergeResult(Result, FSamples.addTotalSamples(NumSamples)); | |||
399 | InlineStack.push_back(&FSamples); | |||
400 | DepthMetadata = 0; | |||
401 | break; | |||
402 | } | |||
403 | case LineType::BodyProfile: { | |||
404 | while (InlineStack.size() > Depth) { | |||
405 | InlineStack.pop_back(); | |||
406 | } | |||
407 | FunctionSamples &FProfile = *InlineStack.back(); | |||
408 | for (const auto &name_count : TargetCountMap) { | |||
409 | MergeResult(Result, FProfile.addCalledTargetSamples( | |||
410 | LineOffset, Discriminator, name_count.first, | |||
411 | name_count.second)); | |||
412 | } | |||
413 | MergeResult(Result, FProfile.addBodySamples(LineOffset, Discriminator, | |||
414 | NumSamples)); | |||
415 | break; | |||
416 | } | |||
417 | case LineType::Metadata: { | |||
418 | FunctionSamples &FProfile = *InlineStack.back(); | |||
419 | if (FunctionHash) { | |||
420 | FProfile.setFunctionHash(FunctionHash); | |||
421 | if (Depth == 1) | |||
422 | ++TopLevelProbeProfileCount; | |||
423 | } | |||
424 | FProfile.getContext().setAllAttributes(Attributes); | |||
425 | if (Attributes & (uint32_t)ContextShouldBeInlined) | |||
426 | ProfileIsPreInlined = true; | |||
427 | DepthMetadata = Depth; | |||
428 | break; | |||
429 | } | |||
430 | } | |||
431 | } | |||
432 | } | |||
433 | ||||
434 | assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&(static_cast <bool> ((CSProfileCount == 0 || CSProfileCount == Profiles.size()) && "Cannot have both context-sensitive and regular profile" ) ? void (0) : __assert_fail ("(CSProfileCount == 0 || CSProfileCount == Profiles.size()) && \"Cannot have both context-sensitive and regular profile\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 435, __extension__ __PRETTY_FUNCTION__)) | |||
435 | "Cannot have both context-sensitive and regular profile")(static_cast <bool> ((CSProfileCount == 0 || CSProfileCount == Profiles.size()) && "Cannot have both context-sensitive and regular profile" ) ? void (0) : __assert_fail ("(CSProfileCount == 0 || CSProfileCount == Profiles.size()) && \"Cannot have both context-sensitive and regular profile\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 435, __extension__ __PRETTY_FUNCTION__)); | |||
436 | ProfileIsCS = (CSProfileCount > 0); | |||
437 | assert((TopLevelProbeProfileCount == 0 ||(static_cast <bool> ((TopLevelProbeProfileCount == 0 || TopLevelProbeProfileCount == Profiles.size()) && "Cannot have both probe-based profiles and regular profiles" ) ? void (0) : __assert_fail ("(TopLevelProbeProfileCount == 0 || TopLevelProbeProfileCount == Profiles.size()) && \"Cannot have both probe-based profiles and regular profiles\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 439, __extension__ __PRETTY_FUNCTION__)) | |||
438 | TopLevelProbeProfileCount == Profiles.size()) &&(static_cast <bool> ((TopLevelProbeProfileCount == 0 || TopLevelProbeProfileCount == Profiles.size()) && "Cannot have both probe-based profiles and regular profiles" ) ? void (0) : __assert_fail ("(TopLevelProbeProfileCount == 0 || TopLevelProbeProfileCount == Profiles.size()) && \"Cannot have both probe-based profiles and regular profiles\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 439, __extension__ __PRETTY_FUNCTION__)) | |||
439 | "Cannot have both probe-based profiles and regular profiles")(static_cast <bool> ((TopLevelProbeProfileCount == 0 || TopLevelProbeProfileCount == Profiles.size()) && "Cannot have both probe-based profiles and regular profiles" ) ? void (0) : __assert_fail ("(TopLevelProbeProfileCount == 0 || TopLevelProbeProfileCount == Profiles.size()) && \"Cannot have both probe-based profiles and regular profiles\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 439, __extension__ __PRETTY_FUNCTION__)); | |||
440 | ProfileIsProbeBased = (TopLevelProbeProfileCount > 0); | |||
441 | FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased; | |||
442 | FunctionSamples::ProfileIsCS = ProfileIsCS; | |||
443 | FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined; | |||
444 | ||||
445 | if (Result == sampleprof_error::success) | |||
446 | computeSummary(); | |||
447 | ||||
448 | return Result; | |||
449 | } | |||
450 | ||||
451 | bool SampleProfileReaderText::hasFormat(const MemoryBuffer &Buffer) { | |||
452 | bool result = false; | |||
453 | ||||
454 | // Check that the first non-comment line is a valid function header. | |||
455 | line_iterator LineIt(Buffer, /*SkipBlanks=*/true, '#'); | |||
456 | if (!LineIt.is_at_eof()) { | |||
457 | if ((*LineIt)[0] != ' ') { | |||
458 | uint64_t NumSamples, NumHeadSamples; | |||
459 | StringRef FName; | |||
460 | result = ParseHead(*LineIt, FName, NumSamples, NumHeadSamples); | |||
461 | } | |||
462 | } | |||
463 | ||||
464 | return result; | |||
465 | } | |||
466 | ||||
467 | template <typename T> ErrorOr<T> SampleProfileReaderBinary::readNumber() { | |||
468 | unsigned NumBytesRead = 0; | |||
469 | std::error_code EC; | |||
470 | uint64_t Val = decodeULEB128(Data, &NumBytesRead); | |||
471 | ||||
472 | if (Val > std::numeric_limits<T>::max()) | |||
473 | EC = sampleprof_error::malformed; | |||
474 | else if (Data + NumBytesRead > End) | |||
475 | EC = sampleprof_error::truncated; | |||
476 | else | |||
477 | EC = sampleprof_error::success; | |||
478 | ||||
479 | if (EC) { | |||
480 | reportError(0, EC.message()); | |||
481 | return EC; | |||
482 | } | |||
483 | ||||
484 | Data += NumBytesRead; | |||
485 | return static_cast<T>(Val); | |||
486 | } | |||
487 | ||||
488 | ErrorOr<StringRef> SampleProfileReaderBinary::readString() { | |||
489 | std::error_code EC; | |||
490 | StringRef Str(reinterpret_cast<const char *>(Data)); | |||
491 | if (Data + Str.size() + 1 > End) { | |||
492 | EC = sampleprof_error::truncated; | |||
493 | reportError(0, EC.message()); | |||
494 | return EC; | |||
495 | } | |||
496 | ||||
497 | Data += Str.size() + 1; | |||
498 | return Str; | |||
499 | } | |||
500 | ||||
501 | template <typename T> | |||
502 | ErrorOr<T> SampleProfileReaderBinary::readUnencodedNumber() { | |||
503 | std::error_code EC; | |||
504 | ||||
505 | if (Data + sizeof(T) > End) { | |||
506 | EC = sampleprof_error::truncated; | |||
507 | reportError(0, EC.message()); | |||
508 | return EC; | |||
509 | } | |||
510 | ||||
511 | using namespace support; | |||
512 | T Val = endian::readNext<T, little, unaligned>(Data); | |||
513 | return Val; | |||
514 | } | |||
515 | ||||
516 | template <typename T> | |||
517 | inline ErrorOr<uint32_t> SampleProfileReaderBinary::readStringIndex(T &Table) { | |||
518 | std::error_code EC; | |||
519 | auto Idx = readNumber<uint32_t>(); | |||
520 | if (std::error_code EC = Idx.getError()) | |||
521 | return EC; | |||
522 | if (*Idx >= Table.size()) | |||
523 | return sampleprof_error::truncated_name_table; | |||
524 | return *Idx; | |||
525 | } | |||
526 | ||||
527 | ErrorOr<StringRef> SampleProfileReaderBinary::readStringFromTable() { | |||
528 | auto Idx = readStringIndex(NameTable); | |||
529 | if (std::error_code EC = Idx.getError()) | |||
530 | return EC; | |||
531 | ||||
532 | return NameTable[*Idx]; | |||
533 | } | |||
534 | ||||
535 | ErrorOr<SampleContext> SampleProfileReaderBinary::readSampleContextFromTable() { | |||
536 | auto FName(readStringFromTable()); | |||
537 | if (std::error_code EC = FName.getError()) | |||
538 | return EC; | |||
539 | return SampleContext(*FName); | |||
540 | } | |||
541 | ||||
542 | ErrorOr<StringRef> SampleProfileReaderExtBinaryBase::readStringFromTable() { | |||
543 | if (!FixedLengthMD5) | |||
544 | return SampleProfileReaderBinary::readStringFromTable(); | |||
545 | ||||
546 | // read NameTable index. | |||
547 | auto Idx = readStringIndex(NameTable); | |||
548 | if (std::error_code EC = Idx.getError()) | |||
549 | return EC; | |||
550 | ||||
551 | // Check whether the name to be accessed has been accessed before, | |||
552 | // if not, read it from memory directly. | |||
553 | StringRef &SR = NameTable[*Idx]; | |||
554 | if (SR.empty()) { | |||
555 | const uint8_t *SavedData = Data; | |||
556 | Data = MD5NameMemStart + ((*Idx) * sizeof(uint64_t)); | |||
557 | auto FID = readUnencodedNumber<uint64_t>(); | |||
558 | if (std::error_code EC = FID.getError()) | |||
559 | return EC; | |||
560 | // Save the string converted from uint64_t in MD5StringBuf. All the | |||
561 | // references to the name are all StringRefs refering to the string | |||
562 | // in MD5StringBuf. | |||
563 | MD5StringBuf->push_back(std::to_string(*FID)); | |||
564 | SR = MD5StringBuf->back(); | |||
565 | Data = SavedData; | |||
566 | } | |||
567 | return SR; | |||
568 | } | |||
569 | ||||
570 | ErrorOr<StringRef> SampleProfileReaderCompactBinary::readStringFromTable() { | |||
571 | auto Idx = readStringIndex(NameTable); | |||
572 | if (std::error_code EC = Idx.getError()) | |||
573 | return EC; | |||
574 | ||||
575 | return StringRef(NameTable[*Idx]); | |||
576 | } | |||
577 | ||||
578 | std::error_code | |||
579 | SampleProfileReaderBinary::readProfile(FunctionSamples &FProfile) { | |||
580 | auto NumSamples = readNumber<uint64_t>(); | |||
581 | if (std::error_code EC = NumSamples.getError()) | |||
582 | return EC; | |||
583 | FProfile.addTotalSamples(*NumSamples); | |||
584 | ||||
585 | // Read the samples in the body. | |||
586 | auto NumRecords = readNumber<uint32_t>(); | |||
587 | if (std::error_code EC = NumRecords.getError()) | |||
588 | return EC; | |||
589 | ||||
590 | for (uint32_t I = 0; I < *NumRecords; ++I) { | |||
591 | auto LineOffset = readNumber<uint64_t>(); | |||
592 | if (std::error_code EC = LineOffset.getError()) | |||
593 | return EC; | |||
594 | ||||
595 | if (!isOffsetLegal(*LineOffset)) { | |||
596 | return std::error_code(); | |||
597 | } | |||
598 | ||||
599 | auto Discriminator = readNumber<uint64_t>(); | |||
600 | if (std::error_code EC = Discriminator.getError()) | |||
601 | return EC; | |||
602 | ||||
603 | auto NumSamples = readNumber<uint64_t>(); | |||
604 | if (std::error_code EC = NumSamples.getError()) | |||
605 | return EC; | |||
606 | ||||
607 | auto NumCalls = readNumber<uint32_t>(); | |||
608 | if (std::error_code EC = NumCalls.getError()) | |||
609 | return EC; | |||
610 | ||||
611 | // Here we handle FS discriminators: | |||
612 | uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask(); | |||
613 | ||||
614 | for (uint32_t J = 0; J < *NumCalls; ++J) { | |||
615 | auto CalledFunction(readStringFromTable()); | |||
616 | if (std::error_code EC = CalledFunction.getError()) | |||
617 | return EC; | |||
618 | ||||
619 | auto CalledFunctionSamples = readNumber<uint64_t>(); | |||
620 | if (std::error_code EC = CalledFunctionSamples.getError()) | |||
621 | return EC; | |||
622 | ||||
623 | FProfile.addCalledTargetSamples(*LineOffset, DiscriminatorVal, | |||
624 | *CalledFunction, *CalledFunctionSamples); | |||
625 | } | |||
626 | ||||
627 | FProfile.addBodySamples(*LineOffset, DiscriminatorVal, *NumSamples); | |||
628 | } | |||
629 | ||||
630 | // Read all the samples for inlined function calls. | |||
631 | auto NumCallsites = readNumber<uint32_t>(); | |||
632 | if (std::error_code EC = NumCallsites.getError()) | |||
633 | return EC; | |||
634 | ||||
635 | for (uint32_t J = 0; J < *NumCallsites; ++J) { | |||
636 | auto LineOffset = readNumber<uint64_t>(); | |||
637 | if (std::error_code EC = LineOffset.getError()) | |||
638 | return EC; | |||
639 | ||||
640 | auto Discriminator = readNumber<uint64_t>(); | |||
641 | if (std::error_code EC = Discriminator.getError()) | |||
642 | return EC; | |||
643 | ||||
644 | auto FName(readStringFromTable()); | |||
645 | if (std::error_code EC = FName.getError()) | |||
646 | return EC; | |||
647 | ||||
648 | // Here we handle FS discriminators: | |||
649 | uint32_t DiscriminatorVal = (*Discriminator) & getDiscriminatorMask(); | |||
650 | ||||
651 | FunctionSamples &CalleeProfile = FProfile.functionSamplesAt( | |||
652 | LineLocation(*LineOffset, DiscriminatorVal))[std::string(*FName)]; | |||
653 | CalleeProfile.setName(*FName); | |||
654 | if (std::error_code EC = readProfile(CalleeProfile)) | |||
655 | return EC; | |||
656 | } | |||
657 | ||||
658 | return sampleprof_error::success; | |||
659 | } | |||
660 | ||||
661 | std::error_code | |||
662 | SampleProfileReaderBinary::readFuncProfile(const uint8_t *Start) { | |||
663 | Data = Start; | |||
664 | auto NumHeadSamples = readNumber<uint64_t>(); | |||
665 | if (std::error_code EC = NumHeadSamples.getError()) | |||
666 | return EC; | |||
667 | ||||
668 | ErrorOr<SampleContext> FContext(readSampleContextFromTable()); | |||
669 | if (std::error_code EC = FContext.getError()) | |||
670 | return EC; | |||
671 | ||||
672 | Profiles[*FContext] = FunctionSamples(); | |||
673 | FunctionSamples &FProfile = Profiles[*FContext]; | |||
674 | FProfile.setContext(*FContext); | |||
675 | FProfile.addHeadSamples(*NumHeadSamples); | |||
676 | ||||
677 | if (FContext->hasContext()) | |||
678 | CSProfileCount++; | |||
679 | ||||
680 | if (std::error_code EC = readProfile(FProfile)) | |||
681 | return EC; | |||
682 | return sampleprof_error::success; | |||
683 | } | |||
684 | ||||
685 | std::error_code SampleProfileReaderBinary::readImpl() { | |||
686 | ProfileIsFS = ProfileIsFSDisciminator; | |||
687 | FunctionSamples::ProfileIsFS = ProfileIsFS; | |||
688 | while (!at_eof()) { | |||
689 | if (std::error_code EC = readFuncProfile(Data)) | |||
690 | return EC; | |||
691 | } | |||
692 | ||||
693 | return sampleprof_error::success; | |||
694 | } | |||
695 | ||||
696 | ErrorOr<SampleContextFrames> | |||
697 | SampleProfileReaderExtBinaryBase::readContextFromTable() { | |||
698 | auto ContextIdx = readNumber<uint32_t>(); | |||
699 | if (std::error_code EC = ContextIdx.getError()) | |||
700 | return EC; | |||
701 | if (*ContextIdx >= CSNameTable->size()) | |||
702 | return sampleprof_error::truncated_name_table; | |||
703 | return (*CSNameTable)[*ContextIdx]; | |||
704 | } | |||
705 | ||||
706 | ErrorOr<SampleContext> | |||
707 | SampleProfileReaderExtBinaryBase::readSampleContextFromTable() { | |||
708 | if (ProfileIsCS) { | |||
709 | auto FContext(readContextFromTable()); | |||
710 | if (std::error_code EC = FContext.getError()) | |||
711 | return EC; | |||
712 | return SampleContext(*FContext); | |||
713 | } else { | |||
714 | auto FName(readStringFromTable()); | |||
715 | if (std::error_code EC = FName.getError()) | |||
716 | return EC; | |||
717 | return SampleContext(*FName); | |||
718 | } | |||
719 | } | |||
720 | ||||
721 | std::error_code SampleProfileReaderExtBinaryBase::readOneSection( | |||
722 | const uint8_t *Start, uint64_t Size, const SecHdrTableEntry &Entry) { | |||
723 | Data = Start; | |||
724 | End = Start + Size; | |||
725 | switch (Entry.Type) { | |||
726 | case SecProfSummary: | |||
727 | if (std::error_code EC = readSummary()) | |||
728 | return EC; | |||
729 | if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial)) | |||
730 | Summary->setPartialProfile(true); | |||
731 | if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFullContext)) | |||
732 | FunctionSamples::ProfileIsCS = ProfileIsCS = true; | |||
733 | if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagIsPreInlined)) | |||
734 | FunctionSamples::ProfileIsPreInlined = ProfileIsPreInlined = true; | |||
735 | if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFSDiscriminator)) | |||
736 | FunctionSamples::ProfileIsFS = ProfileIsFS = true; | |||
737 | break; | |||
738 | case SecNameTable: { | |||
739 | FixedLengthMD5 = | |||
740 | hasSecFlag(Entry, SecNameTableFlags::SecFlagFixedLengthMD5); | |||
741 | bool UseMD5 = hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name); | |||
742 | assert((!FixedLengthMD5 || UseMD5) &&(static_cast <bool> ((!FixedLengthMD5 || UseMD5) && "If FixedLengthMD5 is true, UseMD5 has to be true") ? void ( 0) : __assert_fail ("(!FixedLengthMD5 || UseMD5) && \"If FixedLengthMD5 is true, UseMD5 has to be true\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 743, __extension__ __PRETTY_FUNCTION__)) | |||
743 | "If FixedLengthMD5 is true, UseMD5 has to be true")(static_cast <bool> ((!FixedLengthMD5 || UseMD5) && "If FixedLengthMD5 is true, UseMD5 has to be true") ? void ( 0) : __assert_fail ("(!FixedLengthMD5 || UseMD5) && \"If FixedLengthMD5 is true, UseMD5 has to be true\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 743, __extension__ __PRETTY_FUNCTION__)); | |||
744 | FunctionSamples::HasUniqSuffix = | |||
745 | hasSecFlag(Entry, SecNameTableFlags::SecFlagUniqSuffix); | |||
746 | if (std::error_code EC = readNameTableSec(UseMD5)) | |||
747 | return EC; | |||
748 | break; | |||
749 | } | |||
750 | case SecCSNameTable: { | |||
751 | if (std::error_code EC = readCSNameTableSec()) | |||
752 | return EC; | |||
753 | break; | |||
754 | } | |||
755 | case SecLBRProfile: | |||
756 | if (std::error_code EC = readFuncProfiles()) | |||
757 | return EC; | |||
758 | break; | |||
759 | case SecFuncOffsetTable: | |||
760 | FuncOffsetsOrdered = hasSecFlag(Entry, SecFuncOffsetFlags::SecFlagOrdered); | |||
761 | if (std::error_code EC = readFuncOffsetTable()) | |||
762 | return EC; | |||
763 | break; | |||
764 | case SecFuncMetadata: { | |||
765 | ProfileIsProbeBased = | |||
766 | hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsProbeBased); | |||
767 | FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased; | |||
768 | bool HasAttribute = | |||
769 | hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagHasAttribute); | |||
770 | if (std::error_code EC = readFuncMetadata(HasAttribute)) | |||
771 | return EC; | |||
772 | break; | |||
773 | } | |||
774 | case SecProfileSymbolList: | |||
775 | if (std::error_code EC = readProfileSymbolList()) | |||
776 | return EC; | |||
777 | break; | |||
778 | default: | |||
779 | if (std::error_code EC = readCustomSection(Entry)) | |||
780 | return EC; | |||
781 | break; | |||
782 | } | |||
783 | return sampleprof_error::success; | |||
784 | } | |||
785 | ||||
786 | bool SampleProfileReaderExtBinaryBase::collectFuncsFromModule() { | |||
787 | if (!M) | |||
788 | return false; | |||
789 | FuncsToUse.clear(); | |||
790 | for (auto &F : *M) | |||
791 | FuncsToUse.insert(FunctionSamples::getCanonicalFnName(F)); | |||
792 | return true; | |||
793 | } | |||
794 | ||||
795 | std::error_code SampleProfileReaderExtBinaryBase::readFuncOffsetTable() { | |||
796 | // If there are more than one FuncOffsetTable, the profile read associated | |||
797 | // with previous FuncOffsetTable has to be done before next FuncOffsetTable | |||
798 | // is read. | |||
799 | FuncOffsetTable.clear(); | |||
800 | ||||
801 | auto Size = readNumber<uint64_t>(); | |||
802 | if (std::error_code EC = Size.getError()) | |||
803 | return EC; | |||
804 | ||||
805 | FuncOffsetTable.reserve(*Size); | |||
806 | ||||
807 | if (FuncOffsetsOrdered) { | |||
808 | OrderedFuncOffsets = | |||
809 | std::make_unique<std::vector<std::pair<SampleContext, uint64_t>>>(); | |||
810 | OrderedFuncOffsets->reserve(*Size); | |||
811 | } | |||
812 | ||||
813 | for (uint64_t I = 0; I < *Size; ++I) { | |||
814 | auto FContext(readSampleContextFromTable()); | |||
815 | if (std::error_code EC = FContext.getError()) | |||
816 | return EC; | |||
817 | ||||
818 | auto Offset = readNumber<uint64_t>(); | |||
819 | if (std::error_code EC = Offset.getError()) | |||
820 | return EC; | |||
821 | ||||
822 | FuncOffsetTable[*FContext] = *Offset; | |||
823 | if (FuncOffsetsOrdered) | |||
824 | OrderedFuncOffsets->emplace_back(*FContext, *Offset); | |||
825 | } | |||
826 | ||||
827 | return sampleprof_error::success; | |||
828 | } | |||
829 | ||||
830 | std::error_code SampleProfileReaderExtBinaryBase::readFuncProfiles() { | |||
831 | // Collect functions used by current module if the Reader has been | |||
832 | // given a module. | |||
833 | // collectFuncsFromModule uses FunctionSamples::getCanonicalFnName | |||
834 | // which will query FunctionSamples::HasUniqSuffix, so it has to be | |||
835 | // called after FunctionSamples::HasUniqSuffix is set, i.e. after | |||
836 | // NameTable section is read. | |||
837 | bool LoadFuncsToBeUsed = collectFuncsFromModule(); | |||
838 | ||||
839 | // When LoadFuncsToBeUsed is false, load all the function profiles. | |||
840 | const uint8_t *Start = Data; | |||
841 | if (!LoadFuncsToBeUsed) { | |||
842 | while (Data < End) { | |||
843 | if (std::error_code EC = readFuncProfile(Data)) | |||
844 | return EC; | |||
845 | } | |||
846 | assert(Data == End && "More data is read than expected")(static_cast <bool> (Data == End && "More data is read than expected" ) ? void (0) : __assert_fail ("Data == End && \"More data is read than expected\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 846, __extension__ __PRETTY_FUNCTION__)); | |||
847 | } else { | |||
848 | // Load function profiles on demand. | |||
849 | if (Remapper) { | |||
850 | for (auto Name : FuncsToUse) { | |||
851 | Remapper->insert(Name); | |||
852 | } | |||
853 | } | |||
854 | ||||
855 | if (ProfileIsCS) { | |||
856 | DenseSet<uint64_t> FuncGuidsToUse; | |||
857 | if (useMD5()) { | |||
858 | for (auto Name : FuncsToUse) | |||
859 | FuncGuidsToUse.insert(Function::getGUID(Name)); | |||
860 | } | |||
861 | ||||
862 | // For each function in current module, load all context profiles for | |||
863 | // the function as well as their callee contexts which can help profile | |||
864 | // guided importing for ThinLTO. This can be achieved by walking | |||
865 | // through an ordered context container, where contexts are laid out | |||
866 | // as if they were walked in preorder of a context trie. While | |||
867 | // traversing the trie, a link to the highest common ancestor node is | |||
868 | // kept so that all of its decendants will be loaded. | |||
869 | assert(OrderedFuncOffsets.get() &&(static_cast <bool> (OrderedFuncOffsets.get() && "func offset table should always be sorted in CS profile") ? void (0) : __assert_fail ("OrderedFuncOffsets.get() && \"func offset table should always be sorted in CS profile\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 870, __extension__ __PRETTY_FUNCTION__)) | |||
870 | "func offset table should always be sorted in CS profile")(static_cast <bool> (OrderedFuncOffsets.get() && "func offset table should always be sorted in CS profile") ? void (0) : __assert_fail ("OrderedFuncOffsets.get() && \"func offset table should always be sorted in CS profile\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 870, __extension__ __PRETTY_FUNCTION__)); | |||
871 | const SampleContext *CommonContext = nullptr; | |||
872 | for (const auto &NameOffset : *OrderedFuncOffsets) { | |||
873 | const auto &FContext = NameOffset.first; | |||
874 | auto FName = FContext.getName(); | |||
875 | // For function in the current module, keep its farthest ancestor | |||
876 | // context. This can be used to load itself and its child and | |||
877 | // sibling contexts. | |||
878 | if ((useMD5() && FuncGuidsToUse.count(std::stoull(FName.data()))) || | |||
879 | (!useMD5() && (FuncsToUse.count(FName) || | |||
880 | (Remapper && Remapper->exist(FName))))) { | |||
881 | if (!CommonContext || !CommonContext->IsPrefixOf(FContext)) | |||
882 | CommonContext = &FContext; | |||
883 | } | |||
884 | ||||
885 | if (CommonContext == &FContext || | |||
886 | (CommonContext && CommonContext->IsPrefixOf(FContext))) { | |||
887 | // Load profile for the current context which originated from | |||
888 | // the common ancestor. | |||
889 | const uint8_t *FuncProfileAddr = Start + NameOffset.second; | |||
890 | assert(FuncProfileAddr < End && "out of LBRProfile section")(static_cast <bool> (FuncProfileAddr < End && "out of LBRProfile section") ? void (0) : __assert_fail ("FuncProfileAddr < End && \"out of LBRProfile section\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 890, __extension__ __PRETTY_FUNCTION__)); | |||
891 | if (std::error_code EC = readFuncProfile(FuncProfileAddr)) | |||
892 | return EC; | |||
893 | } | |||
894 | } | |||
895 | } else { | |||
896 | if (useMD5()) { | |||
897 | for (auto Name : FuncsToUse) { | |||
898 | auto GUID = std::to_string(MD5Hash(Name)); | |||
899 | auto iter = FuncOffsetTable.find(StringRef(GUID)); | |||
900 | if (iter == FuncOffsetTable.end()) | |||
901 | continue; | |||
902 | const uint8_t *FuncProfileAddr = Start + iter->second; | |||
903 | assert(FuncProfileAddr < End && "out of LBRProfile section")(static_cast <bool> (FuncProfileAddr < End && "out of LBRProfile section") ? void (0) : __assert_fail ("FuncProfileAddr < End && \"out of LBRProfile section\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 903, __extension__ __PRETTY_FUNCTION__)); | |||
904 | if (std::error_code EC = readFuncProfile(FuncProfileAddr)) | |||
905 | return EC; | |||
906 | } | |||
907 | } else { | |||
908 | for (auto NameOffset : FuncOffsetTable) { | |||
909 | SampleContext FContext(NameOffset.first); | |||
910 | auto FuncName = FContext.getName(); | |||
911 | if (!FuncsToUse.count(FuncName) && | |||
912 | (!Remapper || !Remapper->exist(FuncName))) | |||
913 | continue; | |||
914 | const uint8_t *FuncProfileAddr = Start + NameOffset.second; | |||
915 | assert(FuncProfileAddr < End && "out of LBRProfile section")(static_cast <bool> (FuncProfileAddr < End && "out of LBRProfile section") ? void (0) : __assert_fail ("FuncProfileAddr < End && \"out of LBRProfile section\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 915, __extension__ __PRETTY_FUNCTION__)); | |||
916 | if (std::error_code EC = readFuncProfile(FuncProfileAddr)) | |||
917 | return EC; | |||
918 | } | |||
919 | } | |||
920 | } | |||
921 | Data = End; | |||
922 | } | |||
923 | assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&(static_cast <bool> ((CSProfileCount == 0 || CSProfileCount == Profiles.size()) && "Cannot have both context-sensitive and regular profile" ) ? void (0) : __assert_fail ("(CSProfileCount == 0 || CSProfileCount == Profiles.size()) && \"Cannot have both context-sensitive and regular profile\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 924, __extension__ __PRETTY_FUNCTION__)) | |||
924 | "Cannot have both context-sensitive and regular profile")(static_cast <bool> ((CSProfileCount == 0 || CSProfileCount == Profiles.size()) && "Cannot have both context-sensitive and regular profile" ) ? void (0) : __assert_fail ("(CSProfileCount == 0 || CSProfileCount == Profiles.size()) && \"Cannot have both context-sensitive and regular profile\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 924, __extension__ __PRETTY_FUNCTION__)); | |||
925 | assert((!CSProfileCount || ProfileIsCS) &&(static_cast <bool> ((!CSProfileCount || ProfileIsCS) && "Section flag should be consistent with actual profile") ? void (0) : __assert_fail ("(!CSProfileCount || ProfileIsCS) && \"Section flag should be consistent with actual profile\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 926, __extension__ __PRETTY_FUNCTION__)) | |||
926 | "Section flag should be consistent with actual profile")(static_cast <bool> ((!CSProfileCount || ProfileIsCS) && "Section flag should be consistent with actual profile") ? void (0) : __assert_fail ("(!CSProfileCount || ProfileIsCS) && \"Section flag should be consistent with actual profile\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 926, __extension__ __PRETTY_FUNCTION__)); | |||
927 | return sampleprof_error::success; | |||
928 | } | |||
929 | ||||
930 | std::error_code SampleProfileReaderExtBinaryBase::readProfileSymbolList() { | |||
931 | if (!ProfSymList) | |||
932 | ProfSymList = std::make_unique<ProfileSymbolList>(); | |||
933 | ||||
934 | if (std::error_code EC = ProfSymList->read(Data, End - Data)) | |||
935 | return EC; | |||
936 | ||||
937 | Data = End; | |||
938 | return sampleprof_error::success; | |||
939 | } | |||
940 | ||||
941 | std::error_code SampleProfileReaderExtBinaryBase::decompressSection( | |||
942 | const uint8_t *SecStart, const uint64_t SecSize, | |||
943 | const uint8_t *&DecompressBuf, uint64_t &DecompressBufSize) { | |||
944 | Data = SecStart; | |||
945 | End = SecStart + SecSize; | |||
946 | auto DecompressSize = readNumber<uint64_t>(); | |||
947 | if (std::error_code EC = DecompressSize.getError()) | |||
948 | return EC; | |||
949 | DecompressBufSize = *DecompressSize; | |||
950 | ||||
951 | auto CompressSize = readNumber<uint64_t>(); | |||
952 | if (std::error_code EC = CompressSize.getError()) | |||
953 | return EC; | |||
954 | ||||
955 | if (!llvm::compression::zlib::isAvailable()) | |||
956 | return sampleprof_error::zlib_unavailable; | |||
957 | ||||
958 | uint8_t *Buffer = Allocator.Allocate<uint8_t>(DecompressBufSize); | |||
959 | size_t UCSize = DecompressBufSize; | |||
960 | llvm::Error E = compression::zlib::decompress(ArrayRef(Data, *CompressSize), | |||
961 | Buffer, UCSize); | |||
962 | if (E) | |||
963 | return sampleprof_error::uncompress_failed; | |||
964 | DecompressBuf = reinterpret_cast<const uint8_t *>(Buffer); | |||
965 | return sampleprof_error::success; | |||
966 | } | |||
967 | ||||
968 | std::error_code SampleProfileReaderExtBinaryBase::readImpl() { | |||
969 | const uint8_t *BufStart = | |||
970 | reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()); | |||
971 | ||||
972 | for (auto &Entry : SecHdrTable) { | |||
973 | // Skip empty section. | |||
974 | if (!Entry.Size) | |||
975 | continue; | |||
976 | ||||
977 | // Skip sections without context when SkipFlatProf is true. | |||
978 | if (SkipFlatProf && hasSecFlag(Entry, SecCommonFlags::SecFlagFlat)) | |||
979 | continue; | |||
980 | ||||
981 | const uint8_t *SecStart = BufStart + Entry.Offset; | |||
982 | uint64_t SecSize = Entry.Size; | |||
983 | ||||
984 | // If the section is compressed, decompress it into a buffer | |||
985 | // DecompressBuf before reading the actual data. The pointee of | |||
986 | // 'Data' will be changed to buffer hold by DecompressBuf | |||
987 | // temporarily when reading the actual data. | |||
988 | bool isCompressed = hasSecFlag(Entry, SecCommonFlags::SecFlagCompress); | |||
989 | if (isCompressed) { | |||
990 | const uint8_t *DecompressBuf; | |||
991 | uint64_t DecompressBufSize; | |||
992 | if (std::error_code EC = decompressSection( | |||
993 | SecStart, SecSize, DecompressBuf, DecompressBufSize)) | |||
994 | return EC; | |||
995 | SecStart = DecompressBuf; | |||
996 | SecSize = DecompressBufSize; | |||
997 | } | |||
998 | ||||
999 | if (std::error_code EC = readOneSection(SecStart, SecSize, Entry)) | |||
1000 | return EC; | |||
1001 | if (Data != SecStart + SecSize) | |||
1002 | return sampleprof_error::malformed; | |||
1003 | ||||
1004 | // Change the pointee of 'Data' from DecompressBuf to original Buffer. | |||
1005 | if (isCompressed) { | |||
1006 | Data = BufStart + Entry.Offset; | |||
1007 | End = BufStart + Buffer->getBufferSize(); | |||
1008 | } | |||
1009 | } | |||
1010 | ||||
1011 | return sampleprof_error::success; | |||
1012 | } | |||
1013 | ||||
1014 | std::error_code SampleProfileReaderCompactBinary::readImpl() { | |||
1015 | // Collect functions used by current module if the Reader has been | |||
1016 | // given a module. | |||
1017 | bool LoadFuncsToBeUsed = collectFuncsFromModule(); | |||
1018 | ProfileIsFS = ProfileIsFSDisciminator; | |||
1019 | FunctionSamples::ProfileIsFS = ProfileIsFS; | |||
1020 | std::vector<uint64_t> OffsetsToUse; | |||
1021 | if (!LoadFuncsToBeUsed) { | |||
1022 | // load all the function profiles. | |||
1023 | for (auto FuncEntry : FuncOffsetTable) { | |||
1024 | OffsetsToUse.push_back(FuncEntry.second); | |||
1025 | } | |||
1026 | } else { | |||
1027 | // load function profiles on demand. | |||
1028 | for (auto Name : FuncsToUse) { | |||
1029 | auto GUID = std::to_string(MD5Hash(Name)); | |||
1030 | auto iter = FuncOffsetTable.find(StringRef(GUID)); | |||
1031 | if (iter == FuncOffsetTable.end()) | |||
1032 | continue; | |||
1033 | OffsetsToUse.push_back(iter->second); | |||
1034 | } | |||
1035 | } | |||
1036 | ||||
1037 | for (auto Offset : OffsetsToUse) { | |||
1038 | const uint8_t *SavedData = Data; | |||
1039 | if (std::error_code EC = readFuncProfile( | |||
1040 | reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()) + | |||
1041 | Offset)) | |||
1042 | return EC; | |||
1043 | Data = SavedData; | |||
1044 | } | |||
1045 | return sampleprof_error::success; | |||
1046 | } | |||
1047 | ||||
1048 | std::error_code SampleProfileReaderRawBinary::verifySPMagic(uint64_t Magic) { | |||
1049 | if (Magic == SPMagic()) | |||
1050 | return sampleprof_error::success; | |||
1051 | return sampleprof_error::bad_magic; | |||
1052 | } | |||
1053 | ||||
1054 | std::error_code SampleProfileReaderExtBinary::verifySPMagic(uint64_t Magic) { | |||
1055 | if (Magic == SPMagic(SPF_Ext_Binary)) | |||
1056 | return sampleprof_error::success; | |||
1057 | return sampleprof_error::bad_magic; | |||
1058 | } | |||
1059 | ||||
1060 | std::error_code | |||
1061 | SampleProfileReaderCompactBinary::verifySPMagic(uint64_t Magic) { | |||
1062 | if (Magic == SPMagic(SPF_Compact_Binary)) | |||
1063 | return sampleprof_error::success; | |||
1064 | return sampleprof_error::bad_magic; | |||
1065 | } | |||
1066 | ||||
1067 | std::error_code SampleProfileReaderBinary::readNameTable() { | |||
1068 | auto Size = readNumber<uint32_t>(); | |||
1069 | if (std::error_code EC = Size.getError()) | |||
1070 | return EC; | |||
1071 | NameTable.reserve(*Size + NameTable.size()); | |||
1072 | for (uint32_t I = 0; I < *Size; ++I) { | |||
1073 | auto Name(readString()); | |||
1074 | if (std::error_code EC = Name.getError()) | |||
1075 | return EC; | |||
1076 | NameTable.push_back(*Name); | |||
1077 | } | |||
1078 | ||||
1079 | return sampleprof_error::success; | |||
1080 | } | |||
1081 | ||||
1082 | std::error_code SampleProfileReaderExtBinaryBase::readMD5NameTable() { | |||
1083 | auto Size = readNumber<uint64_t>(); | |||
1084 | if (std::error_code EC = Size.getError()) | |||
1085 | return EC; | |||
1086 | MD5StringBuf = std::make_unique<std::vector<std::string>>(); | |||
1087 | MD5StringBuf->reserve(*Size); | |||
1088 | if (FixedLengthMD5) { | |||
1089 | // Preallocate and initialize NameTable so we can check whether a name | |||
1090 | // index has been read before by checking whether the element in the | |||
1091 | // NameTable is empty, meanwhile readStringIndex can do the boundary | |||
1092 | // check using the size of NameTable. | |||
1093 | NameTable.resize(*Size + NameTable.size()); | |||
1094 | ||||
1095 | MD5NameMemStart = Data; | |||
1096 | Data = Data + (*Size) * sizeof(uint64_t); | |||
1097 | return sampleprof_error::success; | |||
1098 | } | |||
1099 | NameTable.reserve(*Size); | |||
1100 | for (uint64_t I = 0; I < *Size; ++I) { | |||
1101 | auto FID = readNumber<uint64_t>(); | |||
1102 | if (std::error_code EC = FID.getError()) | |||
1103 | return EC; | |||
1104 | MD5StringBuf->push_back(std::to_string(*FID)); | |||
1105 | // NameTable is a vector of StringRef. Here it is pushing back a | |||
1106 | // StringRef initialized with the last string in MD5stringBuf. | |||
1107 | NameTable.push_back(MD5StringBuf->back()); | |||
1108 | } | |||
1109 | return sampleprof_error::success; | |||
1110 | } | |||
1111 | ||||
1112 | std::error_code SampleProfileReaderExtBinaryBase::readNameTableSec(bool IsMD5) { | |||
1113 | if (IsMD5) | |||
1114 | return readMD5NameTable(); | |||
1115 | return SampleProfileReaderBinary::readNameTable(); | |||
1116 | } | |||
1117 | ||||
1118 | // Read in the CS name table section, which basically contains a list of context | |||
1119 | // vectors. Each element of a context vector, aka a frame, refers to the | |||
1120 | // underlying raw function names that are stored in the name table, as well as | |||
1121 | // a callsite identifier that only makes sense for non-leaf frames. | |||
1122 | std::error_code SampleProfileReaderExtBinaryBase::readCSNameTableSec() { | |||
1123 | auto Size = readNumber<uint32_t>(); | |||
1124 | if (std::error_code EC = Size.getError()) | |||
1125 | return EC; | |||
1126 | ||||
1127 | std::vector<SampleContextFrameVector> *PNameVec = | |||
1128 | new std::vector<SampleContextFrameVector>(); | |||
1129 | PNameVec->reserve(*Size); | |||
1130 | for (uint32_t I = 0; I < *Size; ++I) { | |||
1131 | PNameVec->emplace_back(SampleContextFrameVector()); | |||
1132 | auto ContextSize = readNumber<uint32_t>(); | |||
1133 | if (std::error_code EC = ContextSize.getError()) | |||
1134 | return EC; | |||
1135 | for (uint32_t J = 0; J < *ContextSize; ++J) { | |||
1136 | auto FName(readStringFromTable()); | |||
1137 | if (std::error_code EC = FName.getError()) | |||
1138 | return EC; | |||
1139 | auto LineOffset = readNumber<uint64_t>(); | |||
1140 | if (std::error_code EC = LineOffset.getError()) | |||
1141 | return EC; | |||
1142 | ||||
1143 | if (!isOffsetLegal(*LineOffset)) | |||
1144 | return std::error_code(); | |||
1145 | ||||
1146 | auto Discriminator = readNumber<uint64_t>(); | |||
1147 | if (std::error_code EC = Discriminator.getError()) | |||
1148 | return EC; | |||
1149 | ||||
1150 | PNameVec->back().emplace_back( | |||
1151 | FName.get(), LineLocation(LineOffset.get(), Discriminator.get())); | |||
1152 | } | |||
1153 | } | |||
1154 | ||||
1155 | // From this point the underlying object of CSNameTable should be immutable. | |||
1156 | CSNameTable.reset(PNameVec); | |||
1157 | return sampleprof_error::success; | |||
1158 | } | |||
1159 | ||||
1160 | std::error_code | |||
1161 | ||||
1162 | SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute, | |||
1163 | FunctionSamples *FProfile) { | |||
1164 | if (Data < End) { | |||
1165 | if (ProfileIsProbeBased) { | |||
1166 | auto Checksum = readNumber<uint64_t>(); | |||
1167 | if (std::error_code EC = Checksum.getError()) | |||
1168 | return EC; | |||
1169 | if (FProfile) | |||
1170 | FProfile->setFunctionHash(*Checksum); | |||
1171 | } | |||
1172 | ||||
1173 | if (ProfileHasAttribute) { | |||
1174 | auto Attributes = readNumber<uint32_t>(); | |||
1175 | if (std::error_code EC = Attributes.getError()) | |||
1176 | return EC; | |||
1177 | if (FProfile) | |||
1178 | FProfile->getContext().setAllAttributes(*Attributes); | |||
1179 | } | |||
1180 | ||||
1181 | if (!ProfileIsCS) { | |||
1182 | // Read all the attributes for inlined function calls. | |||
1183 | auto NumCallsites = readNumber<uint32_t>(); | |||
1184 | if (std::error_code EC = NumCallsites.getError()) | |||
1185 | return EC; | |||
1186 | ||||
1187 | for (uint32_t J = 0; J < *NumCallsites; ++J) { | |||
1188 | auto LineOffset = readNumber<uint64_t>(); | |||
1189 | if (std::error_code EC = LineOffset.getError()) | |||
1190 | return EC; | |||
1191 | ||||
1192 | auto Discriminator = readNumber<uint64_t>(); | |||
1193 | if (std::error_code EC = Discriminator.getError()) | |||
1194 | return EC; | |||
1195 | ||||
1196 | auto FContext(readSampleContextFromTable()); | |||
1197 | if (std::error_code EC = FContext.getError()) | |||
1198 | return EC; | |||
1199 | ||||
1200 | FunctionSamples *CalleeProfile = nullptr; | |||
1201 | if (FProfile) { | |||
1202 | CalleeProfile = const_cast<FunctionSamples *>( | |||
1203 | &FProfile->functionSamplesAt(LineLocation( | |||
1204 | *LineOffset, | |||
1205 | *Discriminator))[std::string(FContext.get().getName())]); | |||
1206 | } | |||
1207 | if (std::error_code EC = | |||
1208 | readFuncMetadata(ProfileHasAttribute, CalleeProfile)) | |||
1209 | return EC; | |||
1210 | } | |||
1211 | } | |||
1212 | } | |||
1213 | ||||
1214 | return sampleprof_error::success; | |||
1215 | } | |||
1216 | ||||
1217 | std::error_code | |||
1218 | SampleProfileReaderExtBinaryBase::readFuncMetadata(bool ProfileHasAttribute) { | |||
1219 | while (Data < End) { | |||
1220 | auto FContext(readSampleContextFromTable()); | |||
1221 | if (std::error_code EC = FContext.getError()) | |||
1222 | return EC; | |||
1223 | FunctionSamples *FProfile = nullptr; | |||
1224 | auto It = Profiles.find(*FContext); | |||
1225 | if (It != Profiles.end()) | |||
1226 | FProfile = &It->second; | |||
1227 | ||||
1228 | if (std::error_code EC = readFuncMetadata(ProfileHasAttribute, FProfile)) | |||
1229 | return EC; | |||
1230 | } | |||
1231 | ||||
1232 | assert(Data == End && "More data is read than expected")(static_cast <bool> (Data == End && "More data is read than expected" ) ? void (0) : __assert_fail ("Data == End && \"More data is read than expected\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 1232, __extension__ __PRETTY_FUNCTION__)); | |||
1233 | return sampleprof_error::success; | |||
1234 | } | |||
1235 | ||||
1236 | std::error_code SampleProfileReaderCompactBinary::readNameTable() { | |||
1237 | auto Size = readNumber<uint64_t>(); | |||
1238 | if (std::error_code EC = Size.getError()) | |||
1239 | return EC; | |||
1240 | NameTable.reserve(*Size); | |||
1241 | for (uint64_t I = 0; I < *Size; ++I) { | |||
1242 | auto FID = readNumber<uint64_t>(); | |||
1243 | if (std::error_code EC = FID.getError()) | |||
1244 | return EC; | |||
1245 | NameTable.push_back(std::to_string(*FID)); | |||
1246 | } | |||
1247 | return sampleprof_error::success; | |||
1248 | } | |||
1249 | ||||
1250 | std::error_code | |||
1251 | SampleProfileReaderExtBinaryBase::readSecHdrTableEntry(uint32_t Idx) { | |||
1252 | SecHdrTableEntry Entry; | |||
1253 | auto Type = readUnencodedNumber<uint64_t>(); | |||
1254 | if (std::error_code EC = Type.getError()) | |||
1255 | return EC; | |||
1256 | Entry.Type = static_cast<SecType>(*Type); | |||
1257 | ||||
1258 | auto Flags = readUnencodedNumber<uint64_t>(); | |||
1259 | if (std::error_code EC = Flags.getError()) | |||
1260 | return EC; | |||
1261 | Entry.Flags = *Flags; | |||
1262 | ||||
1263 | auto Offset = readUnencodedNumber<uint64_t>(); | |||
1264 | if (std::error_code EC = Offset.getError()) | |||
1265 | return EC; | |||
1266 | Entry.Offset = *Offset; | |||
1267 | ||||
1268 | auto Size = readUnencodedNumber<uint64_t>(); | |||
1269 | if (std::error_code EC = Size.getError()) | |||
1270 | return EC; | |||
1271 | Entry.Size = *Size; | |||
1272 | ||||
1273 | Entry.LayoutIndex = Idx; | |||
1274 | SecHdrTable.push_back(std::move(Entry)); | |||
1275 | return sampleprof_error::success; | |||
1276 | } | |||
1277 | ||||
1278 | std::error_code SampleProfileReaderExtBinaryBase::readSecHdrTable() { | |||
1279 | auto EntryNum = readUnencodedNumber<uint64_t>(); | |||
1280 | if (std::error_code EC = EntryNum.getError()) | |||
1281 | return EC; | |||
1282 | ||||
1283 | for (uint64_t i = 0; i < (*EntryNum); i++) | |||
1284 | if (std::error_code EC = readSecHdrTableEntry(i)) | |||
1285 | return EC; | |||
1286 | ||||
1287 | return sampleprof_error::success; | |||
1288 | } | |||
1289 | ||||
1290 | std::error_code SampleProfileReaderExtBinaryBase::readHeader() { | |||
1291 | const uint8_t *BufStart = | |||
1292 | reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()); | |||
1293 | Data = BufStart; | |||
1294 | End = BufStart + Buffer->getBufferSize(); | |||
1295 | ||||
1296 | if (std::error_code EC = readMagicIdent()) | |||
1297 | return EC; | |||
1298 | ||||
1299 | if (std::error_code EC = readSecHdrTable()) | |||
1300 | return EC; | |||
1301 | ||||
1302 | return sampleprof_error::success; | |||
1303 | } | |||
1304 | ||||
1305 | uint64_t SampleProfileReaderExtBinaryBase::getSectionSize(SecType Type) { | |||
1306 | uint64_t Size = 0; | |||
1307 | for (auto &Entry : SecHdrTable) { | |||
1308 | if (Entry.Type == Type) | |||
1309 | Size += Entry.Size; | |||
1310 | } | |||
1311 | return Size; | |||
1312 | } | |||
1313 | ||||
1314 | uint64_t SampleProfileReaderExtBinaryBase::getFileSize() { | |||
1315 | // Sections in SecHdrTable is not necessarily in the same order as | |||
1316 | // sections in the profile because section like FuncOffsetTable needs | |||
1317 | // to be written after section LBRProfile but needs to be read before | |||
1318 | // section LBRProfile, so we cannot simply use the last entry in | |||
1319 | // SecHdrTable to calculate the file size. | |||
1320 | uint64_t FileSize = 0; | |||
1321 | for (auto &Entry : SecHdrTable) { | |||
1322 | FileSize = std::max(Entry.Offset + Entry.Size, FileSize); | |||
1323 | } | |||
1324 | return FileSize; | |||
1325 | } | |||
1326 | ||||
1327 | static std::string getSecFlagsStr(const SecHdrTableEntry &Entry) { | |||
1328 | std::string Flags; | |||
1329 | if (hasSecFlag(Entry, SecCommonFlags::SecFlagCompress)) | |||
1330 | Flags.append("{compressed,"); | |||
1331 | else | |||
1332 | Flags.append("{"); | |||
1333 | ||||
1334 | if (hasSecFlag(Entry, SecCommonFlags::SecFlagFlat)) | |||
1335 | Flags.append("flat,"); | |||
1336 | ||||
1337 | switch (Entry.Type) { | |||
1338 | case SecNameTable: | |||
1339 | if (hasSecFlag(Entry, SecNameTableFlags::SecFlagFixedLengthMD5)) | |||
1340 | Flags.append("fixlenmd5,"); | |||
1341 | else if (hasSecFlag(Entry, SecNameTableFlags::SecFlagMD5Name)) | |||
1342 | Flags.append("md5,"); | |||
1343 | if (hasSecFlag(Entry, SecNameTableFlags::SecFlagUniqSuffix)) | |||
1344 | Flags.append("uniq,"); | |||
1345 | break; | |||
1346 | case SecProfSummary: | |||
1347 | if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagPartial)) | |||
1348 | Flags.append("partial,"); | |||
1349 | if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFullContext)) | |||
1350 | Flags.append("context,"); | |||
1351 | if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagIsPreInlined)) | |||
1352 | Flags.append("preInlined,"); | |||
1353 | if (hasSecFlag(Entry, SecProfSummaryFlags::SecFlagFSDiscriminator)) | |||
1354 | Flags.append("fs-discriminator,"); | |||
1355 | break; | |||
1356 | case SecFuncOffsetTable: | |||
1357 | if (hasSecFlag(Entry, SecFuncOffsetFlags::SecFlagOrdered)) | |||
1358 | Flags.append("ordered,"); | |||
1359 | break; | |||
1360 | case SecFuncMetadata: | |||
1361 | if (hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagIsProbeBased)) | |||
1362 | Flags.append("probe,"); | |||
1363 | if (hasSecFlag(Entry, SecFuncMetadataFlags::SecFlagHasAttribute)) | |||
1364 | Flags.append("attr,"); | |||
1365 | break; | |||
1366 | default: | |||
1367 | break; | |||
1368 | } | |||
1369 | char &last = Flags.back(); | |||
1370 | if (last == ',') | |||
1371 | last = '}'; | |||
1372 | else | |||
1373 | Flags.append("}"); | |||
1374 | return Flags; | |||
1375 | } | |||
1376 | ||||
1377 | bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) { | |||
1378 | uint64_t TotalSecsSize = 0; | |||
1379 | for (auto &Entry : SecHdrTable) { | |||
1380 | OS << getSecName(Entry.Type) << " - Offset: " << Entry.Offset | |||
1381 | << ", Size: " << Entry.Size << ", Flags: " << getSecFlagsStr(Entry) | |||
1382 | << "\n"; | |||
1383 | ; | |||
1384 | TotalSecsSize += Entry.Size; | |||
1385 | } | |||
1386 | uint64_t HeaderSize = SecHdrTable.front().Offset; | |||
1387 | assert(HeaderSize + TotalSecsSize == getFileSize() &&(static_cast <bool> (HeaderSize + TotalSecsSize == getFileSize () && "Size of 'header + sections' doesn't match the total size of profile" ) ? void (0) : __assert_fail ("HeaderSize + TotalSecsSize == getFileSize() && \"Size of 'header + sections' doesn't match the total size of profile\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 1388, __extension__ __PRETTY_FUNCTION__)) | |||
1388 | "Size of 'header + sections' doesn't match the total size of profile")(static_cast <bool> (HeaderSize + TotalSecsSize == getFileSize () && "Size of 'header + sections' doesn't match the total size of profile" ) ? void (0) : __assert_fail ("HeaderSize + TotalSecsSize == getFileSize() && \"Size of 'header + sections' doesn't match the total size of profile\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 1388, __extension__ __PRETTY_FUNCTION__)); | |||
1389 | ||||
1390 | OS << "Header Size: " << HeaderSize << "\n"; | |||
1391 | OS << "Total Sections Size: " << TotalSecsSize << "\n"; | |||
1392 | OS << "File Size: " << getFileSize() << "\n"; | |||
1393 | return true; | |||
1394 | } | |||
1395 | ||||
1396 | std::error_code SampleProfileReaderBinary::readMagicIdent() { | |||
1397 | // Read and check the magic identifier. | |||
1398 | auto Magic = readNumber<uint64_t>(); | |||
1399 | if (std::error_code EC = Magic.getError()) | |||
1400 | return EC; | |||
1401 | else if (std::error_code EC = verifySPMagic(*Magic)) | |||
1402 | return EC; | |||
1403 | ||||
1404 | // Read the version number. | |||
1405 | auto Version = readNumber<uint64_t>(); | |||
1406 | if (std::error_code EC = Version.getError()) | |||
1407 | return EC; | |||
1408 | else if (*Version != SPVersion()) | |||
1409 | return sampleprof_error::unsupported_version; | |||
1410 | ||||
1411 | return sampleprof_error::success; | |||
1412 | } | |||
1413 | ||||
1414 | std::error_code SampleProfileReaderBinary::readHeader() { | |||
1415 | Data = reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()); | |||
1416 | End = Data + Buffer->getBufferSize(); | |||
1417 | ||||
1418 | if (std::error_code EC = readMagicIdent()) | |||
1419 | return EC; | |||
1420 | ||||
1421 | if (std::error_code EC = readSummary()) | |||
1422 | return EC; | |||
1423 | ||||
1424 | if (std::error_code EC = readNameTable()) | |||
1425 | return EC; | |||
1426 | return sampleprof_error::success; | |||
1427 | } | |||
1428 | ||||
1429 | std::error_code SampleProfileReaderCompactBinary::readHeader() { | |||
1430 | SampleProfileReaderBinary::readHeader(); | |||
1431 | if (std::error_code EC = readFuncOffsetTable()) | |||
1432 | return EC; | |||
1433 | return sampleprof_error::success; | |||
1434 | } | |||
1435 | ||||
1436 | std::error_code SampleProfileReaderCompactBinary::readFuncOffsetTable() { | |||
1437 | auto TableOffset = readUnencodedNumber<uint64_t>(); | |||
1438 | if (std::error_code EC = TableOffset.getError()) | |||
1439 | return EC; | |||
1440 | ||||
1441 | const uint8_t *SavedData = Data; | |||
1442 | const uint8_t *TableStart = | |||
1443 | reinterpret_cast<const uint8_t *>(Buffer->getBufferStart()) + | |||
1444 | *TableOffset; | |||
1445 | Data = TableStart; | |||
1446 | ||||
1447 | auto Size = readNumber<uint64_t>(); | |||
1448 | if (std::error_code EC = Size.getError()) | |||
1449 | return EC; | |||
1450 | ||||
1451 | FuncOffsetTable.reserve(*Size); | |||
1452 | for (uint64_t I = 0; I < *Size; ++I) { | |||
1453 | auto FName(readStringFromTable()); | |||
1454 | if (std::error_code EC = FName.getError()) | |||
1455 | return EC; | |||
1456 | ||||
1457 | auto Offset = readNumber<uint64_t>(); | |||
1458 | if (std::error_code EC = Offset.getError()) | |||
1459 | return EC; | |||
1460 | ||||
1461 | FuncOffsetTable[*FName] = *Offset; | |||
1462 | } | |||
1463 | End = TableStart; | |||
1464 | Data = SavedData; | |||
1465 | return sampleprof_error::success; | |||
1466 | } | |||
1467 | ||||
1468 | bool SampleProfileReaderCompactBinary::collectFuncsFromModule() { | |||
1469 | if (!M) | |||
1470 | return false; | |||
1471 | FuncsToUse.clear(); | |||
1472 | for (auto &F : *M) | |||
1473 | FuncsToUse.insert(FunctionSamples::getCanonicalFnName(F)); | |||
1474 | return true; | |||
1475 | } | |||
1476 | ||||
1477 | std::error_code SampleProfileReaderBinary::readSummaryEntry( | |||
1478 | std::vector<ProfileSummaryEntry> &Entries) { | |||
1479 | auto Cutoff = readNumber<uint64_t>(); | |||
1480 | if (std::error_code EC = Cutoff.getError()) | |||
1481 | return EC; | |||
1482 | ||||
1483 | auto MinBlockCount = readNumber<uint64_t>(); | |||
1484 | if (std::error_code EC = MinBlockCount.getError()) | |||
1485 | return EC; | |||
1486 | ||||
1487 | auto NumBlocks = readNumber<uint64_t>(); | |||
1488 | if (std::error_code EC = NumBlocks.getError()) | |||
1489 | return EC; | |||
1490 | ||||
1491 | Entries.emplace_back(*Cutoff, *MinBlockCount, *NumBlocks); | |||
1492 | return sampleprof_error::success; | |||
1493 | } | |||
1494 | ||||
1495 | std::error_code SampleProfileReaderBinary::readSummary() { | |||
1496 | auto TotalCount = readNumber<uint64_t>(); | |||
1497 | if (std::error_code EC = TotalCount.getError()) | |||
1498 | return EC; | |||
1499 | ||||
1500 | auto MaxBlockCount = readNumber<uint64_t>(); | |||
1501 | if (std::error_code EC = MaxBlockCount.getError()) | |||
1502 | return EC; | |||
1503 | ||||
1504 | auto MaxFunctionCount = readNumber<uint64_t>(); | |||
1505 | if (std::error_code EC = MaxFunctionCount.getError()) | |||
1506 | return EC; | |||
1507 | ||||
1508 | auto NumBlocks = readNumber<uint64_t>(); | |||
1509 | if (std::error_code EC = NumBlocks.getError()) | |||
1510 | return EC; | |||
1511 | ||||
1512 | auto NumFunctions = readNumber<uint64_t>(); | |||
1513 | if (std::error_code EC = NumFunctions.getError()) | |||
1514 | return EC; | |||
1515 | ||||
1516 | auto NumSummaryEntries = readNumber<uint64_t>(); | |||
1517 | if (std::error_code EC = NumSummaryEntries.getError()) | |||
1518 | return EC; | |||
1519 | ||||
1520 | std::vector<ProfileSummaryEntry> Entries; | |||
1521 | for (unsigned i = 0; i < *NumSummaryEntries; i++) { | |||
1522 | std::error_code EC = readSummaryEntry(Entries); | |||
1523 | if (EC != sampleprof_error::success) | |||
1524 | return EC; | |||
1525 | } | |||
1526 | Summary = std::make_unique<ProfileSummary>( | |||
1527 | ProfileSummary::PSK_Sample, Entries, *TotalCount, *MaxBlockCount, 0, | |||
1528 | *MaxFunctionCount, *NumBlocks, *NumFunctions); | |||
1529 | ||||
1530 | return sampleprof_error::success; | |||
1531 | } | |||
1532 | ||||
1533 | bool SampleProfileReaderRawBinary::hasFormat(const MemoryBuffer &Buffer) { | |||
1534 | const uint8_t *Data = | |||
1535 | reinterpret_cast<const uint8_t *>(Buffer.getBufferStart()); | |||
1536 | uint64_t Magic = decodeULEB128(Data); | |||
1537 | return Magic == SPMagic(); | |||
1538 | } | |||
1539 | ||||
1540 | bool SampleProfileReaderExtBinary::hasFormat(const MemoryBuffer &Buffer) { | |||
1541 | const uint8_t *Data = | |||
1542 | reinterpret_cast<const uint8_t *>(Buffer.getBufferStart()); | |||
1543 | uint64_t Magic = decodeULEB128(Data); | |||
1544 | return Magic == SPMagic(SPF_Ext_Binary); | |||
1545 | } | |||
1546 | ||||
1547 | bool SampleProfileReaderCompactBinary::hasFormat(const MemoryBuffer &Buffer) { | |||
1548 | const uint8_t *Data = | |||
1549 | reinterpret_cast<const uint8_t *>(Buffer.getBufferStart()); | |||
1550 | uint64_t Magic = decodeULEB128(Data); | |||
1551 | return Magic == SPMagic(SPF_Compact_Binary); | |||
1552 | } | |||
1553 | ||||
1554 | std::error_code SampleProfileReaderGCC::skipNextWord() { | |||
1555 | uint32_t dummy; | |||
1556 | if (!GcovBuffer.readInt(dummy)) | |||
1557 | return sampleprof_error::truncated; | |||
1558 | return sampleprof_error::success; | |||
1559 | } | |||
1560 | ||||
1561 | template <typename T> ErrorOr<T> SampleProfileReaderGCC::readNumber() { | |||
1562 | if (sizeof(T) <= sizeof(uint32_t)) { | |||
1563 | uint32_t Val; | |||
1564 | if (GcovBuffer.readInt(Val) && Val <= std::numeric_limits<T>::max()) | |||
1565 | return static_cast<T>(Val); | |||
1566 | } else if (sizeof(T) <= sizeof(uint64_t)) { | |||
1567 | uint64_t Val; | |||
1568 | if (GcovBuffer.readInt64(Val) && Val <= std::numeric_limits<T>::max()) | |||
1569 | return static_cast<T>(Val); | |||
1570 | } | |||
1571 | ||||
1572 | std::error_code EC = sampleprof_error::malformed; | |||
1573 | reportError(0, EC.message()); | |||
1574 | return EC; | |||
1575 | } | |||
1576 | ||||
1577 | ErrorOr<StringRef> SampleProfileReaderGCC::readString() { | |||
1578 | StringRef Str; | |||
1579 | if (!GcovBuffer.readString(Str)) | |||
1580 | return sampleprof_error::truncated; | |||
1581 | return Str; | |||
1582 | } | |||
1583 | ||||
1584 | std::error_code SampleProfileReaderGCC::readHeader() { | |||
1585 | // Read the magic identifier. | |||
1586 | if (!GcovBuffer.readGCDAFormat()) | |||
1587 | return sampleprof_error::unrecognized_format; | |||
1588 | ||||
1589 | // Read the version number. Note - the GCC reader does not validate this | |||
1590 | // version, but the profile creator generates v704. | |||
1591 | GCOV::GCOVVersion version; | |||
1592 | if (!GcovBuffer.readGCOVVersion(version)) | |||
1593 | return sampleprof_error::unrecognized_format; | |||
1594 | ||||
1595 | if (version != GCOV::V407) | |||
1596 | return sampleprof_error::unsupported_version; | |||
1597 | ||||
1598 | // Skip the empty integer. | |||
1599 | if (std::error_code EC = skipNextWord()) | |||
1600 | return EC; | |||
1601 | ||||
1602 | return sampleprof_error::success; | |||
1603 | } | |||
1604 | ||||
1605 | std::error_code SampleProfileReaderGCC::readSectionTag(uint32_t Expected) { | |||
1606 | uint32_t Tag; | |||
1607 | if (!GcovBuffer.readInt(Tag)) | |||
1608 | return sampleprof_error::truncated; | |||
1609 | ||||
1610 | if (Tag != Expected) | |||
1611 | return sampleprof_error::malformed; | |||
1612 | ||||
1613 | if (std::error_code EC = skipNextWord()) | |||
1614 | return EC; | |||
1615 | ||||
1616 | return sampleprof_error::success; | |||
1617 | } | |||
1618 | ||||
1619 | std::error_code SampleProfileReaderGCC::readNameTable() { | |||
1620 | if (std::error_code EC = readSectionTag(GCOVTagAFDOFileNames)) | |||
1621 | return EC; | |||
1622 | ||||
1623 | uint32_t Size; | |||
1624 | if (!GcovBuffer.readInt(Size)) | |||
1625 | return sampleprof_error::truncated; | |||
1626 | ||||
1627 | for (uint32_t I = 0; I < Size; ++I) { | |||
1628 | StringRef Str; | |||
1629 | if (!GcovBuffer.readString(Str)) | |||
1630 | return sampleprof_error::truncated; | |||
1631 | Names.push_back(std::string(Str)); | |||
1632 | } | |||
1633 | ||||
1634 | return sampleprof_error::success; | |||
1635 | } | |||
1636 | ||||
1637 | std::error_code SampleProfileReaderGCC::readFunctionProfiles() { | |||
1638 | if (std::error_code EC = readSectionTag(GCOVTagAFDOFunction)) | |||
1639 | return EC; | |||
1640 | ||||
1641 | uint32_t NumFunctions; | |||
1642 | if (!GcovBuffer.readInt(NumFunctions)) | |||
1643 | return sampleprof_error::truncated; | |||
1644 | ||||
1645 | InlineCallStack Stack; | |||
1646 | for (uint32_t I = 0; I < NumFunctions; ++I) | |||
1647 | if (std::error_code EC = readOneFunctionProfile(Stack, true, 0)) | |||
1648 | return EC; | |||
1649 | ||||
1650 | computeSummary(); | |||
1651 | return sampleprof_error::success; | |||
1652 | } | |||
1653 | ||||
1654 | std::error_code SampleProfileReaderGCC::readOneFunctionProfile( | |||
1655 | const InlineCallStack &InlineStack, bool Update, uint32_t Offset) { | |||
1656 | uint64_t HeadCount = 0; | |||
1657 | if (InlineStack.size() == 0) | |||
1658 | if (!GcovBuffer.readInt64(HeadCount)) | |||
1659 | return sampleprof_error::truncated; | |||
1660 | ||||
1661 | uint32_t NameIdx; | |||
1662 | if (!GcovBuffer.readInt(NameIdx)) | |||
1663 | return sampleprof_error::truncated; | |||
1664 | ||||
1665 | StringRef Name(Names[NameIdx]); | |||
1666 | ||||
1667 | uint32_t NumPosCounts; | |||
1668 | if (!GcovBuffer.readInt(NumPosCounts)) | |||
1669 | return sampleprof_error::truncated; | |||
1670 | ||||
1671 | uint32_t NumCallsites; | |||
1672 | if (!GcovBuffer.readInt(NumCallsites)) | |||
1673 | return sampleprof_error::truncated; | |||
1674 | ||||
1675 | FunctionSamples *FProfile = nullptr; | |||
1676 | if (InlineStack.size() == 0) { | |||
1677 | // If this is a top function that we have already processed, do not | |||
1678 | // update its profile again. This happens in the presence of | |||
1679 | // function aliases. Since these aliases share the same function | |||
1680 | // body, there will be identical replicated profiles for the | |||
1681 | // original function. In this case, we simply not bother updating | |||
1682 | // the profile of the original function. | |||
1683 | FProfile = &Profiles[Name]; | |||
1684 | FProfile->addHeadSamples(HeadCount); | |||
1685 | if (FProfile->getTotalSamples() > 0) | |||
1686 | Update = false; | |||
1687 | } else { | |||
1688 | // Otherwise, we are reading an inlined instance. The top of the | |||
1689 | // inline stack contains the profile of the caller. Insert this | |||
1690 | // callee in the caller's CallsiteMap. | |||
1691 | FunctionSamples *CallerProfile = InlineStack.front(); | |||
1692 | uint32_t LineOffset = Offset >> 16; | |||
1693 | uint32_t Discriminator = Offset & 0xffff; | |||
1694 | FProfile = &CallerProfile->functionSamplesAt( | |||
1695 | LineLocation(LineOffset, Discriminator))[std::string(Name)]; | |||
1696 | } | |||
1697 | FProfile->setName(Name); | |||
1698 | ||||
1699 | for (uint32_t I = 0; I < NumPosCounts; ++I) { | |||
1700 | uint32_t Offset; | |||
1701 | if (!GcovBuffer.readInt(Offset)) | |||
1702 | return sampleprof_error::truncated; | |||
1703 | ||||
1704 | uint32_t NumTargets; | |||
1705 | if (!GcovBuffer.readInt(NumTargets)) | |||
1706 | return sampleprof_error::truncated; | |||
1707 | ||||
1708 | uint64_t Count; | |||
1709 | if (!GcovBuffer.readInt64(Count)) | |||
1710 | return sampleprof_error::truncated; | |||
1711 | ||||
1712 | // The line location is encoded in the offset as: | |||
1713 | // high 16 bits: line offset to the start of the function. | |||
1714 | // low 16 bits: discriminator. | |||
1715 | uint32_t LineOffset = Offset >> 16; | |||
1716 | uint32_t Discriminator = Offset & 0xffff; | |||
1717 | ||||
1718 | InlineCallStack NewStack; | |||
1719 | NewStack.push_back(FProfile); | |||
1720 | llvm::append_range(NewStack, InlineStack); | |||
1721 | if (Update) { | |||
1722 | // Walk up the inline stack, adding the samples on this line to | |||
1723 | // the total sample count of the callers in the chain. | |||
1724 | for (auto *CallerProfile : NewStack) | |||
1725 | CallerProfile->addTotalSamples(Count); | |||
1726 | ||||
1727 | // Update the body samples for the current profile. | |||
1728 | FProfile->addBodySamples(LineOffset, Discriminator, Count); | |||
1729 | } | |||
1730 | ||||
1731 | // Process the list of functions called at an indirect call site. | |||
1732 | // These are all the targets that a function pointer (or virtual | |||
1733 | // function) resolved at runtime. | |||
1734 | for (uint32_t J = 0; J < NumTargets; J++) { | |||
1735 | uint32_t HistVal; | |||
1736 | if (!GcovBuffer.readInt(HistVal)) | |||
1737 | return sampleprof_error::truncated; | |||
1738 | ||||
1739 | if (HistVal != HIST_TYPE_INDIR_CALL_TOPN) | |||
1740 | return sampleprof_error::malformed; | |||
1741 | ||||
1742 | uint64_t TargetIdx; | |||
1743 | if (!GcovBuffer.readInt64(TargetIdx)) | |||
1744 | return sampleprof_error::truncated; | |||
1745 | StringRef TargetName(Names[TargetIdx]); | |||
1746 | ||||
1747 | uint64_t TargetCount; | |||
1748 | if (!GcovBuffer.readInt64(TargetCount)) | |||
1749 | return sampleprof_error::truncated; | |||
1750 | ||||
1751 | if (Update) | |||
1752 | FProfile->addCalledTargetSamples(LineOffset, Discriminator, | |||
1753 | TargetName, TargetCount); | |||
1754 | } | |||
1755 | } | |||
1756 | ||||
1757 | // Process all the inlined callers into the current function. These | |||
1758 | // are all the callsites that were inlined into this function. | |||
1759 | for (uint32_t I = 0; I < NumCallsites; I++) { | |||
1760 | // The offset is encoded as: | |||
1761 | // high 16 bits: line offset to the start of the function. | |||
1762 | // low 16 bits: discriminator. | |||
1763 | uint32_t Offset; | |||
1764 | if (!GcovBuffer.readInt(Offset)) | |||
1765 | return sampleprof_error::truncated; | |||
1766 | InlineCallStack NewStack; | |||
1767 | NewStack.push_back(FProfile); | |||
1768 | llvm::append_range(NewStack, InlineStack); | |||
1769 | if (std::error_code EC = readOneFunctionProfile(NewStack, Update, Offset)) | |||
1770 | return EC; | |||
1771 | } | |||
1772 | ||||
1773 | return sampleprof_error::success; | |||
1774 | } | |||
1775 | ||||
1776 | /// Read a GCC AutoFDO profile. | |||
1777 | /// | |||
1778 | /// This format is generated by the Linux Perf conversion tool at | |||
1779 | /// https://github.com/google/autofdo. | |||
1780 | std::error_code SampleProfileReaderGCC::readImpl() { | |||
1781 | assert(!ProfileIsFSDisciminator && "Gcc profiles not support FSDisciminator")(static_cast <bool> (!ProfileIsFSDisciminator && "Gcc profiles not support FSDisciminator") ? void (0) : __assert_fail ("!ProfileIsFSDisciminator && \"Gcc profiles not support FSDisciminator\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 1781, __extension__ __PRETTY_FUNCTION__)); | |||
1782 | // Read the string table. | |||
1783 | if (std::error_code EC = readNameTable()) | |||
1784 | return EC; | |||
1785 | ||||
1786 | // Read the source profile. | |||
1787 | if (std::error_code EC = readFunctionProfiles()) | |||
1788 | return EC; | |||
1789 | ||||
1790 | return sampleprof_error::success; | |||
1791 | } | |||
1792 | ||||
1793 | bool SampleProfileReaderGCC::hasFormat(const MemoryBuffer &Buffer) { | |||
1794 | StringRef Magic(reinterpret_cast<const char *>(Buffer.getBufferStart())); | |||
1795 | return Magic == "adcg*704"; | |||
1796 | } | |||
1797 | ||||
1798 | void SampleProfileReaderItaniumRemapper::applyRemapping(LLVMContext &Ctx) { | |||
1799 | // If the reader uses MD5 to represent string, we can't remap it because | |||
1800 | // we don't know what the original function names were. | |||
1801 | if (Reader.useMD5()) { | |||
1802 | Ctx.diagnose(DiagnosticInfoSampleProfile( | |||
1803 | Reader.getBuffer()->getBufferIdentifier(), | |||
1804 | "Profile data remapping cannot be applied to profile data " | |||
1805 | "in compact format (original mangled names are not available).", | |||
1806 | DS_Warning)); | |||
1807 | return; | |||
1808 | } | |||
1809 | ||||
1810 | // CSSPGO-TODO: Remapper is not yet supported. | |||
1811 | // We will need to remap the entire context string. | |||
1812 | assert(Remappings && "should be initialized while creating remapper")(static_cast <bool> (Remappings && "should be initialized while creating remapper" ) ? void (0) : __assert_fail ("Remappings && \"should be initialized while creating remapper\"" , "llvm/lib/ProfileData/SampleProfReader.cpp", 1812, __extension__ __PRETTY_FUNCTION__)); | |||
1813 | for (auto &Sample : Reader.getProfiles()) { | |||
1814 | DenseSet<StringRef> NamesInSample; | |||
1815 | Sample.second.findAllNames(NamesInSample); | |||
1816 | for (auto &Name : NamesInSample) | |||
1817 | if (auto Key = Remappings->insert(Name)) | |||
1818 | NameMap.insert({Key, Name}); | |||
1819 | } | |||
1820 | ||||
1821 | RemappingApplied = true; | |||
1822 | } | |||
1823 | ||||
1824 | std::optional<StringRef> | |||
1825 | SampleProfileReaderItaniumRemapper::lookUpNameInProfile(StringRef Fname) { | |||
1826 | if (auto Key = Remappings->lookup(Fname)) | |||
1827 | return NameMap.lookup(Key); | |||
1828 | return std::nullopt; | |||
1829 | } | |||
1830 | ||||
1831 | /// Prepare a memory buffer for the contents of \p Filename. | |||
1832 | /// | |||
1833 | /// \returns an error code indicating the status of the buffer. | |||
1834 | static ErrorOr<std::unique_ptr<MemoryBuffer>> | |||
1835 | setupMemoryBuffer(const Twine &Filename, vfs::FileSystem &FS) { | |||
1836 | auto BufferOrErr = Filename.str() == "-" ? MemoryBuffer::getSTDIN() | |||
1837 | : FS.getBufferForFile(Filename); | |||
1838 | if (std::error_code EC = BufferOrErr.getError()) | |||
1839 | return EC; | |||
1840 | auto Buffer = std::move(BufferOrErr.get()); | |||
1841 | ||||
1842 | return std::move(Buffer); | |||
1843 | } | |||
1844 | ||||
1845 | /// Create a sample profile reader based on the format of the input file. | |||
1846 | /// | |||
1847 | /// \param Filename The file to open. | |||
1848 | /// | |||
1849 | /// \param C The LLVM context to use to emit diagnostics. | |||
1850 | /// | |||
1851 | /// \param P The FSDiscriminatorPass. | |||
1852 | /// | |||
1853 | /// \param RemapFilename The file used for profile remapping. | |||
1854 | /// | |||
1855 | /// \returns an error code indicating the status of the created reader. | |||
1856 | ErrorOr<std::unique_ptr<SampleProfileReader>> | |||
1857 | SampleProfileReader::create(const std::string Filename, LLVMContext &C, | |||
1858 | vfs::FileSystem &FS, FSDiscriminatorPass P, | |||
1859 | const std::string RemapFilename) { | |||
1860 | auto BufferOrError = setupMemoryBuffer(Filename, FS); | |||
1861 | if (std::error_code EC = BufferOrError.getError()) | |||
1862 | return EC; | |||
1863 | return create(BufferOrError.get(), C, FS, P, RemapFilename); | |||
1864 | } | |||
1865 | ||||
1866 | /// Create a sample profile remapper from the given input, to remap the | |||
1867 | /// function names in the given profile data. | |||
1868 | /// | |||
1869 | /// \param Filename The file to open. | |||
1870 | /// | |||
1871 | /// \param Reader The profile reader the remapper is going to be applied to. | |||
1872 | /// | |||
1873 | /// \param C The LLVM context to use to emit diagnostics. | |||
1874 | /// | |||
1875 | /// \returns an error code indicating the status of the created reader. | |||
1876 | ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>> | |||
1877 | SampleProfileReaderItaniumRemapper::create(const std::string Filename, | |||
1878 | vfs::FileSystem &FS, | |||
1879 | SampleProfileReader &Reader, | |||
1880 | LLVMContext &C) { | |||
1881 | auto BufferOrError = setupMemoryBuffer(Filename, FS); | |||
1882 | if (std::error_code EC = BufferOrError.getError()) | |||
1883 | return EC; | |||
1884 | return create(BufferOrError.get(), Reader, C); | |||
1885 | } | |||
1886 | ||||
1887 | /// Create a sample profile remapper from the given input, to remap the | |||
1888 | /// function names in the given profile data. | |||
1889 | /// | |||
1890 | /// \param B The memory buffer to create the reader from (assumes ownership). | |||
1891 | /// | |||
1892 | /// \param C The LLVM context to use to emit diagnostics. | |||
1893 | /// | |||
1894 | /// \param Reader The profile reader the remapper is going to be applied to. | |||
1895 | /// | |||
1896 | /// \returns an error code indicating the status of the created reader. | |||
1897 | ErrorOr<std::unique_ptr<SampleProfileReaderItaniumRemapper>> | |||
1898 | SampleProfileReaderItaniumRemapper::create(std::unique_ptr<MemoryBuffer> &B, | |||
1899 | SampleProfileReader &Reader, | |||
1900 | LLVMContext &C) { | |||
1901 | auto Remappings = std::make_unique<SymbolRemappingReader>(); | |||
1902 | if (Error E = Remappings->read(*B)) { | |||
1903 | handleAllErrors( | |||
1904 | std::move(E), [&](const SymbolRemappingParseError &ParseError) { | |||
1905 | C.diagnose(DiagnosticInfoSampleProfile(B->getBufferIdentifier(), | |||
1906 | ParseError.getLineNum(), | |||
1907 | ParseError.getMessage())); | |||
1908 | }); | |||
1909 | return sampleprof_error::malformed; | |||
1910 | } | |||
1911 | ||||
1912 | return std::make_unique<SampleProfileReaderItaniumRemapper>( | |||
1913 | std::move(B), std::move(Remappings), Reader); | |||
1914 | } | |||
1915 | ||||
1916 | /// Create a sample profile reader based on the format of the input data. | |||
1917 | /// | |||
1918 | /// \param B The memory buffer to create the reader from (assumes ownership). | |||
1919 | /// | |||
1920 | /// \param C The LLVM context to use to emit diagnostics. | |||
1921 | /// | |||
1922 | /// \param P The FSDiscriminatorPass. | |||
1923 | /// | |||
1924 | /// \param RemapFilename The file used for profile remapping. | |||
1925 | /// | |||
1926 | /// \returns an error code indicating the status of the created reader. | |||
1927 | ErrorOr<std::unique_ptr<SampleProfileReader>> | |||
1928 | SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C, | |||
1929 | vfs::FileSystem &FS, FSDiscriminatorPass P, | |||
1930 | const std::string RemapFilename) { | |||
1931 | std::unique_ptr<SampleProfileReader> Reader; | |||
1932 | if (SampleProfileReaderRawBinary::hasFormat(*B)) | |||
1933 | Reader.reset(new SampleProfileReaderRawBinary(std::move(B), C)); | |||
1934 | else if (SampleProfileReaderExtBinary::hasFormat(*B)) | |||
1935 | Reader.reset(new SampleProfileReaderExtBinary(std::move(B), C)); | |||
1936 | else if (SampleProfileReaderCompactBinary::hasFormat(*B)) | |||
1937 | Reader.reset(new SampleProfileReaderCompactBinary(std::move(B), C)); | |||
1938 | else if (SampleProfileReaderGCC::hasFormat(*B)) | |||
1939 | Reader.reset(new SampleProfileReaderGCC(std::move(B), C)); | |||
1940 | else if (SampleProfileReaderText::hasFormat(*B)) | |||
1941 | Reader.reset(new SampleProfileReaderText(std::move(B), C)); | |||
1942 | else | |||
1943 | return sampleprof_error::unrecognized_format; | |||
1944 | ||||
1945 | if (!RemapFilename.empty()) { | |||
1946 | auto ReaderOrErr = SampleProfileReaderItaniumRemapper::create( | |||
1947 | RemapFilename, FS, *Reader, C); | |||
1948 | if (std::error_code EC = ReaderOrErr.getError()) { | |||
1949 | std::string Msg = "Could not create remapper: " + EC.message(); | |||
1950 | C.diagnose(DiagnosticInfoSampleProfile(RemapFilename, Msg)); | |||
1951 | return EC; | |||
1952 | } | |||
1953 | Reader->Remapper = std::move(ReaderOrErr.get()); | |||
1954 | } | |||
1955 | ||||
1956 | if (std::error_code EC = Reader->readHeader()) { | |||
1957 | return EC; | |||
1958 | } | |||
1959 | ||||
1960 | Reader->setDiscriminatorMaskedBitFrom(P); | |||
1961 | ||||
1962 | return std::move(Reader); | |||
1963 | } | |||
1964 | ||||
1965 | // For text and GCC file formats, we compute the summary after reading the | |||
1966 | // profile. Binary format has the profile summary in its header. | |||
1967 | void SampleProfileReader::computeSummary() { | |||
1968 | SampleProfileSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs); | |||
1969 | Summary = Builder.computeSummaryForProfiles(Profiles); | |||
1970 | } |