LLVM 19.0.0git
SampleProfWriter.cpp
Go to the documentation of this file.
1//===- SampleProfWriter.cpp - Write LLVM sample profile data --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class that writes LLVM sample profiles. It
10// supports two file formats: text and binary. The textual representation
11// is useful for debugging and testing purposes. The binary representation
12// is more compact, resulting in smaller file sizes. However, they can
13// both be used interchangeably.
14//
15// See lib/ProfileData/SampleProfReader.cpp for documentation on each of the
16// supported formats.
17//
18//===----------------------------------------------------------------------===//
19
21#include "llvm/ADT/StringRef.h"
25#include "llvm/Support/Endian.h"
29#include "llvm/Support/LEB128.h"
30#include "llvm/Support/MD5.h"
32#include <algorithm>
33#include <cmath>
34#include <cstdint>
35#include <memory>
36#include <set>
37#include <system_error>
38#include <utility>
39#include <vector>
40
41#define DEBUG_TYPE "llvm-profdata"
42
43using namespace llvm;
44using namespace sampleprof;
45
46namespace llvm {
47namespace support {
48namespace endian {
49namespace {
50
51// Adapter class to llvm::support::endian::Writer for pwrite().
52struct SeekableWriter {
55 SeekableWriter(raw_pwrite_stream &OS, endianness Endian)
56 : OS(OS), Endian(Endian) {}
57
58 template <typename ValueType>
59 void pwrite(ValueType Val, size_t Offset) {
60 std::string StringBuf;
61 raw_string_ostream SStream(StringBuf);
62 Writer(SStream, Endian).write(Val);
63 OS.pwrite(StringBuf.data(), StringBuf.size(), Offset);
64 }
65};
66
67} // namespace
68} // namespace endian
69} // namespace support
70} // namespace llvm
71
73 SampleProfileMap &ProfileMap, size_t OutputSizeLimit)
74 : FunctionPruningStrategy(ProfileMap, OutputSizeLimit) {
75 sortFuncProfiles(ProfileMap, SortedFunctions);
76}
77
78void DefaultFunctionPruningStrategy::Erase(size_t CurrentOutputSize) {
79 double D = (double)OutputSizeLimit / CurrentOutputSize;
80 size_t NewSize = (size_t)round(ProfileMap.size() * D * D);
81 size_t NumToRemove = ProfileMap.size() - NewSize;
82 if (NumToRemove < 1)
83 NumToRemove = 1;
84
85 assert(NumToRemove <= SortedFunctions.size());
86 for (const NameFunctionSamples &E :
87 llvm::drop_begin(SortedFunctions, SortedFunctions.size() - NumToRemove))
88 ProfileMap.erase(E.first);
89 SortedFunctions.resize(SortedFunctions.size() - NumToRemove);
90}
91
93 SampleProfileMap &ProfileMap, size_t OutputSizeLimit,
94 FunctionPruningStrategy *Strategy) {
95 if (OutputSizeLimit == 0)
96 return write(ProfileMap);
97
98 size_t OriginalFunctionCount = ProfileMap.size();
99
100 std::unique_ptr<raw_ostream> OriginalOutputStream;
101 OutputStream.swap(OriginalOutputStream);
102
103 size_t IterationCount = 0;
104 size_t TotalSize;
105
106 SmallVector<char> StringBuffer;
107 do {
108 StringBuffer.clear();
109 OutputStream.reset(new raw_svector_ostream(StringBuffer));
110 if (std::error_code EC = write(ProfileMap))
111 return EC;
112
113 TotalSize = StringBuffer.size();
114 // On Windows every "\n" is actually written as "\r\n" to disk but not to
115 // memory buffer, this difference should be added when considering the total
116 // output size.
117#ifdef _WIN32
118 if (Format == SPF_Text)
119 TotalSize += LineCount;
120#endif
121 if (TotalSize <= OutputSizeLimit)
122 break;
123
124 Strategy->Erase(TotalSize);
125 IterationCount++;
126 } while (ProfileMap.size() != 0);
127
128 if (ProfileMap.size() == 0)
130
131 OutputStream.swap(OriginalOutputStream);
132 OutputStream->write(StringBuffer.data(), StringBuffer.size());
133 LLVM_DEBUG(dbgs() << "Profile originally has " << OriginalFunctionCount
134 << " functions, reduced to " << ProfileMap.size() << " in "
135 << IterationCount << " iterations\n");
136 // Silence warning on Release build.
137 (void)OriginalFunctionCount;
138 (void)IterationCount;
140}
141
142std::error_code
144 std::vector<NameFunctionSamples> V;
145 sortFuncProfiles(ProfileMap, V);
146 for (const auto &I : V) {
147 if (std::error_code EC = writeSample(*I.second))
148 return EC;
149 }
151}
152
153std::error_code SampleProfileWriter::write(const SampleProfileMap &ProfileMap) {
154 if (std::error_code EC = writeHeader(ProfileMap))
155 return EC;
156
157 if (std::error_code EC = writeFuncProfiles(ProfileMap))
158 return EC;
159
161}
162
163/// Return the current position and prepare to use it as the start
164/// position of a section given the section type \p Type and its position
165/// \p LayoutIdx in SectionHdrLayout.
168 uint32_t LayoutIdx) {
169 uint64_t SectionStart = OutputStream->tell();
170 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
171 const auto &Entry = SectionHdrLayout[LayoutIdx];
172 assert(Entry.Type == Type && "Unexpected section type");
173 // Use LocalBuf as a temporary output for writting data.
175 LocalBufStream.swap(OutputStream);
176 return SectionStart;
177}
178
179std::error_code SampleProfileWriterExtBinaryBase::compressAndOutput() {
182 std::string &UncompressedStrings =
183 static_cast<raw_string_ostream *>(LocalBufStream.get())->str();
184 if (UncompressedStrings.size() == 0)
186 auto &OS = *OutputStream;
187 SmallVector<uint8_t, 128> CompressedStrings;
188 compression::zlib::compress(arrayRefFromStringRef(UncompressedStrings),
189 CompressedStrings,
191 encodeULEB128(UncompressedStrings.size(), OS);
192 encodeULEB128(CompressedStrings.size(), OS);
193 OS << toStringRef(CompressedStrings);
194 UncompressedStrings.clear();
196}
197
198/// Add a new section into section header table given the section type
199/// \p Type, its position \p LayoutIdx in SectionHdrLayout and the
200/// location \p SectionStart where the section should be written to.
202 SecType Type, uint32_t LayoutIdx, uint64_t SectionStart) {
203 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
204 const auto &Entry = SectionHdrLayout[LayoutIdx];
205 assert(Entry.Type == Type && "Unexpected section type");
207 LocalBufStream.swap(OutputStream);
208 if (std::error_code EC = compressAndOutput())
209 return EC;
210 }
211 SecHdrTable.push_back({Type, Entry.Flags, SectionStart - FileStart,
212 OutputStream->tell() - SectionStart, LayoutIdx});
214}
215
216std::error_code
218 // When calling write on a different profile map, existing states should be
219 // cleared.
220 NameTable.clear();
221 CSNameTable.clear();
222 SecHdrTable.clear();
223
224 if (std::error_code EC = writeHeader(ProfileMap))
225 return EC;
226
227 std::string LocalBuf;
228 LocalBufStream = std::make_unique<raw_string_ostream>(LocalBuf);
229 if (std::error_code EC = writeSections(ProfileMap))
230 return EC;
231
232 if (std::error_code EC = writeSecHdrTable())
233 return EC;
234
236}
237
239 const SampleContext &Context) {
240 if (Context.hasContext())
241 return writeCSNameIdx(Context);
242 else
244}
245
246std::error_code
248 const auto &Ret = CSNameTable.find(Context);
249 if (Ret == CSNameTable.end())
251 encodeULEB128(Ret->second, *OutputStream);
253}
254
255std::error_code
257 uint64_t Offset = OutputStream->tell();
258 auto &Context = S.getContext();
259 FuncOffsetTable[Context] = Offset - SecLBRProfileStart;
261 return writeBody(S);
262}
263
265 auto &OS = *OutputStream;
266
267 // Write out the table size.
268 encodeULEB128(FuncOffsetTable.size(), OS);
269
270 // Write out FuncOffsetTable.
271 auto WriteItem = [&](const SampleContext &Context, uint64_t Offset) {
272 if (std::error_code EC = writeContextIdx(Context))
273 return EC;
275 return (std::error_code)sampleprof_error::success;
276 };
277
279 // Sort the contexts before writing them out. This is to help fast load all
280 // context profiles for a function as well as their callee contexts which
281 // can help profile-guided importing for ThinLTO.
282 std::map<SampleContext, uint64_t> OrderedFuncOffsetTable(
283 FuncOffsetTable.begin(), FuncOffsetTable.end());
284 for (const auto &Entry : OrderedFuncOffsetTable) {
285 if (std::error_code EC = WriteItem(Entry.first, Entry.second))
286 return EC;
287 }
289 } else {
290 for (const auto &Entry : FuncOffsetTable) {
291 if (std::error_code EC = WriteItem(Entry.first, Entry.second))
292 return EC;
293 }
294 }
295
296 FuncOffsetTable.clear();
298}
299
301 const FunctionSamples &FunctionProfile) {
302 auto &OS = *OutputStream;
303 if (std::error_code EC = writeContextIdx(FunctionProfile.getContext()))
304 return EC;
305
307 encodeULEB128(FunctionProfile.getFunctionHash(), OS);
309 encodeULEB128(FunctionProfile.getContext().getAllAttributes(), OS);
310 }
311
313 // Recursively emit attributes for all callee samples.
314 uint64_t NumCallsites = 0;
315 for (const auto &J : FunctionProfile.getCallsiteSamples())
316 NumCallsites += J.second.size();
317 encodeULEB128(NumCallsites, OS);
318 for (const auto &J : FunctionProfile.getCallsiteSamples()) {
319 for (const auto &FS : J.second) {
320 LineLocation Loc = J.first;
323 if (std::error_code EC = writeFuncMetadata(FS.second))
324 return EC;
325 }
326 }
327 }
328
330}
331
333 const SampleProfileMap &Profiles) {
337 for (const auto &Entry : Profiles) {
338 if (std::error_code EC = writeFuncMetadata(Entry.second))
339 return EC;
340 }
342}
343
345 if (!UseMD5)
347
348 auto &OS = *OutputStream;
349 std::set<FunctionId> V;
351
352 // Write out the MD5 name table. We wrote unencoded MD5 so reader can
353 // retrieve the name using the name index without having to read the
354 // whole name table.
355 encodeULEB128(NameTable.size(), OS);
357 for (auto N : V)
358 Writer.write(N.getHashCode());
360}
361
363 const SampleProfileMap &ProfileMap) {
364 for (const auto &I : ProfileMap) {
365 addContext(I.second.getContext());
366 addNames(I.second);
367 }
368
369 // If NameTable contains ".__uniq." suffix, set SecFlagUniqSuffix flag
370 // so compiler won't strip the suffix during profile matching after
371 // seeing the flag in the profile.
372 // Original names are unavailable if using MD5, so this option has no use.
373 if (!UseMD5) {
374 for (const auto &I : NameTable) {
375 if (I.first.stringRef().contains(FunctionSamples::UniqSuffix)) {
377 break;
378 }
379 }
380 }
381
382 if (auto EC = writeNameTable())
383 return EC;
385}
386
388 // Sort the names to make CSNameTable deterministic.
389 std::set<SampleContext> OrderedContexts;
390 for (const auto &I : CSNameTable)
391 OrderedContexts.insert(I.first);
392 assert(OrderedContexts.size() == CSNameTable.size() &&
393 "Unmatched ordered and unordered contexts");
394 uint64_t I = 0;
395 for (auto &Context : OrderedContexts)
396 CSNameTable[Context] = I++;
397
398 auto &OS = *OutputStream;
399 encodeULEB128(OrderedContexts.size(), OS);
401 for (auto Context : OrderedContexts) {
402 auto Frames = Context.getContextFrames();
403 encodeULEB128(Frames.size(), OS);
404 for (auto &Callsite : Frames) {
405 if (std::error_code EC = writeNameIdx(Callsite.Func))
406 return EC;
407 encodeULEB128(Callsite.Location.LineOffset, OS);
408 encodeULEB128(Callsite.Location.Discriminator, OS);
409 }
410 }
411
413}
414
415std::error_code
417 if (ProfSymList && ProfSymList->size() > 0)
418 if (std::error_code EC = ProfSymList->write(*OutputStream))
419 return EC;
420
422}
423
425 SecType Type, uint32_t LayoutIdx, const SampleProfileMap &ProfileMap) {
426 // The setting of SecFlagCompress should happen before markSectionStart.
427 if (Type == SecProfileSymbolList && ProfSymList && ProfSymList->toCompress())
431 if (Type == SecFuncMetadata &&
440
441 uint64_t SectionStart = markSectionStart(Type, LayoutIdx);
442 switch (Type) {
443 case SecProfSummary:
444 computeSummary(ProfileMap);
445 if (auto EC = writeSummary())
446 return EC;
447 break;
448 case SecNameTable:
449 if (auto EC = writeNameTableSection(ProfileMap))
450 return EC;
451 break;
452 case SecCSNameTable:
453 if (auto EC = writeCSNameTableSection())
454 return EC;
455 break;
456 case SecLBRProfile:
458 if (std::error_code EC = writeFuncProfiles(ProfileMap))
459 return EC;
460 break;
462 if (auto EC = writeFuncOffsetTable())
463 return EC;
464 break;
465 case SecFuncMetadata:
466 if (std::error_code EC = writeFuncMetadata(ProfileMap))
467 return EC;
468 break;
470 if (auto EC = writeProfileSymbolListSection())
471 return EC;
472 break;
473 default:
474 if (auto EC = writeCustomSection(Type))
475 return EC;
476 break;
477 }
478 if (std::error_code EC = addNewSection(Type, LayoutIdx, SectionStart))
479 return EC;
481}
482
483std::error_code SampleProfileWriterExtBinary::writeDefaultLayout(
484 const SampleProfileMap &ProfileMap) {
485 // The const indices passed to writeOneSection below are specifying the
486 // positions of the sections in SectionHdrLayout. Look at
487 // initSectionHdrLayout to find out where each section is located in
488 // SectionHdrLayout.
489 if (auto EC = writeOneSection(SecProfSummary, 0, ProfileMap))
490 return EC;
491 if (auto EC = writeOneSection(SecNameTable, 1, ProfileMap))
492 return EC;
493 if (auto EC = writeOneSection(SecCSNameTable, 2, ProfileMap))
494 return EC;
495 if (auto EC = writeOneSection(SecLBRProfile, 4, ProfileMap))
496 return EC;
497 if (auto EC = writeOneSection(SecProfileSymbolList, 5, ProfileMap))
498 return EC;
499 if (auto EC = writeOneSection(SecFuncOffsetTable, 3, ProfileMap))
500 return EC;
501 if (auto EC = writeOneSection(SecFuncMetadata, 6, ProfileMap))
502 return EC;
504}
505
506static void splitProfileMapToTwo(const SampleProfileMap &ProfileMap,
507 SampleProfileMap &ContextProfileMap,
508 SampleProfileMap &NoContextProfileMap) {
509 for (const auto &I : ProfileMap) {
510 if (I.second.getCallsiteSamples().size())
511 ContextProfileMap.insert({I.first, I.second});
512 else
513 NoContextProfileMap.insert({I.first, I.second});
514 }
515}
516
517std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout(
518 const SampleProfileMap &ProfileMap) {
519 SampleProfileMap ContextProfileMap, NoContextProfileMap;
520 splitProfileMapToTwo(ProfileMap, ContextProfileMap, NoContextProfileMap);
521
522 if (auto EC = writeOneSection(SecProfSummary, 0, ProfileMap))
523 return EC;
524 if (auto EC = writeOneSection(SecNameTable, 1, ProfileMap))
525 return EC;
526 if (auto EC = writeOneSection(SecLBRProfile, 3, ContextProfileMap))
527 return EC;
528 if (auto EC = writeOneSection(SecFuncOffsetTable, 2, ContextProfileMap))
529 return EC;
530 // Mark the section to have no context. Note section flag needs to be set
531 // before writing the section.
533 if (auto EC = writeOneSection(SecLBRProfile, 5, NoContextProfileMap))
534 return EC;
535 // Mark the section to have no context. Note section flag needs to be set
536 // before writing the section.
538 if (auto EC = writeOneSection(SecFuncOffsetTable, 4, NoContextProfileMap))
539 return EC;
540 if (auto EC = writeOneSection(SecProfileSymbolList, 6, ProfileMap))
541 return EC;
542 if (auto EC = writeOneSection(SecFuncMetadata, 7, ProfileMap))
543 return EC;
544
546}
547
548std::error_code SampleProfileWriterExtBinary::writeSections(
549 const SampleProfileMap &ProfileMap) {
550 std::error_code EC;
552 EC = writeDefaultLayout(ProfileMap);
553 else if (SecLayout == CtxSplitLayout)
554 EC = writeCtxSplitLayout(ProfileMap);
555 else
556 llvm_unreachable("Unsupported layout");
557 return EC;
558}
559
560/// Write samples to a text file.
561///
562/// Note: it may be tempting to implement this in terms of
563/// FunctionSamples::print(). Please don't. The dump functionality is intended
564/// for debugging and has no specified form.
565///
566/// The format used here is more structured and deliberate because
567/// it needs to be parsed by the SampleProfileReaderText class.
569 auto &OS = *OutputStream;
571 OS << "[" << S.getContext().toString() << "]:" << S.getTotalSamples();
572 else
573 OS << S.getFunction() << ":" << S.getTotalSamples();
574
575 if (Indent == 0)
576 OS << ":" << S.getHeadSamples();
577 OS << "\n";
578 LineCount++;
579
581 for (const auto &I : SortedSamples.get()) {
582 LineLocation Loc = I->first;
583 const SampleRecord &Sample = I->second;
584 OS.indent(Indent + 1);
585 if (Loc.Discriminator == 0)
586 OS << Loc.LineOffset << ": ";
587 else
588 OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
589
590 OS << Sample.getSamples();
591
592 for (const auto &J : Sample.getSortedCallTargets())
593 OS << " " << J.first << ":" << J.second;
594 OS << "\n";
595 LineCount++;
596 }
597
600 Indent += 1;
601 for (const auto &I : SortedCallsiteSamples.get())
602 for (const auto &FS : I->second) {
603 LineLocation Loc = I->first;
604 const FunctionSamples &CalleeSamples = FS.second;
605 OS.indent(Indent);
606 if (Loc.Discriminator == 0)
607 OS << Loc.LineOffset << ": ";
608 else
609 OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
610 if (std::error_code EC = writeSample(CalleeSamples))
611 return EC;
612 }
613 Indent -= 1;
614
616 OS.indent(Indent + 1);
617 OS << "!CFGChecksum: " << S.getFunctionHash() << "\n";
618 LineCount++;
619 }
620
621 if (S.getContext().getAllAttributes()) {
622 OS.indent(Indent + 1);
623 OS << "!Attributes: " << S.getContext().getAllAttributes() << "\n";
624 LineCount++;
625 }
626
628}
629
630std::error_code
632 assert(!Context.hasContext() && "cs profile is not supported");
633 return writeNameIdx(Context.getFunction());
634}
635
637 auto &NTable = getNameTable();
638 const auto &Ret = NTable.find(FName);
639 if (Ret == NTable.end())
641 encodeULEB128(Ret->second, *OutputStream);
643}
644
646 auto &NTable = getNameTable();
647 NTable.insert(std::make_pair(FName, 0));
648}
649
651 addName(Context.getFunction());
652}
653
655 // Add all the names in indirect call targets.
656 for (const auto &I : S.getBodySamples()) {
657 const SampleRecord &Sample = I.second;
658 for (const auto &J : Sample.getCallTargets())
659 addName(J.first);
660 }
661
662 // Recursively add all the names for inlined callsites.
663 for (const auto &J : S.getCallsiteSamples())
664 for (const auto &FS : J.second) {
665 const FunctionSamples &CalleeSamples = FS.second;
666 addName(CalleeSamples.getFunction());
667 addNames(CalleeSamples);
668 }
669}
670
672 const SampleContext &Context) {
673 if (Context.hasContext()) {
674 for (auto &Callsite : Context.getContextFrames())
676 CSNameTable.insert(std::make_pair(Context, 0));
677 } else {
679 }
680}
681
683 MapVector<FunctionId, uint32_t> &NameTable, std::set<FunctionId> &V) {
684 // Sort the names to make NameTable deterministic.
685 for (const auto &I : NameTable)
686 V.insert(I.first);
687 int i = 0;
688 for (const FunctionId &N : V)
689 NameTable[N] = i++;
690}
691
693 auto &OS = *OutputStream;
694 std::set<FunctionId> V;
696
697 // Write out the name table.
698 encodeULEB128(NameTable.size(), OS);
699 for (auto N : V) {
700 OS << N;
701 encodeULEB128(0, OS);
702 }
704}
705
706std::error_code
708 auto &OS = *OutputStream;
709 // Write file magic identifier.
713}
714
715std::error_code
717 // When calling write on a different profile map, existing names should be
718 // cleared.
719 NameTable.clear();
720
722
723 computeSummary(ProfileMap);
724 if (auto EC = writeSummary())
725 return EC;
726
727 // Generate the name table for all the functions referenced in the profile.
728 for (const auto &I : ProfileMap) {
729 addContext(I.second.getContext());
730 addNames(I.second);
731 }
732
735}
736
738 for (auto &Entry : SectionHdrLayout)
740}
741
744}
745
746void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {
748
749 Writer.write(static_cast<uint64_t>(SectionHdrLayout.size()));
750 SecHdrTableOffset = OutputStream->tell();
751 for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) {
752 Writer.write(static_cast<uint64_t>(-1));
753 Writer.write(static_cast<uint64_t>(-1));
754 Writer.write(static_cast<uint64_t>(-1));
755 Writer.write(static_cast<uint64_t>(-1));
756 }
757}
758
759std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() {
760 assert(SecHdrTable.size() == SectionHdrLayout.size() &&
761 "SecHdrTable entries doesn't match SectionHdrLayout");
762 SmallVector<uint32_t, 16> IndexMap(SecHdrTable.size(), -1);
763 for (uint32_t TableIdx = 0; TableIdx < SecHdrTable.size(); TableIdx++) {
764 IndexMap[SecHdrTable[TableIdx].LayoutIndex] = TableIdx;
765 }
766
767 // Write the section header table in the order specified in
768 // SectionHdrLayout. SectionHdrLayout specifies the sections
769 // order in which profile reader expect to read, so the section
770 // header table should be written in the order in SectionHdrLayout.
771 // Note that the section order in SecHdrTable may be different
772 // from the order in SectionHdrLayout, for example, SecFuncOffsetTable
773 // needs to be computed after SecLBRProfile (the order in SecHdrTable),
774 // but it needs to be read before SecLBRProfile (the order in
775 // SectionHdrLayout). So we use IndexMap above to switch the order.
776 support::endian::SeekableWriter Writer(
777 static_cast<raw_pwrite_stream &>(*OutputStream),
779 for (uint32_t LayoutIdx = 0; LayoutIdx < SectionHdrLayout.size();
780 LayoutIdx++) {
781 assert(IndexMap[LayoutIdx] < SecHdrTable.size() &&
782 "Incorrect LayoutIdx in SecHdrTable");
783 auto Entry = SecHdrTable[IndexMap[LayoutIdx]];
784 Writer.pwrite(static_cast<uint64_t>(Entry.Type),
785 SecHdrTableOffset + 4 * LayoutIdx * sizeof(uint64_t));
786 Writer.pwrite(static_cast<uint64_t>(Entry.Flags),
787 SecHdrTableOffset + (4 * LayoutIdx + 1) * sizeof(uint64_t));
788 Writer.pwrite(static_cast<uint64_t>(Entry.Offset),
789 SecHdrTableOffset + (4 * LayoutIdx + 2) * sizeof(uint64_t));
790 Writer.pwrite(static_cast<uint64_t>(Entry.Size),
791 SecHdrTableOffset + (4 * LayoutIdx + 3) * sizeof(uint64_t));
792 }
793
795}
796
797std::error_code SampleProfileWriterExtBinaryBase::writeHeader(
798 const SampleProfileMap &ProfileMap) {
799 auto &OS = *OutputStream;
800 FileStart = OS.tell();
802
803 allocSecHdrTable();
805}
806
808 auto &OS = *OutputStream;
809 encodeULEB128(Summary->getTotalCount(), OS);
810 encodeULEB128(Summary->getMaxCount(), OS);
811 encodeULEB128(Summary->getMaxFunctionCount(), OS);
812 encodeULEB128(Summary->getNumCounts(), OS);
813 encodeULEB128(Summary->getNumFunctions(), OS);
814 const std::vector<ProfileSummaryEntry> &Entries =
815 Summary->getDetailedSummary();
816 encodeULEB128(Entries.size(), OS);
817 for (auto Entry : Entries) {
818 encodeULEB128(Entry.Cutoff, OS);
819 encodeULEB128(Entry.MinCount, OS);
820 encodeULEB128(Entry.NumCounts, OS);
821 }
823}
825 auto &OS = *OutputStream;
826 if (std::error_code EC = writeContextIdx(S.getContext()))
827 return EC;
828
830
831 // Emit all the body samples.
832 encodeULEB128(S.getBodySamples().size(), OS);
833 for (const auto &I : S.getBodySamples()) {
834 LineLocation Loc = I.first;
835 const SampleRecord &Sample = I.second;
838 encodeULEB128(Sample.getSamples(), OS);
839 encodeULEB128(Sample.getCallTargets().size(), OS);
840 for (const auto &J : Sample.getSortedCallTargets()) {
841 FunctionId Callee = J.first;
842 uint64_t CalleeSamples = J.second;
843 if (std::error_code EC = writeNameIdx(Callee))
844 return EC;
845 encodeULEB128(CalleeSamples, OS);
846 }
847 }
848
849 // Recursively emit all the callsite samples.
850 uint64_t NumCallsites = 0;
851 for (const auto &J : S.getCallsiteSamples())
852 NumCallsites += J.second.size();
853 encodeULEB128(NumCallsites, OS);
854 for (const auto &J : S.getCallsiteSamples())
855 for (const auto &FS : J.second) {
856 LineLocation Loc = J.first;
857 const FunctionSamples &CalleeSamples = FS.second;
860 if (std::error_code EC = writeBody(CalleeSamples))
861 return EC;
862 }
863
865}
866
867/// Write samples of a top-level function to a binary file.
868///
869/// \returns true if the samples were written successfully, false otherwise.
870std::error_code
873 return writeBody(S);
874}
875
876/// Create a sample profile file writer based on the specified format.
877///
878/// \param Filename The file to create.
879///
880/// \param Format Encoding format for the profile file.
881///
882/// \returns an error code indicating the status of the created writer.
885 std::error_code EC;
886 std::unique_ptr<raw_ostream> OS;
888 OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_None));
889 else
890 OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_TextWithCRLF));
891 if (EC)
892 return EC;
893
894 return create(OS, Format);
895}
896
897/// Create a sample profile stream writer based on the specified format.
898///
899/// \param OS The output stream to store the profile data to.
900///
901/// \param Format Encoding format for the profile file.
902///
903/// \returns an error code indicating the status of the created writer.
905SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
907 std::error_code EC;
908 std::unique_ptr<SampleProfileWriter> Writer;
909
910 // Currently only Text and Extended Binary format are supported for CSSPGO.
914
915 if (Format == SPF_Binary)
916 Writer.reset(new SampleProfileWriterRawBinary(OS));
917 else if (Format == SPF_Ext_Binary)
918 Writer.reset(new SampleProfileWriterExtBinary(OS));
919 else if (Format == SPF_Text)
920 Writer.reset(new SampleProfileWriterText(OS));
921 else if (Format == SPF_GCC)
923 else
925
926 if (EC)
927 return EC;
928
929 Writer->Format = Format;
930 return std::move(Writer);
931}
932
935 Summary = Builder.computeSummaryForProfiles(ProfileMap);
936}
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DEBUG(X)
Definition: Debug.h:101
Provides ErrorOr<T> smart pointer.
#define I(x, y, z)
Definition: MD5.cpp:58
LLVMContext & Context
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
endianness Endian
static void splitProfileMapToTwo(const SampleProfileMap &ProfileMap, SampleProfileMap &ContextProfileMap, SampleProfileMap &NoContextProfileMap)
raw_pwrite_stream & OS
Represents either an error or a value T.
Definition: ErrorOr.h:56
This class implements a map that also provides access to all stored values in a deterministic order.
Definition: MapVector.h:36
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
Definition: ProfileCommon.h:70
std::unique_ptr< ProfileSummary > computeSummaryForProfiles(const sampleprof::SampleProfileMap &Profiles)
size_t size() const
Definition: SmallVector.h:91
pointer data()
Return a pointer to the vector's buffer, even if empty().
Definition: SmallVector.h:299
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:470
uint64_t tell() const
tell - Return the current offset with the file.
Definition: raw_ostream.h:150
raw_ostream & indent(unsigned NumSpaces)
indent - Insert 'NumSpaces' spaces.
An abstract base class for streams implementations that also support a pwrite operation.
Definition: raw_ostream.h:444
void pwrite(const char *Ptr, size_t Size, uint64_t Offset)
Definition: raw_ostream.h:452
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:660
A raw_ostream that writes to an SmallVector or SmallString.
Definition: raw_ostream.h:690
DefaultFunctionPruningStrategy(SampleProfileMap &ProfileMap, size_t OutputSizeLimit)
void Erase(size_t CurrentOutputSize) override
In this default implementation, functions with fewest samples are dropped first.
This class represents a function that is read from a sample profile.
Definition: FunctionId.h:36
When writing a profile with size limit, user may want to use a different strategy to reduce function ...
virtual void Erase(size_t CurrentOutputSize)=0
SampleProfileWriter::writeWithSizeLimit() calls this after every write iteration if the output size s...
Representation of the samples collected for a function.
Definition: SampleProf.h:744
static constexpr const char * UniqSuffix
Definition: SampleProf.h:1095
uint64_t getHeadSamples() const
For top-level functions, return the total number of branch samples that have the function as the bran...
Definition: SampleProf.h:940
FunctionId getFunction() const
Return the function name.
Definition: SampleProf.h:1069
static bool ProfileIsFS
If this profile uses flow sensitive discriminators.
Definition: SampleProf.h:1196
SampleContext & getContext() const
Definition: SampleProf.h:1185
uint64_t getTotalSamples() const
Return the total number of samples collected inside the function.
Definition: SampleProf.h:932
const CallsiteSampleMap & getCallsiteSamples() const
Return all the callsite samples collected in the body of the function.
Definition: SampleProf.h:974
const BodySampleMap & getBodySamples() const
Return all the samples collected in the body of the function.
Definition: SampleProf.h:971
std::error_code write(raw_ostream &OS)
Definition: SampleProf.cpp:388
std::string toString() const
Definition: SampleProf.h:632
This class provides operator overloads to the map container using MD5 as the key type,...
Definition: SampleProf.h:1306
size_t erase(const SampleContext &Ctx)
Definition: SampleProf.h:1327
void stablizeNameTable(MapVector< FunctionId, uint32_t > &NameTable, std::set< FunctionId > &V)
virtual void addContext(const SampleContext &Context)
virtual std::error_code writeMagicIdent(SampleProfileFormat Format)
MapVector< FunctionId, uint32_t > NameTable
virtual std::error_code writeContextIdx(const SampleContext &Context)
std::error_code writeSample(const FunctionSamples &S) override
Write samples of a top-level function to a binary file.
std::error_code writeHeader(const SampleProfileMap &ProfileMap) override
Write a file header for the profile file.
virtual MapVector< FunctionId, uint32_t > & getNameTable()
std::error_code writeBody(const FunctionSamples &S)
std::error_code writeNameIdx(FunctionId FName)
std::error_code writeNameTableSection(const SampleProfileMap &ProfileMap)
SmallVector< SecHdrTableEntry, 8 > SectionHdrLayout
std::error_code writeFuncMetadata(const SampleProfileMap &Profiles)
virtual std::error_code writeCustomSection(SecType Type)=0
virtual std::error_code writeOneSection(SecType Type, uint32_t LayoutIdx, const SampleProfileMap &ProfileMap)
std::error_code writeCSNameIdx(const SampleContext &Context)
virtual std::error_code writeSections(const SampleProfileMap &ProfileMap)=0
void addSectionFlag(SecType Type, SecFlagType Flag)
uint64_t markSectionStart(SecType Type, uint32_t LayoutIdx)
Return the current position and prepare to use it as the start position of a section given the sectio...
void addContext(const SampleContext &Context) override
std::error_code addNewSection(SecType Sec, uint32_t LayoutIdx, uint64_t SectionStart)
Add a new section into section header table given the section type Type, its position LayoutIdx in Se...
std::error_code write(const SampleProfileMap &ProfileMap) override
Write all the sample profiles in the given map of samples.
std::error_code writeContextIdx(const SampleContext &Context) override
std::error_code writeSample(const FunctionSamples &S) override
Write samples of a top-level function to a binary file.
Sample-based profile writer (text format).
std::error_code writeSample(const FunctionSamples &S) override
Write samples to a text file.
std::unique_ptr< ProfileSummary > Summary
Profile summary.
virtual std::error_code writeSample(const FunctionSamples &S)=0
Write sample profiles in S.
SampleProfileFormat Format
Profile format.
std::error_code writeWithSizeLimitInternal(SampleProfileMap &ProfileMap, size_t OutputSizeLimit, FunctionPruningStrategy *Strategy)
void computeSummary(const SampleProfileMap &ProfileMap)
Compute summary for this profile.
virtual std::error_code writeFuncProfiles(const SampleProfileMap &ProfileMap)
std::unique_ptr< raw_ostream > OutputStream
Output stream where to emit the profile to.
size_t LineCount
For writeWithSizeLimit in text mode, each newline takes 1 additional byte on Windows when actually wr...
static ErrorOr< std::unique_ptr< SampleProfileWriter > > create(StringRef Filename, SampleProfileFormat Format)
Profile writer factory.
virtual std::error_code writeHeader(const SampleProfileMap &ProfileMap)=0
Write a file header for the profile file.
virtual std::error_code write(const SampleProfileMap &ProfileMap)
Write all the sample profiles in the given map of samples.
Representation of a single sample record.
Definition: SampleProf.h:325
const CallTargetMap & getCallTargets() const
Definition: SampleProf.h:393
uint64_t getSamples() const
Definition: SampleProf.h:392
const SortedCallTargetSet getSortedCallTargets() const
Definition: SampleProf.h:394
Sort a LocationT->SampleT map by LocationT.
Definition: SampleProf.h:1346
const SamplesWithLocList & get() const
Definition: SampleProf.h:1359
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void compress(ArrayRef< uint8_t > Input, SmallVectorImpl< uint8_t > &CompressedBuffer, int Level=DefaultCompression)
constexpr int BestSizeCompression
Definition: Compression.h:39
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
void sortFuncProfiles(const SampleProfileMap &ProfileMap, std::vector< NameFunctionSamples > &SortedProfiles)
Definition: SampleProf.cpp:202
static uint64_t SPMagic(SampleProfileFormat Format=SPF_Binary)
Definition: SampleProf.h:106
std::pair< hash_code, const FunctionSamples * > NameFunctionSamples
Definition: SampleProf.h:1337
static void addSecFlag(SecHdrTableEntry &Entry, SecFlagType Flag)
Definition: SampleProf.h:248
static bool hasSecFlag(const SecHdrTableEntry &Entry, SecFlagType Flag)
Definition: SampleProf.h:264
@ SecFlagIsPreInlined
SecFlagIsPreInlined means this profile contains ShouldBeInlined contexts thus this is CS preinliner c...
@ SecFlagFSDiscriminator
SecFlagFSDiscriminator means this profile uses flow-sensitive discriminators.
@ SecFlagFullContext
SecFlagContext means this is context-sensitive flat profile for CSSPGO.
static uint64_t SPVersion()
Definition: SampleProf.h:113
@ OF_TextWithCRLF
The file should be opened in text mode and use a carriage linefeed '\r '.
Definition: FileSystem.h:768
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:329
@ Offset
Definition: DWP.cpp:456
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
Definition: LEB128.h:80
endianness
Definition: bit.h:70
#define N
Represents the relative location of an instruction.
Definition: SampleProf.h:280
Adapter to write values to a stream in a particular byte order.
Definition: EndianStream.h:67
void write(ArrayRef< value_type > Val)
Definition: EndianStream.h:71
static uint64_t round(uint64_t Acc, uint64_t Input)
Definition: xxhash.cpp:64