LLVM 17.0.0git
SampleProfWriter.cpp
Go to the documentation of this file.
1//===- SampleProfWriter.cpp - Write LLVM sample profile data --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the class that writes LLVM sample profiles. It
10// supports two file formats: text and binary. The textual representation
11// is useful for debugging and testing purposes. The binary representation
12// is more compact, resulting in smaller file sizes. However, they can
13// both be used interchangeably.
14//
15// See lib/ProfileData/SampleProfReader.cpp for documentation on each of the
16// supported formats.
17//
18//===----------------------------------------------------------------------===//
19
21#include "llvm/ADT/StringRef.h"
25#include "llvm/Support/Endian.h"
29#include "llvm/Support/LEB128.h"
30#include "llvm/Support/MD5.h"
32#include <algorithm>
33#include <cmath>
34#include <cstdint>
35#include <memory>
36#include <set>
37#include <system_error>
38#include <utility>
39#include <vector>
40
41#define DEBUG_TYPE "llvm-profdata"
42
43using namespace llvm;
44using namespace sampleprof;
45
46namespace llvm {
47namespace support {
48namespace endian {
49namespace {
50
51// Adapter class to llvm::support::endian::Writer for pwrite().
52struct SeekableWriter {
55 SeekableWriter(raw_pwrite_stream &OS, endianness Endian)
56 : OS(OS), Endian(Endian) {}
57
58 template <typename ValueType>
59 void pwrite(ValueType Val, size_t Offset) {
60 std::string StringBuf;
61 raw_string_ostream SStream(StringBuf);
62 Writer(SStream, Endian).write(Val);
63 OS.pwrite(StringBuf.data(), StringBuf.size(), Offset);
64 }
65};
66
67} // namespace
68} // namespace endian
69} // namespace support
70} // namespace llvm
71
73 SampleProfileMap &ProfileMap, size_t OutputSizeLimit)
74 : FunctionPruningStrategy(ProfileMap, OutputSizeLimit) {
75 sortFuncProfiles(ProfileMap, SortedFunctions);
76}
77
78void DefaultFunctionPruningStrategy::Erase(size_t CurrentOutputSize) {
79 double D = (double)OutputSizeLimit / CurrentOutputSize;
80 size_t NewSize = (size_t)round(ProfileMap.size() * D * D);
81 size_t NumToRemove = ProfileMap.size() - NewSize;
82 if (NumToRemove < 1)
83 NumToRemove = 1;
84
85 assert(NumToRemove <= SortedFunctions.size());
87 llvm::make_range(SortedFunctions.begin() + SortedFunctions.size() -
88 NumToRemove,
89 SortedFunctions.end()),
90 [&](const NameFunctionSamples &E) { ProfileMap.erase(E.first); });
91 SortedFunctions.resize(SortedFunctions.size() - NumToRemove);
92}
93
95 SampleProfileMap &ProfileMap, size_t OutputSizeLimit,
96 FunctionPruningStrategy *Strategy) {
97 if (OutputSizeLimit == 0)
98 return write(ProfileMap);
99
100 size_t OriginalFunctionCount = ProfileMap.size();
101
102 std::unique_ptr<raw_ostream> OriginalOutputStream;
103 OutputStream.swap(OriginalOutputStream);
104
105 size_t IterationCount = 0;
106 size_t TotalSize;
107
108 SmallVector<char> StringBuffer;
109 do {
110 StringBuffer.clear();
111 OutputStream.reset(new raw_svector_ostream(StringBuffer));
112 if (std::error_code EC = write(ProfileMap))
113 return EC;
114
115 TotalSize = StringBuffer.size();
116 // On Windows every "\n" is actually written as "\r\n" to disk but not to
117 // memory buffer, this difference should be added when considering the total
118 // output size.
119#ifdef _WIN32
120 if (Format == SPF_Text)
121 TotalSize += LineCount;
122#endif
123 if (TotalSize <= OutputSizeLimit)
124 break;
125
126 Strategy->Erase(TotalSize);
127 IterationCount++;
128 } while (ProfileMap.size() != 0);
129
130 if (ProfileMap.size() == 0)
132
133 OutputStream.swap(OriginalOutputStream);
134 OutputStream->write(StringBuffer.data(), StringBuffer.size());
135 LLVM_DEBUG(dbgs() << "Profile originally has " << OriginalFunctionCount
136 << " functions, reduced to " << ProfileMap.size() << " in "
137 << IterationCount << " iterations\n");
138 // Silence warning on Release build.
139 (void)OriginalFunctionCount;
140 (void)IterationCount;
142}
143
144std::error_code
146 std::vector<NameFunctionSamples> V;
147 sortFuncProfiles(ProfileMap, V);
148 for (const auto &I : V) {
149 if (std::error_code EC = writeSample(*I.second))
150 return EC;
151 }
153}
154
155std::error_code SampleProfileWriter::write(const SampleProfileMap &ProfileMap) {
156 if (std::error_code EC = writeHeader(ProfileMap))
157 return EC;
158
159 if (std::error_code EC = writeFuncProfiles(ProfileMap))
160 return EC;
161
163}
164
165/// Return the current position and prepare to use it as the start
166/// position of a section given the section type \p Type and its position
167/// \p LayoutIdx in SectionHdrLayout.
170 uint32_t LayoutIdx) {
171 uint64_t SectionStart = OutputStream->tell();
172 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
173 const auto &Entry = SectionHdrLayout[LayoutIdx];
174 assert(Entry.Type == Type && "Unexpected section type");
175 // Use LocalBuf as a temporary output for writting data.
177 LocalBufStream.swap(OutputStream);
178 return SectionStart;
179}
180
181std::error_code SampleProfileWriterExtBinaryBase::compressAndOutput() {
184 std::string &UncompressedStrings =
185 static_cast<raw_string_ostream *>(LocalBufStream.get())->str();
186 if (UncompressedStrings.size() == 0)
188 auto &OS = *OutputStream;
189 SmallVector<uint8_t, 128> CompressedStrings;
190 compression::zlib::compress(arrayRefFromStringRef(UncompressedStrings),
191 CompressedStrings,
193 encodeULEB128(UncompressedStrings.size(), OS);
194 encodeULEB128(CompressedStrings.size(), OS);
195 OS << toStringRef(CompressedStrings);
196 UncompressedStrings.clear();
198}
199
200/// Add a new section into section header table given the section type
201/// \p Type, its position \p LayoutIdx in SectionHdrLayout and the
202/// location \p SectionStart where the section should be written to.
204 SecType Type, uint32_t LayoutIdx, uint64_t SectionStart) {
205 assert(LayoutIdx < SectionHdrLayout.size() && "LayoutIdx out of range");
206 const auto &Entry = SectionHdrLayout[LayoutIdx];
207 assert(Entry.Type == Type && "Unexpected section type");
209 LocalBufStream.swap(OutputStream);
210 if (std::error_code EC = compressAndOutput())
211 return EC;
212 }
213 SecHdrTable.push_back({Type, Entry.Flags, SectionStart - FileStart,
214 OutputStream->tell() - SectionStart, LayoutIdx});
216}
217
218std::error_code
220 // When calling write on a different profile map, existing states should be
221 // cleared.
222 NameTable.clear();
223 CSNameTable.clear();
224 SecHdrTable.clear();
225
226 if (std::error_code EC = writeHeader(ProfileMap))
227 return EC;
228
229 std::string LocalBuf;
230 LocalBufStream = std::make_unique<raw_string_ostream>(LocalBuf);
231 if (std::error_code EC = writeSections(ProfileMap))
232 return EC;
233
234 if (std::error_code EC = writeSecHdrTable())
235 return EC;
236
238}
239
241 const SampleContext &Context) {
242 if (Context.hasContext())
243 return writeCSNameIdx(Context);
244 else
246}
247
248std::error_code
250 const auto &Ret = CSNameTable.find(Context);
251 if (Ret == CSNameTable.end())
253 encodeULEB128(Ret->second, *OutputStream);
255}
256
257std::error_code
259 uint64_t Offset = OutputStream->tell();
260 auto &Context = S.getContext();
261 FuncOffsetTable[Context] = Offset - SecLBRProfileStart;
263 return writeBody(S);
264}
265
267 auto &OS = *OutputStream;
268
269 // Write out the table size.
270 encodeULEB128(FuncOffsetTable.size(), OS);
271
272 // Write out FuncOffsetTable.
273 auto WriteItem = [&](const SampleContext &Context, uint64_t Offset) {
274 if (std::error_code EC = writeContextIdx(Context))
275 return EC;
277 return (std::error_code)sampleprof_error::success;
278 };
279
281 // Sort the contexts before writing them out. This is to help fast load all
282 // context profiles for a function as well as their callee contexts which
283 // can help profile-guided importing for ThinLTO.
284 std::map<SampleContext, uint64_t> OrderedFuncOffsetTable(
285 FuncOffsetTable.begin(), FuncOffsetTable.end());
286 for (const auto &Entry : OrderedFuncOffsetTable) {
287 if (std::error_code EC = WriteItem(Entry.first, Entry.second))
288 return EC;
289 }
291 } else {
292 for (const auto &Entry : FuncOffsetTable) {
293 if (std::error_code EC = WriteItem(Entry.first, Entry.second))
294 return EC;
295 }
296 }
297
298 FuncOffsetTable.clear();
300}
301
303 const FunctionSamples &FunctionProfile) {
304 auto &OS = *OutputStream;
305 if (std::error_code EC = writeContextIdx(FunctionProfile.getContext()))
306 return EC;
307
309 encodeULEB128(FunctionProfile.getFunctionHash(), OS);
311 encodeULEB128(FunctionProfile.getContext().getAllAttributes(), OS);
312 }
313
315 // Recursively emit attributes for all callee samples.
316 uint64_t NumCallsites = 0;
317 for (const auto &J : FunctionProfile.getCallsiteSamples())
318 NumCallsites += J.second.size();
319 encodeULEB128(NumCallsites, OS);
320 for (const auto &J : FunctionProfile.getCallsiteSamples()) {
321 for (const auto &FS : J.second) {
322 LineLocation Loc = J.first;
325 if (std::error_code EC = writeFuncMetadata(FS.second))
326 return EC;
327 }
328 }
329 }
330
332}
333
335 const SampleProfileMap &Profiles) {
339 for (const auto &Entry : Profiles) {
340 if (std::error_code EC = writeFuncMetadata(Entry.second))
341 return EC;
342 }
344}
345
347 if (!UseMD5)
349
350 auto &OS = *OutputStream;
351 std::set<StringRef> V;
353
354 // Write out the MD5 name table. We wrote unencoded MD5 so reader can
355 // retrieve the name using the name index without having to read the
356 // whole name table.
357 encodeULEB128(NameTable.size(), OS);
359 for (auto N : V)
360 Writer.write(MD5Hash(N));
362}
363
365 const SampleProfileMap &ProfileMap) {
366 for (const auto &I : ProfileMap) {
367 assert(I.first == I.second.getContext() && "Inconsistent profile map");
368 addContext(I.second.getContext());
369 addNames(I.second);
370 }
371
372 // If NameTable contains ".__uniq." suffix, set SecFlagUniqSuffix flag
373 // so compiler won't strip the suffix during profile matching after
374 // seeing the flag in the profile.
375 for (const auto &I : NameTable) {
376 if (I.first.contains(FunctionSamples::UniqSuffix)) {
378 break;
379 }
380 }
381
382 if (auto EC = writeNameTable())
383 return EC;
385}
386
388 // Sort the names to make CSNameTable deterministic.
389 std::set<SampleContext> OrderedContexts;
390 for (const auto &I : CSNameTable)
391 OrderedContexts.insert(I.first);
392 assert(OrderedContexts.size() == CSNameTable.size() &&
393 "Unmatched ordered and unordered contexts");
394 uint64_t I = 0;
395 for (auto &Context : OrderedContexts)
396 CSNameTable[Context] = I++;
397
398 auto &OS = *OutputStream;
399 encodeULEB128(OrderedContexts.size(), OS);
401 for (auto Context : OrderedContexts) {
402 auto Frames = Context.getContextFrames();
403 encodeULEB128(Frames.size(), OS);
404 for (auto &Callsite : Frames) {
405 if (std::error_code EC = writeNameIdx(Callsite.FuncName))
406 return EC;
407 encodeULEB128(Callsite.Location.LineOffset, OS);
408 encodeULEB128(Callsite.Location.Discriminator, OS);
409 }
410 }
411
413}
414
415std::error_code
417 if (ProfSymList && ProfSymList->size() > 0)
418 if (std::error_code EC = ProfSymList->write(*OutputStream))
419 return EC;
420
422}
423
425 SecType Type, uint32_t LayoutIdx, const SampleProfileMap &ProfileMap) {
426 // The setting of SecFlagCompress should happen before markSectionStart.
427 if (Type == SecProfileSymbolList && ProfSymList && ProfSymList->toCompress())
431 if (Type == SecFuncMetadata &&
440
441 uint64_t SectionStart = markSectionStart(Type, LayoutIdx);
442 switch (Type) {
443 case SecProfSummary:
444 computeSummary(ProfileMap);
445 if (auto EC = writeSummary())
446 return EC;
447 break;
448 case SecNameTable:
449 if (auto EC = writeNameTableSection(ProfileMap))
450 return EC;
451 break;
452 case SecCSNameTable:
453 if (auto EC = writeCSNameTableSection())
454 return EC;
455 break;
456 case SecLBRProfile:
458 if (std::error_code EC = writeFuncProfiles(ProfileMap))
459 return EC;
460 break;
462 if (auto EC = writeFuncOffsetTable())
463 return EC;
464 break;
465 case SecFuncMetadata:
466 if (std::error_code EC = writeFuncMetadata(ProfileMap))
467 return EC;
468 break;
470 if (auto EC = writeProfileSymbolListSection())
471 return EC;
472 break;
473 default:
474 if (auto EC = writeCustomSection(Type))
475 return EC;
476 break;
477 }
478 if (std::error_code EC = addNewSection(Type, LayoutIdx, SectionStart))
479 return EC;
481}
482
483std::error_code SampleProfileWriterExtBinary::writeDefaultLayout(
484 const SampleProfileMap &ProfileMap) {
485 // The const indices passed to writeOneSection below are specifying the
486 // positions of the sections in SectionHdrLayout. Look at
487 // initSectionHdrLayout to find out where each section is located in
488 // SectionHdrLayout.
489 if (auto EC = writeOneSection(SecProfSummary, 0, ProfileMap))
490 return EC;
491 if (auto EC = writeOneSection(SecNameTable, 1, ProfileMap))
492 return EC;
493 if (auto EC = writeOneSection(SecCSNameTable, 2, ProfileMap))
494 return EC;
495 if (auto EC = writeOneSection(SecLBRProfile, 4, ProfileMap))
496 return EC;
497 if (auto EC = writeOneSection(SecProfileSymbolList, 5, ProfileMap))
498 return EC;
499 if (auto EC = writeOneSection(SecFuncOffsetTable, 3, ProfileMap))
500 return EC;
501 if (auto EC = writeOneSection(SecFuncMetadata, 6, ProfileMap))
502 return EC;
504}
505
506static void splitProfileMapToTwo(const SampleProfileMap &ProfileMap,
507 SampleProfileMap &ContextProfileMap,
508 SampleProfileMap &NoContextProfileMap) {
509 for (const auto &I : ProfileMap) {
510 if (I.second.getCallsiteSamples().size())
511 ContextProfileMap.insert({I.first, I.second});
512 else
513 NoContextProfileMap.insert({I.first, I.second});
514 }
515}
516
517std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout(
518 const SampleProfileMap &ProfileMap) {
519 SampleProfileMap ContextProfileMap, NoContextProfileMap;
520 splitProfileMapToTwo(ProfileMap, ContextProfileMap, NoContextProfileMap);
521
522 if (auto EC = writeOneSection(SecProfSummary, 0, ProfileMap))
523 return EC;
524 if (auto EC = writeOneSection(SecNameTable, 1, ProfileMap))
525 return EC;
526 if (auto EC = writeOneSection(SecLBRProfile, 3, ContextProfileMap))
527 return EC;
528 if (auto EC = writeOneSection(SecFuncOffsetTable, 2, ContextProfileMap))
529 return EC;
530 // Mark the section to have no context. Note section flag needs to be set
531 // before writing the section.
533 if (auto EC = writeOneSection(SecLBRProfile, 5, NoContextProfileMap))
534 return EC;
535 // Mark the section to have no context. Note section flag needs to be set
536 // before writing the section.
538 if (auto EC = writeOneSection(SecFuncOffsetTable, 4, NoContextProfileMap))
539 return EC;
540 if (auto EC = writeOneSection(SecProfileSymbolList, 6, ProfileMap))
541 return EC;
542 if (auto EC = writeOneSection(SecFuncMetadata, 7, ProfileMap))
543 return EC;
544
546}
547
548std::error_code SampleProfileWriterExtBinary::writeSections(
549 const SampleProfileMap &ProfileMap) {
550 std::error_code EC;
552 EC = writeDefaultLayout(ProfileMap);
553 else if (SecLayout == CtxSplitLayout)
554 EC = writeCtxSplitLayout(ProfileMap);
555 else
556 llvm_unreachable("Unsupported layout");
557 return EC;
558}
559
560std::error_code
562 if (std::error_code EC = SampleProfileWriter::write(ProfileMap))
563 return EC;
564 if (std::error_code EC = writeFuncOffsetTable())
565 return EC;
567}
568
569/// Write samples to a text file.
570///
571/// Note: it may be tempting to implement this in terms of
572/// FunctionSamples::print(). Please don't. The dump functionality is intended
573/// for debugging and has no specified form.
574///
575/// The format used here is more structured and deliberate because
576/// it needs to be parsed by the SampleProfileReaderText class.
578 auto &OS = *OutputStream;
580 OS << "[" << S.getContext().toString() << "]:" << S.getTotalSamples();
581 else
582 OS << S.getName() << ":" << S.getTotalSamples();
583
584 if (Indent == 0)
585 OS << ":" << S.getHeadSamples();
586 OS << "\n";
587 LineCount++;
588
590 for (const auto &I : SortedSamples.get()) {
591 LineLocation Loc = I->first;
592 const SampleRecord &Sample = I->second;
593 OS.indent(Indent + 1);
594 if (Loc.Discriminator == 0)
595 OS << Loc.LineOffset << ": ";
596 else
597 OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
598
599 OS << Sample.getSamples();
600
601 for (const auto &J : Sample.getSortedCallTargets())
602 OS << " " << J.first << ":" << J.second;
603 OS << "\n";
604 LineCount++;
605 }
606
609 Indent += 1;
610 for (const auto &I : SortedCallsiteSamples.get())
611 for (const auto &FS : I->second) {
612 LineLocation Loc = I->first;
613 const FunctionSamples &CalleeSamples = FS.second;
614 OS.indent(Indent);
615 if (Loc.Discriminator == 0)
616 OS << Loc.LineOffset << ": ";
617 else
618 OS << Loc.LineOffset << "." << Loc.Discriminator << ": ";
619 if (std::error_code EC = writeSample(CalleeSamples))
620 return EC;
621 }
622 Indent -= 1;
623
625 OS.indent(Indent + 1);
626 OS << "!CFGChecksum: " << S.getFunctionHash() << "\n";
627 LineCount++;
628 }
629
630 if (S.getContext().getAllAttributes()) {
631 OS.indent(Indent + 1);
632 OS << "!Attributes: " << S.getContext().getAllAttributes() << "\n";
633 LineCount++;
634 }
635
637}
638
639std::error_code
641 assert(!Context.hasContext() && "cs profile is not supported");
642 return writeNameIdx(Context.getName());
643}
644
646 auto &NTable = getNameTable();
647 const auto &Ret = NTable.find(FName);
648 if (Ret == NTable.end())
650 encodeULEB128(Ret->second, *OutputStream);
652}
653
655 auto &NTable = getNameTable();
656 NTable.insert(std::make_pair(FName, 0));
657}
658
660 addName(Context.getName());
661}
662
664 // Add all the names in indirect call targets.
665 for (const auto &I : S.getBodySamples()) {
666 const SampleRecord &Sample = I.second;
667 for (const auto &J : Sample.getCallTargets())
668 addName(J.first());
669 }
670
671 // Recursively add all the names for inlined callsites.
672 for (const auto &J : S.getCallsiteSamples())
673 for (const auto &FS : J.second) {
674 const FunctionSamples &CalleeSamples = FS.second;
675 addName(CalleeSamples.getName());
676 addNames(CalleeSamples);
677 }
678}
679
681 const SampleContext &Context) {
682 if (Context.hasContext()) {
683 for (auto &Callsite : Context.getContextFrames())
684 SampleProfileWriterBinary::addName(Callsite.FuncName);
685 CSNameTable.insert(std::make_pair(Context, 0));
686 } else {
688 }
689}
690
692 MapVector<StringRef, uint32_t> &NameTable, std::set<StringRef> &V) {
693 // Sort the names to make NameTable deterministic.
694 for (const auto &I : NameTable)
695 V.insert(I.first);
696 int i = 0;
697 for (const StringRef &N : V)
698 NameTable[N] = i++;
699}
700
702 auto &OS = *OutputStream;
703 std::set<StringRef> V;
705
706 // Write out the name table.
707 encodeULEB128(NameTable.size(), OS);
708 for (auto N : V) {
709 OS << N;
710 encodeULEB128(0, OS);
711 }
713}
714
716 auto &OS = *OutputStream;
717
718 // Fill the slot remembered by TableOffset with the offset of FuncOffsetTable.
719 uint64_t FuncOffsetTableStart = OS.tell();
720 support::endian::SeekableWriter Writer(static_cast<raw_pwrite_stream &>(OS),
722 Writer.pwrite(FuncOffsetTableStart, TableOffset);
723
724 // Write out the table size.
726
727 // Write out FuncOffsetTable.
728 for (auto Entry : FuncOffsetTable) {
729 if (std::error_code EC = writeNameIdx(Entry.first))
730 return EC;
731 encodeULEB128(Entry.second, OS);
732 }
733 FuncOffsetTable.clear();
735}
736
738 auto &OS = *OutputStream;
739 std::set<StringRef> V;
741
742 // Write out the name table.
743 encodeULEB128(NameTable.size(), OS);
744 for (auto N : V) {
746 }
748}
749
750std::error_code
752 auto &OS = *OutputStream;
753 // Write file magic identifier.
757}
758
759std::error_code
761 // When calling write on a different profile map, existing names should be
762 // cleared.
763 NameTable.clear();
764
766
767 computeSummary(ProfileMap);
768 if (auto EC = writeSummary())
769 return EC;
770
771 // Generate the name table for all the functions referenced in the profile.
772 for (const auto &I : ProfileMap) {
773 assert(I.first == I.second.getContext() && "Inconsistent profile map");
774 addContext(I.first);
775 addNames(I.second);
776 }
777
780}
781
783 for (auto &Entry : SectionHdrLayout)
785}
786
789}
790
791void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {
793
794 Writer.write(static_cast<uint64_t>(SectionHdrLayout.size()));
795 SecHdrTableOffset = OutputStream->tell();
796 for (uint32_t i = 0; i < SectionHdrLayout.size(); i++) {
797 Writer.write(static_cast<uint64_t>(-1));
798 Writer.write(static_cast<uint64_t>(-1));
799 Writer.write(static_cast<uint64_t>(-1));
800 Writer.write(static_cast<uint64_t>(-1));
801 }
802}
803
804std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() {
805 assert(SecHdrTable.size() == SectionHdrLayout.size() &&
806 "SecHdrTable entries doesn't match SectionHdrLayout");
807 SmallVector<uint32_t, 16> IndexMap(SecHdrTable.size(), -1);
808 for (uint32_t TableIdx = 0; TableIdx < SecHdrTable.size(); TableIdx++) {
809 IndexMap[SecHdrTable[TableIdx].LayoutIndex] = TableIdx;
810 }
811
812 // Write the section header table in the order specified in
813 // SectionHdrLayout. SectionHdrLayout specifies the sections
814 // order in which profile reader expect to read, so the section
815 // header table should be written in the order in SectionHdrLayout.
816 // Note that the section order in SecHdrTable may be different
817 // from the order in SectionHdrLayout, for example, SecFuncOffsetTable
818 // needs to be computed after SecLBRProfile (the order in SecHdrTable),
819 // but it needs to be read before SecLBRProfile (the order in
820 // SectionHdrLayout). So we use IndexMap above to switch the order.
821 support::endian::SeekableWriter Writer(
823 for (uint32_t LayoutIdx = 0; LayoutIdx < SectionHdrLayout.size();
824 LayoutIdx++) {
825 assert(IndexMap[LayoutIdx] < SecHdrTable.size() &&
826 "Incorrect LayoutIdx in SecHdrTable");
827 auto Entry = SecHdrTable[IndexMap[LayoutIdx]];
828 Writer.pwrite(static_cast<uint64_t>(Entry.Type),
829 SecHdrTableOffset + 4 * LayoutIdx * sizeof(uint64_t));
830 Writer.pwrite(static_cast<uint64_t>(Entry.Flags),
831 SecHdrTableOffset + (4 * LayoutIdx + 1) * sizeof(uint64_t));
832 Writer.pwrite(static_cast<uint64_t>(Entry.Offset),
833 SecHdrTableOffset + (4 * LayoutIdx + 2) * sizeof(uint64_t));
834 Writer.pwrite(static_cast<uint64_t>(Entry.Size),
835 SecHdrTableOffset + (4 * LayoutIdx + 3) * sizeof(uint64_t));
836 }
837
839}
840
841std::error_code SampleProfileWriterExtBinaryBase::writeHeader(
842 const SampleProfileMap &ProfileMap) {
843 auto &OS = *OutputStream;
844 FileStart = OS.tell();
846
847 allocSecHdrTable();
849}
850
852 const SampleProfileMap &ProfileMap) {
854 if (auto EC = SampleProfileWriterBinary::writeHeader(ProfileMap))
855 return EC;
856
857 // Reserve a slot for the offset of function offset table. The slot will
858 // be populated with the offset of FuncOffsetTable later.
859 TableOffset = OutputStream->tell();
860 Writer.write(static_cast<uint64_t>(-2));
862}
863
865 auto &OS = *OutputStream;
866 encodeULEB128(Summary->getTotalCount(), OS);
867 encodeULEB128(Summary->getMaxCount(), OS);
868 encodeULEB128(Summary->getMaxFunctionCount(), OS);
869 encodeULEB128(Summary->getNumCounts(), OS);
870 encodeULEB128(Summary->getNumFunctions(), OS);
871 const std::vector<ProfileSummaryEntry> &Entries =
872 Summary->getDetailedSummary();
873 encodeULEB128(Entries.size(), OS);
874 for (auto Entry : Entries) {
875 encodeULEB128(Entry.Cutoff, OS);
876 encodeULEB128(Entry.MinCount, OS);
877 encodeULEB128(Entry.NumCounts, OS);
878 }
880}
882 auto &OS = *OutputStream;
883 if (std::error_code EC = writeContextIdx(S.getContext()))
884 return EC;
885
887
888 // Emit all the body samples.
889 encodeULEB128(S.getBodySamples().size(), OS);
890 for (const auto &I : S.getBodySamples()) {
891 LineLocation Loc = I.first;
892 const SampleRecord &Sample = I.second;
895 encodeULEB128(Sample.getSamples(), OS);
896 encodeULEB128(Sample.getCallTargets().size(), OS);
897 for (const auto &J : Sample.getSortedCallTargets()) {
898 StringRef Callee = J.first;
899 uint64_t CalleeSamples = J.second;
900 if (std::error_code EC = writeNameIdx(Callee))
901 return EC;
902 encodeULEB128(CalleeSamples, OS);
903 }
904 }
905
906 // Recursively emit all the callsite samples.
907 uint64_t NumCallsites = 0;
908 for (const auto &J : S.getCallsiteSamples())
909 NumCallsites += J.second.size();
910 encodeULEB128(NumCallsites, OS);
911 for (const auto &J : S.getCallsiteSamples())
912 for (const auto &FS : J.second) {
913 LineLocation Loc = J.first;
914 const FunctionSamples &CalleeSamples = FS.second;
917 if (std::error_code EC = writeBody(CalleeSamples))
918 return EC;
919 }
920
922}
923
924/// Write samples of a top-level function to a binary file.
925///
926/// \returns true if the samples were written successfully, false otherwise.
927std::error_code
930 return writeBody(S);
931}
932
933std::error_code
935 uint64_t Offset = OutputStream->tell();
936 StringRef Name = S.getName();
939 return writeBody(S);
940}
941
942/// Create a sample profile file writer based on the specified format.
943///
944/// \param Filename The file to create.
945///
946/// \param Format Encoding format for the profile file.
947///
948/// \returns an error code indicating the status of the created writer.
951 std::error_code EC;
952 std::unique_ptr<raw_ostream> OS;
953 if (Format == SPF_Binary || Format == SPF_Ext_Binary ||
955 OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_None));
956 else
957 OS.reset(new raw_fd_ostream(Filename, EC, sys::fs::OF_TextWithCRLF));
958 if (EC)
959 return EC;
960
961 return create(OS, Format);
962}
963
964/// Create a sample profile stream writer based on the specified format.
965///
966/// \param OS The output stream to store the profile data to.
967///
968/// \param Format Encoding format for the profile file.
969///
970/// \returns an error code indicating the status of the created writer.
972SampleProfileWriter::create(std::unique_ptr<raw_ostream> &OS,
974 std::error_code EC;
975 std::unique_ptr<SampleProfileWriter> Writer;
976
977 // Currently only Text and Extended Binary format are supported for CSSPGO.
981
982 if (Format == SPF_Binary)
983 Writer.reset(new SampleProfileWriterRawBinary(OS));
984 else if (Format == SPF_Ext_Binary)
985 Writer.reset(new SampleProfileWriterExtBinary(OS));
986 else if (Format == SPF_Compact_Binary)
987 Writer.reset(new SampleProfileWriterCompactBinary(OS));
988 else if (Format == SPF_Text)
989 Writer.reset(new SampleProfileWriterText(OS));
990 else if (Format == SPF_GCC)
992 else
994
995 if (EC)
996 return EC;
997
998 Writer->Format = Format;
999 return std::move(Writer);
1000}
1001
1004 Summary = Builder.computeSummaryForProfiles(ProfileMap);
1005}
amdgpu Simplify well known AMD library false FunctionCallee Callee
assume Assume Builder
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DEBUG(X)
Definition: Debug.h:101
std::string Name
Provides ErrorOr<T> smart pointer.
#define I(x, y, z)
Definition: MD5.cpp:58
LLVMContext & Context
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
endianness Endian
static void splitProfileMapToTwo(const SampleProfileMap &ProfileMap, SampleProfileMap &ContextProfileMap, SampleProfileMap &NoContextProfileMap)
raw_pwrite_stream & OS
Represents either an error or a value T.
Definition: ErrorOr.h:56
This class implements a map that also provides access to all stored values in a deterministic order.
Definition: MapVector.h:37
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
Definition: ProfileCommon.h:65
size_t size() const
Definition: SmallVector.h:91
pointer data()
Return a pointer to the vector's buffer, even if empty().
Definition: SmallVector.h:289
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
unsigned size() const
Definition: StringMap.h:95
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:454
uint64_t tell() const
tell - Return the current offset with the file.
Definition: raw_ostream.h:134
raw_ostream & indent(unsigned NumSpaces)
indent - Insert 'NumSpaces' spaces.
An abstract base class for streams implementations that also support a pwrite operation.
Definition: raw_ostream.h:428
void pwrite(const char *Ptr, size_t Size, uint64_t Offset)
Definition: raw_ostream.h:436
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:642
A raw_ostream that writes to an SmallVector or SmallString.
Definition: raw_ostream.h:672
DefaultFunctionPruningStrategy(SampleProfileMap &ProfileMap, size_t OutputSizeLimit)
void Erase(size_t CurrentOutputSize) override
In this default implementation, functions with fewest samples are dropped first.
When writing a profile with size limit, user may want to use a different strategy to reduce function ...
virtual void Erase(size_t CurrentOutputSize)=0
SampleProfileWriter::writeWithSizeLimit() calls this after every write iteration if the output size s...
Representation of the samples collected for a function.
Definition: SampleProf.h:726
static constexpr const char * UniqSuffix
Definition: SampleProf.h:1046
uint64_t getHeadSamples() const
For top-level functions, return the total number of branch samples that have the function as the bran...
Definition: SampleProf.h:899
static bool ProfileIsFS
If this profile uses flow sensitive discriminators.
Definition: SampleProf.h:1143
SampleContext & getContext() const
Definition: SampleProf.h:1132
uint64_t getTotalSamples() const
Return the total number of samples collected inside the function.
Definition: SampleProf.h:891
const CallsiteSampleMap & getCallsiteSamples() const
Return all the callsite samples collected in the body of the function.
Definition: SampleProf.h:933
StringRef getName() const
Return the function name.
Definition: SampleProf.h:1025
const BodySampleMap & getBodySamples() const
Return all the samples collected in the body of the function.
Definition: SampleProf.h:930
std::error_code write(raw_ostream &OS)
Definition: SampleProf.cpp:439
std::string toString() const
Definition: SampleProf.h:621
void stablizeNameTable(MapVector< StringRef, uint32_t > &NameTable, std::set< StringRef > &V)
virtual void addContext(const SampleContext &Context)
virtual std::error_code writeMagicIdent(SampleProfileFormat Format)
std::error_code writeNameIdx(StringRef FName)
virtual std::error_code writeContextIdx(const SampleContext &Context)
std::error_code writeSample(const FunctionSamples &S) override
Write samples of a top-level function to a binary file.
std::error_code writeHeader(const SampleProfileMap &ProfileMap) override
Write a file header for the profile file.
virtual MapVector< StringRef, uint32_t > & getNameTable()
MapVector< StringRef, uint32_t > NameTable
std::error_code writeBody(const FunctionSamples &S)
uint64_t TableOffset
The offset of the slot to be filled with the offset of FuncOffsetTable towards profile start.
std::error_code writeSample(const FunctionSamples &S) override
Write samples of a top-level function to a binary file.
std::error_code writeHeader(const SampleProfileMap &ProfileMap) override
Write a file header for the profile file.
std::error_code write(const SampleProfileMap &ProfileMap) override
Write all the sample profiles in the given map of samples.
MapVector< StringRef, uint64_t > FuncOffsetTable
The table mapping from function name to the offset of its FunctionSample towards profile start.
std::error_code writeNameTableSection(const SampleProfileMap &ProfileMap)
SmallVector< SecHdrTableEntry, 8 > SectionHdrLayout
std::error_code writeFuncMetadata(const SampleProfileMap &Profiles)
virtual std::error_code writeCustomSection(SecType Type)=0
virtual std::error_code writeOneSection(SecType Type, uint32_t LayoutIdx, const SampleProfileMap &ProfileMap)
std::error_code writeCSNameIdx(const SampleContext &Context)
virtual std::error_code writeSections(const SampleProfileMap &ProfileMap)=0
void addSectionFlag(SecType Type, SecFlagType Flag)
uint64_t markSectionStart(SecType Type, uint32_t LayoutIdx)
Return the current position and prepare to use it as the start position of a section given the sectio...
void addContext(const SampleContext &Context) override
std::error_code addNewSection(SecType Sec, uint32_t LayoutIdx, uint64_t SectionStart)
Add a new section into section header table given the section type Type, its position LayoutIdx in Se...
std::error_code write(const SampleProfileMap &ProfileMap) override
Write all the sample profiles in the given map of samples.
std::error_code writeContextIdx(const SampleContext &Context) override
std::error_code writeSample(const FunctionSamples &S) override
Write samples of a top-level function to a binary file.
Sample-based profile writer (text format).
std::error_code writeSample(const FunctionSamples &S) override
Write samples to a text file.
std::unique_ptr< ProfileSummary > Summary
Profile summary.
virtual std::error_code writeSample(const FunctionSamples &S)=0
Write sample profiles in S.
SampleProfileFormat Format
Profile format.
std::error_code writeWithSizeLimitInternal(SampleProfileMap &ProfileMap, size_t OutputSizeLimit, FunctionPruningStrategy *Strategy)
void computeSummary(const SampleProfileMap &ProfileMap)
Compute summary for this profile.
virtual std::error_code writeFuncProfiles(const SampleProfileMap &ProfileMap)
std::unique_ptr< raw_ostream > OutputStream
Output stream where to emit the profile to.
size_t LineCount
For writeWithSizeLimit in text mode, each newline takes 1 additional byte on Windows when actually wr...
static ErrorOr< std::unique_ptr< SampleProfileWriter > > create(StringRef Filename, SampleProfileFormat Format)
Profile writer factory.
virtual std::error_code writeHeader(const SampleProfileMap &ProfileMap)=0
Write a file header for the profile file.
virtual std::error_code write(const SampleProfileMap &ProfileMap)
Write all the sample profiles in the given map of samples.
Representation of a single sample record.
Definition: SampleProf.h:325
const CallTargetMap & getCallTargets() const
Definition: SampleProf.h:393
uint64_t getSamples() const
Definition: SampleProf.h:392
const SortedCallTargetSet getSortedCallTargets() const
Definition: SampleProf.h:394
Sort a LocationT->SampleT map by LocationT.
Definition: SampleProf.h:1233
const SamplesWithLocList & get() const
Definition: SampleProf.h:1246
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
void compress(ArrayRef< uint8_t > Input, SmallVectorImpl< uint8_t > &CompressedBuffer, int Level=DefaultCompression)
constexpr int BestSizeCompression
Definition: Compression.h:39
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
void sortFuncProfiles(const SampleProfileMap &ProfileMap, std::vector< NameFunctionSamples > &SortedProfiles)
Definition: SampleProf.cpp:201
std::unordered_map< SampleContext, FunctionSamples, SampleContext::Hash > SampleProfileMap
Definition: SampleProf.h:1222
static uint64_t SPMagic(SampleProfileFormat Format=SPF_Binary)
Definition: SampleProf.h:99
std::pair< SampleContext, const FunctionSamples * > NameFunctionSamples
Definition: SampleProf.h:1224
static void addSecFlag(SecHdrTableEntry &Entry, SecFlagType Flag)
Definition: SampleProf.h:251
static bool hasSecFlag(const SecHdrTableEntry &Entry, SecFlagType Flag)
Definition: SampleProf.h:267
@ SecFlagIsPreInlined
SecFlagIsPreInlined means this profile contains ShouldBeInlined contexts thus this is CS preinliner c...
@ SecFlagFSDiscriminator
SecFlagFSDiscriminator means this profile uses flow-sensitive discriminators.
@ SecFlagFullContext
SecFlagContext means this is context-sensitive flat profile for CSSPGO.
static uint64_t SPVersion()
Definition: SampleProf.h:116
@ OF_TextWithCRLF
The file should be opened in text mode and use a carriage linefeed '\r '.
Definition: FileSystem.h:770
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:406
UnaryFunction for_each(R &&Range, UnaryFunction F)
Provide wrappers to std::for_each which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1775
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
uint64_t MD5Hash(StringRef Str)
Helper to compute and return lower 64 bits of the given string's MD5 hash.
Definition: MD5.h:109
unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)
Utility function to encode a ULEB128 value to an output stream.
Definition: LEB128.h:80
#define N
Represents the relative location of an instruction.
Definition: SampleProf.h:283
Adapter to write values to a stream in a particular byte order.
Definition: EndianStream.h:52
void write(ArrayRef< value_type > Val)
Definition: EndianStream.h:56
static uint64_t round(uint64_t Acc, uint64_t Input)
Definition: xxhash.cpp:56