LLVM 19.0.0git
InstrProfWriter.cpp
Go to the documentation of this file.
1//===- InstrProfWriter.cpp - Instrumented profiling writer ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains support for writing profiling data for clang's
10// instrumentation based PGO and coverage.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/STLExtras.h"
16#include "llvm/ADT/SetVector.h"
17#include "llvm/ADT/StringRef.h"
22#include "llvm/Support/Endian.h"
24#include "llvm/Support/Error.h"
28#include <cstdint>
29#include <memory>
30#include <string>
31#include <tuple>
32#include <utility>
33#include <vector>
34
35using namespace llvm;
36
37// A struct to define how the data stream should be patched. For Indexed
38// profiling, only uint64_t data type is needed.
39struct PatchItem {
40 uint64_t Pos; // Where to patch.
41 uint64_t *D; // Pointer to an array of source data.
42 int N; // Number of elements in \c D array.
43};
44
45namespace llvm {
46
47// A wrapper class to abstract writer stream with support of bytes
48// back patching.
50public:
52 : IsFDOStream(true), OS(FD), LE(FD, llvm::endianness::little) {}
54 : IsFDOStream(false), OS(STR), LE(STR, llvm::endianness::little) {}
55
56 uint64_t tell() { return OS.tell(); }
57 void write(uint64_t V) { LE.write<uint64_t>(V); }
58 void writeByte(uint8_t V) { LE.write<uint8_t>(V); }
59
60 // \c patch can only be called when all data is written and flushed.
61 // For raw_string_ostream, the patch is done on the target string
62 // directly and it won't be reflected in the stream's internal buffer.
63 void patch(PatchItem *P, int NItems) {
64 using namespace support;
65
66 if (IsFDOStream) {
67 raw_fd_ostream &FDOStream = static_cast<raw_fd_ostream &>(OS);
68 const uint64_t LastPos = FDOStream.tell();
69 for (int K = 0; K < NItems; K++) {
70 FDOStream.seek(P[K].Pos);
71 for (int I = 0; I < P[K].N; I++)
72 write(P[K].D[I]);
73 }
74 // Reset the stream to the last position after patching so that users
75 // don't accidentally overwrite data. This makes it consistent with
76 // the string stream below which replaces the data directly.
77 FDOStream.seek(LastPos);
78 } else {
79 raw_string_ostream &SOStream = static_cast<raw_string_ostream &>(OS);
80 std::string &Data = SOStream.str(); // with flush
81 for (int K = 0; K < NItems; K++) {
82 for (int I = 0; I < P[K].N; I++) {
83 uint64_t Bytes =
84 endian::byte_swap<uint64_t, llvm::endianness::little>(P[K].D[I]);
85 Data.replace(P[K].Pos + I * sizeof(uint64_t), sizeof(uint64_t),
86 (const char *)&Bytes, sizeof(uint64_t));
87 }
88 }
89 }
90 }
91
92 // If \c OS is an instance of \c raw_fd_ostream, this field will be
93 // true. Otherwise, \c OS will be an raw_string_ostream.
97};
98
100public:
103
106
109
113
115
118 }
119
120 static std::pair<offset_type, offset_type>
122 using namespace support;
123
125
126 offset_type N = K.size();
127 LE.write<offset_type>(N);
128
129 offset_type M = 0;
130 for (const auto &ProfileData : *V) {
131 const InstrProfRecord &ProfRecord = ProfileData.second;
132 M += sizeof(uint64_t); // The function hash
133 M += sizeof(uint64_t); // The size of the Counts vector
134 M += ProfRecord.Counts.size() * sizeof(uint64_t);
135 M += sizeof(uint64_t); // The size of the Bitmap vector
136 M += ProfRecord.BitmapBytes.size() * sizeof(uint64_t);
137
138 // Value data
139 M += ValueProfData::getSize(ProfileData.second);
140 }
141 LE.write<offset_type>(M);
142
143 return std::make_pair(N, M);
144 }
145
147 Out.write(K.data(), N);
148 }
149
151 using namespace support;
152
154 for (const auto &ProfileData : *V) {
155 const InstrProfRecord &ProfRecord = ProfileData.second;
156 if (NamedInstrProfRecord::hasCSFlagInHash(ProfileData.first))
157 CSSummaryBuilder->addRecord(ProfRecord);
158 else
159 SummaryBuilder->addRecord(ProfRecord);
160
161 LE.write<uint64_t>(ProfileData.first); // Function hash
162 LE.write<uint64_t>(ProfRecord.Counts.size());
163 for (uint64_t I : ProfRecord.Counts)
164 LE.write<uint64_t>(I);
165
166 LE.write<uint64_t>(ProfRecord.BitmapBytes.size());
167 for (uint64_t I : ProfRecord.BitmapBytes)
168 LE.write<uint64_t>(I);
169
170 // Write value data
171 std::unique_ptr<ValueProfData> VDataPtr =
172 ValueProfData::serializeFrom(ProfileData.second);
173 uint32_t S = VDataPtr->getSize();
174 VDataPtr->swapBytesFromHost(ValueProfDataEndianness);
175 Out.write((const char *)VDataPtr.get(), S);
176 }
177 }
178};
179
180} // end namespace llvm
181
183 uint64_t TemporalProfTraceReservoirSize,
184 uint64_t MaxTemporalProfTraceLength)
185 : Sparse(Sparse), MaxTemporalProfTraceLength(MaxTemporalProfTraceLength),
186 TemporalProfTraceReservoirSize(TemporalProfTraceReservoirSize),
187 InfoObj(new InstrProfRecordWriterTrait()) {}
188
190
191// Internal interface for testing purpose only.
193 InfoObj->ValueProfDataEndianness = Endianness;
194}
195
197 this->Sparse = Sparse;
198}
199
201 function_ref<void(Error)> Warn) {
202 auto Name = I.Name;
203 auto Hash = I.Hash;
204 addRecord(Name, Hash, std::move(I), Weight, Warn);
205}
206
208 OverlapStats &Overlap,
209 OverlapStats &FuncLevelOverlap,
210 const OverlapFuncFilters &FuncFilter) {
211 auto Name = Other.Name;
212 auto Hash = Other.Hash;
213 Other.accumulateCounts(FuncLevelOverlap.Test);
214 if (!FunctionData.contains(Name)) {
215 Overlap.addOneUnique(FuncLevelOverlap.Test);
216 return;
217 }
218 if (FuncLevelOverlap.Test.CountSum < 1.0f) {
219 Overlap.Overlap.NumEntries += 1;
220 return;
221 }
222 auto &ProfileDataMap = FunctionData[Name];
223 bool NewFunc;
225 std::tie(Where, NewFunc) =
226 ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord()));
227 if (NewFunc) {
228 Overlap.addOneMismatch(FuncLevelOverlap.Test);
229 return;
230 }
231 InstrProfRecord &Dest = Where->second;
232
233 uint64_t ValueCutoff = FuncFilter.ValueCutoff;
234 if (!FuncFilter.NameFilter.empty() && Name.contains(FuncFilter.NameFilter))
235 ValueCutoff = 0;
236
237 Dest.overlap(Other, Overlap, FuncLevelOverlap, ValueCutoff);
238}
239
241 InstrProfRecord &&I, uint64_t Weight,
242 function_ref<void(Error)> Warn) {
243 auto &ProfileDataMap = FunctionData[Name];
244
245 bool NewFunc;
247 std::tie(Where, NewFunc) =
248 ProfileDataMap.insert(std::make_pair(Hash, InstrProfRecord()));
249 InstrProfRecord &Dest = Where->second;
250
251 auto MapWarn = [&](instrprof_error E) {
252 Warn(make_error<InstrProfError>(E));
253 };
254
255 if (NewFunc) {
256 // We've never seen a function with this name and hash, add it.
257 Dest = std::move(I);
258 if (Weight > 1)
259 Dest.scale(Weight, 1, MapWarn);
260 } else {
261 // We're updating a function we've seen before.
262 Dest.merge(I, Weight, MapWarn);
263 }
264
265 Dest.sortValueData();
266}
267
270 auto Result = MemProfRecordData.insert({Id, Record});
271 // If we inserted a new record then we are done.
272 if (Result.second) {
273 return;
274 }
275 memprof::IndexedMemProfRecord &Existing = Result.first->second;
276 Existing.merge(Record);
277}
278
280 const memprof::Frame &Frame,
281 function_ref<void(Error)> Warn) {
282 auto Result = MemProfFrameData.insert({Id, Frame});
283 // If a mapping already exists for the current frame id and it does not
284 // match the new mapping provided then reset the existing contents and bail
285 // out. We don't support the merging of memprof data whose Frame -> Id
286 // mapping across profiles is inconsistent.
287 if (!Result.second && Result.first->second != Frame) {
288 Warn(make_error<InstrProfError>(instrprof_error::malformed,
289 "frame to id mapping mismatch"));
290 return false;
291 }
292 return true;
293}
294
296 llvm::append_range(BinaryIds, BIs);
297}
298
299void InstrProfWriter::addTemporalProfileTrace(TemporalProfTraceTy Trace) {
300 if (Trace.FunctionNameRefs.size() > MaxTemporalProfTraceLength)
301 Trace.FunctionNameRefs.resize(MaxTemporalProfTraceLength);
302 if (Trace.FunctionNameRefs.empty())
303 return;
304
305 if (TemporalProfTraceStreamSize < TemporalProfTraceReservoirSize) {
306 // Simply append the trace if we have not yet hit our reservoir size limit.
307 TemporalProfTraces.push_back(std::move(Trace));
308 } else {
309 // Otherwise, replace a random trace in the stream.
310 std::uniform_int_distribution<uint64_t> Distribution(
311 0, TemporalProfTraceStreamSize);
312 uint64_t RandomIndex = Distribution(RNG);
313 if (RandomIndex < TemporalProfTraces.size())
314 TemporalProfTraces[RandomIndex] = std::move(Trace);
315 }
316 ++TemporalProfTraceStreamSize;
317}
318
320 SmallVectorImpl<TemporalProfTraceTy> &SrcTraces, uint64_t SrcStreamSize) {
321 // Assume that the source has the same reservoir size as the destination to
322 // avoid needing to record it in the indexed profile format.
323 bool IsDestSampled =
324 (TemporalProfTraceStreamSize > TemporalProfTraceReservoirSize);
325 bool IsSrcSampled = (SrcStreamSize > TemporalProfTraceReservoirSize);
326 if (!IsDestSampled && IsSrcSampled) {
327 // If one of the traces are sampled, ensure that it belongs to Dest.
328 std::swap(TemporalProfTraces, SrcTraces);
329 std::swap(TemporalProfTraceStreamSize, SrcStreamSize);
330 std::swap(IsDestSampled, IsSrcSampled);
331 }
332 if (!IsSrcSampled) {
333 // If the source stream is not sampled, we add each source trace normally.
334 for (auto &Trace : SrcTraces)
335 addTemporalProfileTrace(std::move(Trace));
336 return;
337 }
338 // Otherwise, we find the traces that would have been removed if we added
339 // the whole source stream.
340 SmallSetVector<uint64_t, 8> IndicesToReplace;
341 for (uint64_t I = 0; I < SrcStreamSize; I++) {
342 std::uniform_int_distribution<uint64_t> Distribution(
343 0, TemporalProfTraceStreamSize);
344 uint64_t RandomIndex = Distribution(RNG);
345 if (RandomIndex < TemporalProfTraces.size())
346 IndicesToReplace.insert(RandomIndex);
347 ++TemporalProfTraceStreamSize;
348 }
349 // Then we insert a random sample of the source traces.
350 llvm::shuffle(SrcTraces.begin(), SrcTraces.end(), RNG);
351 for (const auto &[Index, Trace] : llvm::zip(IndicesToReplace, SrcTraces))
352 TemporalProfTraces[Index] = std::move(Trace);
353}
354
356 function_ref<void(Error)> Warn) {
357 for (auto &I : IPW.FunctionData)
358 for (auto &Func : I.getValue())
359 addRecord(I.getKey(), Func.first, std::move(Func.second), 1, Warn);
360
361 BinaryIds.reserve(BinaryIds.size() + IPW.BinaryIds.size());
362 for (auto &I : IPW.BinaryIds)
364
365 addTemporalProfileTraces(IPW.TemporalProfTraces,
366 IPW.TemporalProfTraceStreamSize);
367
368 MemProfFrameData.reserve(IPW.MemProfFrameData.size());
369 for (auto &I : IPW.MemProfFrameData) {
370 // If we weren't able to add the frame mappings then it doesn't make sense
371 // to try to merge the records from this profile.
372 if (!addMemProfFrame(I.first, I.second, Warn))
373 return;
374 }
375
376 MemProfRecordData.reserve(IPW.MemProfRecordData.size());
377 for (auto &I : IPW.MemProfRecordData) {
378 addMemProfRecord(I.first, I.second);
379 }
380}
381
382bool InstrProfWriter::shouldEncodeData(const ProfilingData &PD) {
383 if (!Sparse)
384 return true;
385 for (const auto &Func : PD) {
386 const InstrProfRecord &IPR = Func.second;
387 if (llvm::any_of(IPR.Counts, [](uint64_t Count) { return Count > 0; }))
388 return true;
389 if (llvm::any_of(IPR.BitmapBytes, [](uint8_t Byte) { return Byte > 0; }))
390 return true;
391 }
392 return false;
393}
394
395static void setSummary(IndexedInstrProf::Summary *TheSummary,
396 ProfileSummary &PS) {
397 using namespace IndexedInstrProf;
398
399 const std::vector<ProfileSummaryEntry> &Res = PS.getDetailedSummary();
400 TheSummary->NumSummaryFields = Summary::NumKinds;
401 TheSummary->NumCutoffEntries = Res.size();
402 TheSummary->set(Summary::MaxFunctionCount, PS.getMaxFunctionCount());
403 TheSummary->set(Summary::MaxBlockCount, PS.getMaxCount());
404 TheSummary->set(Summary::MaxInternalBlockCount, PS.getMaxInternalCount());
405 TheSummary->set(Summary::TotalBlockCount, PS.getTotalCount());
406 TheSummary->set(Summary::TotalNumBlocks, PS.getNumCounts());
407 TheSummary->set(Summary::TotalNumFunctions, PS.getNumFunctions());
408 for (unsigned I = 0; I < Res.size(); I++)
409 TheSummary->setEntry(I, Res[I]);
410}
411
412Error InstrProfWriter::writeImpl(ProfOStream &OS) {
413 using namespace IndexedInstrProf;
414 using namespace support;
415
417
419 InfoObj->SummaryBuilder = &ISB;
421 InfoObj->CSSummaryBuilder = &CSISB;
422
423 // Populate the hash table generator.
425 for (const auto &I : FunctionData)
426 if (shouldEncodeData(I.getValue()))
427 OrderedData.emplace_back((I.getKey()), &I.getValue());
428 llvm::sort(OrderedData, less_first());
429 for (const auto &I : OrderedData)
430 Generator.insert(I.first, I.second);
431
432 // Write the header.
434 Header.Magic = IndexedInstrProf::Magic;
436 if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
437 Header.Version |= VARIANT_MASK_IR_PROF;
438 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
439 Header.Version |= VARIANT_MASK_CSIR_PROF;
440 if (static_cast<bool>(ProfileKind &
442 Header.Version |= VARIANT_MASK_INSTR_ENTRY;
443 if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
444 Header.Version |= VARIANT_MASK_BYTE_COVERAGE;
445 if (static_cast<bool>(ProfileKind & InstrProfKind::FunctionEntryOnly))
446 Header.Version |= VARIANT_MASK_FUNCTION_ENTRY_ONLY;
447 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf))
448 Header.Version |= VARIANT_MASK_MEMPROF;
449 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
450 Header.Version |= VARIANT_MASK_TEMPORAL_PROF;
451
452 Header.Unused = 0;
453 Header.HashType = static_cast<uint64_t>(IndexedInstrProf::HashType);
454 Header.HashOffset = 0;
455 Header.MemProfOffset = 0;
456 Header.BinaryIdOffset = 0;
457 Header.TemporalProfTracesOffset = 0;
458 Header.VTableNamesOffset = 0;
459
460 // Only write out the first four fields. We need to remember the offset of the
461 // remaining fields to allow back patching later.
462 for (int I = 0; I < 4; I++)
463 OS.write(reinterpret_cast<uint64_t *>(&Header)[I]);
464
465 // Save the location of Header.HashOffset field in \c OS.
466 uint64_t HashTableStartFieldOffset = OS.tell();
467 // Reserve the space for HashOffset field.
468 OS.write(0);
469
470 // Save the location of MemProf profile data. This is stored in two parts as
471 // the schema and as a separate on-disk chained hashtable.
472 uint64_t MemProfSectionOffset = OS.tell();
473 // Reserve space for the MemProf table field to be patched later if this
474 // profile contains memory profile information.
475 OS.write(0);
476
477 // Save the location of binary ids section.
478 uint64_t BinaryIdSectionOffset = OS.tell();
479 // Reserve space for the BinaryIdOffset field to be patched later if this
480 // profile contains binary ids.
481 OS.write(0);
482
483 uint64_t TemporalProfTracesOffset = OS.tell();
484 OS.write(0);
485
486 uint64_t VTableNamesOffset = OS.tell();
487 OS.write(0);
488
489 // Reserve space to write profile summary data.
491 uint32_t SummarySize = Summary::getSize(Summary::NumKinds, NumEntries);
492 // Remember the summary offset.
493 uint64_t SummaryOffset = OS.tell();
494 for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
495 OS.write(0);
496 uint64_t CSSummaryOffset = 0;
497 uint64_t CSSummarySize = 0;
498 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
499 CSSummaryOffset = OS.tell();
500 CSSummarySize = SummarySize / sizeof(uint64_t);
501 for (unsigned I = 0; I < CSSummarySize; I++)
502 OS.write(0);
503 }
504
505 // Write the hash table.
506 uint64_t HashTableStart = Generator.Emit(OS.OS, *InfoObj);
507
508 // Write the MemProf profile data if we have it. This includes a simple schema
509 // with the format described below followed by the hashtable:
510 // uint64_t RecordTableOffset = RecordTableGenerator.Emit
511 // uint64_t FramePayloadOffset = Stream offset before emitting the frame table
512 // uint64_t FrameTableOffset = FrameTableGenerator.Emit
513 // uint64_t Num schema entries
514 // uint64_t Schema entry 0
515 // uint64_t Schema entry 1
516 // ....
517 // uint64_t Schema entry N - 1
518 // OnDiskChainedHashTable MemProfRecordData
519 // OnDiskChainedHashTable MemProfFrameData
520 uint64_t MemProfSectionStart = 0;
521 if (static_cast<bool>(ProfileKind & InstrProfKind::MemProf)) {
522 MemProfSectionStart = OS.tell();
523 OS.write(0ULL); // Reserve space for the memprof record table offset.
524 OS.write(0ULL); // Reserve space for the memprof frame payload offset.
525 OS.write(0ULL); // Reserve space for the memprof frame table offset.
526
528 OS.write(static_cast<uint64_t>(Schema.size()));
529 for (const auto Id : Schema) {
530 OS.write(static_cast<uint64_t>(Id));
531 }
532
533 auto RecordWriter = std::make_unique<memprof::RecordWriterTrait>();
534 RecordWriter->Schema = &Schema;
536 RecordTableGenerator;
537 for (auto &I : MemProfRecordData) {
538 // Insert the key (func hash) and value (memprof record).
539 RecordTableGenerator.insert(I.first, I.second);
540 }
541 // Release the memory of this MapVector as it is no longer needed.
542 MemProfRecordData.clear();
543
544 // The call to Emit invokes RecordWriterTrait::EmitData which destructs
545 // the memprof record copies owned by the RecordTableGenerator. This works
546 // because the RecordTableGenerator is not used after this point.
547 uint64_t RecordTableOffset =
548 RecordTableGenerator.Emit(OS.OS, *RecordWriter);
549
550 uint64_t FramePayloadOffset = OS.tell();
551
552 auto FrameWriter = std::make_unique<memprof::FrameWriterTrait>();
554 FrameTableGenerator;
555 for (auto &I : MemProfFrameData) {
556 // Insert the key (frame id) and value (frame contents).
557 FrameTableGenerator.insert(I.first, I.second);
558 }
559 // Release the memory of this MapVector as it is no longer needed.
560 MemProfFrameData.clear();
561
562 uint64_t FrameTableOffset = FrameTableGenerator.Emit(OS.OS, *FrameWriter);
563
564 PatchItem PatchItems[] = {
565 {MemProfSectionStart, &RecordTableOffset, 1},
566 {MemProfSectionStart + sizeof(uint64_t), &FramePayloadOffset, 1},
567 {MemProfSectionStart + 2 * sizeof(uint64_t), &FrameTableOffset, 1},
568 };
569 OS.patch(PatchItems, 3);
570 }
571
572 // BinaryIdSection has two parts:
573 // 1. uint64_t BinaryIdsSectionSize
574 // 2. list of binary ids that consist of:
575 // a. uint64_t BinaryIdLength
576 // b. uint8_t BinaryIdData
577 // c. uint8_t Padding (if necessary)
578 uint64_t BinaryIdSectionStart = OS.tell();
579 // Calculate size of binary section.
580 uint64_t BinaryIdsSectionSize = 0;
581
582 // Remove duplicate binary ids.
583 llvm::sort(BinaryIds);
584 BinaryIds.erase(std::unique(BinaryIds.begin(), BinaryIds.end()),
585 BinaryIds.end());
586
587 for (auto BI : BinaryIds) {
588 // Increment by binary id length data type size.
589 BinaryIdsSectionSize += sizeof(uint64_t);
590 // Increment by binary id data length, aligned to 8 bytes.
591 BinaryIdsSectionSize += alignToPowerOf2(BI.size(), sizeof(uint64_t));
592 }
593 // Write binary ids section size.
594 OS.write(BinaryIdsSectionSize);
595
596 for (auto BI : BinaryIds) {
597 uint64_t BILen = BI.size();
598 // Write binary id length.
599 OS.write(BILen);
600 // Write binary id data.
601 for (unsigned K = 0; K < BILen; K++)
602 OS.writeByte(BI[K]);
603 // Write padding if necessary.
604 uint64_t PaddingSize = alignToPowerOf2(BILen, sizeof(uint64_t)) - BILen;
605 for (unsigned K = 0; K < PaddingSize; K++)
606 OS.writeByte(0);
607 }
608
609 uint64_t VTableNamesSectionStart = OS.tell();
610
611 // Use a dummy (and uncompressed) string as compressed vtable names and get
612 // the necessary profile format change in place for version 12.
613 // TODO: Store the list of vtable names in InstrProfWriter and use the
614 // real compressed name.
615 std::string CompressedVTableNames = "VTableNames";
616
617 uint64_t CompressedStringLen = CompressedVTableNames.length();
618
619 // Record the length of compressed string.
620 OS.write(CompressedStringLen);
621
622 // Write the chars in compressed strings.
623 for (auto &c : CompressedVTableNames)
624 OS.writeByte(static_cast<uint8_t>(c));
625
626 // Pad up to a multiple of 8.
627 // InstrProfReader would read bytes according to 'CompressedStringLen'.
628 uint64_t PaddedLength = alignTo(CompressedStringLen, 8);
629
630 for (uint64_t K = CompressedStringLen; K < PaddedLength; K++) {
631 OS.writeByte(0);
632 }
633
634 uint64_t TemporalProfTracesSectionStart = 0;
635 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile)) {
636 TemporalProfTracesSectionStart = OS.tell();
637 OS.write(TemporalProfTraces.size());
638 OS.write(TemporalProfTraceStreamSize);
639 for (auto &Trace : TemporalProfTraces) {
640 OS.write(Trace.Weight);
641 OS.write(Trace.FunctionNameRefs.size());
642 for (auto &NameRef : Trace.FunctionNameRefs)
643 OS.write(NameRef);
644 }
645 }
646
647 // Allocate space for data to be serialized out.
648 std::unique_ptr<IndexedInstrProf::Summary> TheSummary =
650 // Compute the Summary and copy the data to the data
651 // structure to be serialized out (to disk or buffer).
652 std::unique_ptr<ProfileSummary> PS = ISB.getSummary();
653 setSummary(TheSummary.get(), *PS);
654 InfoObj->SummaryBuilder = nullptr;
655
656 // For Context Sensitive summary.
657 std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary = nullptr;
658 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive)) {
659 TheCSSummary = IndexedInstrProf::allocSummary(SummarySize);
660 std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary();
661 setSummary(TheCSSummary.get(), *CSPS);
662 }
663 InfoObj->CSSummaryBuilder = nullptr;
664
665 // Now do the final patch:
666 PatchItem PatchItems[] = {
667 // Patch the Header.HashOffset field.
668 {HashTableStartFieldOffset, &HashTableStart, 1},
669 // Patch the Header.MemProfOffset (=0 for profiles without MemProf
670 // data).
671 {MemProfSectionOffset, &MemProfSectionStart, 1},
672 // Patch the Header.BinaryIdSectionOffset.
673 {BinaryIdSectionOffset, &BinaryIdSectionStart, 1},
674 // Patch the Header.TemporalProfTracesOffset (=0 for profiles without
675 // traces).
676 {TemporalProfTracesOffset, &TemporalProfTracesSectionStart, 1},
677 {VTableNamesOffset, &VTableNamesSectionStart, 1},
678 // Patch the summary data.
679 {SummaryOffset, reinterpret_cast<uint64_t *>(TheSummary.get()),
680 (int)(SummarySize / sizeof(uint64_t))},
681 {CSSummaryOffset, reinterpret_cast<uint64_t *>(TheCSSummary.get()),
682 (int)CSSummarySize}};
683
684 OS.patch(PatchItems, std::size(PatchItems));
685
686 for (const auto &I : FunctionData)
687 for (const auto &F : I.getValue())
688 if (Error E = validateRecord(F.second))
689 return E;
690
691 return Error::success();
692}
693
695 // Write the hash table.
696 ProfOStream POS(OS);
697 return writeImpl(POS);
698}
699
701 ProfOStream POS(OS);
702 return writeImpl(POS);
703}
704
705std::unique_ptr<MemoryBuffer> InstrProfWriter::writeBuffer() {
706 std::string Data;
708 // Write the hash table.
709 if (Error E = write(OS))
710 return nullptr;
711 // Return this in an aligned memory buffer.
713}
714
715static const char *ValueProfKindStr[] = {
716#define VALUE_PROF_KIND(Enumerator, Value, Descr) #Enumerator,
718};
719
721 for (uint32_t VK = 0; VK <= IPVK_Last; VK++) {
722 uint32_t NS = Func.getNumValueSites(VK);
723 if (!NS)
724 continue;
725 for (uint32_t S = 0; S < NS; S++) {
726 uint32_t ND = Func.getNumValueDataForSite(VK, S);
727 std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
728 DenseSet<uint64_t> SeenValues;
729 for (uint32_t I = 0; I < ND; I++)
730 if ((VK != IPVK_IndirectCallTarget && VK != IPVK_VTableTarget) &&
731 !SeenValues.insert(VD[I].Value).second)
732 return make_error<InstrProfError>(instrprof_error::invalid_prof);
733 }
734 }
735
736 return Error::success();
737}
738
740 const InstrProfRecord &Func,
741 InstrProfSymtab &Symtab,
743 OS << Name << "\n";
744 OS << "# Func Hash:\n" << Hash << "\n";
745 OS << "# Num Counters:\n" << Func.Counts.size() << "\n";
746 OS << "# Counter Values:\n";
747 for (uint64_t Count : Func.Counts)
748 OS << Count << "\n";
749
750 if (Func.BitmapBytes.size() > 0) {
751 OS << "# Num Bitmap Bytes:\n$" << Func.BitmapBytes.size() << "\n";
752 OS << "# Bitmap Byte Values:\n";
753 for (uint8_t Byte : Func.BitmapBytes) {
754 OS << "0x";
755 OS.write_hex(Byte);
756 OS << "\n";
757 }
758 OS << "\n";
759 }
760
761 uint32_t NumValueKinds = Func.getNumValueKinds();
762 if (!NumValueKinds) {
763 OS << "\n";
764 return;
765 }
766
767 OS << "# Num Value Kinds:\n" << Func.getNumValueKinds() << "\n";
768 for (uint32_t VK = 0; VK < IPVK_Last + 1; VK++) {
769 uint32_t NS = Func.getNumValueSites(VK);
770 if (!NS)
771 continue;
772 OS << "# ValueKind = " << ValueProfKindStr[VK] << ":\n" << VK << "\n";
773 OS << "# NumValueSites:\n" << NS << "\n";
774 for (uint32_t S = 0; S < NS; S++) {
775 uint32_t ND = Func.getNumValueDataForSite(VK, S);
776 OS << ND << "\n";
777 std::unique_ptr<InstrProfValueData[]> VD = Func.getValueForSite(VK, S);
778 for (uint32_t I = 0; I < ND; I++) {
779 if (VK == IPVK_IndirectCallTarget || VK == IPVK_VTableTarget)
780 OS << Symtab.getFuncOrVarNameIfDefined(VD[I].Value) << ":"
781 << VD[I].Count << "\n";
782 else
783 OS << VD[I].Value << ":" << VD[I].Count << "\n";
784 }
785 }
786 }
787
788 OS << "\n";
789}
790
792 // Check CS first since it implies an IR level profile.
793 if (static_cast<bool>(ProfileKind & InstrProfKind::ContextSensitive))
794 OS << "# CSIR level Instrumentation Flag\n:csir\n";
795 else if (static_cast<bool>(ProfileKind & InstrProfKind::IRInstrumentation))
796 OS << "# IR level Instrumentation Flag\n:ir\n";
797
798 if (static_cast<bool>(ProfileKind &
800 OS << "# Always instrument the function entry block\n:entry_first\n";
801 if (static_cast<bool>(ProfileKind & InstrProfKind::SingleByteCoverage))
802 OS << "# Instrument block coverage\n:single_byte_coverage\n";
803 InstrProfSymtab Symtab;
804
806 using RecordType = std::pair<StringRef, FuncPair>;
807 SmallVector<RecordType, 4> OrderedFuncData;
808
809 for (const auto &I : FunctionData) {
810 if (shouldEncodeData(I.getValue())) {
811 if (Error E = Symtab.addFuncName(I.getKey()))
812 return E;
813 for (const auto &Func : I.getValue())
814 OrderedFuncData.push_back(std::make_pair(I.getKey(), Func));
815 }
816 }
817
818 if (static_cast<bool>(ProfileKind & InstrProfKind::TemporalProfile))
820
821 llvm::sort(OrderedFuncData, [](const RecordType &A, const RecordType &B) {
822 return std::tie(A.first, A.second.first) <
823 std::tie(B.first, B.second.first);
824 });
825
826 for (const auto &record : OrderedFuncData) {
827 const StringRef &Name = record.first;
828 const FuncPair &Func = record.second;
829 writeRecordInText(Name, Func.first, Func.second, Symtab, OS);
830 }
831
832 for (const auto &record : OrderedFuncData) {
833 const FuncPair &Func = record.second;
834 if (Error E = validateRecord(Func.second))
835 return E;
836 }
837
838 return Error::success();
839}
840
842 InstrProfSymtab &Symtab) {
843 OS << ":temporal_prof_traces\n";
844 OS << "# Num Temporal Profile Traces:\n" << TemporalProfTraces.size() << "\n";
845 OS << "# Temporal Profile Trace Stream Size:\n"
846 << TemporalProfTraceStreamSize << "\n";
847 for (auto &Trace : TemporalProfTraces) {
848 OS << "# Weight:\n" << Trace.Weight << "\n";
849 for (auto &NameRef : Trace.FunctionNameRefs)
850 OS << Symtab.getFuncOrVarName(NameRef) << ",";
851 OS << "\n";
852 }
853 OS << "\n";
854}
basic Basic Alias true
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
std::string Name
static void setSummary(IndexedInstrProf::Summary *TheSummary, ProfileSummary &PS)
static const char * ValueProfKindStr[]
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Defines facilities for reading and writing on-disk hash tables.
#define P(N)
This file contains some templates that are useful if you are working with the STL at all.
raw_pwrite_stream & OS
This file implements a set that has insertion order iteration characteristics.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
DenseMapIterator< KeyT, ValueT, KeyInfoT, BucketT > iterator
Definition: DenseMap.h:71
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:334
static std::pair< offset_type, offset_type > EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V)
const InstrProfWriter::ProfilingData *const data_type_ref
InstrProfSummaryBuilder * SummaryBuilder
static hash_value_type ComputeHash(key_type_ref K)
InstrProfSummaryBuilder * CSSummaryBuilder
void EmitKey(raw_ostream &Out, key_type_ref K, offset_type N)
void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type)
const InstrProfWriter::ProfilingData *const data_type
void addRecord(const InstrProfRecord &)
A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...
Definition: InstrProf.h:429
StringRef getFuncOrVarName(uint64_t ValMD5Hash)
Return name of functions or global variables from the name's md5 hash value.
Definition: InstrProf.h:588
Error addFuncName(StringRef FuncName)
Update the symtab by adding FuncName to the table.
Definition: InstrProf.h:503
StringRef getFuncOrVarNameIfDefined(uint64_t ValMD5Hash)
Just like getFuncOrVarName, except that it will return literal string 'External Symbol' if the functi...
Definition: InstrProf.h:581
Error write(raw_fd_ostream &OS)
Write the profile to OS.
void addTemporalProfileTraces(SmallVectorImpl< TemporalProfTraceTy > &SrcTraces, uint64_t SrcStreamSize)
Add SrcTraces using reservoir sampling where SrcStreamSize is the total number of temporal profiling ...
void overlapRecord(NamedInstrProfRecord &&Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, const OverlapFuncFilters &FuncFilter)
Error writeText(raw_fd_ostream &OS)
Write the profile in text format to OS.
InstrProfWriter(bool Sparse=false, uint64_t TemporalProfTraceReservoirSize=0, uint64_t MaxTemporalProfTraceLength=0)
void addBinaryIds(ArrayRef< llvm::object::BuildID > BIs)
void addMemProfRecord(const GlobalValue::GUID Id, const memprof::IndexedMemProfRecord &Record)
Add a memprof record for a function identified by its Id.
static void writeRecordInText(StringRef Name, uint64_t Hash, const InstrProfRecord &Counters, InstrProfSymtab &Symtab, raw_fd_ostream &OS)
Write Record in text format to OS.
void setValueProfDataEndianness(llvm::endianness Endianness)
void addRecord(NamedInstrProfRecord &&I, uint64_t Weight, function_ref< void(Error)> Warn)
Add function counts for the given function.
void mergeRecordsFromWriter(InstrProfWriter &&IPW, function_ref< void(Error)> Warn)
Merge existing function counts from the given writer.
void writeTextTemporalProfTraceData(raw_fd_ostream &OS, InstrProfSymtab &Symtab)
Write temporal profile trace data to the header in text format to OS.
std::unique_ptr< MemoryBuffer > writeBuffer()
Write the profile, returning the raw data. For testing.
bool addMemProfFrame(const memprof::FrameId, const memprof::Frame &F, function_ref< void(Error)> Warn)
Add a memprof frame identified by the hash of the contents of the frame in FrameId.
void setOutputSparse(bool Sparse)
Error validateRecord(const InstrProfRecord &Func)
static std::unique_ptr< MemoryBuffer > getMemBufferCopy(StringRef InputData, const Twine &BufferName="")
Open the specified memory range as a MemoryBuffer, copying the contents and taking ownership of it.
Generates an on disk hash table.
offset_type Emit(raw_ostream &Out)
Emit the table to Out, which must not be at offset 0.
void patch(PatchItem *P, int NItems)
void writeByte(uint8_t V)
ProfOStream(raw_string_ostream &STR)
support::endian::Writer LE
ProfOStream(raw_fd_ostream &FD)
void write(uint64_t V)
static const ArrayRef< uint32_t > DefaultCutoffs
A vector of useful cutoff values for detailed summary.
Definition: ProfileCommon.h:70
uint64_t getTotalCount() const
uint64_t getMaxCount() const
const SummaryEntryVector & getDetailedSummary()
uint32_t getNumCounts() const
uint64_t getMaxInternalCount() const
uint64_t getMaxFunctionCount() const
uint32_t getNumFunctions() const
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:370
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:950
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
bool empty() const
Definition: Trace.h:96
unsigned size() const
Definition: Trace.h:95
LLVM Value Representation.
Definition: Value.h:74
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:470
uint64_t seek(uint64_t off)
Flushes the stream and repositions the underlying file descriptor position to the offset specified fr...
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
uint64_t tell() const
tell - Return the current offset with the file.
Definition: raw_ostream.h:150
raw_ostream & write_hex(unsigned long long N)
Output N in hexadecimal, without any prefix or padding.
raw_ostream & write(unsigned char C)
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:660
std::string & str()
Returns the string's reference.
Definition: raw_ostream.h:678
std::unique_ptr< Summary > allocSummary(uint32_t TotalSize)
Definition: InstrProf.h:1163
uint64_t ComputeHash(StringRef K)
Definition: InstrProf.h:1051
const uint64_t Magic
Definition: InstrProf.h:1008
const HashT HashType
Definition: InstrProf.h:1049
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
Definition: STLExtras.h:862
uint64_t alignToPowerOf2(uint64_t Value, uint64_t Align)
Definition: MathExtras.h:382
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2053
void shuffle(Iterator first, Iterator last, RNG &&g)
Definition: STLExtras.h:1550
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1738
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1656
@ Other
Any other memory.
instrprof_error
Definition: InstrProf.h:324
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:155
endianness
Definition: bit.h:70
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
uint64_t Pos
uint64_t * D
Helper object to track which of three possible relocation mechanisms are used for a particular value ...
void set(SummaryFieldKind K, uint64_t V)
Definition: InstrProf.h:1149
void setEntry(uint32_t I, const ProfileSummaryEntry &E)
Definition: InstrProf.h:1155
Profiling information for a single function.
Definition: InstrProf.h:704
std::vector< uint64_t > Counts
Definition: InstrProf.h:705
void merge(InstrProfRecord &Other, uint64_t Weight, function_ref< void(instrprof_error)> Warn)
Merge the counts in Other into this one.
Definition: InstrProf.cpp:812
void overlap(InstrProfRecord &Other, OverlapStats &Overlap, OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff)
Compute the overlap b/w this IntrprofRecord and Other.
Definition: InstrProf.cpp:708
void sortValueData()
Sort value profile data (per site) by count.
Definition: InstrProf.h:779
std::vector< uint8_t > BitmapBytes
Definition: InstrProf.h:706
void scale(uint64_t N, uint64_t D, function_ref< void(instrprof_error)> Warn)
Scale up profile counts (including value profile data) by a factor of (N / D).
Definition: InstrProf.cpp:875
static bool hasCSFlagInHash(uint64_t FuncHash)
Definition: InstrProf.h:910
const std::string NameFilter
Definition: InstrProf.h:669
void addOneMismatch(const CountSumOrPercent &MismatchFunc)
Definition: InstrProf.cpp:1407
CountSumOrPercent Overlap
Definition: InstrProf.h:631
void addOneUnique(const CountSumOrPercent &UniqueFunc)
Definition: InstrProf.cpp:1417
CountSumOrPercent Test
Definition: InstrProf.h:629
An ordered list of functions identified by their NameRef found in INSTR_PROF_DATA.
Definition: InstrProf.h:355
Function object to check whether the first component of a container supported by std::get (like std::...
Definition: STLExtras.h:1459
void merge(const IndexedMemProfRecord &Other)
Definition: MemProf.h:343
static MemProfSchema getSchema()
Definition: MemProf.h:101
Adapter to write values to a stream in a particular byte order.
Definition: EndianStream.h:67
void write(ArrayRef< value_type > Val)
Definition: EndianStream.h:71