1#ifndef LLVM_PROFILEDATA_MEMPROF_H_
2#define LLVM_PROFILEDATA_MEMPROF_H_
44#define MIBEntryDef(NameTag, Name, Type) NameTag,
65 for (
const Meta Id : IncomingSchema)
67#define MIBEntryDef(NameTag, Name, Type) Name = Block.Name;
79 const unsigned char *
Ptr) {
80 using namespace support;
83 for (
const Meta Id : IncomingSchema) {
85#define MIBEntryDef(NameTag, Name, Type) \
87 Name = endian::readNext<Type, llvm::endianness::little>(Ptr); \
93 "a newer version of the runtime?");
103 using namespace support;
106 for (
const Meta Id : Schema) {
108#define MIBEntryDef(NameTag, Name, Type) \
110 LE.write<Type>(Name); \
122 OS <<
" MemInfoBlock:\n";
123#define MIBEntryDef(NameTag, Name, Type) \
124 OS << " " << #Name << ": " << Name << "\n";
127 if (AccessHistogramSize > 0) {
128 OS <<
" " <<
"AccessHistogramValues" <<
":";
129 for (
uint32_t I = 0;
I < AccessHistogramSize; ++
I) {
142#define MIBEntryDef(NameTag, Name, Type) \
143 Type get##Name() const { \
144 assert(Schema[llvm::to_underlying(Meta::Name)]); \
153 if (
Other.Schema != Schema)
156#define MIBEntryDef(NameTag, Name, Type) \
157 if (Schema[llvm::to_underlying(Meta::Name)] && \
158 Other.get##Name() != get##Name()) \
172 for (
const Meta Id : Schema) {
174#define MIBEntryDef(NameTag, Name, Type) \
176 Result += sizeof(Type); \
192#define MIBEntryDef(NameTag, Name, Type) Type Name = Type();
222 ? std::make_unique<std::string>(*
Other.SymbolName)
242 ? std::make_unique<std::string>(*
Other.SymbolName)
265 using namespace support;
271 static_assert(std::is_same<GlobalValue::GUID, uint64_t>::value,
272 "Expect GUID to be uint64_t.");
282 using namespace support;
285 endian::readNext<uint64_t, llvm::endianness::little>(
Ptr);
287 endian::readNext<uint32_t, llvm::endianness::little>(
Ptr);
289 endian::readNext<uint32_t, llvm::endianness::little>(
Ptr);
290 const bool I = endian::readNext<bool, llvm::endianness::little>(
Ptr);
304 <<
" Function: " <<
Function <<
"\n"
307 <<
" Column: " <<
Column <<
"\n"
315 auto HashCombine = [](
auto Value,
size_t Seed) {
316 std::hash<
decltype(
Value)> Hasher;
320 return Hasher(
Value) + 0x9e3779b97f4a7c15 + (
Seed << 6) + (
Seed >> 2);
324 Result ^= HashCombine(
Function, Result);
326 Result ^= HashCombine(
Column, Result);
328 return static_cast<FrameId>(Result);
353 const MemInfoBlock &MB,
388 CallStack.push_back(IdToFrameCallback(Id));
395 OS <<
" Callstack:\n";
451 *MemProfCallStackIndexes =
nullptr)
const;
455 const unsigned char *Buffer,
483 AllocSites.emplace_back(IndexedAI, IdToFrameCallback);
486 std::vector<Frame> Frames;
487 for (
const FrameId Id : Site) {
488 Frames.push_back(IdToFrameCallback(Id));
497 OS <<
" AllocSites:\n";
503 OS <<
" CallSites:\n";
504 for (
const std::vector<Frame> &Frames :
CallSites) {
505 for (
const Frame &
F : Frames) {
532 : Version(V), Schema(S) {}
540 static std::pair<offset_type, offset_type>
542 using namespace support;
545 endian::readNext<offset_type, llvm::endianness::little>(
D);
547 endian::readNext<offset_type, llvm::endianness::little>(
D);
548 return std::make_pair(KeyLen, DataLen);
552 using namespace support;
553 return endian::readNext<external_key_type, llvm::endianness::little>(
D);
598 : Schema(Schema), Version(V),
599 MemProfCallStackIndexes(MemProfCallStackIndexes) {}
603 std::pair<offset_type, offset_type>
605 using namespace support;
610 offset_type M = V.serializedSize(*Schema, Version);
612 return std::make_pair(
N, M);
616 using namespace support;
623 assert(Schema !=
nullptr &&
"MemProf schema is not initialized!");
624 V.serialize(*Schema, Out, Version, MemProfCallStackIndexes);
647 static std::pair<offset_type, offset_type>
649 using namespace support;
655 return std::make_pair(
N, M);
659 using namespace support;
687 static std::pair<offset_type, offset_type>
689 using namespace support;
692 endian::readNext<offset_type, llvm::endianness::little>(
D);
694 endian::readNext<offset_type, llvm::endianness::little>(
D);
695 return std::make_pair(KeyLen, DataLen);
699 using namespace support;
700 return endian::readNext<external_key_type, llvm::endianness::little>(
D);
723 static std::pair<offset_type, offset_type>
725 using namespace support;
731 return std::make_pair(
N, M);
735 using namespace support;
742 using namespace support;
768 static std::pair<offset_type, offset_type>
770 using namespace support;
775 endian::readNext<offset_type, llvm::endianness::little>(
D);
776 return std::make_pair(KeyLen, DataLen);
780 using namespace support;
781 return endian::readNext<external_key_type, llvm::endianness::little>(
D);
785 using namespace support;
791 for (
size_t I = 0;
I != NumFrames; ++
I) {
792 FrameId F = endian::readNext<FrameId, llvm::endianness::little>(
D);
806template <
typename value_type,
typename IterTy>
809 if constexpr (std::is_same_v<deref_type, value_type>)
831 auto Iter =
Map.find(Id);
832 if (Iter ==
Map.end()) {
834 return Frame(0, 0, 0,
false);
836 return detail::DerefIterator<Frame>(Iter);
858 std::vector<Frame> Frames;
859 auto CSIter =
Map.find(CSId);
860 if (CSIter ==
Map.end()) {
864 detail::DerefIterator<llvm::SmallVector<FrameId>>(CSIter);
900 std::vector<Frame> Frames;
902 const unsigned char *
Ptr =
906 support::endian::readNext<uint32_t, llvm::endianness::little>(
Ptr);
907 Frames.reserve(NumFrames);
908 for (; NumFrames; --NumFrames) {
910 support::endian::read<LinearFrameId, llvm::endianness::little>(
Ptr);
913 if (
static_cast<std::make_signed_t<LinearFrameId>
>(Elem) < 0) {
916 support::endian::read<LinearFrameId, llvm::endianness::little>(
Ptr);
919 assert(
static_cast<std::make_signed_t<LinearFrameId>
>(Elem) >= 0);
952 &MemProfCallStackData);
1011 std::vector<LinearFrameId> RadixArray;
1034 std::vector<LinearCallStackId> Indexes;
1036 using CSIdPair = std::pair<CallStackId, llvm::SmallVector<FrameId>>;
1050 &&MemProfCallStackData,
1057 return std::move(CallStackPos);
1071 &FunctionProfileData);
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
This file implements a map that provides insertion order iteration.
static ManagedStatic< cl::opt< uint64_t >, CreateSeed > Seed
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains library features backported from future STL versions.
This file defines the SmallVector class.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Tagged union holding either a T or a Error.
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
This class implements a map that also provides access to all stored values in a deterministic order.
void reserve(size_type N)
void push_back(const T &Elt)
StringRef - Represent a constant reference to a string, i.e.
LLVM Value Representation.
An efficient, type-erasing, non-owning reference to a callable.
static uint64_t GetInternalKey(internal_key_type K)
hash_value_type ComputeHash(internal_key_type K)
static bool EqualKey(internal_key_type A, internal_key_type B)
static std::pair< offset_type, offset_type > ReadKeyDataLength(const unsigned char *&D)
CallStackId external_key_type
uint64_t ReadKey(const unsigned char *D, offset_type)
data_type ReadData(uint64_t K, const unsigned char *D, offset_type Length)
static uint64_t GetExternalKey(external_key_type K)
CallStackRadixTreeBuilder()=default
void build(llvm::MapVector< CallStackId, llvm::SmallVector< FrameId > > &&MemProfCallStackData, const llvm::DenseMap< FrameId, LinearFrameId > &MemProfFrameIndexes, llvm::DenseMap< FrameId, FrameStat > &FrameHistogram)
llvm::DenseMap< CallStackId, LinearCallStackId > takeCallStackPos()
const std::vector< LinearFrameId > & getRadixArray() const
static hash_value_type ComputeHash(key_type_ref K)
static std::pair< offset_type, offset_type > EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V)
void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type)
void EmitKey(raw_ostream &Out, key_type_ref K, offset_type)
Helper class to iterate through stack ids in both metadata (memprof MIB and callsite) and the corresp...
data_type ReadData(uint64_t K, const unsigned char *D, offset_type)
static uint64_t GetExternalKey(external_key_type K)
static bool EqualKey(internal_key_type A, internal_key_type B)
static uint64_t GetInternalKey(internal_key_type K)
static std::pair< offset_type, offset_type > ReadKeyDataLength(const unsigned char *&D)
hash_value_type ComputeHash(internal_key_type K)
uint64_t ReadKey(const unsigned char *D, offset_type)
static hash_value_type ComputeHash(key_type_ref K)
void EmitKey(raw_ostream &Out, key_type_ref K, offset_type)
void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type)
static std::pair< offset_type, offset_type > EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V)
static bool EqualKey(uint64_t A, uint64_t B)
uint64_t ReadKey(const unsigned char *D, offset_type)
data_type ReadData(uint64_t K, const unsigned char *D, offset_type)
static std::pair< offset_type, offset_type > ReadKeyDataLength(const unsigned char *&D)
static uint64_t GetInternalKey(uint64_t K)
hash_value_type ComputeHash(uint64_t K)
static uint64_t GetExternalKey(uint64_t K)
RecordLookupTrait(IndexedVersion V, const MemProfSchema &S)
RecordLookupTrait()=delete
static hash_value_type ComputeHash(key_type_ref K)
RecordWriterTrait(const MemProfSchema *Schema, IndexedVersion V, llvm::DenseMap< CallStackId, LinearCallStackId > *MemProfCallStackIndexes)
std::pair< offset_type, offset_type > EmitKeyDataLength(raw_ostream &Out, key_type_ref K, data_type_ref V)
RecordWriterTrait()=delete
void EmitKey(raw_ostream &Out, key_type_ref K, offset_type)
void EmitData(raw_ostream &Out, key_type_ref, data_type_ref V, offset_type)
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
value_type DerefIterator(IterTy Iter)
void verifyIndexedMemProfRecord(const IndexedMemProfRecord &Record)
constexpr uint64_t MaximumSupportedVersion
MemProfSchema getHotColdSchema()
uint32_t LinearCallStackId
llvm::DenseMap< FrameId, FrameStat > computeFrameHistogram(llvm::MapVector< CallStackId, llvm::SmallVector< FrameId > > &MemProfCallStackData)
CallStackId hashCallStack(ArrayRef< FrameId > CS)
constexpr uint64_t MinimumSupportedVersion
MemProfSchema getFullSchema()
Expected< MemProfSchema > readMemProfSchema(const unsigned char *&Buffer)
void verifyFunctionProfileData(const llvm::MapVector< GlobalValue::GUID, IndexedMemProfRecord > &FunctionProfileData)
This is an optimization pass for GlobalISel generic memory operations.
constexpr std::underlying_type_t< Enum > to_underlying(Enum E)
Returns underlying integer value of an enum.
typename llvm::remove_cvref< T >::type remove_cvref_t
std::vector< Frame > CallStack
AllocationInfo(const IndexedAllocationInfo &IndexedAI, llvm::function_ref< const Frame(const FrameId)> IdToFrameCallback)
PortableMemInfoBlock Info
void printYAML(raw_ostream &OS) const
CallStackIdConverter & operator=(const CallStackIdConverter &)=delete
CallStackIdConverter()=delete
llvm::function_ref< Frame(FrameId)> FrameIdToFrame
std::optional< CallStackId > LastUnmappedId
CallStackIdConverter(MapTy &Map, llvm::function_ref< Frame(FrameId)> FrameIdToFrame)
std::vector< Frame > operator()(CallStackId CSId)
CallStackIdConverter(const CallStackIdConverter &)=delete
FrameIdConverter(const FrameIdConverter &)=delete
Frame operator()(FrameId Id)
FrameIdConverter(MapTy &Map)
std::optional< FrameId > LastUnmappedId
FrameIdConverter()=delete
FrameIdConverter & operator=(const FrameIdConverter &)=delete
Frame & operator=(const Frame &Other)
Frame(const Frame &Other)
static constexpr size_t serializedSize()
void printYAML(raw_ostream &OS) const
GlobalValue::GUID Function
void serialize(raw_ostream &OS) const
bool hasSymbolName() const
std::string getSymbolNameOr(StringRef Alt) const
StringRef getSymbolName() const
bool operator==(const Frame &Other) const
bool operator!=(const Frame &Other) const
static Frame deserialize(const unsigned char *Ptr)
Frame(GlobalValue::GUID Hash, uint32_t Off, uint32_t Col, bool Inline)
std::unique_ptr< std::string > SymbolName
IndexedAllocationInfo(ArrayRef< FrameId > CS, CallStackId CSId, const MemInfoBlock &MB, const MemProfSchema &Schema=getFullSchema())
bool operator==(const IndexedAllocationInfo &Other) const
bool operator!=(const IndexedAllocationInfo &Other) const
PortableMemInfoBlock Info
size_t serializedSize(const MemProfSchema &Schema, IndexedVersion Version) const
llvm::SmallVector< FrameId > CallStack
IndexedAllocationInfo()=default
llvm::MapVector< CallStackId, llvm::SmallVector< FrameId > > CallStacks
llvm::MapVector< GlobalValue::GUID, IndexedMemProfRecord > Records
llvm::MapVector< FrameId, Frame > Frames
llvm::SmallVector< CallStackId > CallSiteIds
llvm::SmallVector< IndexedAllocationInfo > AllocSites
size_t serializedSize(const MemProfSchema &Schema, IndexedVersion Version) const
void serialize(const MemProfSchema &Schema, raw_ostream &OS, IndexedVersion Version, llvm::DenseMap< CallStackId, LinearCallStackId > *MemProfCallStackIndexes=nullptr) const
static IndexedMemProfRecord deserialize(const MemProfSchema &Schema, const unsigned char *Buffer, IndexedVersion Version)
llvm::SmallVector< llvm::SmallVector< FrameId > > CallSites
bool operator==(const IndexedMemProfRecord &Other) const
static GlobalValue::GUID getGUID(const StringRef FunctionName)
void merge(const IndexedMemProfRecord &Other)
MemProfRecord toMemProfRecord(llvm::function_ref< std::vector< Frame >(const CallStackId)> Callback) const
const unsigned char * CallStackBase
std::function< Frame(LinearFrameId)> FrameIdToFrame
LinearCallStackIdConverter(const unsigned char *CallStackBase, std::function< Frame(LinearFrameId)> FrameIdToFrame)
LinearCallStackIdConverter()=delete
std::vector< Frame > operator()(LinearCallStackId LinearCSId)
LinearFrameIdConverter()=delete
LinearFrameIdConverter(const unsigned char *FrameBase)
const unsigned char * FrameBase
Frame operator()(LinearFrameId LinearId)
llvm::SmallVector< std::vector< Frame > > CallSites
llvm::SmallVector< AllocationInfo > AllocSites
void print(llvm::raw_ostream &OS) const
MemProfRecord(const IndexedMemProfRecord &Record, llvm::function_ref< const Frame(const FrameId Id)> IdToFrameCallback)
bool operator!=(const PortableMemInfoBlock &Other) const
void deserialize(const MemProfSchema &IncomingSchema, const unsigned char *Ptr)
PortableMemInfoBlock(const MemProfSchema &Schema, const unsigned char *Ptr)
PortableMemInfoBlock()=default
std::bitset< llvm::to_underlying(Meta::Size)> getSchema() const
static size_t serializedSize(const MemProfSchema &Schema)
void printYAML(raw_ostream &OS) const
void serialize(const MemProfSchema &Schema, raw_ostream &OS) const
PortableMemInfoBlock(const MemInfoBlock &Block, const MemProfSchema &IncomingSchema)
bool operator==(const PortableMemInfoBlock &Other) const
Adapter to write values to a stream in a particular byte order.