LLVM 24.0.0git
MemProfContextDisambiguation.cpp
Go to the documentation of this file.
1//==-- MemProfContextDisambiguation.cpp - Disambiguate contexts -------------=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements support for context disambiguation of allocation
10// calls for profile-guided memory optimization (MemPGO). Specifically, it uses
11// MemProf profiles which indicate context specific allocation behavior
12// (currently distinguishing cold vs hot memory allocations). Cloning is
13// performed to expose the cold allocation call contexts, and the allocation
14// calls are subsequently annotated with an attribute for later transformation.
15//
16// The transformations can be performed either directly on IR (regular LTO), or
17// on a ThinLTO index (and later applied to the IR during the ThinLTO backend).
18// Both types of LTO operate on a the same base graph representation, which
19// uses CRTP to support either IR or Index formats.
20//
21//===----------------------------------------------------------------------===//
22
24#include "llvm/ADT/DenseMap.h"
25#include "llvm/ADT/DenseSet.h"
26#include "llvm/ADT/MapVector.h"
29#include "llvm/ADT/SmallSet.h"
31#include "llvm/ADT/Statistic.h"
38#include "llvm/IR/Module.h"
40#include "llvm/Pass.h"
43#include "llvm/Support/SHA1.h"
45#include "llvm/Transforms/IPO.h"
49#include <deque>
50#include <sstream>
51#include <vector>
52using namespace llvm;
53using namespace llvm::memprof;
54
55#define DEBUG_TYPE "memprof-context-disambiguation"
56
57STATISTIC(FunctionClonesAnalysis,
58 "Number of function clones created during whole program analysis");
59STATISTIC(FunctionClonesThinBackend,
60 "Number of function clones created during ThinLTO backend");
61STATISTIC(FunctionsClonedThinBackend,
62 "Number of functions that had clones created during ThinLTO backend");
64 FunctionCloneDuplicatesThinBackend,
65 "Number of function clone duplicates detected during ThinLTO backend");
66STATISTIC(AllocTypeNotCold, "Number of not cold static allocations (possibly "
67 "cloned) during whole program analysis");
68STATISTIC(AllocTypeCold, "Number of cold static allocations (possibly cloned) "
69 "during whole program analysis");
70STATISTIC(AllocTypeAmbiguous,
71 "Number of ambiguous static allocations (possibly cloned) during "
72 "whole program analysis");
73STATISTIC(AllocTypeNotColdThinBackend,
74 "Number of not cold static allocations (possibly cloned) during "
75 "ThinLTO backend");
76STATISTIC(AllocTypeColdThinBackend, "Number of cold static allocations "
77 "(possibly cloned) during ThinLTO backend");
78STATISTIC(AllocTypeAmbiguousThinBackend,
79 "Number of ambiguous static allocations (possibly cloned) during "
80 "ThinLTO backend");
81STATISTIC(OrigAllocsThinBackend,
82 "Number of original (not cloned) allocations with memprof profiles "
83 "during ThinLTO backend");
85 AllocVersionsThinBackend,
86 "Number of allocation versions (including clones) during ThinLTO backend");
87STATISTIC(MaxAllocVersionsThinBackend,
88 "Maximum number of allocation versions created for an original "
89 "allocation during ThinLTO backend");
90STATISTIC(UnclonableAllocsThinBackend,
91 "Number of unclonable ambigous allocations during ThinLTO backend");
92STATISTIC(RemovedEdgesWithMismatchedCallees,
93 "Number of edges removed due to mismatched callees (profiled vs IR)");
94STATISTIC(FoundProfiledCalleeCount,
95 "Number of profiled callees found via tail calls");
96STATISTIC(FoundProfiledCalleeDepth,
97 "Aggregate depth of profiled callees found via tail calls");
98STATISTIC(FoundProfiledCalleeMaxDepth,
99 "Maximum depth of profiled callees found via tail calls");
100STATISTIC(FoundProfiledCalleeNonUniquelyCount,
101 "Number of profiled callees found via multiple tail call chains");
102STATISTIC(FoundProfiledCalleeIndirectTailCallCount,
103 "Number of tail calls in chains to profiled callees that are "
104 "indirect (thin link)");
105STATISTIC(IndirectTailCallsRecordedForICPThinBackend,
106 "Number of indirect tail calls recorded for ICP for cloning during "
107 "ThinLTO backend");
108STATISTIC(DeferredBackedges, "Number of backedges with deferred cloning");
109STATISTIC(NewMergedNodes, "Number of new nodes created during merging");
110STATISTIC(NonNewMergedNodes, "Number of non new nodes used during merging");
111STATISTIC(MissingAllocForContextId,
112 "Number of missing alloc nodes for context ids");
113STATISTIC(SkippedCallsCloning,
114 "Number of calls skipped during cloning due to unexpected operand");
115STATISTIC(MismatchedCloneAssignments,
116 "Number of callsites assigned to call multiple non-matching clones");
117STATISTIC(TotalMergeInvokes, "Number of merge invocations for nodes");
118STATISTIC(TotalMergeIters, "Number of merge iterations for nodes");
119STATISTIC(MaxMergeIters, "Max merge iterations for nodes");
120STATISTIC(NumImportantContextIds, "Number of important context ids");
121STATISTIC(NumFixupEdgeIdsInserted, "Number of fixup edge ids inserted");
122STATISTIC(NumFixupEdgesAdded, "Number of fixup edges added");
123STATISTIC(NumFixedContexts, "Number of contexts with fixed edges");
124STATISTIC(NumImplicitCallersAssigned,
125 "Number of callsite clones without an edge to a callee's clones "
126 "assigned the callee function clone of their original node");
127STATISTIC(NumImplicitCallersSourced,
128 "Number of calls in function clones whose callee was taken from a "
129 "callsite clone an implicit caller requires");
130STATISTIC(NumTrimmedContextIds,
131 "Number of context ids with call stacks trimmed at a callsite");
132STATISTIC(NumImplicitCallersUnresolved,
133 "Number of callsite clones without an edge to a callee's clones "
134 "left in a function clone calling a conflicting callee clone");
135STATISTIC(AliaseesPrevailingInDiffModuleFromAlias,
136 "Number of aliasees prevailing in a different module than its alias");
137STATISTIC(NumClonedCalleeCallsites,
138 "Number of summary callsites assigned to call a cloned callee (thin "
139 "link)");
140STATISTIC(NumClonedCalleeIndirectCallsites,
141 "Number of summary callsites assigned to call a cloned callee with "
142 "an indirect call edge (thin link)");
143STATISTIC(NumClonedCalleeEdgesMarkedRequiresDeclImport,
144 "Number of summary indirect call edges to cloned callees marked as "
145 "requiring a declaration import (thin link)");
146STATISTIC(NumCallsAssignedTrimmedInlinedContexts,
147 "Number of calls assigned contexts trimmed within their inlined "
148 "stack id sequence");
149STATISTIC(NumTrimmedInlinedContextIds,
150 "Number of (duplicated) context ids for contexts trimmed within a "
151 "call's inlined stack id sequence");
152
154 "memprof-dot-file-path-prefix", cl::init(""), cl::Hidden,
155 cl::value_desc("filename"),
156 cl::desc("Specify the path prefix of the MemProf dot files."));
157
158static cl::opt<bool> ExportToDot("memprof-export-to-dot", cl::init(false),
160 cl::desc("Export graph to dot files."));
161
162// TODO: Remove this option once new handling is validated more widely.
164 "memprof-merge-iteration", cl::init(true), cl::Hidden,
165 cl::desc("Iteratively apply merging on a node to catch new callers"));
166
167// How much of the graph to export to dot.
169 All, // The full CCG graph.
170 Alloc, // Only contexts for the specified allocation.
171 Context, // Only the specified context.
172};
173
175 "memprof-dot-scope", cl::desc("Scope of graph to export to dot"),
178 clEnumValN(DotScope::All, "all", "Export full callsite graph"),
180 "Export only nodes with contexts feeding given "
181 "-memprof-dot-alloc-id"),
182 clEnumValN(DotScope::Context, "context",
183 "Export only nodes with given -memprof-dot-context-id")));
184
186 AllocIdForDot("memprof-dot-alloc-id", cl::init(0), cl::Hidden,
187 cl::desc("Id of alloc to export if -memprof-dot-scope=alloc "
188 "or to highlight if -memprof-dot-scope=all"));
189
191 "memprof-dot-context-id", cl::init(0), cl::Hidden,
192 cl::desc("Id of context to export if -memprof-dot-scope=context or to "
193 "highlight otherwise"));
194
195static cl::opt<bool>
196 DumpCCG("memprof-dump-ccg", cl::init(false), cl::Hidden,
197 cl::desc("Dump CallingContextGraph to stdout after each stage."));
198
199static cl::opt<bool>
200 VerifyCCG("memprof-verify-ccg", cl::init(false), cl::Hidden,
201 cl::desc("Perform verification checks on CallingContextGraph."));
202
203static cl::opt<bool>
204 VerifyNodes("memprof-verify-nodes", cl::init(false), cl::Hidden,
205 cl::desc("Perform frequent verification checks on nodes."));
206
208 "memprof-import-summary",
209 cl::desc("Import summary to use for testing the ThinLTO backend via opt"),
210 cl::Hidden);
211
213 TailCallSearchDepth("memprof-tail-call-search-depth", cl::init(5),
215 cl::desc("Max depth to recursively search for missing "
216 "frames through tail calls."));
217
218// Whether the ThinLTO search for frames missing due to tail calls should look
219// through indirect tail calls, via the summary call edges to their profiled
220// targets.
222 "memprof-indirect-tail-call-search", cl::init(true), cl::Hidden,
223 cl::desc("Search for missing frames through indirect tail calls to their "
224 "profiled targets (ThinLTO)."));
225
226// Optionally enable cloning of callsites involved with recursive cycles
228 "memprof-allow-recursive-callsites", cl::init(true), cl::Hidden,
229 cl::desc("Allow cloning of callsites involved in recursive cycles"));
230
232 "memprof-clone-recursive-contexts", cl::init(true), cl::Hidden,
233 cl::desc("Allow cloning of contexts through recursive cycles"));
234
235// Generally this is needed for correct assignment of allocation clones to
236// function clones, however, allow it to be disabled for debugging while the
237// functionality is new and being tested more widely.
238static cl::opt<bool>
239 MergeClones("memprof-merge-clones", cl::init(true), cl::Hidden,
240 cl::desc("Merge clones before assigning functions"));
241
242// When disabled, try to detect and prevent cloning of recursive contexts.
243// This is only necessary until we support cloning through recursive cycles.
244// Leave on by default for now, as disabling requires a little bit of compile
245// time overhead and doesn't affect correctness, it will just inflate the cold
246// hinted bytes reporting a bit when -memprof-report-hinted-sizes is enabled.
248 "memprof-allow-recursive-contexts", cl::init(true), cl::Hidden,
249 cl::desc("Allow cloning of contexts having recursive cycles"));
250
251// When updating the stack nodes for callsites, a call with inlined frames is
252// assigned the contexts that include all of its stack ids that have context
253// nodes. The profile may have trimmed some contexts at an earlier stack id in
254// that sequence (because they had a single allocation type from that point
255// on); when a later stack id has a node due to other contexts, those trimmed
256// contexts are not assigned to the call, only to calls whose last stack id
257// with a node is the earlier id (e.g. an out-of-line copy of the inlined
258// callee). When enabled, also assign them to the call with the inlined frames,
259// via duplicates of their context ids (the other calls keep the originals).
260// TODO: Remove this option once the new handling is validated more widely.
262 "memprof-assign-trimmed-inlined-contexts", cl::init(true), cl::Hidden,
263 cl::desc("Assign contexts trimmed within a call's inlined stack id "
264 "sequence to that call"));
265
266// Set the minimum absolute count threshold for allowing inlining of indirect
267// calls promoted during cloning.
269 "memprof-icp-noinline-threshold", cl::init(0), cl::Hidden,
270 cl::desc("Minimum absolute count for promoted target to be inlinable"));
271
272namespace llvm {
274 "enable-memprof-context-disambiguation", cl::Hidden,
275 cl::desc("Enable MemProf context disambiguation"));
276
277// Indicate we are linking with an allocator that supports hot/cold operator
278// new interfaces.
280 "supports-hot-cold-new", cl::init(false), cl::Hidden,
281 cl::desc("Linking with hot/cold operator new interfaces"));
282
284 "memprof-require-definition-for-promotion", cl::init(false), cl::Hidden,
285 cl::desc(
286 "Require target function definition when promoting indirect calls"));
287
288// Applying the thin link cloning decisions to indirect calls requires the
289// ThinLTO backend to perform ICP to the profiled target, which requires that
290// the target be available in the caller's module (or in any module the caller
291// is imported into). When the target is defined in a different module it must
292// be imported, but the profiled targets are often synthesized from the MemProf
293// profile with a small count, so their summary edges are not hot and large
294// targets are often not imported. Since the thin link cloning runs before the
295// function importing analysis, we can mark the indirect call edges to callees
296// of callsites that were assigned to call a cloned callee as requiring a
297// declaration import, in which case the function importer imports at least a
298// declaration of the callee into any module containing the caller that
299// doesn't define the callee. Direct calls don't require importing the callee,
300// since the ThinLTO backend can simply call a declaration of the callee clone,
301// so we only do this for edges marked as being from indirect calls in the
302// summary. Note that unlike regular ICP, which by default requires the
303// target's definition (see -icp-allow-decls), MemProf ICP can promote to a
304// declaration of the target (see -memprof-require-definition-for-promotion),
305// since the purpose is to call the assigned clone rather than to inline. This
306// is therefore not done under -memprof-require-definition-for-promotion.
308 "memprof-import-cloned-indirect-targets", cl::init(true), cl::Hidden,
309 cl::desc("Import at least a declaration of the indirect call targets of "
310 "callsites assigned to call a cloned callee (needed for MemProf "
311 "ICP in the ThinLTO backend)"));
312
316
317// Appends a per-context diagnostic suffix to each -memprof-report-hinted-sizes
318// line: the allocation function clone, how far the context matched, and the
319// post-cloning call paths from the outermost matched caller to the reached
320// allocation clone(s) and their final hints. Requires
321// -memprof-report-hinted-sizes, and adds some memory and time overhead.
323 "memprof-report-hinted-sizes-detail", cl::init(false), cl::Hidden,
324 cl::desc("Append post-cloning call path and matching details to each "
325 "-memprof-report-hinted-sizes line"));
326
327// Post-cloning call paths in the detail suffix longer than this (in functions)
328// have runs of uncloned (or cloned) functions in their middle elided to
329// "...", keeping the ends and every cloned/uncloned transition.
331 "memprof-report-hinted-sizes-detail-max-path-len", cl::init(7), cl::Hidden,
332 cl::desc("Maximum number of functions printed in full for a post-cloning "
333 "call path in the -memprof-report-hinted-sizes-detail suffix; "
334 "longer paths are elided in the middle"));
335
337 "memprof-top-n-important", cl::init(10), cl::Hidden,
338 cl::desc("Number of largest cold contexts to consider important"));
339
341 "memprof-fixup-important", cl::init(true), cl::Hidden,
342 cl::desc("Enables edge fixup for important contexts"));
343
344} // namespace llvm
345
346namespace {
347
348/// CRTP base for graphs built from either IR or ThinLTO summary index.
349///
350/// The graph represents the call contexts in all memprof metadata on allocation
351/// calls, with nodes for the allocations themselves, as well as for the calls
352/// in each context. The graph is initially built from the allocation memprof
353/// metadata (or summary) MIBs. It is then updated to match calls with callsite
354/// metadata onto the nodes, updating it to reflect any inlining performed on
355/// those calls.
356///
357/// Each MIB (representing an allocation's call context with allocation
358/// behavior) is assigned a unique context id during the graph build. The edges
359/// and nodes in the graph are decorated with the context ids they carry. This
360/// is used to correctly update the graph when cloning is performed so that we
361/// can uniquify the context for a single (possibly cloned) allocation.
362template <typename DerivedCCG, typename FuncTy, typename CallTy>
363class CallsiteContextGraph {
364public:
365 CallsiteContextGraph() = default;
366 CallsiteContextGraph(const CallsiteContextGraph &) = default;
367 CallsiteContextGraph(CallsiteContextGraph &&) = default;
368
369 /// Main entry point to perform analysis and transformations on graph.
370 bool process(function_ref<void(StringRef, StringRef, const Twine &)>
371 EmitRemark = nullptr,
372 bool AllowExtraAnalysis = false);
373
374 /// Perform cloning on the graph necessary to uniquely identify the allocation
375 /// behavior of an allocation based on its context.
376 void identifyClones();
377
378 /// Assign callsite clones to functions, cloning functions as needed to
379 /// accommodate the combinations of their callsite clones reached by callers.
380 /// For regular LTO this clones functions and callsites in the IR, but for
381 /// ThinLTO the cloning decisions are noted in the summaries and later applied
382 /// in applyImport.
383 bool assignFunctions();
384
385 void dump() const;
386 void print(raw_ostream &OS) const;
387 void printTotalSizes(raw_ostream &OS,
388 function_ref<void(StringRef, StringRef, const Twine &)>
389 EmitRemark = nullptr) const;
390
392 const CallsiteContextGraph &CCG) {
393 CCG.print(OS);
394 return OS;
395 }
396
397 friend struct GraphTraits<
398 const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>;
399 friend struct DOTGraphTraits<
400 const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>;
401
402 void exportToDot(std::string Label) const;
403
404 /// Represents a function clone via FuncTy pointer and clone number pair.
405 struct FuncInfo final
406 : public std::pair<FuncTy *, unsigned /*Clone number*/> {
407 using Base = std::pair<FuncTy *, unsigned>;
408 FuncInfo(const Base &B) : Base(B) {}
409 FuncInfo(FuncTy *F = nullptr, unsigned CloneNo = 0) : Base(F, CloneNo) {}
410 explicit operator bool() const { return this->first != nullptr; }
411 FuncTy *func() const { return this->first; }
412 unsigned cloneNo() const { return this->second; }
413 };
414
415 /// Represents a callsite clone via CallTy and clone number pair.
416 struct CallInfo final : public std::pair<CallTy, unsigned /*Clone number*/> {
417 using Base = std::pair<CallTy, unsigned>;
418 CallInfo(const Base &B) : Base(B) {}
419 CallInfo(CallTy Call = nullptr, unsigned CloneNo = 0)
420 : Base(Call, CloneNo) {}
421 explicit operator bool() const { return (bool)this->first; }
422 CallTy call() const { return this->first; }
423 unsigned cloneNo() const { return this->second; }
424 void setCloneNo(unsigned N) { this->second = N; }
425 void print(raw_ostream &OS) const {
426 if (!operator bool()) {
427 assert(!cloneNo());
428 OS << "null Call";
429 return;
430 }
431 call()->print(OS);
432 OS << "\t(clone " << cloneNo() << ")";
433 }
434 void dump() const {
435 print(dbgs());
436 dbgs() << "\n";
437 }
438 friend raw_ostream &operator<<(raw_ostream &OS, const CallInfo &Call) {
439 Call.print(OS);
440 return OS;
441 }
442 };
443
444 struct ContextEdge;
445
446 /// Node in the Callsite Context Graph
447 struct ContextNode {
448 // Assigned to nodes as they are created, useful for debugging.
449 unsigned NodeId = 0;
450
451 // Keep this for now since in the IR case where we have an Instruction* it
452 // is not as immediately discoverable. Used for printing richer information
453 // when dumping graph.
454 bool IsAllocation;
455
456 // Keeps track of when the Call was reset to null because there was
457 // recursion.
458 bool Recursive = false;
459
460 // This will be formed by ORing together the AllocationType enum values
461 // for contexts including this node.
462 uint8_t AllocTypes = 0;
463
464 // The corresponding allocation or interior call. This is the primary call
465 // for which we have created this node.
466 CallInfo Call;
467
468 // List of other calls that can be treated the same as the primary call
469 // through cloning. I.e. located in the same function and have the same
470 // (possibly trimmed) stack ids. They will be updated the same way as the
471 // primary call when assigning to function clones.
472 SmallVector<CallInfo, 0> MatchingCalls;
473
474 // For alloc nodes this is a unique id assigned when constructed, and for
475 // callsite stack nodes it is the original stack id when the node is
476 // constructed from the memprof MIB metadata on the alloc nodes. Note that
477 // this is only used when matching callsite metadata onto the stack nodes
478 // created when processing the allocation memprof MIBs, and for labeling
479 // nodes in the dot graph. Therefore we don't bother to assign a value for
480 // clones.
481 uint64_t OrigStackOrAllocId = 0;
482
483 // Edges to all callees in the profiled call stacks.
484 // TODO: Should this be a map (from Callee node) for more efficient lookup?
485 std::vector<std::shared_ptr<ContextEdge>> CalleeEdges;
486
487 // Edges to all callers in the profiled call stacks.
488 // TODO: Should this be a map (from Caller node) for more efficient lookup?
489 std::vector<std::shared_ptr<ContextEdge>> CallerEdges;
490
491 // Returns true if we need to look at the callee edges for determining the
492 // node context ids and allocation type.
493 bool useCallerEdgesForContextInfo() const {
494 // Typically if the callee edges are empty either the caller edges are
495 // also empty, or this is an allocation (leaf node). However, if we are
496 // allowing recursive callsites and contexts this will be violated for
497 // incompletely cloned recursive cycles.
498 assert(!CalleeEdges.empty() || CallerEdges.empty() || IsAllocation ||
500 // When cloning for a recursive context, during cloning we might be in the
501 // midst of cloning for a recurrence and have moved context ids off of a
502 // caller edge onto the clone but not yet off of the incoming caller
503 // (back) edge. If we don't look at those we miss the fact that this node
504 // still has context ids of interest.
505 return IsAllocation || CloneRecursiveContexts;
506 }
507
508 // Compute the context ids for this node from the union of its edge context
509 // ids.
510 DenseSet<uint32_t> getContextIds() const {
511 unsigned Count = 0;
512 // Compute the number of ids for reserve below. In general we only need to
513 // look at one set of edges, typically the callee edges, since other than
514 // allocations and in some cases during recursion cloning, all the context
515 // ids on the callers should also flow out via callee edges.
516 for (auto &Edge : CalleeEdges.empty() ? CallerEdges : CalleeEdges)
517 Count += Edge->getContextIds().size();
518 DenseSet<uint32_t> ContextIds;
519 ContextIds.reserve(Count);
521 CalleeEdges, useCallerEdgesForContextInfo()
522 ? CallerEdges
523 : std::vector<std::shared_ptr<ContextEdge>>());
524 for (const auto &Edge : Edges)
525 ContextIds.insert_range(Edge->getContextIds());
526 return ContextIds;
527 }
528
529 // Compute the allocation type for this node from the OR of its edge
530 // allocation types.
531 uint8_t computeAllocType() const {
532 uint8_t BothTypes =
533 (uint8_t)AllocationType::Cold | (uint8_t)AllocationType::NotCold;
534 uint8_t AllocType = (uint8_t)AllocationType::None;
536 CalleeEdges, useCallerEdgesForContextInfo()
537 ? CallerEdges
538 : std::vector<std::shared_ptr<ContextEdge>>());
539 for (const auto &Edge : Edges) {
540 AllocType |= Edge->AllocTypes;
541 // Bail early if alloc type reached both, no further refinement.
542 if (AllocType == BothTypes)
543 return AllocType;
544 }
545 return AllocType;
546 }
547
548 // The context ids set for this node is empty if its edge context ids are
549 // also all empty.
550 bool emptyContextIds() const {
552 CalleeEdges, useCallerEdgesForContextInfo()
553 ? CallerEdges
554 : std::vector<std::shared_ptr<ContextEdge>>());
555 for (const auto &Edge : Edges) {
556 if (!Edge->getContextIds().empty())
557 return false;
558 }
559 return true;
560 }
561
562 // Whether the given context id is among this node's context ids.
563 bool hasContextId(uint32_t Id) const {
565 CalleeEdges, useCallerEdgesForContextInfo()
566 ? CallerEdges
567 : std::vector<std::shared_ptr<ContextEdge>>());
568 for (const auto &Edge : Edges) {
569 if (Edge->getContextIds().contains(Id))
570 return true;
571 }
572 return false;
573 }
574
575 // Removes from Ids any that are not on this node (i.e. on none of the
576 // edges its context ids are computed from). Walks the edges once for all
577 // ids, stopping early once every id has been found.
578 void removeContextIdsNotPresent(DenseSet<uint32_t> &Ids) const {
579 DenseSet<uint32_t> NotFound = Ids;
581 CalleeEdges, useCallerEdgesForContextInfo()
582 ? CallerEdges
583 : std::vector<std::shared_ptr<ContextEdge>>());
584 for (const auto &Edge : Edges) {
585 const auto &EdgeIds = Edge->getContextIds();
586 // Iterate over the smaller of the two sets.
587 if (EdgeIds.size() < NotFound.size()) {
588 for (auto Id : EdgeIds)
589 NotFound.erase(Id);
590 } else {
591 NotFound.remove_if(
592 [&EdgeIds](uint32_t Id) { return EdgeIds.contains(Id); });
593 }
594 if (NotFound.empty())
595 return;
596 }
597 set_subtract(Ids, NotFound);
598 }
599
600 // List of clones of this ContextNode, initially empty.
601 std::vector<ContextNode *> Clones;
602
603 // If a clone, points to the original uncloned node.
604 ContextNode *CloneOf = nullptr;
605
606 ContextNode(bool IsAllocation) : IsAllocation(IsAllocation), Call() {}
607
608 ContextNode(bool IsAllocation, CallInfo C)
609 : IsAllocation(IsAllocation), Call(C) {}
610
611 void addClone(ContextNode *Clone) {
612 if (CloneOf) {
613 CloneOf->Clones.push_back(Clone);
614 Clone->CloneOf = CloneOf;
615 } else {
616 Clones.push_back(Clone);
617 assert(!Clone->CloneOf);
618 Clone->CloneOf = this;
619 }
620 }
621
622 ContextNode *getOrigNode() {
623 if (!CloneOf)
624 return this;
625 return CloneOf;
626 }
627
628 const ContextNode *getOrigNode() const {
629 if (!CloneOf)
630 return this;
631 return CloneOf;
632 }
633
634 void addOrUpdateCallerEdge(ContextNode *Caller, AllocationType AllocType,
635 unsigned int ContextId);
636
637 ContextEdge *findEdgeFromCallee(const ContextNode *Callee);
638 ContextEdge *findEdgeFromCaller(const ContextNode *Caller);
639 void eraseCalleeEdge(const ContextEdge *Edge);
640 void eraseCallerEdge(const ContextEdge *Edge);
641
642 void setCall(CallInfo C) { Call = std::move(C); }
643
644 bool hasCall() const { return (bool)Call.call(); }
645
646 void printCall(raw_ostream &OS) const { Call.print(OS); }
647
648 // True if this node was effectively removed from the graph, in which case
649 // it should have an allocation type of None and empty context ids.
650 bool isRemoved() const {
651 // Typically if the callee edges are empty either the caller edges are
652 // also empty, or this is an allocation (leaf node). However, if we are
653 // allowing recursive callsites and contexts this will be violated for
654 // incompletely cloned recursive cycles.
656 (AllocTypes == (uint8_t)AllocationType::None) ==
657 emptyContextIds());
658 return AllocTypes == (uint8_t)AllocationType::None;
659 }
660
661 void dump() const;
662 void print(raw_ostream &OS) const;
663
664 friend raw_ostream &operator<<(raw_ostream &OS, const ContextNode &Node) {
665 Node.print(OS);
666 return OS;
667 }
668 };
669
670 /// Edge in the Callsite Context Graph from a ContextNode N to a caller or
671 /// callee.
672 struct ContextEdge {
673 ContextNode *Callee;
674 ContextNode *Caller;
675
676 // This will be formed by ORing together the AllocationType enum values
677 // for contexts including this edge.
678 uint8_t AllocTypes = 0;
679
680 // Set just before initiating cloning when cloning of recursive contexts is
681 // enabled. Used to defer cloning of backedges until we have done cloning of
682 // the callee node for non-backedge caller edges. This exposes cloning
683 // opportunities through the backedge of the cycle.
684 // TODO: Note that this is not updated during cloning, and it is unclear
685 // whether that would be needed.
686 bool IsBackedge = false;
687
688 // The set of IDs for contexts including this edge.
689 DenseSet<uint32_t> ContextIds;
690
691 ContextEdge(ContextNode *Callee, ContextNode *Caller, uint8_t AllocType,
692 DenseSet<uint32_t> ContextIds)
693 : Callee(Callee), Caller(Caller), AllocTypes(AllocType),
694 ContextIds(std::move(ContextIds)) {}
695
696 DenseSet<uint32_t> &getContextIds() { return ContextIds; }
697
698 // Helper to clear the fields of this edge when we are removing it from the
699 // graph.
700 inline void clear() {
701 ContextIds.clear();
702 AllocTypes = (uint8_t)AllocationType::None;
703 Caller = nullptr;
704 Callee = nullptr;
705 }
706
707 // Check if edge was removed from the graph. This is useful while iterating
708 // over a copy of edge lists when performing operations that mutate the
709 // graph in ways that might remove one of the edges.
710 inline bool isRemoved() const {
711 if (Callee || Caller)
712 return false;
713 // Any edges that have been removed from the graph but are still in a
714 // shared_ptr somewhere should have all fields null'ed out by clear()
715 // above.
716 assert(AllocTypes == (uint8_t)AllocationType::None);
717 assert(ContextIds.empty());
718 return true;
719 }
720
721 void dump() const;
722 void print(raw_ostream &OS) const;
723
724 friend raw_ostream &operator<<(raw_ostream &OS, const ContextEdge &Edge) {
725 Edge.print(OS);
726 return OS;
727 }
728 };
729
730 /// Helpers to remove edges that have allocation type None (due to not
731 /// carrying any context ids) after transformations.
732 void removeNoneTypeCalleeEdges(ContextNode *Node);
733 void removeNoneTypeCallerEdges(ContextNode *Node);
734 void
735 recursivelyRemoveNoneTypeCalleeEdges(ContextNode *Node,
736 DenseSet<const ContextNode *> &Visited);
737
738protected:
739 /// Get the list of context nodes for the stack ids in the given callsite
740 /// context, stopping at the first stack id that does not have a node.
741 template <class NodeT, class IteratorT>
742 std::vector<ContextNode *>
743 getStackNodesForCallsite(CallStack<NodeT, IteratorT> &CallsiteContext);
744
745 /// Adds nodes for the given allocation and any stack ids on its memprof MIB
746 /// metadata (or summary).
747 ContextNode *addAllocNode(CallInfo Call, const FuncTy *F);
748
749 /// Adds nodes for the given MIB stack ids.
750 template <class NodeT, class IteratorT>
751 void addStackNodesForMIB(
752 ContextNode *AllocNode, CallStack<NodeT, IteratorT> &StackContext,
753 CallStack<NodeT, IteratorT> &CallsiteContext, AllocationType AllocType,
754 ArrayRef<ContextTotalSize> ContextSizeInfo,
755 std::map<uint64_t, uint32_t> &TotalSizeToContextIdTopNCold);
756
757 /// Matches all callsite metadata (or summary) to the nodes created for
758 /// allocation memprof MIB metadata, synthesizing new nodes to reflect any
759 /// inlining performed on those callsite instructions.
760 void updateStackNodes();
761
762 /// Optionally fixup edges for the N largest cold contexts to better enable
763 /// cloning. This is particularly helpful if the context includes recursion
764 /// as well as inlining, resulting in a single stack node for multiple stack
765 /// ids in the context. With recursion it is particularly difficult to get the
766 /// edge updates correct as in the general case we have lost the original
767 /// stack id ordering for the context. Do more expensive fixup for the largest
768 /// contexts, controlled by MemProfTopNImportant and MemProfFixupImportant.
769 void fixupImportantContexts();
770
771 /// Update graph to conservatively handle any callsite stack nodes that target
772 /// multiple different callee target functions.
773 void handleCallsitesWithMultipleTargets();
774
775 /// Mark backedges via the standard DFS based backedge algorithm.
776 void markBackedges();
777
778 /// Merge clones generated during cloning for different allocations but that
779 /// are called by the same caller node, to ensure proper function assignment.
780 void mergeClones();
781
782 // Try to partition calls on the given node (already placed into the AllCalls
783 // array) by callee function, creating new copies of Node as needed to hold
784 // calls with different callees, and moving the callee edges appropriately.
785 // Returns true if partitioning was successful.
786 bool partitionCallsByCallee(
787 ContextNode *Node, ArrayRef<CallInfo> AllCalls,
788 std::vector<std::pair<CallInfo, ContextNode *>> &NewCallToNode);
789
790 /// Save lists of calls with MemProf metadata in each function, for faster
791 /// iteration.
792 MapVector<FuncTy *, std::vector<CallInfo>> FuncToCallsWithMetadata;
793
794 /// Map from callsite node to the enclosing caller function.
795 std::map<const ContextNode *, const FuncTy *> NodeToCallingFunc;
796
797 const FuncTy *getCallingFunc(const ContextNode *N) const {
798 auto It = NodeToCallingFunc.find(N);
799 return It != NodeToCallingFunc.end() ? It->second : nullptr;
800 }
801
802 /// Functions that are referenced (called directly, or address-taken) in the
803 /// module or summary index. Only populated, at the start of reporting, when
804 /// MemProfReportHintedSizesDetail is enabled; used to distinguish allocation
805 /// functions with no references at all from those whose callers simply did
806 /// not match the profile.
807 DenseSet<const FuncTy *> FunctionsWithCallers;
808
809 // When exporting to dot, and an allocation id is specified, contains the
810 // context ids on that allocation.
811 DenseSet<uint32_t> DotAllocContextIds;
812
813private:
814 using EdgeIter = typename std::vector<std::shared_ptr<ContextEdge>>::iterator;
815
816 // Structure to keep track of information for each call as we are matching
817 // non-allocation callsites onto context nodes created from the allocation
818 // call metadata / summary contexts.
819 struct CallContextInfo {
820 // The callsite we're trying to match.
821 CallTy Call;
822 // The context nodes for the callsite's stack ids that have one, saved to
823 // avoid repeatedly looking them up. The stack ids are available via the
824 // nodes' OrigStackOrAllocId. Since each stack id has its own node, and the
825 // stack id to node map is not modified while matching calls, two calls
826 // have the same stack ids iff they have the same nodes.
827 std::vector<ContextNode *> StackNodes;
828 // The function containing this callsite.
829 const FuncTy *Func;
830 // Initially empty, if needed this will be updated to contain the context
831 // ids for use in a new context node created for this callsite.
832 DenseSet<uint32_t> ContextIds;
833 // Duplicates, created for this callsite, of the ids of contexts trimmed at
834 // a stack id before the last one in StackNodes (see
835 // -memprof-assign-trimmed-inlined-contexts). Any new node created for this
836 // callsite gets these in addition to ContextIds.
837 DenseSet<uint32_t> TrimmedInlinedDupIds;
838 };
839
840 /// Helper to remove edge from graph, updating edge iterator if it is provided
841 /// (in which case CalleeIter indicates which edge list is being iterated).
842 /// This will also perform the necessary clearing of the ContextEdge members
843 /// to enable later checking if the edge has been removed (since we may have
844 /// other copies of the shared_ptr in existence, and in fact rely on this to
845 /// enable removal while iterating over a copy of a node's edge list).
846 void removeEdgeFromGraph(ContextEdge *Edge, EdgeIter *EI = nullptr,
847 bool CalleeIter = true);
848
849 /// Assigns the given Node to calls at or inlined into the location with
850 /// the Node's stack id, after post order traversing and processing its
851 /// caller nodes. Uses the call information recorded in the given
852 /// StackIdToMatchingCalls map, and creates new nodes for inlined sequences
853 /// as needed. Called by updateStackNodes which sets up the given
854 /// StackIdToMatchingCalls map.
855 void assignStackNodesPostOrder(
856 ContextNode *Node, DenseSet<const ContextNode *> &Visited,
857 DenseMap<uint64_t, std::vector<CallContextInfo>> &StackIdToMatchingCalls,
858 DenseMap<CallInfo, CallInfo> &CallToMatchingCall,
859 const DenseSet<uint32_t> &ImportantContextIds);
860
861 /// Duplicates the given set of context ids, updating the provided
862 /// map from each original id with the newly generated context ids,
863 /// and returning the new duplicated id set. If provided, OnDuplicate is
864 /// invoked with each original id and its new duplicate.
865 DenseSet<uint32_t> duplicateContextIds(
866 const DenseSet<uint32_t> &StackSequenceContextIds,
867 DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds,
868 function_ref<void(uint32_t OldId, uint32_t NewId)> OnDuplicate = nullptr);
869
870 /// Propagates all duplicated context ids across the graph.
871 void propagateDuplicateContextIds(
872 const DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds);
873
874 /// Cache of the ids of contexts trimmed at each node (i.e. that are on the
875 /// node but not on any of its caller edges). The sets are stored in a vector
876 /// and looked up by index, so that references to them are not invalidated
877 /// by adding more entries.
878 struct TrimmedAtNodeCache {
879 DenseMap<const ContextNode *, unsigned> NodeToIndex;
880 std::vector<DenseSet<uint32_t>> TrimmedAtNodeIds;
881 };
882
883 /// Returns the ids of contexts that were trimmed at a stack node before the
884 /// last one in the given call's inlined stack node sequence (see the
885 /// definition for details).
886 DenseSet<uint32_t>
887 computeTrimmedInlinedContextIds(ArrayRef<ContextNode *> StackNodes,
888 TrimmedAtNodeCache &Cache);
889
890 /// Connect the NewNode to OrigNode's callees if TowardsCallee is true,
891 /// else to its callers. Also updates OrigNode's edges to remove any context
892 /// ids moved to the newly created edge.
893 void connectNewNode(ContextNode *NewNode, ContextNode *OrigNode,
894 bool TowardsCallee, const DenseSet<uint32_t> &ContextIds);
895
896 /// Get the stack id corresponding to the given Id or Index (for IR this will
897 /// return itself, for a summary index this will return the id recorded in the
898 /// index for that stack id index value).
899 uint64_t getStackId(uint64_t IdOrIndex) const {
900 return static_cast<const DerivedCCG *>(this)->getStackId(IdOrIndex);
901 }
902
903 /// Returns true if the given call targets the callee of the given edge, or if
904 /// we were able to identify the call chain through intermediate tail calls.
905 /// In the latter case new context nodes are added to the graph for the
906 /// identified tail calls, and their synthesized nodes are added to
907 /// TailCallToContextNodeMap. The EdgeIter is updated in the latter case for
908 /// the updated edges and to prepare it for an increment in the caller.
909 bool
910 calleesMatch(CallTy Call, EdgeIter &EI,
911 MapVector<CallInfo, ContextNode *> &TailCallToContextNodeMap);
912
913 // Return the callee function of the given call, or nullptr if it can't be
914 // determined
915 const FuncTy *getCalleeFunc(CallTy Call) {
916 return static_cast<DerivedCCG *>(this)->getCalleeFunc(Call);
917 }
918
919 /// Returns true if the given call targets the given function, or if we were
920 /// able to identify the call chain through intermediate tail calls (in which
921 /// case FoundCalleeChain will be populated).
922 bool calleeMatchesFunc(
923 CallTy Call, const FuncTy *Func, const FuncTy *CallerFunc,
924 std::vector<std::pair<CallTy, FuncTy *>> &FoundCalleeChain) {
925 return static_cast<DerivedCCG *>(this)->calleeMatchesFunc(
926 Call, Func, CallerFunc, FoundCalleeChain);
927 }
928
929 /// Returns true if both call instructions have the same callee.
930 bool sameCallee(CallTy Call1, CallTy Call2) {
931 return static_cast<DerivedCCG *>(this)->sameCallee(Call1, Call2);
932 }
933
934 /// Get the list of context nodes for the stack ids in the given callsite's
935 /// context, stopping at the first stack id that does not have a node.
936 std::vector<ContextNode *> getStackNodesForCall(CallTy Call) {
937 return static_cast<DerivedCCG *>(this)->getStackNodesForCall(Call);
938 }
939
940 /// Get the last stack id in the context for callsite.
941 uint64_t getLastStackId(CallTy Call) {
942 return static_cast<DerivedCCG *>(this)->getLastStackId(Call);
943 }
944
945 /// Update the allocation call to record type of allocated memory.
946 void updateAllocationCall(CallInfo &Call, AllocationType AllocType) {
947 if (AllocType == AllocationType::Cold)
948 AllocTypeCold++;
949 else if (isAmbiguousAllocType((uint8_t)AllocType))
950 AllocTypeAmbiguous++;
951 else
952 AllocTypeNotCold++;
953 static_cast<DerivedCCG *>(this)->updateAllocationCall(Call, AllocType);
954 }
955
956 /// Get the AllocationType assigned to the given allocation instruction clone.
957 AllocationType getAllocationCallType(const CallInfo &Call) const {
958 return static_cast<const DerivedCCG *>(this)->getAllocationCallType(Call);
959 }
960
961 /// Update non-allocation call to invoke (possibly cloned) function
962 /// CalleeFunc.
963 void updateCall(CallInfo &CallerCall, FuncInfo CalleeFunc) {
964 static_cast<DerivedCCG *>(this)->updateCall(CallerCall, CalleeFunc);
965 }
966
967 /// Clone the given function for the given callsite, recording mapping of all
968 /// of the functions tracked calls to their new versions in the CallMap.
969 /// Assigns new clones to clone number CloneNo.
970 FuncInfo cloneFunctionForCallsite(
971 FuncInfo &Func, CallInfo &Call, DenseMap<CallInfo, CallInfo> &CallMap,
972 std::vector<CallInfo> &CallsWithMetadataInFunc, unsigned CloneNo) {
973 return static_cast<DerivedCCG *>(this)->cloneFunctionForCallsite(
974 Func, Call, CallMap, CallsWithMetadataInFunc, CloneNo);
975 }
976
977 /// Gets a label to use in the dot graph for the given call clone in the given
978 /// function.
979 std::string getLabel(const FuncTy *Func, const CallTy Call,
980 unsigned CloneNo) const {
981 return static_cast<const DerivedCCG *>(this)->getLabel(Func, Call, CloneNo);
982 }
983
984 /// Gets the function name (with optional clone suffix) for reporting.
985 std::string getFuncName(const FuncTy *Func, unsigned CloneNo = 0) const {
986 return static_cast<const DerivedCCG *>(this)->getFuncName(Func, CloneNo);
987 }
988
989 /// Returns true if the given function was marked dead by ThinLTO dead symbol
990 /// analysis.
991 bool isFuncDead(const FuncTy *Func) const {
992 return static_cast<const DerivedCCG *>(this)->isFuncDead(Func);
993 }
994
995 /// Populates FunctionsWithCallers (MemProfReportHintedSizesDetail only).
996 void populateFunctionsWithCallers() {
997 static_cast<DerivedCCG *>(this)->populateFunctionsWithCallers();
998 }
999
1000 /// Records, before identifyClones(), the chain of (uncloned) nodes of every
1001 /// context and how far it matched (MemProfReportHintedSizesDetail only).
1002 void capturePreCloningContextInfo();
1003
1004 /// Formats the detail suffix for the report line of context Id, whose
1005 /// (possibly cloned) allocation node is Node (MemProfReportHintedSizesDetail
1006 /// only). See the option description for the format.
1007 std::string formatContextReportSuffix(const ContextNode *Node,
1008 uint32_t Id) const;
1009
1010 // Create and return a new ContextNode.
1011 ContextNode *createNewNode(bool IsAllocation, const FuncTy *F = nullptr,
1012 CallInfo C = CallInfo()) {
1013 NodeOwner.push_back(std::make_unique<ContextNode>(IsAllocation, C));
1014 auto *NewNode = NodeOwner.back().get();
1015 if (F)
1016 NodeToCallingFunc[NewNode] = F;
1017 NewNode->NodeId = NodeOwner.size();
1018 return NewNode;
1019 }
1020
1021 /// Helpers to find the node corresponding to the given call or stackid.
1022 ContextNode *getNodeForInst(const CallInfo &C);
1023 ContextNode *getNodeForAlloc(const CallInfo &C);
1024 ContextNode *getNodeForStackId(uint64_t StackId);
1025
1026 /// Computes the alloc type corresponding to the given context ids, by
1027 /// unioning their recorded alloc types.
1028 uint8_t computeAllocType(DenseSet<uint32_t> &ContextIds) const;
1029
1030 /// Returns the allocation type of the intersection of the contexts of two
1031 /// nodes (based on their provided context id sets), optimized for the case
1032 /// when Node1Ids is smaller than Node2Ids.
1033 uint8_t intersectAllocTypesImpl(const DenseSet<uint32_t> &Node1Ids,
1034 const DenseSet<uint32_t> &Node2Ids) const;
1035
1036 /// Returns the allocation type of the intersection of the contexts of two
1037 /// nodes (based on their provided context id sets).
1038 uint8_t intersectAllocTypes(const DenseSet<uint32_t> &Node1Ids,
1039 const DenseSet<uint32_t> &Node2Ids) const;
1040
1041 /// Create a clone of Edge's callee and move Edge to that new callee node,
1042 /// performing the necessary context id and allocation type updates.
1043 /// If ContextIdsToMove is non-empty, only that subset of Edge's ids are
1044 /// moved to an edge to the new callee.
1045 ContextNode *
1046 moveEdgeToNewCalleeClone(const std::shared_ptr<ContextEdge> &Edge,
1047 DenseSet<uint32_t> ContextIdsToMove = {});
1048
1049 /// Change the callee of Edge to existing callee clone NewCallee, performing
1050 /// the necessary context id and allocation type updates.
1051 /// If ContextIdsToMove is non-empty, only that subset of Edge's ids are
1052 /// moved to an edge to the new callee.
1053 void moveEdgeToExistingCalleeClone(const std::shared_ptr<ContextEdge> &Edge,
1054 ContextNode *NewCallee,
1055 bool NewClone = false,
1056 DenseSet<uint32_t> ContextIdsToMove = {});
1057
1058 /// Change the caller of the edge at the given callee edge iterator to be
1059 /// NewCaller, performing the necessary context id and allocation type
1060 /// updates. This is similar to the above moveEdgeToExistingCalleeClone, but
1061 /// a simplified version of it as we always move the given edge and all of its
1062 /// context ids.
1063 void moveCalleeEdgeToNewCaller(const std::shared_ptr<ContextEdge> &Edge,
1064 ContextNode *NewCaller);
1065
1066 /// Recursive helper for marking backedges via DFS.
1067 void markBackedges(ContextNode *Node, DenseSet<const ContextNode *> &Visited,
1068 DenseSet<const ContextNode *> &CurrentStack);
1069
1070 /// Recursive helper for merging clones.
1071 void
1072 mergeClones(ContextNode *Node, DenseSet<const ContextNode *> &Visited,
1073 DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode);
1074 /// Main worker for merging callee clones for a given node.
1075 void mergeNodeCalleeClones(
1076 ContextNode *Node, DenseSet<const ContextNode *> &Visited,
1077 DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode);
1078 /// Helper to find other callers of the given set of callee edges that can
1079 /// share the same callee merge node.
1080 void findOtherCallersToShareMerge(
1081 ContextNode *Node, std::vector<std::shared_ptr<ContextEdge>> &CalleeEdges,
1082 DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode,
1083 DenseSet<ContextNode *> &OtherCallersToShareMerge);
1084
1085 /// Recursively perform cloning on the graph for the given Node and its
1086 /// callers, in order to uniquely identify the allocation behavior of an
1087 /// allocation given its context. The context ids of the allocation being
1088 /// processed are given in AllocContextIds.
1089 void identifyClones(ContextNode *Node, DenseSet<const ContextNode *> &Visited,
1090 const DenseSet<uint32_t> &AllocContextIds);
1091
1092 /// Records which clone of a callsite node an implicit caller requires (see
1093 /// assignImplicitCallers). The call to that callsite in a given function
1094 /// clone (the map key below) must call the same callee version as Source.
1095 struct ImplicitSource {
1096 /// The required clone of the callsite node.
1097 const ContextNode *Source;
1098 /// One of the trimmed context ids that made Source required. If Source is
1099 /// later emptied by further cloning, this id is used to find the clone
1100 /// that now carries the trimmed contexts (see resolveImplicitSource).
1101 uint32_t ContextId;
1102 };
1103 /// Map from (original callsite node, function clone number) to the clone of
1104 /// that callsite node that the call in that function clone must follow.
1105 using ImplicitSourceMap =
1106 DenseMap<std::pair<const ContextNode *, unsigned>, ImplicitSource>;
1107
1108 /// Assign the implicit callers of the clones of Node (callsite clones without
1109 /// an edge to any clone of Node which contexts trimmed at Node's frame would
1110 /// have reached) the function clone called by the caller carrying those
1111 /// contexts, and record the clone of Node their calls must follow. See
1112 /// assignFunctions for the maps.
1113 void assignImplicitCallers(
1114 ContextNode *Node, const DenseSet<uint32_t> &TrimmedContextIds,
1115 DenseMap<ContextNode *, FuncInfo> &CallsiteToCalleeFuncCloneMap,
1116 const std::map<FuncInfo, ContextNode *> &FuncCloneToCurNodeCloneMap,
1117 ImplicitSourceMap &ImplicitSources);
1118
1119 /// Returns the clone of Node (the original callsite node) that the implicit
1120 /// source Src refers to: Src.Source unless it has since been emptied, in
1121 /// which case the clone that now carries Src.ContextId (null if none does,
1122 /// which is not expected).
1123 static const ContextNode *resolveImplicitSource(const ContextNode *Node,
1124 const ImplicitSource &Src);
1125
1126 /// Map from each context ID to the AllocationType assigned to that context.
1127 DenseMap<uint32_t, AllocationType> ContextIdToAllocationType;
1128
1129 /// Map from each contextID to the profiled full contexts and their total
1130 /// sizes (there may be more than one due to context trimming),
1131 /// optionally populated when requested (via MemProfReportHintedSizes or
1132 /// MinClonedColdBytePercent).
1133 DenseMap<uint32_t, std::vector<ContextTotalSize>> ContextIdToContextSizeInfos;
1134
1135 /// Identifies the context node created for a stack id when adding the MIB
1136 /// contexts to the graph. This is used to locate the context nodes when
1137 /// trying to assign the corresponding callsites with those stack ids to these
1138 /// nodes.
1139 DenseMap<uint64_t, ContextNode *> StackEntryIdToContextNodeMap;
1140
1141 /// Saves information for the contexts identified as important (the largest
1142 /// cold contexts up to MemProfTopNImportant).
1143 struct ImportantContextInfo {
1144 // The original list of leaf first stack ids corresponding to this context.
1145 std::vector<uint64_t> StackIds;
1146 // Max length of stack ids corresponding to a single stack ContextNode for
1147 // this context (i.e. the max length of a key in StackIdsToNode below).
1148 unsigned MaxLength = 0;
1149 // Mapping of slices of the stack ids to the corresponding ContextNode
1150 // (there can be multiple stack ids due to inlining). Populated when
1151 // updating stack nodes while matching them to the IR or summary.
1152 std::map<std::vector<uint64_t>, ContextNode *> StackIdsToNode;
1153 };
1154
1155 // Map of important full context ids to information about each.
1156 DenseMap<uint32_t, ImportantContextInfo> ImportantContextIdInfo;
1157
1158 // For each important context id found in Node (if any), records the list of
1159 // stack ids (those of the given StackNodes) that corresponded to the given
1160 // callsite Node. There can be more than one in the case of inlining.
1161 void recordStackNode(ArrayRef<ContextNode *> StackNodes, ContextNode *Node,
1162 // We pass in the Node's context ids to avoid the
1163 // overhead of computing them as the caller already has
1164 // them in some cases.
1165 const DenseSet<uint32_t> &NodeContextIds,
1166 const DenseSet<uint32_t> &ImportantContextIds) {
1167 if (!MemProfTopNImportant) {
1168 assert(ImportantContextIds.empty());
1169 return;
1170 }
1171 DenseSet<uint32_t> Ids =
1172 set_intersection(NodeContextIds, ImportantContextIds);
1173 if (Ids.empty())
1174 return;
1175 std::vector<uint64_t> StackIds;
1176 StackIds.reserve(StackNodes.size());
1177 for (const ContextNode *N : StackNodes)
1178 StackIds.push_back(N->OrigStackOrAllocId);
1179 auto Size = StackIds.size();
1180 for (auto Id : Ids) {
1181 auto &Entry = ImportantContextIdInfo[Id];
1182 Entry.StackIdsToNode[StackIds] = Node;
1183 // Keep track of the max to simplify later analysis.
1184 if (Size > Entry.MaxLength)
1185 Entry.MaxLength = Size;
1186 }
1187 }
1188
1189 /// Maps to track the calls to their corresponding nodes in the graph.
1190 MapVector<CallInfo, ContextNode *> AllocationCallToContextNodeMap;
1191 MapVector<CallInfo, ContextNode *> NonAllocationCallToContextNodeMap;
1192
1193 /// Owner of all ContextNode unique_ptrs.
1194 std::vector<std::unique_ptr<ContextNode>> NodeOwner;
1195
1196 /// Perform sanity checks on graph when requested.
1197 void check() const;
1198
1199 /// Keeps track of the last unique context id assigned.
1200 unsigned int LastContextId = 0;
1201
1202 /// Per-context information captured before identifyClones() when
1203 /// MemProfReportHintedSizesDetail is enabled, so that the report can relate
1204 /// the final (cloned) state back to the original context. "Frames" below are
1205 /// CCG nodes, not profile stack frames: a node matched to a call covers all
1206 /// of that call's inlined stack frames, so a chain can be shorter than the
1207 /// profiled stack.
1208 struct PreCloningContextInfo {
1209 // Ordered chain of uncloned ContextNodes for this context ID from the leaf
1210 // allocation node (Chain[0]) up to the outermost node in the CCG for this
1211 // context.
1212 std::vector<const ContextNode *> Chain;
1213 // Index in Chain of the outermost matched callsite node, i.e. the
1214 // outermost node such that it and all nodes below it
1215 // (Chain[1..MaxMatchedFrame]) were matched to a call in the IR/summary
1216 // (hasCall() == true). If the whole context matched, this is the
1217 // context's outermost node (possibly where the context was trimmed);
1218 // otherwise it is the last matched node below the first unmatched one. The
1219 // function containing it (the "outermost matched caller") is where the
1220 // reported post-cloning paths start. 0 if no caller of the allocation
1221 // matched.
1222 unsigned MaxMatchedFrame = 0;
1223 // True if the outermost matched node had caller edges for other contexts,
1224 // i.e. the context was trimmed below callers that other contexts reach.
1225 bool OutermostMatchedNodeHasOtherCallers = false;
1226 // If MaxMatchedFrame + 1 < Chain.size(), why did Chain[MaxMatchedFrame + 1]
1227 // not have a call: it was a recursive cycle node, or its calling function
1228 // was found but the call there did not match the callee (mismatched
1229 // callee); otherwise the caller had no !callsite for this stack id.
1230 bool StoppedAtRecursive = false;
1231 bool StoppedAtMismatchedCallee = false;
1232 // Stack id and (when known) calling function of the first unmatched node.
1233 uint64_t UnmatchedStackId = 0;
1234 const FuncTy *UnmatchedCallingFunc = nullptr;
1235 };
1236 /// Map from context id to its pre-cloning information.
1237 DenseMap<uint32_t, PreCloningContextInfo> PreCloningContextInfoMap;
1238 /// Allocation nodes (uncloned) that had at least one matched caller before
1239 /// cloning.
1240 DenseSet<const ContextNode *> PreCloningAllocHasMatchedCaller;
1241 /// Map from function to the number of function clones assigned to it
1242 /// (including the original), recorded during assignFunctions().
1243 DenseMap<const FuncTy *, unsigned> FuncToNumClones;
1244 /// Map from (original callsite node, clone number of the function clone
1245 /// containing the call) to the clone number of the callee function the call
1246 /// was updated to in assignFunctions(). Used to follow the post-cloning call
1247 /// path from a caller clone down to the allocation clone it reaches.
1248 DenseMap<std::pair<const ContextNode *, unsigned>, unsigned>
1249 OrigNodeAndCloneToCalleeClone;
1250 /// Map from (original allocation node, clone number of the function clone
1251 /// containing the allocation) to the allocation node in that clone, whose
1252 /// call carries the final hint.
1253 DenseMap<std::pair<const ContextNode *, unsigned>, const ContextNode *>
1254 OrigAllocAndCloneToNode;
1255};
1256
1257template <typename DerivedCCG, typename FuncTy, typename CallTy>
1258using ContextNode =
1259 typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode;
1260template <typename DerivedCCG, typename FuncTy, typename CallTy>
1261using ContextEdge =
1262 typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge;
1263template <typename DerivedCCG, typename FuncTy, typename CallTy>
1264using FuncInfo =
1265 typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::FuncInfo;
1266template <typename DerivedCCG, typename FuncTy, typename CallTy>
1267using CallInfo =
1268 typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::CallInfo;
1269
1270/// CRTP derived class for graphs built from IR (regular LTO).
1271class ModuleCallsiteContextGraph
1272 : public CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
1273 Instruction *> {
1274public:
1275 ModuleCallsiteContextGraph(
1276 Module &M,
1277 llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter);
1278
1279private:
1280 friend CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
1281 Instruction *>;
1282
1283 uint64_t getStackId(uint64_t IdOrIndex) const;
1284 const Function *getCalleeFunc(Instruction *Call);
1285 bool calleeMatchesFunc(
1286 Instruction *Call, const Function *Func, const Function *CallerFunc,
1287 std::vector<std::pair<Instruction *, Function *>> &FoundCalleeChain);
1288 bool sameCallee(Instruction *Call1, Instruction *Call2);
1289 bool findProfiledCalleeThroughTailCalls(
1290 const Function *ProfiledCallee, Value *CurCallee, unsigned Depth,
1291 std::vector<std::pair<Instruction *, Function *>> &FoundCalleeChain,
1292 bool &FoundMultipleCalleeChains);
1293 uint64_t getLastStackId(Instruction *Call);
1294 std::vector<ContextNode *> getStackNodesForCall(Instruction *Call);
1295 void updateAllocationCall(CallInfo &Call, AllocationType AllocType);
1296 AllocationType getAllocationCallType(const CallInfo &Call) const;
1297 void updateCall(CallInfo &CallerCall, FuncInfo CalleeFunc);
1298 CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
1299 Instruction *>::FuncInfo
1300 cloneFunctionForCallsite(FuncInfo &Func, CallInfo &Call,
1301 DenseMap<CallInfo, CallInfo> &CallMap,
1302 std::vector<CallInfo> &CallsWithMetadataInFunc,
1303 unsigned CloneNo);
1304 std::string getLabel(const Function *Func, const Instruction *Call,
1305 unsigned CloneNo) const;
1306 std::string getFuncName(const Function *Func, unsigned CloneNo = 0) const;
1307 bool isFuncDead(const Function *Func) const { return false; }
1308 void populateFunctionsWithCallers();
1309
1310 const Module &Mod;
1311 llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter;
1312};
1313
1314/// Represents a call in the summary index graph, which can either be an
1315/// allocation or an interior callsite node in an allocation's context.
1316/// Holds a pointer to the corresponding data structure in the index.
1317struct IndexCall : public PointerUnion<CallsiteInfo *, AllocInfo *> {
1318 IndexCall() : PointerUnion() {}
1319 IndexCall(std::nullptr_t) : IndexCall() {}
1320 IndexCall(CallsiteInfo *StackNode) : PointerUnion(StackNode) {}
1321 IndexCall(AllocInfo *AllocNode) : PointerUnion(AllocNode) {}
1322 IndexCall(PointerUnion PT) : PointerUnion(PT) {}
1323
1324 IndexCall *operator->() { return this; }
1325
1326 void print(raw_ostream &OS) const {
1327 PointerUnion<CallsiteInfo *, AllocInfo *> Base = *this;
1329 OS << *AI;
1330 } else {
1332 assert(CI);
1333 OS << *CI;
1334 }
1335 }
1336};
1337} // namespace
1338
1339namespace llvm {
1340template <> struct simplify_type<IndexCall> {
1342 static SimpleType getSimplifiedValue(IndexCall &Val) { return Val; }
1343};
1344template <> struct simplify_type<const IndexCall> {
1346 static SimpleType getSimplifiedValue(const IndexCall &Val) { return Val; }
1347};
1348} // namespace llvm
1349
1350namespace {
1351/// CRTP derived class for graphs built from summary index (ThinLTO).
1352class IndexCallsiteContextGraph
1353 : public CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
1354 IndexCall> {
1355public:
1356 IndexCallsiteContextGraph(
1357 ModuleSummaryIndex &Index,
1358 llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1359 isPrevailing);
1360
1361 ~IndexCallsiteContextGraph() {
1362 // Now that we are done with the graph it is safe to add the new
1363 // CallsiteInfo structs to the function summary vectors. The graph nodes
1364 // point into locations within these vectors, so we don't want to add them
1365 // any earlier.
1366 for (auto &I : FunctionCalleesToSynthesizedCallsiteInfos) {
1367 auto *FS = I.first;
1368 for (auto &Callsite : I.second)
1369 FS->addCallsite(std::move(*Callsite.second));
1370 }
1371 }
1372
1373private:
1374 friend CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
1375 IndexCall>;
1376
1377 uint64_t getStackId(uint64_t IdOrIndex) const;
1378 const FunctionSummary *getCalleeFunc(IndexCall &Call);
1379 bool calleeMatchesFunc(
1380 IndexCall &Call, const FunctionSummary *Func,
1381 const FunctionSummary *CallerFunc,
1382 std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain);
1383 bool sameCallee(IndexCall &Call1, IndexCall &Call2);
1384 bool findProfiledCalleeThroughTailCalls(
1385 ValueInfo ProfiledCallee, ValueInfo CurCallee, unsigned Depth,
1386 std::vector<std::pair<ValueInfo, FunctionSummary *>> &FoundCalleeChain,
1387 bool &FoundMultipleCalleeChains);
1388 uint64_t getLastStackId(IndexCall &Call);
1389 std::vector<ContextNode *> getStackNodesForCall(IndexCall &Call);
1390 void updateAllocationCall(CallInfo &Call, AllocationType AllocType);
1391 AllocationType getAllocationCallType(const CallInfo &Call) const;
1392 void updateCall(CallInfo &CallerCall, FuncInfo CalleeFunc);
1393 CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
1394 IndexCall>::FuncInfo
1395 cloneFunctionForCallsite(FuncInfo &Func, CallInfo &Call,
1396 DenseMap<CallInfo, CallInfo> &CallMap,
1397 std::vector<CallInfo> &CallsWithMetadataInFunc,
1398 unsigned CloneNo);
1399 std::string getLabel(const FunctionSummary *Func, const IndexCall &Call,
1400 unsigned CloneNo) const;
1401 std::string getFuncName(const FunctionSummary *Func,
1402 unsigned CloneNo = 0) const;
1403 bool isFuncDead(const FunctionSummary *Func) const;
1404 void populateFunctionsWithCallers();
1405 DenseSet<GlobalValue::GUID> findAliaseeGUIDsPrevailingInDifferentModule();
1406
1407 // Saves mapping from function summaries containing memprof records back to
1408 // its VI, for use in checking and debugging.
1409 std::map<const FunctionSummary *, ValueInfo> FSToVIMap;
1410
1411 const ModuleSummaryIndex &Index;
1412 llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
1413 isPrevailing;
1414
1415 // Saves/owns the callsite info structures synthesized for missing tail call
1416 // frames that we discover while building the graph.
1417 // It maps from the summary of the function making the tail call, to a map
1418 // of callee ValueInfo to corresponding synthesized callsite info.
1419 DenseMap<FunctionSummary *,
1420 std::map<ValueInfo, std::unique_ptr<CallsiteInfo>>>
1421 FunctionCalleesToSynthesizedCallsiteInfos;
1422};
1423} // namespace
1424
1425template <>
1426struct llvm::DenseMapInfo<CallsiteContextGraph<
1427 ModuleCallsiteContextGraph, Function, Instruction *>::CallInfo>
1429template <>
1430struct llvm::DenseMapInfo<CallsiteContextGraph<
1431 IndexCallsiteContextGraph, FunctionSummary, IndexCall>::CallInfo>
1432 : public DenseMapInfo<std::pair<IndexCall, unsigned>> {};
1433template <>
1434struct llvm::DenseMapInfo<IndexCall>
1435 : public DenseMapInfo<PointerUnion<CallsiteInfo *, AllocInfo *>> {};
1436
1437namespace {
1438
1439// Map the uint8_t alloc types (which may contain NotCold|Cold) to the alloc
1440// type we should actually use on the corresponding allocation.
1441// If we can't clone a node that has NotCold+Cold alloc type, we will fall
1442// back to using NotCold. So don't bother cloning to distinguish NotCold+Cold
1443// from NotCold.
1444AllocationType allocTypeToUse(uint8_t AllocTypes) {
1445 assert(AllocTypes != (uint8_t)AllocationType::None);
1446 if (AllocTypes ==
1449 else
1450 return (AllocationType)AllocTypes;
1451}
1452
1453// Helper to check if the alloc types for all edges recorded in the
1454// InAllocTypes vector match the alloc types for all edges in the Edges
1455// vector.
1456template <typename DerivedCCG, typename FuncTy, typename CallTy>
1457bool allocTypesMatch(
1458 const std::vector<uint8_t> &InAllocTypes,
1459 const std::vector<std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>>>
1460 &Edges) {
1461 // This should be called only when the InAllocTypes vector was computed for
1462 // this set of Edges. Make sure the sizes are the same.
1463 assert(InAllocTypes.size() == Edges.size());
1464 return std::equal(
1465 InAllocTypes.begin(), InAllocTypes.end(), Edges.begin(), Edges.end(),
1466 [](const uint8_t &l,
1467 const std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>> &r) {
1468 // Can share if one of the edges is None type - don't
1469 // care about the type along that edge as it doesn't
1470 // exist for those context ids.
1471 if (l == (uint8_t)AllocationType::None ||
1472 r->AllocTypes == (uint8_t)AllocationType::None)
1473 return true;
1474 return allocTypeToUse(l) == allocTypeToUse(r->AllocTypes);
1475 });
1476}
1477
1478// Helper to check if the alloc types for all edges recorded in the
1479// InAllocTypes vector match the alloc types for callee edges in the given
1480// clone. Because the InAllocTypes were computed from the original node's callee
1481// edges, and other cloning could have happened after this clone was created, we
1482// need to find the matching clone callee edge, which may or may not exist.
1483template <typename DerivedCCG, typename FuncTy, typename CallTy>
1484bool allocTypesMatchClone(
1485 const std::vector<uint8_t> &InAllocTypes,
1486 const ContextNode<DerivedCCG, FuncTy, CallTy> *Clone) {
1487 const ContextNode<DerivedCCG, FuncTy, CallTy> *Node = Clone->CloneOf;
1488 assert(Node);
1489 // InAllocTypes should have been computed for the original node's callee
1490 // edges.
1491 assert(InAllocTypes.size() == Node->CalleeEdges.size());
1492 // First create a map of the clone callee edge callees to the edge alloc type.
1494 EdgeCalleeMap;
1495 for (const auto &E : Clone->CalleeEdges) {
1496 assert(!EdgeCalleeMap.contains(E->Callee));
1497 EdgeCalleeMap[E->Callee] = E->AllocTypes;
1498 }
1499 // Next, walk the original node's callees, and look for the corresponding
1500 // clone edge to that callee.
1501 for (unsigned I = 0; I < Node->CalleeEdges.size(); I++) {
1502 auto Iter = EdgeCalleeMap.find(Node->CalleeEdges[I]->Callee);
1503 // Not found is ok, we will simply add an edge if we use this clone.
1504 if (Iter == EdgeCalleeMap.end())
1505 continue;
1506 // Can share if one of the edges is None type - don't
1507 // care about the type along that edge as it doesn't
1508 // exist for those context ids.
1509 if (InAllocTypes[I] == (uint8_t)AllocationType::None ||
1510 Iter->second == (uint8_t)AllocationType::None)
1511 continue;
1512 if (allocTypeToUse(Iter->second) != allocTypeToUse(InAllocTypes[I]))
1513 return false;
1514 }
1515 return true;
1516}
1517
1518} // end anonymous namespace
1519
1520template <typename DerivedCCG, typename FuncTy, typename CallTy>
1521typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1522CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getNodeForInst(
1523 const CallInfo &C) {
1524 ContextNode *Node = getNodeForAlloc(C);
1525 if (Node)
1526 return Node;
1527
1528 return NonAllocationCallToContextNodeMap.lookup(C);
1529}
1530
1531template <typename DerivedCCG, typename FuncTy, typename CallTy>
1532typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1533CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getNodeForAlloc(
1534 const CallInfo &C) {
1535 return AllocationCallToContextNodeMap.lookup(C);
1536}
1537
1538template <typename DerivedCCG, typename FuncTy, typename CallTy>
1539typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1540CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getNodeForStackId(
1541 uint64_t StackId) {
1542 auto StackEntryNode = StackEntryIdToContextNodeMap.find(StackId);
1543 if (StackEntryNode != StackEntryIdToContextNodeMap.end())
1544 return StackEntryNode->second;
1545 return nullptr;
1546}
1547
1548template <typename DerivedCCG, typename FuncTy, typename CallTy>
1549void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1550 addOrUpdateCallerEdge(ContextNode *Caller, AllocationType AllocType,
1551 unsigned int ContextId) {
1552 for (auto &Edge : CallerEdges) {
1553 if (Edge->Caller == Caller) {
1554 Edge->AllocTypes |= (uint8_t)AllocType;
1555 Edge->getContextIds().insert(ContextId);
1556 return;
1557 }
1558 }
1559 std::shared_ptr<ContextEdge> Edge = std::make_shared<ContextEdge>(
1560 this, Caller, (uint8_t)AllocType, DenseSet<uint32_t>({ContextId}));
1561 CallerEdges.push_back(Edge);
1562 Caller->CalleeEdges.push_back(Edge);
1563}
1564
1565template <typename DerivedCCG, typename FuncTy, typename CallTy>
1566void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::removeEdgeFromGraph(
1567 ContextEdge *Edge, EdgeIter *EI, bool CalleeIter) {
1568 assert(!EI || (*EI)->get() == Edge);
1569 assert(!Edge->isRemoved());
1570 // Save the Caller and Callee pointers so we can erase Edge from their edge
1571 // lists after clearing Edge below. We do the clearing first in case it is
1572 // destructed after removing from the edge lists (if those were the last
1573 // shared_ptr references to Edge).
1574 auto *Callee = Edge->Callee;
1575 auto *Caller = Edge->Caller;
1576
1577 // Make sure the edge fields are cleared out so we can properly detect
1578 // removed edges if Edge is not destructed because there is still a shared_ptr
1579 // reference.
1580 Edge->clear();
1581
1582#ifndef NDEBUG
1583 auto CalleeCallerCount = Callee->CallerEdges.size();
1584 auto CallerCalleeCount = Caller->CalleeEdges.size();
1585#endif
1586 if (!EI) {
1587 Callee->eraseCallerEdge(Edge);
1588 Caller->eraseCalleeEdge(Edge);
1589 } else if (CalleeIter) {
1590 Callee->eraseCallerEdge(Edge);
1591 *EI = Caller->CalleeEdges.erase(*EI);
1592 } else {
1593 Caller->eraseCalleeEdge(Edge);
1594 *EI = Callee->CallerEdges.erase(*EI);
1595 }
1596 assert(Callee->CallerEdges.size() < CalleeCallerCount);
1597 assert(Caller->CalleeEdges.size() < CallerCalleeCount);
1598}
1599
1600template <typename DerivedCCG, typename FuncTy, typename CallTy>
1601void CallsiteContextGraph<
1602 DerivedCCG, FuncTy, CallTy>::removeNoneTypeCalleeEdges(ContextNode *Node) {
1603 for (auto EI = Node->CalleeEdges.begin(); EI != Node->CalleeEdges.end();) {
1604 auto Edge = *EI;
1605 if (Edge->AllocTypes == (uint8_t)AllocationType::None) {
1606 assert(Edge->ContextIds.empty());
1607 removeEdgeFromGraph(Edge.get(), &EI, /*CalleeIter=*/true);
1608 } else
1609 ++EI;
1610 }
1611}
1612
1613template <typename DerivedCCG, typename FuncTy, typename CallTy>
1614void CallsiteContextGraph<
1615 DerivedCCG, FuncTy, CallTy>::removeNoneTypeCallerEdges(ContextNode *Node) {
1616 for (auto EI = Node->CallerEdges.begin(); EI != Node->CallerEdges.end();) {
1617 auto Edge = *EI;
1618 if (Edge->AllocTypes == (uint8_t)AllocationType::None) {
1619 assert(Edge->ContextIds.empty());
1620 Edge->Caller->eraseCalleeEdge(Edge.get());
1621 EI = Node->CallerEdges.erase(EI);
1622 } else
1623 ++EI;
1624 }
1625}
1626
1627template <typename DerivedCCG, typename FuncTy, typename CallTy>
1628typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge *
1629CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1630 findEdgeFromCallee(const ContextNode *Callee) {
1631 for (const auto &Edge : CalleeEdges)
1632 if (Edge->Callee == Callee)
1633 return Edge.get();
1634 return nullptr;
1635}
1636
1637template <typename DerivedCCG, typename FuncTy, typename CallTy>
1638typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge *
1639CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1640 findEdgeFromCaller(const ContextNode *Caller) {
1641 for (const auto &Edge : CallerEdges)
1642 if (Edge->Caller == Caller)
1643 return Edge.get();
1644 return nullptr;
1645}
1646
1647template <typename DerivedCCG, typename FuncTy, typename CallTy>
1648void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1649 eraseCalleeEdge(const ContextEdge *Edge) {
1650 auto EI = llvm::find_if(
1651 CalleeEdges, [Edge](const std::shared_ptr<ContextEdge> &CalleeEdge) {
1652 return CalleeEdge.get() == Edge;
1653 });
1654 assert(EI != CalleeEdges.end());
1655 CalleeEdges.erase(EI);
1656}
1657
1658template <typename DerivedCCG, typename FuncTy, typename CallTy>
1659void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::
1660 eraseCallerEdge(const ContextEdge *Edge) {
1661 auto EI = llvm::find_if(
1662 CallerEdges, [Edge](const std::shared_ptr<ContextEdge> &CallerEdge) {
1663 return CallerEdge.get() == Edge;
1664 });
1665 assert(EI != CallerEdges.end());
1666 CallerEdges.erase(EI);
1667}
1668
1669template <typename DerivedCCG, typename FuncTy, typename CallTy>
1670uint8_t CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::computeAllocType(
1671 DenseSet<uint32_t> &ContextIds) const {
1672 uint8_t BothTypes =
1673 (uint8_t)AllocationType::Cold | (uint8_t)AllocationType::NotCold;
1674 uint8_t AllocType = (uint8_t)AllocationType::None;
1675 for (auto Id : ContextIds) {
1676 AllocType |= (uint8_t)ContextIdToAllocationType.at(Id);
1677 // Bail early if alloc type reached both, no further refinement.
1678 if (AllocType == BothTypes)
1679 return AllocType;
1680 }
1681 return AllocType;
1682}
1683
1684template <typename DerivedCCG, typename FuncTy, typename CallTy>
1685uint8_t
1686CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::intersectAllocTypesImpl(
1687 const DenseSet<uint32_t> &Node1Ids,
1688 const DenseSet<uint32_t> &Node2Ids) const {
1689 uint8_t BothTypes =
1690 (uint8_t)AllocationType::Cold | (uint8_t)AllocationType::NotCold;
1691 uint8_t AllocType = (uint8_t)AllocationType::None;
1692 for (auto Id : Node1Ids) {
1693 if (!Node2Ids.count(Id))
1694 continue;
1695 AllocType |= (uint8_t)ContextIdToAllocationType.at(Id);
1696 // Bail early if alloc type reached both, no further refinement.
1697 if (AllocType == BothTypes)
1698 return AllocType;
1699 }
1700 return AllocType;
1701}
1702
1703template <typename DerivedCCG, typename FuncTy, typename CallTy>
1704uint8_t CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::intersectAllocTypes(
1705 const DenseSet<uint32_t> &Node1Ids,
1706 const DenseSet<uint32_t> &Node2Ids) const {
1707 if (Node1Ids.size() < Node2Ids.size())
1708 return intersectAllocTypesImpl(Node1Ids, Node2Ids);
1709 else
1710 return intersectAllocTypesImpl(Node2Ids, Node1Ids);
1711}
1712
1713template <typename DerivedCCG, typename FuncTy, typename CallTy>
1714typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
1715CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::addAllocNode(
1716 CallInfo Call, const FuncTy *F) {
1717 assert(!getNodeForAlloc(Call));
1718 ContextNode *AllocNode = createNewNode(/*IsAllocation=*/true, F, Call);
1719 AllocationCallToContextNodeMap[Call] = AllocNode;
1720 // Use LastContextId as a uniq id for MIB allocation nodes.
1721 AllocNode->OrigStackOrAllocId = LastContextId;
1722 // Alloc type should be updated as we add in the MIBs. We should assert
1723 // afterwards that it is not still None.
1724 AllocNode->AllocTypes = (uint8_t)AllocationType::None;
1725
1726 return AllocNode;
1727}
1728
1729static std::string getAllocTypeString(uint8_t AllocTypes) {
1730 if (!AllocTypes)
1731 return "None";
1732 std::string Str;
1733 if (AllocTypes & (uint8_t)AllocationType::NotCold)
1734 Str += "NotCold";
1735 if (AllocTypes & (uint8_t)AllocationType::Cold)
1736 Str += "Cold";
1737 return Str;
1738}
1739
1740template <typename DerivedCCG, typename FuncTy, typename CallTy>
1741template <class NodeT, class IteratorT>
1742void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::addStackNodesForMIB(
1743 ContextNode *AllocNode, CallStack<NodeT, IteratorT> &StackContext,
1744 CallStack<NodeT, IteratorT> &CallsiteContext, AllocationType AllocType,
1745 ArrayRef<ContextTotalSize> ContextSizeInfo,
1746 std::map<uint64_t, uint32_t> &TotalSizeToContextIdTopNCold) {
1747 // Treating the hot alloc type as NotCold before the disambiguation for "hot"
1748 // is done.
1749 if (AllocType == AllocationType::Hot)
1750 AllocType = AllocationType::NotCold;
1751
1752 ContextIdToAllocationType[++LastContextId] = AllocType;
1753
1754 bool IsImportant = false;
1755 if (!ContextSizeInfo.empty()) {
1756 auto &Entry = ContextIdToContextSizeInfos[LastContextId];
1757 // If this is a cold allocation, and we are collecting non-zero largest
1758 // contexts, see if this is a candidate.
1759 if (AllocType == AllocationType::Cold && MemProfTopNImportant > 0) {
1760 uint64_t TotalCold = 0;
1761 for (auto &CSI : ContextSizeInfo)
1762 TotalCold += CSI.TotalSize;
1763 // Record this context if either we haven't found the first top-n largest
1764 // yet, or if it is larger than the smallest already recorded.
1765 if (TotalSizeToContextIdTopNCold.size() < MemProfTopNImportant ||
1766 // Since TotalSizeToContextIdTopNCold is a std::map, it is implicitly
1767 // sorted in ascending size of its key which is the size.
1768 TotalCold > TotalSizeToContextIdTopNCold.begin()->first) {
1769 if (TotalSizeToContextIdTopNCold.size() == MemProfTopNImportant) {
1770 // Remove old one and its associated entries.
1771 auto IdToRemove = TotalSizeToContextIdTopNCold.begin()->second;
1772 TotalSizeToContextIdTopNCold.erase(
1773 TotalSizeToContextIdTopNCold.begin());
1774 assert(ImportantContextIdInfo.count(IdToRemove));
1775 ImportantContextIdInfo.erase(IdToRemove);
1776 }
1777 TotalSizeToContextIdTopNCold[TotalCold] = LastContextId;
1778 IsImportant = true;
1779 }
1780 }
1781 Entry.insert(Entry.begin(), ContextSizeInfo.begin(), ContextSizeInfo.end());
1782 }
1783
1784 // Update alloc type and context ids for this MIB.
1785 AllocNode->AllocTypes |= (uint8_t)AllocType;
1786
1787 // Now add or update nodes for each stack id in alloc's context.
1788 // Later when processing the stack ids on non-alloc callsites we will adjust
1789 // for any inlining in the context.
1790 ContextNode *PrevNode = AllocNode;
1791 // Look for recursion (direct recursion should have been collapsed by
1792 // module summary analysis, here we should just be detecting mutual
1793 // recursion). Mark these nodes so we don't try to clone.
1794 SmallSet<uint64_t, 8> StackIdSet;
1795 // Skip any on the allocation call (inlining).
1796 for (auto ContextIter = StackContext.beginAfterSharedPrefix(CallsiteContext);
1797 ContextIter != StackContext.end(); ++ContextIter) {
1798 auto StackId = getStackId(*ContextIter);
1799 if (IsImportant)
1800 ImportantContextIdInfo[LastContextId].StackIds.push_back(StackId);
1801 ContextNode *StackNode = getNodeForStackId(StackId);
1802 if (!StackNode) {
1803 StackNode = createNewNode(/*IsAllocation=*/false);
1804 StackEntryIdToContextNodeMap[StackId] = StackNode;
1805 StackNode->OrigStackOrAllocId = StackId;
1806 }
1807 // Marking a node recursive will prevent its cloning completely, even for
1808 // non-recursive contexts flowing through it.
1810 auto Ins = StackIdSet.insert(StackId);
1811 if (!Ins.second)
1812 StackNode->Recursive = true;
1813 }
1814 StackNode->AllocTypes |= (uint8_t)AllocType;
1815 PrevNode->addOrUpdateCallerEdge(StackNode, AllocType, LastContextId);
1816 PrevNode = StackNode;
1817 }
1818}
1819
1820template <typename DerivedCCG, typename FuncTy, typename CallTy>
1821DenseSet<uint32_t>
1822CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::duplicateContextIds(
1823 const DenseSet<uint32_t> &StackSequenceContextIds,
1824 DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds,
1825 function_ref<void(uint32_t OldId, uint32_t NewId)> OnDuplicate) {
1826 DenseSet<uint32_t> NewContextIds;
1827 NewContextIds.reserve(StackSequenceContextIds.size());
1828 for (auto OldId : StackSequenceContextIds) {
1829 NewContextIds.insert(++LastContextId);
1830 OldToNewContextIds[OldId].insert(LastContextId);
1831 if (OnDuplicate)
1832 OnDuplicate(OldId, LastContextId);
1833 assert(ContextIdToAllocationType.count(OldId));
1834 // The new context has the same allocation type and size info as original.
1835 ContextIdToAllocationType[LastContextId] = ContextIdToAllocationType[OldId];
1836 auto CSI = ContextIdToContextSizeInfos.find(OldId);
1837 if (CSI != ContextIdToContextSizeInfos.end())
1838 ContextIdToContextSizeInfos[LastContextId] = CSI->second;
1839 if (DotAllocContextIds.contains(OldId))
1840 DotAllocContextIds.insert(LastContextId);
1841 }
1842 return NewContextIds;
1843}
1844
1845template <typename DerivedCCG, typename FuncTy, typename CallTy>
1846void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
1847 propagateDuplicateContextIds(
1848 const DenseMap<uint32_t, DenseSet<uint32_t>> &OldToNewContextIds) {
1849 // Build a set of duplicated context ids corresponding to the input id set.
1850 auto GetNewIds = [&OldToNewContextIds](const DenseSet<uint32_t> &ContextIds) {
1851 DenseSet<uint32_t> NewIds;
1852 for (auto Id : ContextIds)
1853 if (auto NewId = OldToNewContextIds.find(Id);
1854 NewId != OldToNewContextIds.end())
1855 NewIds.insert_range(NewId->second);
1856 return NewIds;
1857 };
1858
1859 // Recursively update context ids sets along caller edges.
1860 auto UpdateCallers = [&](ContextNode *Node,
1861 DenseSet<const ContextEdge *> &Visited,
1862 auto &&UpdateCallers) -> void {
1863 for (const auto &Edge : Node->CallerEdges) {
1864 auto Inserted = Visited.insert(Edge.get());
1865 if (!Inserted.second)
1866 continue;
1867 ContextNode *NextNode = Edge->Caller;
1868 DenseSet<uint32_t> NewIdsToAdd = GetNewIds(Edge->getContextIds());
1869 // Only need to recursively iterate to NextNode via this caller edge if
1870 // it resulted in any added ids to NextNode.
1871 if (!NewIdsToAdd.empty()) {
1872 Edge->getContextIds().insert_range(NewIdsToAdd);
1873 UpdateCallers(NextNode, Visited, UpdateCallers);
1874 }
1875 }
1876 };
1877
1878 DenseSet<const ContextEdge *> Visited;
1879 for (auto &Entry : AllocationCallToContextNodeMap) {
1880 auto *Node = Entry.second;
1881 UpdateCallers(Node, Visited, UpdateCallers);
1882 }
1883}
1884
1885template <typename DerivedCCG, typename FuncTy, typename CallTy>
1886void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::connectNewNode(
1887 ContextNode *NewNode, ContextNode *OrigNode, bool TowardsCallee,
1888 const DenseSet<uint32_t> &ContextIds) {
1889 auto &OrigEdges =
1890 TowardsCallee ? OrigNode->CalleeEdges : OrigNode->CallerEdges;
1891 // Increment iterator in loop so that we can remove edges as needed.
1892 for (auto EI = OrigEdges.begin(); EI != OrigEdges.end();) {
1893 auto Edge = *EI;
1894 DenseSet<uint32_t> NewEdgeContextIds;
1895 // Remove any matching context ids from Edge, return set that were found and
1896 // removed, these are the new edge's context ids.
1897 set_subtract(Edge->getContextIds(), ContextIds, NewEdgeContextIds);
1898 // If no matching context ids for this edge, skip it.
1899 if (NewEdgeContextIds.empty()) {
1900 ++EI;
1901 continue;
1902 }
1903 if (TowardsCallee) {
1904 uint8_t NewAllocType = computeAllocType(NewEdgeContextIds);
1905 auto NewEdge = std::make_shared<ContextEdge>(
1906 Edge->Callee, NewNode, NewAllocType, std::move(NewEdgeContextIds));
1907 NewNode->CalleeEdges.push_back(NewEdge);
1908 NewEdge->Callee->CallerEdges.push_back(NewEdge);
1909 } else {
1910 uint8_t NewAllocType = computeAllocType(NewEdgeContextIds);
1911 auto NewEdge = std::make_shared<ContextEdge>(
1912 NewNode, Edge->Caller, NewAllocType, std::move(NewEdgeContextIds));
1913 NewNode->CallerEdges.push_back(NewEdge);
1914 NewEdge->Caller->CalleeEdges.push_back(NewEdge);
1915 }
1916 // Remove old edge if context ids empty, otherwise update its alloc type to
1917 // reflect the removed context ids.
1918 if (Edge->getContextIds().empty()) {
1919 removeEdgeFromGraph(Edge.get(), &EI, TowardsCallee);
1920 continue;
1921 }
1922 Edge->AllocTypes = computeAllocType(Edge->getContextIds());
1923 ++EI;
1924 }
1925}
1926
1927template <typename DerivedCCG, typename FuncTy, typename CallTy>
1928static void checkEdge(
1929 const std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>> &Edge) {
1930 // Confirm that alloc type is not None and that we have at least one context
1931 // id.
1932 assert(Edge->AllocTypes != (uint8_t)AllocationType::None);
1933 assert(!Edge->ContextIds.empty());
1934}
1935
1936template <typename DerivedCCG, typename FuncTy, typename CallTy>
1937static void checkNode(const ContextNode<DerivedCCG, FuncTy, CallTy> *Node,
1938 bool CheckEdges = true) {
1939 if (Node->isRemoved())
1940 return;
1941#ifndef NDEBUG
1942 // Compute node's context ids once for use in asserts.
1943 auto NodeContextIds = Node->getContextIds();
1944#endif
1945 // Node's context ids should be the union of both its callee and caller edge
1946 // context ids.
1947 if (Node->CallerEdges.size()) {
1948 DenseSet<uint32_t> CallerEdgeContextIds(
1949 Node->CallerEdges.front()->ContextIds);
1950 for (const auto &Edge : llvm::drop_begin(Node->CallerEdges)) {
1951 if (CheckEdges)
1953 set_union(CallerEdgeContextIds, Edge->ContextIds);
1954 }
1955 // Node can have more context ids than callers if some contexts terminate at
1956 // node and some are longer. If we are allowing recursive callsites and
1957 // contexts this will be violated for incompletely cloned recursive cycles,
1958 // so skip the checking in that case.
1960 NodeContextIds == CallerEdgeContextIds ||
1961 set_is_subset(CallerEdgeContextIds, NodeContextIds));
1962 }
1963 if (Node->CalleeEdges.size()) {
1964 DenseSet<uint32_t> CalleeEdgeContextIds(
1965 Node->CalleeEdges.front()->ContextIds);
1966 for (const auto &Edge : llvm::drop_begin(Node->CalleeEdges)) {
1967 if (CheckEdges)
1969 set_union(CalleeEdgeContextIds, Edge->getContextIds());
1970 }
1971 // If we are allowing recursive callsites and contexts this will be violated
1972 // for incompletely cloned recursive cycles, so skip the checking in that
1973 // case.
1975 NodeContextIds == CalleeEdgeContextIds);
1976 }
1977 // FIXME: Since this checking is only invoked under an option, we should
1978 // change the error checking from using assert to something that will trigger
1979 // an error on a release build.
1980#ifndef NDEBUG
1981 // Make sure we don't end up with duplicate edges between the same caller and
1982 // callee.
1984 for (const auto &E : Node->CalleeEdges)
1985 NodeSet.insert(E->Callee);
1986 assert(NodeSet.size() == Node->CalleeEdges.size());
1987#endif
1988}
1989
1990template <typename DerivedCCG, typename FuncTy, typename CallTy>
1991void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
1992 assignStackNodesPostOrder(ContextNode *Node,
1993 DenseSet<const ContextNode *> &Visited,
1994 DenseMap<uint64_t, std::vector<CallContextInfo>>
1995 &StackIdToMatchingCalls,
1996 DenseMap<CallInfo, CallInfo> &CallToMatchingCall,
1997 const DenseSet<uint32_t> &ImportantContextIds) {
1998 auto Inserted = Visited.insert(Node);
1999 if (!Inserted.second)
2000 return;
2001 // Post order traversal. Iterate over a copy since we may add nodes and
2002 // therefore new callers during the recursive call, invalidating any
2003 // iterator over the original edge vector. We don't need to process these
2004 // new nodes as they were already processed on creation.
2005 auto CallerEdges = Node->CallerEdges;
2006 for (auto &Edge : CallerEdges) {
2007 // Skip any that have been removed during the recursion.
2008 if (Edge->isRemoved()) {
2009 assert(!is_contained(Node->CallerEdges, Edge));
2010 continue;
2011 }
2012 assignStackNodesPostOrder(Edge->Caller, Visited, StackIdToMatchingCalls,
2013 CallToMatchingCall, ImportantContextIds);
2014 }
2015
2016 // If this node's stack id is in the map, update the graph to contain new
2017 // nodes representing any inlining at interior callsites. Note we move the
2018 // associated context ids over to the new nodes.
2019
2020 // Ignore this node if it is for an allocation or we didn't record any
2021 // stack id lists ending at it.
2022 if (Node->IsAllocation ||
2023 !StackIdToMatchingCalls.count(Node->OrigStackOrAllocId))
2024 return;
2025
2026 auto &Calls = StackIdToMatchingCalls[Node->OrigStackOrAllocId];
2027 // Handle the simple case first. A single call with a single stack id.
2028 // In this case there is no need to create any new context nodes, simply
2029 // assign the context node for stack id to this Call.
2030 if (Calls.size() == 1) {
2031 auto &[Call, StackNodes, Func, SavedContextIds, TrimmedInlinedDupIds] =
2032 Calls[0];
2033 if (StackNodes.size() == 1) {
2034 assert(SavedContextIds.empty());
2035 // Only calls with inlined frames get these.
2036 assert(TrimmedInlinedDupIds.empty());
2037 // It should be this Node
2038 assert(Node == StackNodes[0]);
2039 if (Node->Recursive)
2040 return;
2041 Node->setCall(Call);
2042 NonAllocationCallToContextNodeMap[Call] = Node;
2043 NodeToCallingFunc[Node] = Func;
2044 recordStackNode(StackNodes, Node, Node->getContextIds(),
2045 ImportantContextIds);
2046 return;
2047 }
2048 }
2049
2050#ifndef NDEBUG
2051 // Find the node for the last stack id, which should be the same
2052 // across all calls recorded for this id, and is this node's id.
2053 uint64_t LastId = Node->OrigStackOrAllocId;
2054 ContextNode *LastNode = getNodeForStackId(LastId);
2055 // We should only have kept stack ids that had nodes.
2056 assert(LastNode);
2057 assert(LastNode == Node);
2058#else
2059 ContextNode *LastNode = Node;
2060#endif
2061
2062 // Compute the last node's context ids once, as it is shared by all calls in
2063 // this entry.
2064 DenseSet<uint32_t> LastNodeContextIds = LastNode->getContextIds();
2065
2066 [[maybe_unused]] bool PrevIterCreatedNode = false;
2067 bool CreatedNode = false;
2068 for (unsigned I = 0; I < Calls.size();
2069 I++, PrevIterCreatedNode = CreatedNode) {
2070 CreatedNode = false;
2071 auto &[Call, StackNodes, Func, SavedContextIds, TrimmedInlinedDupIds] =
2072 Calls[I];
2073 // Skip any for which we didn't assign any ids, these don't get a node in
2074 // the graph.
2075 if (SavedContextIds.empty() && TrimmedInlinedDupIds.empty()) {
2076 // If this call has a matching call (located in the same function and
2077 // having the same stack ids), simply add it to the context node created
2078 // for its matching call earlier. These can be treated the same through
2079 // cloning and get updated at the same time.
2080 if (!CallToMatchingCall.contains(Call))
2081 continue;
2082 auto MatchingCall = CallToMatchingCall[Call];
2083 if (!NonAllocationCallToContextNodeMap.contains(MatchingCall)) {
2084 // This should only happen if we had a prior iteration, and it didn't
2085 // create a node because of the below recomputation of context ids
2086 // finding none remaining and continuing early.
2087 assert(I > 0 && !PrevIterCreatedNode);
2088 continue;
2089 }
2090 NonAllocationCallToContextNodeMap[MatchingCall]->MatchingCalls.push_back(
2091 Call);
2092 continue;
2093 }
2094
2095 assert(LastNode == StackNodes.back());
2096
2097 // Recompute the context ids for this stack id sequence (the
2098 // intersection of the context ids of the corresponding nodes).
2099 // Start with the ids we saved in the map for this call, which could be
2100 // duplicated context ids. We have to recompute as we might have overlap
2101 // overlap between the saved context ids for different last nodes, and
2102 // removed them already during the post order traversal.
2103 set_intersect(SavedContextIds, LastNodeContextIds);
2104 ContextNode *PrevNode = LastNode;
2105 bool Skip = SavedContextIds.empty();
2106 // Iterate backwards through the stack nodes, starting after the last one
2107 // in the list, which was handled once outside for all Calls.
2108 for (auto NodeIter = StackNodes.rbegin() + 1;
2109 !Skip && NodeIter != StackNodes.rend(); NodeIter++) {
2110 ContextNode *CurNode = *NodeIter;
2111 // We should only have kept stack ids that had nodes and weren't
2112 // recursive.
2113 assert(CurNode);
2114 assert(!CurNode->Recursive);
2115
2116 auto *Edge = CurNode->findEdgeFromCaller(PrevNode);
2117 if (!Edge) {
2118 Skip = true;
2119 break;
2120 }
2121 PrevNode = CurNode;
2122
2123 // Update the context ids, which is the intersection of the ids along
2124 // all edges in the sequence.
2125 set_intersect(SavedContextIds, Edge->getContextIds());
2126
2127 // If we now have no context ids for clone, skip this call.
2128 if (SavedContextIds.empty()) {
2129 Skip = true;
2130 break;
2131 }
2132 }
2133 if (Skip)
2134 SavedContextIds.clear();
2135 // Add the duplicates created for this call (see updateStackNodes) that are
2136 // still on its first node. Other calls given them end below that node and
2137 // leave them on it, but a recursive context may traverse this call's
2138 // initial nodes more than once with different continuations, in which
2139 // case a call assigned one of the other traversals may have been processed
2140 // first and claimed them (as happens to the original ids too).
2141 if (!TrimmedInlinedDupIds.empty()) {
2142 StackNodes[0]->removeContextIdsNotPresent(TrimmedInlinedDupIds);
2143 SavedContextIds.insert_range(TrimmedInlinedDupIds);
2144 }
2145 if (SavedContextIds.empty())
2146 continue;
2147
2148 // Create new context node.
2149 ContextNode *NewNode = createNewNode(/*IsAllocation=*/false, Func, Call);
2150 NonAllocationCallToContextNodeMap[Call] = NewNode;
2151 CreatedNode = true;
2152 NewNode->AllocTypes = computeAllocType(SavedContextIds);
2153
2154 ContextNode *FirstNode = StackNodes[0];
2155 assert(FirstNode);
2156
2157 // Connect to callees of innermost stack frame in inlined call chain.
2158 // This updates context ids for FirstNode's callee's to reflect those
2159 // moved to NewNode.
2160 connectNewNode(NewNode, FirstNode, /*TowardsCallee=*/true, SavedContextIds);
2161
2162 // Connect to callers of outermost stack frame in inlined call chain.
2163 // This updates context ids for FirstNode's caller's to reflect those
2164 // moved to NewNode.
2165 connectNewNode(NewNode, LastNode, /*TowardsCallee=*/false, SavedContextIds);
2166
2167 // Now we need to remove context ids from edges/nodes between First and
2168 // Last Node.
2169 PrevNode = nullptr;
2170 for (ContextNode *CurNode : StackNodes) {
2171 // We should only have kept stack ids that had nodes.
2172 assert(CurNode);
2173
2174 // Remove the context ids moved to NewNode from CurNode, and the
2175 // edge from the prior node.
2176 if (PrevNode) {
2177 auto *PrevEdge = CurNode->findEdgeFromCallee(PrevNode);
2178 // If the sequence contained recursion, we might have already removed
2179 // some edges during the connectNewNode calls above.
2180 if (!PrevEdge) {
2181 PrevNode = CurNode;
2182 continue;
2183 }
2184 set_subtract(PrevEdge->getContextIds(), SavedContextIds);
2185 if (PrevEdge->getContextIds().empty())
2186 removeEdgeFromGraph(PrevEdge);
2187 else
2188 PrevEdge->AllocTypes = computeAllocType(PrevEdge->getContextIds());
2189 }
2190 // Since we update the edges from leaf to tail, only look at the callee
2191 // edges. This isn't an alloc node, so if there are no callee edges, the
2192 // alloc type is None.
2193 CurNode->AllocTypes = CurNode->CalleeEdges.empty()
2194 ? (uint8_t)AllocationType::None
2195 : CurNode->computeAllocType();
2196 PrevNode = CurNode;
2197 }
2198
2199 recordStackNode(StackNodes, NewNode, SavedContextIds, ImportantContextIds);
2200
2201 if (VerifyNodes) {
2202 checkNode<DerivedCCG, FuncTy, CallTy>(NewNode, /*CheckEdges=*/true);
2203 for (ContextNode *CurNode : StackNodes) {
2204 // We should only have kept stack ids that had nodes.
2205 assert(CurNode);
2206 checkNode<DerivedCCG, FuncTy, CallTy>(CurNode, /*CheckEdges=*/true);
2207 }
2208 }
2209 }
2210}
2211
2212template <typename DerivedCCG, typename FuncTy, typename CallTy>
2213void CallsiteContextGraph<DerivedCCG, FuncTy,
2214 CallTy>::fixupImportantContexts() {
2215 if (ImportantContextIdInfo.empty())
2216 return;
2217
2218 // Update statistics as we are done building this map at this point.
2219 NumImportantContextIds = ImportantContextIdInfo.size();
2220
2222 return;
2223
2224 if (ExportToDot)
2225 exportToDot("beforestackfixup");
2226
2227 // For each context we identified as important, walk through the saved context
2228 // stack ids in order from leaf upwards, and make sure all edges are correct.
2229 // These can be difficult to get right when updating the graph while mapping
2230 // nodes onto summary or IR, especially when there is recursion. In
2231 // particular, when we have created new nodes to reflect inlining, it is
2232 // sometimes impossible to know exactly how to update the edges in the face of
2233 // recursion, as we have lost the original ordering of the stack ids in the
2234 // contexts.
2235 // TODO: Consider only doing this if we detect the context has recursive
2236 // cycles.
2237 //
2238 // I.e. assume we have a context with stack ids like: {A B A C A D E}
2239 // and let's say A was inlined into B, C, and D. The original graph will have
2240 // multiple recursive cycles through A. When we match the original context
2241 // nodes onto the IR or summary, we will merge {A B} into one context node,
2242 // {A C} onto another, and {A D} onto another. Looking at the stack sequence
2243 // above, we should end up with a non-cyclic set of edges like:
2244 // {AB} <- {AC} <- {AD} <- E. However, because we normally have lost the
2245 // original ordering, we won't get the edges correct initially (it's
2246 // impossible without the original ordering). Here we do the fixup (add and
2247 // removing edges where necessary) for this context. In the
2248 // ImportantContextInfo struct in this case we should have a MaxLength = 2,
2249 // and map entries for {A B}, {A C}, {A D}, and {E}.
2250 for (auto &[CurContextId, Info] : ImportantContextIdInfo) {
2251 if (Info.StackIdsToNode.empty())
2252 continue;
2253 bool Changed = false;
2254 ContextNode *PrevNode = nullptr;
2255 ContextNode *CurNode = nullptr;
2256 DenseSet<const ContextEdge *> VisitedEdges;
2257 ArrayRef<uint64_t> AllStackIds(Info.StackIds);
2258 // Try to identify what callsite ContextNode maps to which slice of the
2259 // context's ordered stack ids.
2260 for (unsigned I = 0; I < AllStackIds.size(); I++, PrevNode = CurNode) {
2261 // We will do this greedily, trying up to MaxLength stack ids in a row, to
2262 // see if we recorded a context node for that sequence.
2263 auto Len = Info.MaxLength;
2264 auto LenToEnd = AllStackIds.size() - I;
2265 if (Len > LenToEnd)
2266 Len = LenToEnd;
2267 CurNode = nullptr;
2268 // Try to find a recorded context node starting with the longest length
2269 // recorded, and on down until we check for just a single stack node.
2270 for (; Len > 0; Len--) {
2271 // Get the slice of the original stack id sequence to check.
2272 auto CheckStackIds = AllStackIds.slice(I, Len);
2273 auto EntryIt = Info.StackIdsToNode.find(CheckStackIds);
2274 if (EntryIt == Info.StackIdsToNode.end())
2275 continue;
2276 CurNode = EntryIt->second;
2277 // Skip forward so we don't try to look for the ones we just matched.
2278 // We increment by Len - 1, because the outer for loop will increment I.
2279 I += Len - 1;
2280 break;
2281 }
2282 // Give up if we couldn't find a node. Since we need to clone from the
2283 // leaf allocation upwards, no sense in doing anymore fixup further up
2284 // the context if we couldn't match part of the original stack context
2285 // onto a callsite node.
2286 if (!CurNode)
2287 break;
2288 // No edges to fix up until we have a pair of nodes that should be
2289 // adjacent in the graph.
2290 if (!PrevNode)
2291 continue;
2292 // See if we already have a call edge from CurNode to PrevNode.
2293 auto *CurEdge = PrevNode->findEdgeFromCaller(CurNode);
2294 if (CurEdge) {
2295 // We already have an edge. Make sure it contains this context id.
2296 if (CurEdge->getContextIds().insert(CurContextId).second) {
2297 NumFixupEdgeIdsInserted++;
2298 Changed = true;
2299 }
2300 } else {
2301 // No edge exists - add one.
2302 NumFixupEdgesAdded++;
2303 DenseSet<uint32_t> ContextIds({CurContextId});
2304 auto AllocType = computeAllocType(ContextIds);
2305 auto NewEdge = std::make_shared<ContextEdge>(
2306 PrevNode, CurNode, AllocType, std::move(ContextIds));
2307 PrevNode->CallerEdges.push_back(NewEdge);
2308 CurNode->CalleeEdges.push_back(NewEdge);
2309 // Save the new edge for the below handling.
2310 CurEdge = NewEdge.get();
2311 Changed = true;
2312 }
2313 VisitedEdges.insert(CurEdge);
2314 // Now remove this context id from any other caller edges calling
2315 // PrevNode.
2316 for (auto &Edge : PrevNode->CallerEdges) {
2317 // Skip the edge updating/created above and edges we have already
2318 // visited (due to recursion).
2319 if (Edge.get() != CurEdge && !VisitedEdges.contains(Edge.get()))
2320 Edge->getContextIds().erase(CurContextId);
2321 }
2322 }
2323 if (Changed)
2324 NumFixedContexts++;
2325 }
2326}
2327
2328// Returns the ids of contexts whose stacks end partway through the given
2329// call's inlined sequence of stack nodes: those that follow the sequence from
2330// StackNodes[0] (the innermost inlined callee frame) through some node before
2331// the last, and were trimmed at that node, i.e. have no caller edge out of it.
2332// For example, given nodes [A, B, C], a context whose stack ends at A, or that
2333// follows A->B and ends at B, is returned, but one that follows A->B->C (which
2334// is assigned to the call normally) or A->D is not.
2335//
2336// Nothing is returned if the sequence contains a recursive node before the
2337// last, as updateStackNodes does not assign such calls a node at all.
2338template <typename DerivedCCG, typename FuncTy, typename CallTy>
2339DenseSet<uint32_t> CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
2340 computeTrimmedInlinedContextIds(ArrayRef<ContextNode *> StackNodes,
2341 TrimmedAtNodeCache &Cache) {
2342 assert(StackNodes.size() >= 2);
2343 DenseSet<uint32_t> Result;
2344
2345 // Returns the index in the cache of the ids of contexts trimmed at the given
2346 // node. This only depends on the node, which is often shared by many calls'
2347 // inlined sequences, so it is computed once per node.
2348 auto GetTrimmedAtNodeIndex = [&Cache](const ContextNode *Node) {
2349 auto [It, Inserted] =
2350 Cache.NodeToIndex.try_emplace(Node, Cache.TrimmedAtNodeIds.size());
2351 unsigned Index = It->second;
2352 if (!Inserted)
2353 return Index;
2354 // All of the node's context ids flow in through its callee edges. Those
2355 // not flowing out through a caller edge were trimmed here.
2356 DenseSet<uint32_t> &Trimmed = Cache.TrimmedAtNodeIds.emplace_back();
2357 for (const auto &CalleeEdge : Node->CalleeEdges)
2358 Trimmed.insert_range(CalleeEdge->getContextIds());
2359 for (const auto &CallerEdge : Node->CallerEdges) {
2360 set_subtract(Trimmed, CallerEdge->getContextIds());
2361 if (Trimmed.empty())
2362 break;
2363 }
2364 return Index;
2365 };
2366
2367 // Collect the cache index for each node but the last. Skip the walk
2368 // entirely if no contexts were trimmed at any of them (the common case).
2369 SmallVector<unsigned, 8> TrimmedAtNodeIdsIndices;
2370 bool AnyTrimmed = false;
2371 for (ContextNode *Node : StackNodes.drop_back()) {
2372 assert(Node);
2373 if (Node->Recursive)
2374 return Result;
2375 unsigned Index = GetTrimmedAtNodeIndex(Node);
2376 AnyTrimmed |= !Cache.TrimmedAtNodeIds[Index].empty();
2377 TrimmedAtNodeIdsIndices.push_back(Index);
2378 }
2379 if (!AnyTrimmed)
2380 return Result;
2381
2382 // Walk the sequence from the innermost node outward, tracking in
2383 // PrefixContextIds the contexts that follow the call's stack nodes from the
2384 // first through the current one. Any of those contexts that were trimmed at
2385 // the current node end partway through the call's sequence, so add them to
2386 // Result. All contexts on the first node trivially follow the sequence up to
2387 // it, so start with all those trimmed there, and start PrefixContextIds with
2388 // the ids on the edge out of the first node (rather than the possibly very
2389 // large set of ids on the first node itself).
2390 //
2391 // If an edge is missing between two of the nodes, the contexts trimmed at
2392 // the nodes before it (in particular all those trimmed at the first node)
2393 // are deliberately kept, even though updateStackNodes does not assign such
2394 // a call any contexts otherwise: a trimmed context is unambiguous for every
2395 // continuation, including an inlined sequence never observed in the profile.
2396 Result = Cache.TrimmedAtNodeIds[TrimmedAtNodeIdsIndices[0]];
2397 DenseSet<uint32_t> PrefixContextIds;
2398 for (unsigned I = 1; I < TrimmedAtNodeIdsIndices.size(); I++) {
2399 auto *Edge = StackNodes[I]->findEdgeFromCallee(StackNodes[I - 1]);
2400 if (!Edge)
2401 break;
2402 if (I == 1)
2403 PrefixContextIds = Edge->getContextIds();
2404 else
2405 set_intersect(PrefixContextIds, Edge->getContextIds());
2406 if (PrefixContextIds.empty())
2407 break;
2408 Result.insert_range(set_intersection(
2409 PrefixContextIds, Cache.TrimmedAtNodeIds[TrimmedAtNodeIdsIndices[I]]));
2410 }
2411 return Result;
2412}
2413
2414template <typename DerivedCCG, typename FuncTy, typename CallTy>
2415void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::updateStackNodes() {
2416 // Map of stack id to all calls with that as the last (outermost caller)
2417 // callsite id that has a context node (some might not due to trimming
2418 // performed during matching of the allocation profile contexts).
2419 // The CallContextInfo contains the Call and a list of the context nodes for
2420 // its stack ids, the function containing Call, and the set of context ids
2421 // the analysis will eventually identify for use in any new node created
2422 // for that callsite.
2423 DenseMap<uint64_t, std::vector<CallContextInfo>> StackIdToMatchingCalls;
2424 for (auto &[Func, CallsWithMetadata] : FuncToCallsWithMetadata) {
2425 for (auto &Call : CallsWithMetadata) {
2426 // Ignore allocations, already handled.
2427 if (AllocationCallToContextNodeMap.count(Call))
2428 continue;
2429 std::vector<ContextNode *> StackNodes = getStackNodesForCall(Call.call());
2430 // If there were no nodes created for MIBs on allocs (maybe this was in
2431 // the unambiguous part of the MIB stack that was trimmed), ignore.
2432 if (StackNodes.empty())
2433 continue;
2434 // Otherwise, record this Call along with the list of nodes for the last
2435 // (outermost caller) stack id with a node.
2436 uint64_t LastId = StackNodes.back()->OrigStackOrAllocId;
2437 StackIdToMatchingCalls[LastId].push_back(
2438 {Call.call(), std::move(StackNodes), Func, {}, {}});
2439 }
2440 }
2441
2442 // In order to do the best and maximal matching of inlined calls to context
2443 // node sequences, sort the calls with each last stack id by the length of
2444 // their stack node sequences in descending order, and within each length,
2445 // lexicographically by stack id. The latter is so that we can specially
2446 // handle calls that have identical stack id sequences (either due to cloning
2447 // or artificially because of the MIB context trimming). Those with the same
2448 // stack ids are then sorted by function to facilitate efficiently mapping
2449 // them to the same context node. Because the functions are pointers, to
2450 // ensure a stable sort first assign each function pointer to its first index
2451 // in the Calls array, and then use that to sort by. The stack nodes are
2452 // compared by stack id rather than by pointer, for a deterministic order.
2453 auto StackIdsLess = [](ArrayRef<ContextNode *> A, ArrayRef<ContextNode *> B) {
2454 return std::lexicographical_compare(
2455 A.begin(), A.end(), B.begin(), B.end(),
2456 [](const ContextNode *X, const ContextNode *Y) {
2457 return X->OrigStackOrAllocId < Y->OrigStackOrAllocId;
2458 });
2459 };
2460 // Save a map from each call to any that are found to match it. I.e. located
2461 // in the same function and have the same (possibly trimmed) stack ids. We use
2462 // this to avoid creating extra graph nodes as they can be treated the same.
2463 DenseMap<CallInfo, CallInfo> CallToMatchingCall;
2464 for (auto &Entry : StackIdToMatchingCalls) {
2465 auto &Calls = Entry.second;
2466 if (Calls.size() == 1)
2467 continue;
2468 DenseMap<const FuncTy *, unsigned> FuncToIndex;
2469 for (const auto &[Idx, CallCtxInfo] : enumerate(Calls))
2470 FuncToIndex.insert({CallCtxInfo.Func, Idx});
2471 auto CallLess = [&FuncToIndex, &StackIdsLess](const CallContextInfo &A,
2472 const CallContextInfo &B) {
2473 return A.StackNodes.size() > B.StackNodes.size() ||
2474 (A.StackNodes.size() == B.StackNodes.size() &&
2475 (StackIdsLess(A.StackNodes, B.StackNodes) ||
2476 (A.StackNodes == B.StackNodes &&
2477 FuncToIndex[A.Func] < FuncToIndex[B.Func])));
2478 };
2479 llvm::stable_sort(Calls, CallLess);
2480 // Matching calls are now adjacent. Record each run's calls after the first
2481 // as matching it. Comparing the stack nodes is equivalent to comparing the
2482 // stack ids, as each stack id has its own node and the stack id to node
2483 // map is not modified here.
2484 unsigned FirstInRun = 0;
2485 for (unsigned I = 1; I < Calls.size(); I++) {
2486 if (Calls[I].Func != Calls[FirstInRun].Func ||
2487 Calls[I].StackNodes != Calls[FirstInRun].StackNodes) {
2488 FirstInRun = I;
2489 continue;
2490 }
2491 CallToMatchingCall[Calls[I].Call] = Calls[FirstInRun].Call;
2492 }
2493 }
2494
2495 // First make a pass through all stack ids that correspond to a call,
2496 // as identified in the above loop. Compute the context ids corresponding to
2497 // each of these calls when they correspond to multiple stack ids due to
2498 // due to inlining. Perform any duplication of context ids required when
2499 // there is more than one call with the same stack ids. Their (possibly newly
2500 // duplicated) context ids are saved in the StackIdToMatchingCalls map.
2501 DenseMap<uint32_t, DenseSet<uint32_t>> OldToNewContextIds;
2502
2503 // First, give each call with inlined frames duplicates of the ids of any
2504 // contexts trimmed at a stack id before the last one in its sequence (see
2505 // computeTrimmedInlinedContextIds), saved in its TrimmedInlinedDupIds.
2506 // Those contexts are also assigned, with their original ids, to calls whose
2507 // last stack id with a node is the earlier id, so this call gets its own
2508 // copies. This pass only reads the graph, as does the main loop below (the
2509 // duplicates are only added to the graph afterwards, by
2510 // propagateDuplicateContextIds), so it can run first; that lets the main
2511 // loop hand these duplicates to the other calls that should also get them
2512 // (see AddTrimmedInlinedDups).
2513
2514 // The stack nodes of each call that duplicates were created for. These
2515 // reference the vectors in StackIdToMatchingCalls, which is not modified
2516 // after this point.
2517 std::vector<ArrayRef<ContextNode *>> TrimmedInlinedDupCreatorNodes;
2518 // Map from each original trimmed context id to its duplicates, each paired
2519 // with the index in TrimmedInlinedDupCreatorNodes of the creating call.
2520 DenseMap<uint32_t, SmallVector<std::pair<uint32_t, unsigned>, 1>>
2521 OrigToTrimmedInlinedDups;
2523 TrimmedAtNodeCache Cache;
2524 for (auto &Entry : StackIdToMatchingCalls) {
2525 auto &Calls = Entry.second;
2526 // The main loop below skips these, see the comments there. All calls in
2527 // this entry share the same last node.
2528 if (Calls[0].StackNodes.back()->Recursive)
2529 continue;
2530 for (unsigned I = 0; I < Calls.size(); I++) {
2531 auto &CallCtxInfo = Calls[I];
2532 auto &StackNodes = CallCtxInfo.StackNodes;
2533 // Calls are sorted longest sequence first, so no later call has
2534 // inlined frames either.
2535 if (StackNodes.size() < 2)
2536 break;
2537 // Calls matching an earlier call in the same function will share its
2538 // context node, so only give duplicates to the first.
2539 if (CallToMatchingCall.contains(CallCtxInfo.Call))
2540 continue;
2541 auto Trimmed = computeTrimmedInlinedContextIds(StackNodes, Cache);
2542 if (Trimmed.empty())
2543 continue;
2544 unsigned CreatorIdx = TrimmedInlinedDupCreatorNodes.size();
2545 TrimmedInlinedDupCreatorNodes.push_back(StackNodes);
2546 CallCtxInfo.TrimmedInlinedDupIds = duplicateContextIds(
2547 Trimmed, OldToNewContextIds, [&](uint32_t OldId, uint32_t NewId) {
2548 OrigToTrimmedInlinedDups[OldId].push_back({NewId, CreatorIdx});
2549 });
2550 NumCallsAssignedTrimmedInlinedContexts++;
2551 NumTrimmedInlinedContextIds += Trimmed.size();
2552 }
2553 }
2554 }
2555
2556 // Adds to the given saved context ids, for a call with the given last stack
2557 // node, the trimmed inlined duplicates of any of those ids created for other
2558 // calls. The saved context ids are computed from the graph, which does not
2559 // include the duplicates until propagateDuplicateContextIds below adds them
2560 // to all of the same nodes and edges as the originals. A call assigned an
2561 // original should also be assigned its duplicates; otherwise, when that
2562 // call's new node takes the originals, the duplicates would be left behind
2563 // on the original stack nodes, which have no call to clone.
2564 //
2565 // TODO: The saved context ids may themselves be duplicates (created for a
2566 // call with the same stack ids in a different function), in which case the
2567 // original context id is not found here and the trimmed inlined duplicates
2568 // are not added. In the future, handle that by looking up the originals of
2569 // the saved ids.
2570 auto AddTrimmedInlinedDups = [&](DenseSet<uint32_t> &SavedContextIds,
2571 const ContextNode *LastNode) {
2572 SmallVector<uint32_t> DupIdsToAdd;
2573 // Collects the duplicates of one original id that this call should get.
2574 auto AddDups = [&](ArrayRef<std::pair<uint32_t, unsigned>> Dups) {
2575 for (auto [DupId, CreatorIdx] : Dups) {
2576 ArrayRef<ContextNode *> CreatorNodes =
2577 TrimmedInlinedDupCreatorNodes[CreatorIdx];
2578 // Skip the duplicate if this call ends at any node of the creating
2579 // call's sequence. Such a call overlaps the stretch of the context
2580 // the creating call needs the duplicate on, and could claim the
2581 // duplicate first (the post order processes the creating call first
2582 // only when all edges along its sequence exist); being assigned the
2583 // original there is enough for it. Calls ending below the creating
2584 // call's first node only cover the callee side of the context, and
2585 // leave the duplicate on that first node for the creating call.
2586 if (is_contained(CreatorNodes, LastNode))
2587 continue;
2588 DupIdsToAdd.push_back(DupId);
2589 }
2590 };
2591 // Iterate over the smaller of the two.
2592 if (OrigToTrimmedInlinedDups.size() < SavedContextIds.size()) {
2593 for (auto &[OrigId, Dups] : OrigToTrimmedInlinedDups)
2594 if (SavedContextIds.contains(OrigId))
2595 AddDups(Dups);
2596 } else {
2597 for (auto Id : SavedContextIds) {
2598 auto It = OrigToTrimmedInlinedDups.find(Id);
2599 if (It != OrigToTrimmedInlinedDups.end())
2600 AddDups(It->second);
2601 }
2602 }
2603 SavedContextIds.insert_range(DupIdsToAdd);
2604 };
2605 for (auto &It : StackIdToMatchingCalls) {
2606 auto &Calls = It.getSecond();
2607 // Skip single calls with a single stack id. These don't need a new node.
2608 if (Calls.size() == 1) {
2609 auto &StackNodes = Calls[0].StackNodes;
2610 if (StackNodes.size() == 1)
2611 continue;
2612 }
2613
2614 // Find the node for the last stack id, which should be the same
2615 // across all calls recorded for this id, and is the id for this
2616 // entry in the StackIdToMatchingCalls map.
2617 uint64_t LastId = It.getFirst();
2618 // All calls in this entry share the same last node.
2619 ContextNode *LastNode = Calls[0].StackNodes.back();
2620 // We should only have kept stack ids that had nodes.
2621 assert(LastNode);
2622 assert(LastNode == getNodeForStackId(LastId));
2623
2624 if (LastNode->Recursive)
2625 continue;
2626
2627 // Initialize the context ids with the last node's. We will subsequently
2628 // refine the context ids by computing the intersection along all edges.
2629 DenseSet<uint32_t> LastNodeContextIds = LastNode->getContextIds();
2630 assert(!LastNodeContextIds.empty());
2631
2632#ifndef NDEBUG
2633 // Save the set of functions seen for a particular set of the same stack
2634 // ids. This is used to ensure that they have been correctly sorted to be
2635 // adjacent in the Calls list, since we rely on that to efficiently place
2636 // all such matching calls onto the same context node.
2637 DenseSet<const FuncTy *> MatchingIdsFuncSet;
2638#endif
2639
2640 for (unsigned I = 0; I < Calls.size(); I++) {
2641 auto &[Call, StackNodes, Func, SavedContextIds, TrimmedInlinedDupIds] =
2642 Calls[I];
2643 assert(SavedContextIds.empty());
2644 assert(LastNode == StackNodes.back());
2645
2646#ifndef NDEBUG
2647 // If this call has a different set of ids than the last one, clear the
2648 // set used to ensure they are sorted properly.
2649 if (I > 0 && StackNodes != Calls[I - 1].StackNodes)
2650 MatchingIdsFuncSet.clear();
2651#endif
2652
2653 // Skip the calls matching this one (recorded in CallToMatchingCall when
2654 // sorting above), which are adjacent to it, so that they get assigned to
2655 // the same context node. Also note whether a call in a different
2656 // function has the same stack ids.
2657 bool DuplicateContextIds = false;
2658 for (unsigned J = I + 1; J < Calls.size(); J++) {
2659 auto &CallCtxInfo = Calls[J];
2660 if (CallCtxInfo.StackNodes != StackNodes)
2661 break;
2662 auto *NextFunc = CallCtxInfo.Func;
2663 if (NextFunc != Func) {
2664 // We have another Call with the same ids but that cannot share this
2665 // node, must duplicate ids for it.
2666 DuplicateContextIds = true;
2667 break;
2668 }
2669 assert(CallToMatchingCall.lookup(CallCtxInfo.Call).call() == Call);
2670 // Update I so that it gets incremented correctly to skip this call.
2671 I = J;
2672 }
2673
2674 // First compute the context ids for this stack id sequence (the
2675 // intersection of the context ids of the corresponding nodes).
2676 // Start with the remaining saved ids for the last node.
2677 assert(!LastNodeContextIds.empty());
2678 DenseSet<uint32_t> StackSequenceContextIds = LastNodeContextIds;
2679
2680 ContextNode *PrevNode = LastNode;
2681 ContextNode *CurNode = LastNode;
2682 bool Skip = false;
2683
2684 // Iterate backwards through the stack nodes, starting after the last
2685 // one in the list, which was handled once outside for all Calls.
2686 for (auto NodeIter = StackNodes.rbegin() + 1;
2687 NodeIter != StackNodes.rend(); NodeIter++) {
2688 CurNode = *NodeIter;
2689 // We should only have kept stack ids that had nodes.
2690 assert(CurNode);
2691
2692 if (CurNode->Recursive) {
2693 Skip = true;
2694 break;
2695 }
2696
2697 auto *Edge = CurNode->findEdgeFromCaller(PrevNode);
2698 // If there is no edge then the nodes belong to different MIB contexts,
2699 // and we should skip this inlined context sequence. For example, this
2700 // particular inlined context may include stack ids A->B, and we may
2701 // indeed have nodes for both A and B, but it is possible that they were
2702 // never profiled in sequence in a single MIB for any allocation (i.e.
2703 // we might have profiled an allocation that involves the callsite A,
2704 // but through a different one of its callee callsites, and we might
2705 // have profiled an allocation that involves callsite B, but reached
2706 // from a different caller callsite).
2707 if (!Edge) {
2708 Skip = true;
2709 break;
2710 }
2711 PrevNode = CurNode;
2712
2713 // Update the context ids, which is the intersection of the ids along
2714 // all edges in the sequence.
2715 set_intersect(StackSequenceContextIds, Edge->getContextIds());
2716
2717 // If we now have no context ids for clone, skip this call.
2718 if (StackSequenceContextIds.empty()) {
2719 Skip = true;
2720 break;
2721 }
2722 }
2723 if (Skip)
2724 continue;
2725
2726 // If some of this call's stack ids did not have corresponding nodes (due
2727 // to trimming), don't include any context ids for contexts that extend
2728 // beyond these nodes. Otherwise we would be matching part of unrelated /
2729 // not fully matching stack contexts. To do this, subtract any context ids
2730 // found in caller nodes of the last node found above.
2731 if (LastId != getLastStackId(Call)) {
2732 for (const auto &PE : LastNode->CallerEdges) {
2733 set_subtract(StackSequenceContextIds, PE->getContextIds());
2734 if (StackSequenceContextIds.empty())
2735 break;
2736 }
2737 // If we now have no context ids for clone, skip this call.
2738 if (StackSequenceContextIds.empty())
2739 continue;
2740 }
2741
2742#ifndef NDEBUG
2743 // If the prior call had the same stack ids this set would not be empty.
2744 // Check if we already have a call that "matches" because it is located
2745 // in the same function. If the Calls list was sorted properly we should
2746 // not encounter this situation as all such entries should be adjacent
2747 // and processed in bulk further below.
2748 assert(!MatchingIdsFuncSet.contains(Func));
2749
2750 MatchingIdsFuncSet.insert(Func);
2751#endif
2752
2753 // If we don't have duplicate context ids, then we can assign all the
2754 // context ids computed for the original node sequence to this call.
2755 // If there are duplicate calls with the same stack ids then we synthesize
2756 // new context ids that are duplicates of the originals. These are
2757 // assigned to SavedContextIds, which is a reference into the map entry
2758 // for this call, allowing us to access these ids later on.
2759 OldToNewContextIds.reserve(OldToNewContextIds.size() +
2760 StackSequenceContextIds.size());
2761 SavedContextIds =
2762 DuplicateContextIds
2763 ? duplicateContextIds(StackSequenceContextIds, OldToNewContextIds)
2764 : StackSequenceContextIds;
2765 assert(!SavedContextIds.empty());
2766 if (!OrigToTrimmedInlinedDups.empty())
2767 AddTrimmedInlinedDups(SavedContextIds, LastNode);
2768
2769 if (!DuplicateContextIds) {
2770 // Update saved last node's context ids to remove those that are
2771 // assigned to other calls, so that it is ready for the next call at
2772 // this stack id.
2773 set_subtract(LastNodeContextIds, StackSequenceContextIds);
2774 if (LastNodeContextIds.empty())
2775 break;
2776 }
2777 }
2778 }
2779
2780 // Propagate the duplicate context ids over the graph.
2781 propagateDuplicateContextIds(OldToNewContextIds);
2782
2783 if (VerifyCCG)
2784 check();
2785
2786 // Now perform a post-order traversal over the graph, starting with the
2787 // allocation nodes, essentially processing nodes from callers to callees.
2788 // For any that contains an id in the map, update the graph to contain new
2789 // nodes representing any inlining at interior callsites. Note we move the
2790 // associated context ids over to the new nodes.
2791 DenseSet<const ContextNode *> Visited;
2792 DenseSet<uint32_t> ImportantContextIds(llvm::from_range,
2793 ImportantContextIdInfo.keys());
2794 for (auto &Entry : AllocationCallToContextNodeMap)
2795 assignStackNodesPostOrder(Entry.second, Visited, StackIdToMatchingCalls,
2796 CallToMatchingCall, ImportantContextIds);
2797
2798 fixupImportantContexts();
2799
2800 if (VerifyCCG)
2801 check();
2802}
2803
2804uint64_t ModuleCallsiteContextGraph::getLastStackId(Instruction *Call) {
2805 CallStack<MDNode, MDNode::op_iterator> CallsiteContext(
2806 Call->getMetadata(LLVMContext::MD_callsite));
2807 return CallsiteContext.back();
2808}
2809
2810uint64_t IndexCallsiteContextGraph::getLastStackId(IndexCall &Call) {
2812 CallStack<CallsiteInfo, SmallVector<unsigned>::const_iterator>
2813 CallsiteContext(dyn_cast_if_present<CallsiteInfo *>(Call));
2814 // Need to convert index into stack id.
2815 return Index.getStackIdAtIndex(CallsiteContext.back());
2816}
2817
2818static std::string getMemProfFuncName(Twine Base, unsigned CloneNo) {
2819 // We use CloneNo == 0 to refer to the original version, which doesn't get
2820 // renamed with a suffix.
2821 if (!CloneNo)
2822 return Base.str();
2823 return (Base + MemProfCloneSuffix + Twine(CloneNo)).str();
2824}
2825
2826// Return the clone number of the given function by extracting it from the
2827// memprof suffix. Assumes the caller has already confirmed it is a memprof
2828// clone.
2829static unsigned getMemProfCloneNum(const Function &F) {
2831 auto Pos = F.getName().find_last_of('.');
2832 assert(Pos > 0);
2833 unsigned CloneNo;
2834 bool Err = F.getName().drop_front(Pos + 1).getAsInteger(10, CloneNo);
2835 assert(!Err);
2836 (void)Err;
2837 return CloneNo;
2838}
2839
2840std::string ModuleCallsiteContextGraph::getLabel(const Function *Func,
2841 const Instruction *Call,
2842 unsigned CloneNo) const {
2843 return (Twine(Call->getFunction()->getName()) + " -> " +
2844 cast<CallBase>(Call)->getCalledFunction()->getName())
2845 .str();
2846}
2847
2848std::string IndexCallsiteContextGraph::getLabel(const FunctionSummary *Func,
2849 const IndexCall &Call,
2850 unsigned CloneNo) const {
2851 auto VI = FSToVIMap.find(Func);
2852 assert(VI != FSToVIMap.end());
2853 std::string CallerName = getMemProfFuncName(VI->second.name(), CloneNo);
2855 return CallerName + " -> alloc";
2856 else {
2857 auto *Callsite = dyn_cast_if_present<CallsiteInfo *>(Call);
2858 return CallerName + " -> " +
2859 getMemProfFuncName(Callsite->Callee.name(),
2860 Callsite->Clones[CloneNo]);
2861 }
2862}
2863
2864std::string ModuleCallsiteContextGraph::getFuncName(const Function *Func,
2865 unsigned CloneNo) const {
2866 if (!Func)
2867 return "<unknown>";
2868 return getMemProfFuncName(Func->getName(), CloneNo);
2869}
2870
2871void ModuleCallsiteContextGraph::populateFunctionsWithCallers() {
2872 // Any use (direct call, address taken, alias, etc.) means the function is
2873 // referenced and may be reached by callers above the profiled context.
2874 for (const auto &F : Mod)
2875 if (!F.use_empty())
2876 FunctionsWithCallers.insert(&F);
2877}
2878
2879std::string IndexCallsiteContextGraph::getFuncName(const FunctionSummary *Func,
2880 unsigned CloneNo) const {
2881 if (!Func)
2882 return "<unknown>";
2883 auto VI = FSToVIMap.find(Func);
2884 if (VI != FSToVIMap.end()) {
2885 if (!VI->second.name().empty())
2886 return getMemProfFuncName(VI->second.name(), CloneNo);
2887 return getMemProfFuncName("GUID:" + std::to_string(VI->second.getGUID()),
2888 CloneNo);
2889 }
2890 return getMemProfFuncName("<unknown>", CloneNo);
2891}
2892
2893bool IndexCallsiteContextGraph::isFuncDead(const FunctionSummary *Func) const {
2894 return Index.withGlobalValueDeadStripping() && Func && !Func->isLive();
2895}
2896
2897void IndexCallsiteContextGraph::populateFunctionsWithCallers() {
2898 for (const auto &I : Index) {
2899 auto VI = Index.getValueInfo(I);
2900 for (const auto &S : VI.getSummaryList()) {
2901 if (!GlobalValue::isLocalLinkage(S->linkage()) &&
2902 !isPrevailing(VI.getGUID(), S.get()))
2903 continue;
2904 if (Index.withGlobalValueDeadStripping() && !S->isLive())
2905 continue;
2906 auto AddReferenced = [&](ValueInfo RefVI) {
2907 for (const auto &RefS : RefVI.getSummaryList()) {
2908 const GlobalValueSummary *Base = RefS->getBaseObject();
2909 if (const auto *RefFS = dyn_cast<FunctionSummary>(Base))
2910 FunctionsWithCallers.insert(RefFS);
2911 }
2912 };
2913 // Address-taken references (e.g. from vtables or function pointers).
2914 for (const auto &RefVI : S->refs())
2915 AddReferenced(RefVI);
2916 // Direct calls.
2917 if (const auto *FS = dyn_cast<FunctionSummary>(S.get()))
2918 for (const auto &CallEdge : FS->calls())
2919 AddReferenced(CallEdge.first);
2920 }
2921 }
2922}
2923
2924std::vector<ModuleCallsiteContextGraph::ContextNode *>
2925ModuleCallsiteContextGraph::getStackNodesForCall(Instruction *Call) {
2926 CallStack<MDNode, MDNode::op_iterator> CallsiteContext(
2927 Call->getMetadata(LLVMContext::MD_callsite));
2928 return getStackNodesForCallsite<MDNode, MDNode::op_iterator>(CallsiteContext);
2929}
2930
2931std::vector<IndexCallsiteContextGraph::ContextNode *>
2932IndexCallsiteContextGraph::getStackNodesForCall(IndexCall &Call) {
2934 CallStack<CallsiteInfo, SmallVector<unsigned>::const_iterator>
2935 CallsiteContext(dyn_cast_if_present<CallsiteInfo *>(Call));
2936 return getStackNodesForCallsite<CallsiteInfo,
2937 SmallVector<unsigned>::const_iterator>(
2938 CallsiteContext);
2939}
2940
2941template <typename DerivedCCG, typename FuncTy, typename CallTy>
2942template <class NodeT, class IteratorT>
2943std::vector<
2944 typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *>
2945CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::getStackNodesForCallsite(
2946 CallStack<NodeT, IteratorT> &CallsiteContext) {
2947 std::vector<ContextNode *> StackNodes;
2948 for (auto IdOrIndex : CallsiteContext) {
2949 auto StackId = getStackId(IdOrIndex);
2950 ContextNode *Node = getNodeForStackId(StackId);
2951 if (!Node)
2952 break;
2953 StackNodes.push_back(Node);
2954 }
2955 return StackNodes;
2956}
2957
2958ModuleCallsiteContextGraph::ModuleCallsiteContextGraph(
2959 Module &M,
2960 llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter)
2961 : Mod(M), OREGetter(OREGetter) {
2962 // Map for keeping track of the largest cold contexts up to the number given
2963 // by MemProfTopNImportant. Must be a std::map (not DenseMap) because keys
2964 // must be sorted.
2965 std::map<uint64_t, uint32_t> TotalSizeToContextIdTopNCold;
2966 for (auto &F : M) {
2967 std::vector<CallInfo> CallsWithMetadata;
2968 for (auto &BB : F) {
2969 for (auto &I : BB) {
2970 if (!isa<CallBase>(I))
2971 continue;
2972 if (auto *MemProfMD = I.getMetadata(LLVMContext::MD_memprof)) {
2973 CallsWithMetadata.push_back(&I);
2974 auto *AllocNode = addAllocNode(&I, &F);
2975 auto *CallsiteMD = I.getMetadata(LLVMContext::MD_callsite);
2976 assert(CallsiteMD);
2977 CallStack<MDNode, MDNode::op_iterator> CallsiteContext(CallsiteMD);
2978 // Add all of the MIBs and their stack nodes.
2979 for (auto &MDOp : MemProfMD->operands()) {
2980 auto *MIBMD = cast<const MDNode>(MDOp);
2981 std::vector<ContextTotalSize> ContextSizeInfo;
2982 // Collect the context size information if it exists.
2983 if (MIBMD->getNumOperands() > 2) {
2984 for (unsigned I = 2; I < MIBMD->getNumOperands(); I++) {
2985 MDNode *ContextSizePair =
2986 dyn_cast<MDNode>(MIBMD->getOperand(I));
2987 assert(ContextSizePair->getNumOperands() == 2);
2989 ContextSizePair->getOperand(0))
2990 ->getZExtValue();
2992 ContextSizePair->getOperand(1))
2993 ->getZExtValue();
2994 ContextSizeInfo.push_back({FullStackId, TotalSize});
2995 }
2996 }
3000 addStackNodesForMIB<MDNode, MDNode::op_iterator>(
3001 AllocNode, StackContext, CallsiteContext,
3002 getMIBAllocType(MIBMD), ContextSizeInfo,
3003 TotalSizeToContextIdTopNCold);
3004 }
3005 // If exporting the graph to dot and an allocation id of interest was
3006 // specified, record all the context ids for this allocation node.
3007 if (ExportToDot && AllocNode->OrigStackOrAllocId == AllocIdForDot)
3008 DotAllocContextIds = AllocNode->getContextIds();
3009 assert(AllocNode->AllocTypes != (uint8_t)AllocationType::None);
3010 // Memprof and callsite metadata on memory allocations no longer
3011 // needed.
3012 I.setMetadata(LLVMContext::MD_memprof, nullptr);
3013 I.setMetadata(LLVMContext::MD_callsite, nullptr);
3014 }
3015 // For callsite metadata, add to list for this function for later use.
3016 else if (I.getMetadata(LLVMContext::MD_callsite)) {
3017 CallsWithMetadata.push_back(&I);
3018 }
3019 }
3020 }
3021 if (!CallsWithMetadata.empty())
3022 FuncToCallsWithMetadata[&F] = CallsWithMetadata;
3023 }
3024
3025 if (DumpCCG) {
3026 dbgs() << "CCG before updating call stack chains:\n";
3027 dbgs() << *this;
3028 }
3029
3030 if (ExportToDot)
3031 exportToDot("prestackupdate");
3032
3033 updateStackNodes();
3034
3035 if (ExportToDot)
3036 exportToDot("poststackupdate");
3037
3038 handleCallsitesWithMultipleTargets();
3039
3040 markBackedges();
3041
3042 // Strip off remaining callsite metadata, no longer needed.
3043 for (auto &FuncEntry : FuncToCallsWithMetadata)
3044 for (auto &Call : FuncEntry.second)
3045 Call.call()->setMetadata(LLVMContext::MD_callsite, nullptr);
3046}
3047
3048// Finds the set of GUIDs for weak aliasees that are prevailing in different
3049// modules than any of their aliases. We need to handle these specially.
3051IndexCallsiteContextGraph::findAliaseeGUIDsPrevailingInDifferentModule() {
3052 DenseSet<GlobalValue::GUID> AliaseeGUIDs;
3053 for (auto &I : Index) {
3054 auto VI = Index.getValueInfo(I);
3055 for (auto &S : VI.getSummaryList()) {
3056 // We only care about aliases to functions.
3057 auto *AS = dyn_cast<AliasSummary>(S.get());
3058 if (!AS)
3059 continue;
3060 auto *AliaseeSummary = &AS->getAliasee();
3061 auto *AliaseeFS = dyn_cast<FunctionSummary>(AliaseeSummary);
3062 if (!AliaseeFS)
3063 continue;
3064 // Skip this summary if it is not for the prevailing symbol for this GUID.
3065 // The linker doesn't resolve local linkage values so don't check whether
3066 // those are prevailing.
3067 if (!GlobalValue::isLocalLinkage(S->linkage()) &&
3068 !isPrevailing(VI.getGUID(), S.get()))
3069 continue;
3070 // Prevailing aliasee could be in a different module only if it is weak.
3071 if (!GlobalValue::isWeakForLinker(AliaseeSummary->linkage()))
3072 continue;
3073 auto AliaseeGUID = AS->getAliaseeGUID();
3074 // If the aliasee copy in this module is not prevailing, record it.
3075 if (!isPrevailing(AliaseeGUID, AliaseeSummary))
3076 AliaseeGUIDs.insert(AliaseeGUID);
3077 }
3078 }
3079 AliaseesPrevailingInDiffModuleFromAlias += AliaseeGUIDs.size();
3080 return AliaseeGUIDs;
3081}
3082
3083IndexCallsiteContextGraph::IndexCallsiteContextGraph(
3084 ModuleSummaryIndex &Index,
3085 llvm::function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
3086 isPrevailing)
3087 : Index(Index), isPrevailing(isPrevailing) {
3088 // Since we use the aliasee summary info to create the necessary clones for
3089 // its aliases, conservatively skip recording the aliasee function's callsites
3090 // in the CCG for any that are prevailing in a different module than one of
3091 // its aliases. We could record the necessary information to do this in the
3092 // summary, but this case should not be common.
3093 DenseSet<GlobalValue::GUID> GUIDsToSkip =
3094 findAliaseeGUIDsPrevailingInDifferentModule();
3095 // Map for keeping track of the largest cold contexts up to the number given
3096 // by MemProfTopNImportant. Must be a std::map (not DenseMap) because keys
3097 // must be sorted.
3098 std::map<uint64_t, uint32_t> TotalSizeToContextIdTopNCold;
3099 // Sort by GUID for deterministic graph construction order.
3100 // TODO: This sort has a measurable cost on the thin link when memprof is
3101 // enabled. Investigate gating it behind an option that is only enabled for
3102 // tests that check internal state.
3103 for (const auto &I : Index.sortedGlobalValueSummariesRange()) {
3104 auto VI = Index.getValueInfo(I);
3105 if (GUIDsToSkip.contains(VI.getGUID()))
3106 continue;
3107 for (auto &S : VI.getSummaryList()) {
3108 // We should only add the prevailing nodes. Otherwise we may try to clone
3109 // in a weak copy that won't be linked (and may be different than the
3110 // prevailing version).
3111 // We only keep the memprof summary on the prevailing copy now when
3112 // building the combined index, as a space optimization, however don't
3113 // rely on this optimization. The linker doesn't resolve local linkage
3114 // values so don't check whether those are prevailing.
3115 if (!GlobalValue::isLocalLinkage(S->linkage()) &&
3116 !isPrevailing(VI.getGUID(), S.get()))
3117 continue;
3118 auto *FS = dyn_cast<FunctionSummary>(S.get());
3119 if (!FS)
3120 continue;
3121 std::vector<CallInfo> CallsWithMetadata;
3122 if (!FS->allocs().empty()) {
3123 for (auto &AN : FS->mutableAllocs()) {
3124 // This can happen because of recursion elimination handling that
3125 // currently exists in ModuleSummaryAnalysis. Skip these for now.
3126 // We still added them to the summary because we need to be able to
3127 // correlate properly in applyImport in the backends.
3128 if (AN.MIBs.empty())
3129 continue;
3130 IndexCall AllocCall(&AN);
3131 CallsWithMetadata.push_back(AllocCall);
3132 auto *AllocNode = addAllocNode(AllocCall, FS);
3133 // Pass an empty CallStack to the CallsiteContext (second)
3134 // parameter, since for ThinLTO we already collapsed out the inlined
3135 // stack ids on the allocation call during ModuleSummaryAnalysis.
3137 EmptyContext;
3138 unsigned I = 0;
3140 AN.ContextSizeInfos.size() == AN.MIBs.size());
3141 // Now add all of the MIBs and their stack nodes.
3142 for (auto &MIB : AN.MIBs) {
3144 StackContext(&MIB);
3145 std::vector<ContextTotalSize> ContextSizeInfo;
3146 if (!AN.ContextSizeInfos.empty()) {
3147 for (auto [FullStackId, TotalSize] : AN.ContextSizeInfos[I])
3148 ContextSizeInfo.push_back({FullStackId, TotalSize});
3149 }
3150 addStackNodesForMIB<MIBInfo, SmallVector<unsigned>::const_iterator>(
3151 AllocNode, StackContext, EmptyContext, MIB.AllocType,
3152 ContextSizeInfo, TotalSizeToContextIdTopNCold);
3153 I++;
3154 }
3155 // If exporting the graph to dot and an allocation id of interest was
3156 // specified, record all the context ids for this allocation node.
3157 if (ExportToDot && AllocNode->OrigStackOrAllocId == AllocIdForDot)
3158 DotAllocContextIds = AllocNode->getContextIds();
3159 assert(AllocNode->AllocTypes != (uint8_t)AllocationType::None);
3160 // Initialize version 0 on the summary alloc node to the current alloc
3161 // type, unless it has both types in which case make it default, so
3162 // that in the case where we aren't able to clone the original version
3163 // always ends up with the default allocation behavior.
3164 AN.Versions[0] = (uint8_t)allocTypeToUse(AllocNode->AllocTypes);
3165 }
3166 }
3167 // For callsite metadata, add to list for this function for later use.
3168 if (!FS->callsites().empty())
3169 for (auto &SN : FS->mutableCallsites()) {
3170 IndexCall StackNodeCall(&SN);
3171 CallsWithMetadata.push_back(StackNodeCall);
3172 }
3173
3174 if (!CallsWithMetadata.empty())
3175 FuncToCallsWithMetadata[FS] = CallsWithMetadata;
3176
3177 if (!FS->allocs().empty() || !FS->callsites().empty())
3178 FSToVIMap[FS] = VI;
3179 }
3180 }
3181
3182 if (DumpCCG) {
3183 dbgs() << "CCG before updating call stack chains:\n";
3184 dbgs() << *this;
3185 }
3186
3187 if (ExportToDot)
3188 exportToDot("prestackupdate");
3189
3190 updateStackNodes();
3191
3192 if (ExportToDot)
3193 exportToDot("poststackupdate");
3194
3195 handleCallsitesWithMultipleTargets();
3196
3197 markBackedges();
3198}
3199
3200template <typename DerivedCCG, typename FuncTy, typename CallTy>
3201void CallsiteContextGraph<DerivedCCG, FuncTy,
3202 CallTy>::handleCallsitesWithMultipleTargets() {
3203 // Look for and workaround callsites that call multiple functions.
3204 // This can happen for indirect calls, which needs better handling, and in
3205 // more rare cases (e.g. macro expansion).
3206 // TODO: To fix this for indirect calls we will want to perform speculative
3207 // devirtualization using either the normal PGO info with ICP, or using the
3208 // information in the profiled MemProf contexts. We can do this prior to
3209 // this transformation for regular LTO, and for ThinLTO we can simulate that
3210 // effect in the summary and perform the actual speculative devirtualization
3211 // while cloning in the ThinLTO backend.
3212
3213 // Keep track of the new nodes synthesized for discovered tail calls missing
3214 // from the profiled contexts.
3215 MapVector<CallInfo, ContextNode *> TailCallToContextNodeMap;
3216
3217 std::vector<std::pair<CallInfo, ContextNode *>> NewCallToNode;
3218 for (auto &Entry : NonAllocationCallToContextNodeMap) {
3219 auto *Node = Entry.second;
3220 assert(Node->Clones.empty());
3221 // Check all node callees and see if in the same function.
3222 // We need to check all of the calls recorded in this Node, because in some
3223 // cases we may have had multiple calls with the same debug info calling
3224 // different callees. This can happen, for example, when an object is
3225 // constructed in the paramter list - the destructor call of the object has
3226 // the same debug info (line/col) as the call the object was passed to.
3227 // Here we will prune any that don't match all callee nodes.
3228 std::vector<CallInfo> AllCalls;
3229 AllCalls.reserve(Node->MatchingCalls.size() + 1);
3230 AllCalls.push_back(Node->Call);
3231 llvm::append_range(AllCalls, Node->MatchingCalls);
3232
3233 // First see if we can partition the calls by callee function, creating new
3234 // nodes to host each set of calls calling the same callees. This is
3235 // necessary for support indirect calls with ThinLTO, for which we
3236 // synthesized CallsiteInfo records for each target. They will all have the
3237 // same callsite stack ids and would be sharing a context node at this
3238 // point. We need to perform separate cloning for each, which will be
3239 // applied along with speculative devirtualization in the ThinLTO backends
3240 // as needed. Note this does not currently support looking through tail
3241 // calls, it is unclear if we need that for indirect call targets.
3242 // First partition calls by callee func. Map indexed by func, value is
3243 // struct with list of matching calls, assigned node.
3244 if (partitionCallsByCallee(Node, AllCalls, NewCallToNode))
3245 continue;
3246
3247 auto It = AllCalls.begin();
3248 // Iterate through the calls until we find the first that matches.
3249 for (; It != AllCalls.end(); ++It) {
3250 auto ThisCall = *It;
3251 bool Match = true;
3252 for (auto EI = Node->CalleeEdges.begin(); EI != Node->CalleeEdges.end();
3253 ++EI) {
3254 auto Edge = *EI;
3255 if (!Edge->Callee->hasCall())
3256 continue;
3257 assert(NodeToCallingFunc.count(Edge->Callee));
3258 // Check if the called function matches that of the callee node.
3259 if (!calleesMatch(ThisCall.call(), EI, TailCallToContextNodeMap)) {
3260 Match = false;
3261 break;
3262 }
3263 }
3264 // Found a call that matches the callee nodes, we can quit now.
3265 if (Match) {
3266 // If the first match is not the primary call on the Node, update it
3267 // now. We will update the list of matching calls further below.
3268 if (Node->Call != ThisCall) {
3269 Node->setCall(ThisCall);
3270 // We need to update the NonAllocationCallToContextNodeMap, but don't
3271 // want to do this during iteration over that map, so save the calls
3272 // that need updated entries.
3273 NewCallToNode.push_back({ThisCall, Node});
3274 }
3275 break;
3276 }
3277 }
3278 // We will update this list below (or leave it cleared if there was no
3279 // match found above).
3280 Node->MatchingCalls.clear();
3281 // If we hit the end of the AllCalls vector, no call matching the callee
3282 // nodes was found, clear the call information in the node.
3283 if (It == AllCalls.end()) {
3284 RemovedEdgesWithMismatchedCallees++;
3285 // Work around by setting Node to have a null call, so it gets
3286 // skipped during cloning. Otherwise assignFunctions will assert
3287 // because its data structures are not designed to handle this case.
3288 Node->setCall(CallInfo());
3289 continue;
3290 }
3291 // Now add back any matching calls that call the same function as the
3292 // matching primary call on Node.
3293 for (++It; It != AllCalls.end(); ++It) {
3294 auto ThisCall = *It;
3295 if (!sameCallee(Node->Call.call(), ThisCall.call()))
3296 continue;
3297 Node->MatchingCalls.push_back(ThisCall);
3298 }
3299 }
3300
3301 // Remove all mismatched nodes identified in the above loop from the node map
3302 // (checking whether they have a null call which is set above). For a
3303 // MapVector like NonAllocationCallToContextNodeMap it is much more efficient
3304 // to do the removal via remove_if than by individually erasing entries above.
3305 // Also remove any entries if we updated the node's primary call above.
3306 NonAllocationCallToContextNodeMap.remove_if([](const auto &it) {
3307 return !it.second->hasCall() || it.second->Call != it.first;
3308 });
3309
3310 // Add entries for any new primary calls recorded above.
3311 for (auto &[Call, Node] : NewCallToNode)
3312 NonAllocationCallToContextNodeMap[Call] = Node;
3313
3314 // Add the new nodes after the above loop so that the iteration is not
3315 // invalidated.
3316 for (auto &[Call, Node] : TailCallToContextNodeMap)
3317 NonAllocationCallToContextNodeMap[Call] = Node;
3318}
3319
3320template <typename DerivedCCG, typename FuncTy, typename CallTy>
3321bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::partitionCallsByCallee(
3322 ContextNode *Node, ArrayRef<CallInfo> AllCalls,
3323 std::vector<std::pair<CallInfo, ContextNode *>> &NewCallToNode) {
3324 // Struct to keep track of all the calls having the same callee function,
3325 // and the node we eventually assign to them. Eventually we will record the
3326 // context node assigned to this group of calls.
3327 struct CallsWithSameCallee {
3328 std::vector<CallInfo> Calls;
3329 ContextNode *Node = nullptr;
3330 };
3331
3332 // First partition calls by callee function. Build map from each function
3333 // to the list of matching calls.
3335 for (auto ThisCall : AllCalls) {
3336 auto *F = getCalleeFunc(ThisCall.call());
3337 if (F)
3338 CalleeFuncToCallInfo[F].Calls.push_back(ThisCall);
3339 }
3340
3341 // Next, walk through all callee edges. For each callee node, get its
3342 // containing function and see if it was recorded in the above map (meaning we
3343 // have at least one matching call). Build another map from each callee node
3344 // with a matching call to the structure instance created above containing all
3345 // the calls.
3347 for (const auto &Edge : Node->CalleeEdges) {
3348 if (!Edge->Callee->hasCall())
3349 continue;
3350 const FuncTy *ProfiledCalleeFunc = NodeToCallingFunc[Edge->Callee];
3351 if (CalleeFuncToCallInfo.contains(ProfiledCalleeFunc))
3352 CalleeNodeToCallInfo[Edge->Callee] =
3353 &CalleeFuncToCallInfo[ProfiledCalleeFunc];
3354 }
3355
3356 // If there are entries in the second map, then there were no matching
3357 // calls/callees, nothing to do here. Return so we can go to the handling that
3358 // looks through tail calls.
3359 if (CalleeNodeToCallInfo.empty())
3360 return false;
3361
3362 // Walk through all callee edges again. Any and all callee edges that didn't
3363 // match any calls (callee not in the CalleeNodeToCallInfo map) are moved to a
3364 // new caller node (UnmatchedCalleesNode) which gets a null call so that it is
3365 // ignored during cloning. If it is in the map, then we use the node recorded
3366 // in that entry (creating it if needed), and move the callee edge to it.
3367 // The first callee will use the original node instead of creating a new one.
3368 // Note that any of the original calls on this node (in AllCalls) that didn't
3369 // have a callee function automatically get dropped from the node as part of
3370 // this process.
3371 ContextNode *UnmatchedCalleesNode = nullptr;
3372 // Track whether we already assigned original node to a callee.
3373 bool UsedOrigNode = false;
3374 assert(NodeToCallingFunc[Node]);
3375 // Iterate over a copy of Node's callee edges, since we may need to remove
3376 // edges in moveCalleeEdgeToNewCaller, and this simplifies the handling and
3377 // makes it less error-prone.
3378 auto CalleeEdges = Node->CalleeEdges;
3379 for (auto &Edge : CalleeEdges) {
3380 if (!Edge->Callee->hasCall())
3381 continue;
3382
3383 // Will be updated below to point to whatever (caller) node this callee edge
3384 // should be moved to.
3385 ContextNode *CallerNodeToUse = nullptr;
3386
3387 // Handle the case where there were no matching calls first. Move this
3388 // callee edge to the UnmatchedCalleesNode, creating it if needed.
3389 if (!CalleeNodeToCallInfo.contains(Edge->Callee)) {
3390 if (!UnmatchedCalleesNode)
3391 UnmatchedCalleesNode =
3392 createNewNode(/*IsAllocation=*/false, NodeToCallingFunc[Node]);
3393 CallerNodeToUse = UnmatchedCalleesNode;
3394 } else {
3395 // Look up the information recorded for this callee node, and use the
3396 // recorded caller node (creating it if needed).
3397 auto *Info = CalleeNodeToCallInfo[Edge->Callee];
3398 if (!Info->Node) {
3399 // If we haven't assigned any callees to the original node use it.
3400 if (!UsedOrigNode) {
3401 Info->Node = Node;
3402 // Clear the set of matching calls which will be updated below.
3403 Node->MatchingCalls.clear();
3404 UsedOrigNode = true;
3405 } else
3406 Info->Node =
3407 createNewNode(/*IsAllocation=*/false, NodeToCallingFunc[Node]);
3408 assert(!Info->Calls.empty());
3409 // The first call becomes the primary call for this caller node, and the
3410 // rest go in the matching calls list.
3411 Info->Node->setCall(Info->Calls.front());
3412 llvm::append_range(Info->Node->MatchingCalls,
3413 llvm::drop_begin(Info->Calls));
3414 // Save the primary call to node correspondence so that we can update
3415 // the NonAllocationCallToContextNodeMap, which is being iterated in the
3416 // caller of this function.
3417 NewCallToNode.push_back({Info->Node->Call, Info->Node});
3418 }
3419 CallerNodeToUse = Info->Node;
3420 }
3421
3422 // Don't need to move edge if we are using the original node;
3423 if (CallerNodeToUse == Node)
3424 continue;
3425
3426 moveCalleeEdgeToNewCaller(Edge, CallerNodeToUse);
3427 }
3428 // Now that we are done moving edges, clean up any caller edges that ended
3429 // up with no type or context ids. During moveCalleeEdgeToNewCaller all
3430 // caller edges from Node are replicated onto the new callers, and it
3431 // simplifies the handling to leave them until we have moved all
3432 // edges/context ids.
3433 for (auto &I : CalleeNodeToCallInfo)
3434 removeNoneTypeCallerEdges(I.second->Node);
3435 if (UnmatchedCalleesNode)
3436 removeNoneTypeCallerEdges(UnmatchedCalleesNode);
3437 removeNoneTypeCallerEdges(Node);
3438
3439 return true;
3440}
3441
3442uint64_t ModuleCallsiteContextGraph::getStackId(uint64_t IdOrIndex) const {
3443 // In the Module (IR) case this is already the Id.
3444 return IdOrIndex;
3445}
3446
3447uint64_t IndexCallsiteContextGraph::getStackId(uint64_t IdOrIndex) const {
3448 // In the Index case this is an index into the stack id list in the summary
3449 // index, convert it to an Id.
3450 return Index.getStackIdAtIndex(IdOrIndex);
3451}
3452
3453template <typename DerivedCCG, typename FuncTy, typename CallTy>
3454bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::calleesMatch(
3455 CallTy Call, EdgeIter &EI,
3456 MapVector<CallInfo, ContextNode *> &TailCallToContextNodeMap) {
3457 auto Edge = *EI;
3458 const FuncTy *ProfiledCalleeFunc = NodeToCallingFunc[Edge->Callee];
3459 const FuncTy *CallerFunc = NodeToCallingFunc[Edge->Caller];
3460 // Will be populated in order of callee to caller if we find a chain of tail
3461 // calls between the profiled caller and callee.
3462 std::vector<std::pair<CallTy, FuncTy *>> FoundCalleeChain;
3463 if (!calleeMatchesFunc(Call, ProfiledCalleeFunc, CallerFunc,
3464 FoundCalleeChain))
3465 return false;
3466
3467 // The usual case where the profiled callee matches that of the IR/summary.
3468 if (FoundCalleeChain.empty())
3469 return true;
3470
3471 auto AddEdge = [Edge, &EI](ContextNode *Caller, ContextNode *Callee) {
3472 auto *CurEdge = Callee->findEdgeFromCaller(Caller);
3473 // If there is already an edge between these nodes, simply update it and
3474 // return.
3475 if (CurEdge) {
3476 CurEdge->ContextIds.insert_range(Edge->ContextIds);
3477 CurEdge->AllocTypes |= Edge->AllocTypes;
3478 return;
3479 }
3480 // Otherwise, create a new edge and insert it into the caller and callee
3481 // lists.
3482 auto NewEdge = std::make_shared<ContextEdge>(
3483 Callee, Caller, Edge->AllocTypes, Edge->ContextIds);
3484 Callee->CallerEdges.push_back(NewEdge);
3485 if (Caller == Edge->Caller) {
3486 // If we are inserting the new edge into the current edge's caller, insert
3487 // the new edge before the current iterator position, and then increment
3488 // back to the current edge.
3489 EI = Caller->CalleeEdges.insert(EI, NewEdge);
3490 ++EI;
3491 assert(*EI == Edge &&
3492 "Iterator position not restored after insert and increment");
3493 } else
3494 Caller->CalleeEdges.push_back(NewEdge);
3495 };
3496
3497 // Create new nodes for each found callee and connect in between the profiled
3498 // caller and callee.
3499 auto *CurCalleeNode = Edge->Callee;
3500 for (auto &[NewCall, Func] : FoundCalleeChain) {
3501 ContextNode *NewNode = nullptr;
3502 // First check if we have already synthesized a node for this tail call.
3503 if (TailCallToContextNodeMap.count(NewCall)) {
3504 NewNode = TailCallToContextNodeMap[NewCall];
3505 NewNode->AllocTypes |= Edge->AllocTypes;
3506 } else {
3507 FuncToCallsWithMetadata[Func].push_back({NewCall});
3508 // Create Node and record node info.
3509 NewNode = createNewNode(/*IsAllocation=*/false, Func, NewCall);
3510 TailCallToContextNodeMap[NewCall] = NewNode;
3511 NewNode->AllocTypes = Edge->AllocTypes;
3512 }
3513
3514 // Hook up node to its callee node
3515 AddEdge(NewNode, CurCalleeNode);
3516
3517 CurCalleeNode = NewNode;
3518 }
3519
3520 // Hook up edge's original caller to new callee node.
3521 AddEdge(Edge->Caller, CurCalleeNode);
3522
3523#ifndef NDEBUG
3524 // Save this because Edge's fields get cleared below when removed.
3525 auto *Caller = Edge->Caller;
3526#endif
3527
3528 // Remove old edge
3529 removeEdgeFromGraph(Edge.get(), &EI, /*CalleeIter=*/true);
3530
3531 // To simplify the increment of EI in the caller, subtract one from EI.
3532 // In the final AddEdge call we would have either added a new callee edge,
3533 // to Edge->Caller, or found an existing one. Either way we are guaranteed
3534 // that there is at least one callee edge.
3535 assert(!Caller->CalleeEdges.empty());
3536 --EI;
3537
3538 return true;
3539}
3540
3541bool ModuleCallsiteContextGraph::findProfiledCalleeThroughTailCalls(
3542 const Function *ProfiledCallee, Value *CurCallee, unsigned Depth,
3543 std::vector<std::pair<Instruction *, Function *>> &FoundCalleeChain,
3544 bool &FoundMultipleCalleeChains) {
3545 // Stop recursive search if we have already explored the maximum specified
3546 // depth.
3548 return false;
3549
3550 auto SaveCallsiteInfo = [&](Instruction *Callsite, Function *F) {
3551 FoundCalleeChain.push_back({Callsite, F});
3552 };
3553
3554 auto *CalleeFunc = dyn_cast<Function>(CurCallee);
3555 if (!CalleeFunc) {
3556 auto *Alias = dyn_cast<GlobalAlias>(CurCallee);
3557 assert(Alias);
3558 CalleeFunc = dyn_cast<Function>(Alias->getAliasee());
3559 assert(CalleeFunc);
3560 }
3561
3562 // Look for tail calls in this function, and check if they either call the
3563 // profiled callee directly, or indirectly (via a recursive search).
3564 // Only succeed if there is a single unique tail call chain found between the
3565 // profiled caller and callee, otherwise we could perform incorrect cloning.
3566 bool FoundSingleCalleeChain = false;
3567 for (auto &BB : *CalleeFunc) {
3568 for (auto &I : BB) {
3569 auto *CB = dyn_cast<CallBase>(&I);
3570 if (!CB || !CB->isTailCall())
3571 continue;
3572 auto *CalledValue = CB->getCalledOperand();
3573 auto *CalledFunction = CB->getCalledFunction();
3574 if (CalledValue && !CalledFunction) {
3575 CalledValue = CalledValue->stripPointerCasts();
3576 // Stripping pointer casts can reveal a called function.
3577 CalledFunction = dyn_cast<Function>(CalledValue);
3578 }
3579 // Check if this is an alias to a function. If so, get the
3580 // called aliasee for the checks below.
3581 if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
3582 assert(!CalledFunction &&
3583 "Expected null called function in callsite for alias");
3584 CalledFunction = dyn_cast<Function>(GA->getAliaseeObject());
3585 }
3586 if (!CalledFunction)
3587 continue;
3588 if (CalledFunction == ProfiledCallee) {
3589 if (FoundSingleCalleeChain) {
3590 FoundMultipleCalleeChains = true;
3591 return false;
3592 }
3593 FoundSingleCalleeChain = true;
3594 FoundProfiledCalleeCount++;
3595 FoundProfiledCalleeDepth += Depth;
3596 if (Depth > FoundProfiledCalleeMaxDepth)
3597 FoundProfiledCalleeMaxDepth = Depth;
3598 SaveCallsiteInfo(&I, CalleeFunc);
3599 } else if (findProfiledCalleeThroughTailCalls(
3600 ProfiledCallee, CalledFunction, Depth + 1,
3601 FoundCalleeChain, FoundMultipleCalleeChains)) {
3602 // findProfiledCalleeThroughTailCalls should not have returned
3603 // true if FoundMultipleCalleeChains.
3604 assert(!FoundMultipleCalleeChains);
3605 if (FoundSingleCalleeChain) {
3606 FoundMultipleCalleeChains = true;
3607 return false;
3608 }
3609 FoundSingleCalleeChain = true;
3610 SaveCallsiteInfo(&I, CalleeFunc);
3611 } else if (FoundMultipleCalleeChains)
3612 return false;
3613 }
3614 }
3615
3616 return FoundSingleCalleeChain;
3617}
3618
3619const Function *ModuleCallsiteContextGraph::getCalleeFunc(Instruction *Call) {
3620 auto *CB = dyn_cast<CallBase>(Call);
3621 if (!CB->getCalledOperand() || CB->isIndirectCall())
3622 return nullptr;
3623 auto *CalleeVal = CB->getCalledOperand()->stripPointerCasts();
3624 auto *Alias = dyn_cast<GlobalAlias>(CalleeVal);
3625 if (Alias)
3626 return dyn_cast<Function>(Alias->getAliasee());
3627 return dyn_cast<Function>(CalleeVal);
3628}
3629
3630bool ModuleCallsiteContextGraph::calleeMatchesFunc(
3631 Instruction *Call, const Function *Func, const Function *CallerFunc,
3632 std::vector<std::pair<Instruction *, Function *>> &FoundCalleeChain) {
3633 auto *CB = dyn_cast<CallBase>(Call);
3634 if (!CB->getCalledOperand() || CB->isIndirectCall())
3635 return false;
3636 auto *CalleeVal = CB->getCalledOperand()->stripPointerCasts();
3637 auto *CalleeFunc = dyn_cast<Function>(CalleeVal);
3638 if (CalleeFunc == Func)
3639 return true;
3640 auto *Alias = dyn_cast<GlobalAlias>(CalleeVal);
3641 if (Alias && Alias->getAliasee() == Func)
3642 return true;
3643
3644 // Recursively search for the profiled callee through tail calls starting with
3645 // the actual Callee. The discovered tail call chain is saved in
3646 // FoundCalleeChain, and we will fixup the graph to include these callsites
3647 // after returning.
3648 // FIXME: We will currently redo the same recursive walk if we find the same
3649 // mismatched callee from another callsite. We can improve this with more
3650 // bookkeeping of the created chain of new nodes for each mismatch.
3651 unsigned Depth = 1;
3652 bool FoundMultipleCalleeChains = false;
3653 if (!findProfiledCalleeThroughTailCalls(Func, CalleeVal, Depth,
3654 FoundCalleeChain,
3655 FoundMultipleCalleeChains)) {
3656 LLVM_DEBUG(dbgs() << "Not found through unique tail call chain: "
3657 << Func->getName() << " from " << CallerFunc->getName()
3658 << " that actually called " << CalleeVal->getName()
3659 << (FoundMultipleCalleeChains
3660 ? " (found multiple possible chains)"
3661 : "")
3662 << "\n");
3663 if (FoundMultipleCalleeChains)
3664 FoundProfiledCalleeNonUniquelyCount++;
3665 return false;
3666 }
3667
3668 return true;
3669}
3670
3671bool ModuleCallsiteContextGraph::sameCallee(Instruction *Call1,
3672 Instruction *Call2) {
3673 auto *CB1 = cast<CallBase>(Call1);
3674 if (!CB1->getCalledOperand() || CB1->isIndirectCall())
3675 return false;
3676 auto *CalleeVal1 = CB1->getCalledOperand()->stripPointerCasts();
3677 auto *CalleeFunc1 = dyn_cast<Function>(CalleeVal1);
3678 auto *CB2 = cast<CallBase>(Call2);
3679 if (!CB2->getCalledOperand() || CB2->isIndirectCall())
3680 return false;
3681 auto *CalleeVal2 = CB2->getCalledOperand()->stripPointerCasts();
3682 auto *CalleeFunc2 = dyn_cast<Function>(CalleeVal2);
3683 return CalleeFunc1 == CalleeFunc2;
3684}
3685
3686bool IndexCallsiteContextGraph::findProfiledCalleeThroughTailCalls(
3687 ValueInfo ProfiledCallee, ValueInfo CurCallee, unsigned Depth,
3688 std::vector<std::pair<ValueInfo, FunctionSummary *>> &FoundCalleeChain,
3689 bool &FoundMultipleCalleeChains) {
3690 // Stop recursive search if we have already explored the maximum specified
3691 // depth.
3693 return false;
3694
3695 // Record each discovered tail call (callee and calling function) in the
3696 // chain. The CallsiteInfo records are only synthesized by the caller once
3697 // the search succeeds, since the chain is discarded if the profiled callee
3698 // is found through multiple chains.
3699 auto SaveTailCall = [&](ValueInfo Callee, FunctionSummary *FS) {
3700 FoundCalleeChain.push_back({Callee, FS});
3701 };
3702
3703 // Returns true if the given tail call callee is the profiled callee, or is
3704 // an alias of it. Only the aliasee's function summary gets a context node
3705 // (see calleeMatchesFunc), so the profiled callee is always the aliasee. The
3706 // synthesized callsite record is still keyed by the alias (the callee of the
3707 // call edge), which is what the ThinLTO backend looks up for the tail call
3708 // (see findValueInfoForGV).
3709 auto IsProfiledCallee = [&](ValueInfo Callee) {
3710 if (Callee == ProfiledCallee)
3711 return true;
3712 for (auto &S : Callee.getSummaryList()) {
3713 if (!GlobalValue::isLocalLinkage(S->linkage()) &&
3714 !isPrevailing(Callee.getGUID(), S.get()))
3715 continue;
3716 auto *AS = dyn_cast<AliasSummary>(S.get());
3717 return AS && AS->getAliaseeVI() == ProfiledCallee;
3718 }
3719 return false;
3720 };
3721
3722 // Look for tail calls in this function, and check if they either call the
3723 // profiled callee directly, or indirectly (via a recursive search).
3724 // Only succeed if there is a single unique tail call chain found between the
3725 // profiled caller and callee, otherwise we could perform incorrect cloning.
3726 bool FoundSingleCalleeChain = false;
3727 for (auto &S : CurCallee.getSummaryList()) {
3728 if (!GlobalValue::isLocalLinkage(S->linkage()) &&
3729 !isPrevailing(CurCallee.getGUID(), S.get()))
3730 continue;
3731 auto *FS = dyn_cast<FunctionSummary>(S->getBaseObject());
3732 if (!FS)
3733 continue;
3734 auto FSVI = CurCallee;
3735 auto *AS = dyn_cast<AliasSummary>(S.get());
3736 if (AS)
3737 FSVI = AS->getAliaseeVI();
3738 for (auto &CallEdge : FS->calls()) {
3739 if (!CallEdge.second.hasTailCall())
3740 continue;
3741 // Edges marked indirect are the profiled targets of indirect calls.
3742 // If the tail call is indirect, the ThinLTO backend will perform
3743 // ICP to the target in order to call a clone of it.
3744 if (CallEdge.second.hasIndirectCall() && !IndirectTailCallSearch)
3745 continue;
3746 if (IsProfiledCallee(CallEdge.first)) {
3747 if (FoundSingleCalleeChain) {
3748 FoundMultipleCalleeChains = true;
3749 return false;
3750 }
3751 FoundSingleCalleeChain = true;
3752 FoundProfiledCalleeCount++;
3753 FoundProfiledCalleeDepth += Depth;
3754 if (Depth > FoundProfiledCalleeMaxDepth)
3755 FoundProfiledCalleeMaxDepth = Depth;
3756 if (CallEdge.second.hasIndirectCall())
3757 FoundProfiledCalleeIndirectTailCallCount++;
3758 SaveTailCall(CallEdge.first, FS);
3759 // Add FS to FSToVIMap in case it isn't already there.
3760 assert(!FSToVIMap.count(FS) || FSToVIMap[FS] == FSVI);
3761 FSToVIMap[FS] = FSVI;
3762 } else if (findProfiledCalleeThroughTailCalls(
3763 ProfiledCallee, CallEdge.first, Depth + 1,
3764 FoundCalleeChain, FoundMultipleCalleeChains)) {
3765 // findProfiledCalleeThroughTailCalls should not have returned
3766 // true if FoundMultipleCalleeChains.
3767 assert(!FoundMultipleCalleeChains);
3768 if (FoundSingleCalleeChain) {
3769 FoundMultipleCalleeChains = true;
3770 return false;
3771 }
3772 FoundSingleCalleeChain = true;
3773 if (CallEdge.second.hasIndirectCall())
3774 FoundProfiledCalleeIndirectTailCallCount++;
3775 SaveTailCall(CallEdge.first, FS);
3776 // Add FS to FSToVIMap in case it isn't already there.
3777 assert(!FSToVIMap.count(FS) || FSToVIMap[FS] == FSVI);
3778 FSToVIMap[FS] = FSVI;
3779 } else if (FoundMultipleCalleeChains)
3780 return false;
3781 }
3782 }
3783
3784 return FoundSingleCalleeChain;
3785}
3786
3787const FunctionSummary *
3788IndexCallsiteContextGraph::getCalleeFunc(IndexCall &Call) {
3789 ValueInfo Callee = dyn_cast_if_present<CallsiteInfo *>(Call)->Callee;
3790 if (Callee.getSummaryList().empty())
3791 return nullptr;
3792 return dyn_cast<FunctionSummary>(Callee.getSummaryList()[0]->getBaseObject());
3793}
3794
3795bool IndexCallsiteContextGraph::calleeMatchesFunc(
3796 IndexCall &Call, const FunctionSummary *Func,
3797 const FunctionSummary *CallerFunc,
3798 std::vector<std::pair<IndexCall, FunctionSummary *>> &FoundCalleeChain) {
3799 ValueInfo Callee = dyn_cast_if_present<CallsiteInfo *>(Call)->Callee;
3800 // If there is no summary list then this is a call to an externally defined
3801 // symbol.
3802 AliasSummary *Alias =
3803 Callee.getSummaryList().empty()
3804 ? nullptr
3805 : dyn_cast<AliasSummary>(Callee.getSummaryList()[0].get());
3806 assert(FSToVIMap.count(Func));
3807 auto FuncVI = FSToVIMap[Func];
3808 if (Callee == FuncVI ||
3809 // If callee is an alias, check the aliasee, since only function
3810 // summary base objects will contain the stack node summaries and thus
3811 // get a context node.
3812 (Alias && Alias->getAliaseeVI() == FuncVI))
3813 return true;
3814
3815 // Recursively search for the profiled callee through tail calls starting with
3816 // the actual Callee. The discovered tail call chain is saved in
3817 // FoundCalleeChain, and we will fixup the graph to include these callsites
3818 // after returning.
3819 // FIXME: We will currently redo the same recursive walk if we find the same
3820 // mismatched callee from another callsite. We can improve this with more
3821 // bookkeeping of the created chain of new nodes for each mismatch.
3822 unsigned Depth = 1;
3823 bool FoundMultipleCalleeChains = false;
3824 std::vector<std::pair<ValueInfo, FunctionSummary *>> FoundCalleeVIChain;
3825 if (!findProfiledCalleeThroughTailCalls(FuncVI, Callee, Depth,
3826 FoundCalleeVIChain,
3827 FoundMultipleCalleeChains)) {
3828 LLVM_DEBUG(dbgs() << "Not found through unique tail call chain: " << FuncVI
3829 << " from " << FSToVIMap[CallerFunc]
3830 << " that actually called " << Callee
3831 << (FoundMultipleCalleeChains
3832 ? " (found multiple possible chains)"
3833 : "")
3834 << "\n");
3835 if (FoundMultipleCalleeChains)
3836 FoundProfiledCalleeNonUniquelyCount++;
3837 return false;
3838 }
3839
3840 // Now that we know the chain is unique, synthesize a CallsiteInfo for each
3841 // tail call in it, if one hasn't already been synthesized (e.g. for a chain
3842 // to a different profiled callee through the same tail call). These are only
3843 // added to the function summaries once the graph is destroyed (see
3844 // ~IndexCallsiteContextGraph).
3845 for (auto &[CalleeVI, FS] : FoundCalleeVIChain) {
3846 auto &CallsiteInfoPtr =
3847 FunctionCalleesToSynthesizedCallsiteInfos[FS][CalleeVI];
3848 if (!CallsiteInfoPtr)
3849 // StackIds is empty (we don't have debug info available in the index for
3850 // these callsites)
3851 CallsiteInfoPtr =
3852 std::make_unique<CallsiteInfo>(CalleeVI, SmallVector<unsigned>());
3853 FoundCalleeChain.push_back({CallsiteInfoPtr.get(), FS});
3854 }
3855
3856 return true;
3857}
3858
3859bool IndexCallsiteContextGraph::sameCallee(IndexCall &Call1, IndexCall &Call2) {
3860 ValueInfo Callee1 = dyn_cast_if_present<CallsiteInfo *>(Call1)->Callee;
3861 ValueInfo Callee2 = dyn_cast_if_present<CallsiteInfo *>(Call2)->Callee;
3862 return Callee1 == Callee2;
3863}
3864
3865template <typename DerivedCCG, typename FuncTy, typename CallTy>
3866void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::dump()
3867 const {
3868 print(dbgs());
3869 dbgs() << "\n";
3870}
3871
3872template <typename DerivedCCG, typename FuncTy, typename CallTy>
3873void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode::print(
3874 raw_ostream &OS) const {
3875 OS << "Node " << this << "\n";
3876 OS << "\t";
3877 printCall(OS);
3878 if (Recursive)
3879 OS << " (recursive)";
3880 OS << "\n";
3881 if (!MatchingCalls.empty()) {
3882 OS << "\tMatchingCalls:\n";
3883 for (auto &MatchingCall : MatchingCalls) {
3884 OS << "\t";
3885 MatchingCall.print(OS);
3886 OS << "\n";
3887 }
3888 }
3889 OS << "\tNodeId: " << NodeId << "\n";
3890 OS << "\tAllocTypes: " << getAllocTypeString(AllocTypes) << "\n";
3891 OS << "\tContextIds:";
3892 // Make a copy of the computed context ids that we can sort for stability.
3893 auto ContextIds = getContextIds();
3894 std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
3895 std::sort(SortedIds.begin(), SortedIds.end());
3896 for (auto Id : SortedIds)
3897 OS << " " << Id;
3898 OS << "\n";
3899 OS << "\tCalleeEdges:\n";
3900 for (auto &Edge : CalleeEdges)
3901 OS << "\t\t" << *Edge << " (Callee NodeId: " << Edge->Callee->NodeId
3902 << ")\n";
3903 OS << "\tCallerEdges:\n";
3904 for (auto &Edge : CallerEdges)
3905 OS << "\t\t" << *Edge << " (Caller NodeId: " << Edge->Caller->NodeId
3906 << ")\n";
3907 if (!Clones.empty()) {
3908 OS << "\tClones: ";
3909 ListSeparator LS;
3910 for (auto *C : Clones)
3911 OS << LS << C << " NodeId: " << C->NodeId;
3912 OS << "\n";
3913 } else if (CloneOf) {
3914 OS << "\tClone of " << CloneOf << " NodeId: " << CloneOf->NodeId << "\n";
3915 }
3916}
3917
3918template <typename DerivedCCG, typename FuncTy, typename CallTy>
3919void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge::dump()
3920 const {
3921 print(dbgs());
3922 dbgs() << "\n";
3923}
3924
3925template <typename DerivedCCG, typename FuncTy, typename CallTy>
3926void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextEdge::print(
3927 raw_ostream &OS) const {
3928 OS << "Edge from Callee " << Callee << " to Caller: " << Caller
3929 << (IsBackedge ? " (BE)" : "")
3930 << " AllocTypes: " << getAllocTypeString(AllocTypes);
3931 OS << " ContextIds:";
3932 std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
3933 std::sort(SortedIds.begin(), SortedIds.end());
3934 for (auto Id : SortedIds)
3935 OS << " " << Id;
3936}
3937
3938template <typename DerivedCCG, typename FuncTy, typename CallTy>
3939void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::dump() const {
3940 print(dbgs());
3941}
3942
3943template <typename DerivedCCG, typename FuncTy, typename CallTy>
3944void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::print(
3945 raw_ostream &OS) const {
3946 OS << "Callsite Context Graph:\n";
3947 using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
3948 for (const auto Node : nodes<GraphType>(this)) {
3949 if (Node->isRemoved())
3950 continue;
3951 Node->print(OS);
3952 OS << "\n";
3953 }
3954}
3955
3956template <typename DerivedCCG, typename FuncTy, typename CallTy>
3957void CallsiteContextGraph<DerivedCCG, FuncTy,
3958 CallTy>::capturePreCloningContextInfo() {
3959 for (const auto &Entry : AllocationCallToContextNodeMap) {
3960 const ContextNode *AllocNode = Entry.second;
3961 // Remember whether any caller of this allocation matched a call, so that
3962 // the report can distinguish "no matched callers" from an allocation whose
3963 // callers are simply not part of this context.
3964 for (const auto &Edge : AllocNode->CallerEdges) {
3965 if (!Edge->isRemoved() && Edge->Caller->hasCall()) {
3966 PreCloningAllocHasMatchedCaller.insert(AllocNode);
3967 break;
3968 }
3969 }
3970 DenseSet<uint32_t> ContextIds = AllocNode->getContextIds();
3971 for (uint32_t Id : ContextIds) {
3972 PreCloningContextInfo Info;
3973 // Walk the caller edges carrying this context id from the allocation
3974 // node outwards to build the chain of nodes for this context. Each node
3975 // is visited once; backedges (recursive cycles) are only followed when
3976 // no other caller edge carries the id, so that the chain reflects the
3977 // context's stack order rather than looping.
3978 Info.Chain.push_back(AllocNode);
3979 SmallPtrSet<const ContextNode *, 16> Visited;
3980 Visited.insert(AllocNode);
3981 const ContextNode *Cur = AllocNode;
3982 while (true) {
3983 const ContextNode *Next = nullptr;
3984 for (const auto &Edge : Cur->CallerEdges) {
3985 if (Edge->isRemoved() || Edge->IsBackedge)
3986 continue;
3987 if (Edge->getContextIds().contains(Id) &&
3988 !Visited.contains(Edge->Caller)) {
3989 Next = Edge->Caller;
3990 break;
3991 }
3992 }
3993 if (!Next) {
3994 for (const auto &Edge : Cur->CallerEdges) {
3995 if (Edge->isRemoved())
3996 continue;
3997 if (Edge->getContextIds().contains(Id) &&
3998 !Visited.contains(Edge->Caller)) {
3999 Next = Edge->Caller;
4000 break;
4001 }
4002 }
4003 }
4004 if (!Next)
4005 break;
4006 Visited.insert(Next);
4007 Info.Chain.push_back(Next);
4008 Cur = Next;
4009 }
4010
4011 // Find the outermost node (walking from the allocation) such that it
4012 // and all nodes below it matched a call, and record why the first
4013 // unmatched node (if any) did not match.
4014 Info.MaxMatchedFrame = 0;
4015 for (unsigned I = 1; I < Info.Chain.size(); ++I) {
4016 if (Info.Chain[I]->hasCall()) {
4017 Info.MaxMatchedFrame = I;
4018 } else {
4019 const ContextNode *Unmatched = Info.Chain[I];
4020 Info.UnmatchedStackId = Unmatched->OrigStackOrAllocId;
4021 Info.StoppedAtRecursive = Unmatched->Recursive;
4022 // A calling function is known for the unmatched node only when a
4023 // call in that function was considered and rejected because its
4024 // callee did not match the node's callee.
4025 if (auto It = NodeToCallingFunc.find(Unmatched);
4026 It != NodeToCallingFunc.end()) {
4027 Info.StoppedAtMismatchedCallee = true;
4028 Info.UnmatchedCallingFunc = It->second;
4029 }
4030 break;
4031 }
4032 }
4033
4034 // If the whole context matched, check whether its outermost node has
4035 // matched callers reached by other contexts: then this context was
4036 // trimmed below them, and clones of those callers can change which
4037 // allocation clone it reaches.
4038 if (Info.MaxMatchedFrame > 0 &&
4039 Info.MaxMatchedFrame + 1 == Info.Chain.size()) {
4040 const ContextNode *OutermostMatchedNode =
4041 Info.Chain[Info.MaxMatchedFrame];
4042 for (const auto &Edge : OutermostMatchedNode->CallerEdges) {
4043 if (!Edge->isRemoved() && Edge->Caller->hasCall()) {
4044 Info.OutermostMatchedNodeHasOtherCallers = true;
4045 break;
4046 }
4047 }
4048 }
4049
4050 PreCloningContextInfoMap[Id] = std::move(Info);
4051 }
4052 }
4053}
4054
4055template <typename DerivedCCG, typename FuncTy, typename CallTy>
4056std::string
4057CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::formatContextReportSuffix(
4058 const ContextNode *Node, uint32_t Id) const {
4059 auto InfoIt = PreCloningContextInfoMap.find(Id);
4060 if (InfoIt == PreCloningContextInfoMap.end())
4061 return "";
4062 const PreCloningContextInfo &Info = InfoIt->second;
4063
4064 // 1. The allocation function (clone) the context's allocation node ended up
4065 // in, with a qualifier when it is dead, or when the context has callers in
4066 // the profile but none of the allocation's callers matched (either because
4067 // nothing references the function at all, or because the references did not
4068 // match the profiled callsites).
4069 const ContextNode *OrigAlloc = Node->getOrigNode();
4070 const FuncTy *AllocFunc = getCallingFunc(OrigAlloc);
4071 unsigned AllocCloneNo = Node->Call.cloneNo();
4072 std::string AllocFuncStr = getFuncName(AllocFunc, AllocCloneNo);
4073 if (isFuncDead(AllocFunc)) {
4074 AllocFuncStr += " (dead)";
4075 } else if (!PreCloningAllocHasMatchedCaller.contains(OrigAlloc) &&
4076 Info.Chain.size() > 1) {
4077 // Distinguish a function with no references in the IR/summary (it may
4078 // still be referenced from native objects) from one whose references did
4079 // not match.
4080 AllocFuncStr += AllocFunc && !FunctionsWithCallers.contains(AllocFunc)
4081 ? " (no references in IR/summary)"
4082 : " (no matched callers)";
4083 }
4084
4085 // 2. How far the context matched: all of it ("matched N frames to F", or
4086 // "trimmed at frame N in F" when F has callers above that the context does
4087 // not cover), or up to an unmatched node (with the reason and stack id).
4088 unsigned TotalFrames = Info.Chain.size() > 0 ? Info.Chain.size() - 1 : 0;
4089 unsigned Matched = Info.MaxMatchedFrame;
4090 std::string StatusStr;
4091 if (TotalFrames == 0) {
4092 StatusStr = "inlined into leaf";
4093 } else if (Matched < TotalFrames) {
4094 // Where and why matching stopped: the first unmatched node was either a
4095 // recursive callsite (not matched when recursive callsites are disabled),
4096 // a callsite whose IR callee did not match the profiled callee, or a
4097 // callsite with no matching call at all.
4098 if (Info.StoppedAtRecursive)
4099 StatusStr = "unmatched recursive callsite";
4100 else if (Info.StoppedAtMismatchedCallee)
4101 StatusStr = "unmatched callsite (mismatched callee in " +
4102 getFuncName(Info.UnmatchedCallingFunc, 0) + ")";
4103 else
4104 StatusStr = "unmatched callsite";
4105 StatusStr += " at frame " + std::to_string(Matched + 1) + "/" +
4106 std::to_string(TotalFrames) + " (stack id " +
4107 std::to_string(Info.UnmatchedStackId) + ")";
4108 } else {
4109 const ContextNode *OutermostMatchedNode = Info.Chain[Matched];
4110 const FuncTy *OutermostMatchedFunc = getCallingFunc(OutermostMatchedNode);
4111 unsigned NumOutermostMatchedFuncClones =
4112 std::max<unsigned>(1, FuncToNumClones.lookup(OutermostMatchedFunc));
4113 bool HasCallersAbove =
4114 NumOutermostMatchedFuncClones > 1 ||
4115 Info.OutermostMatchedNodeHasOtherCallers ||
4116 (OutermostMatchedFunc &&
4117 FunctionsWithCallers.contains(OutermostMatchedFunc));
4118 if (HasCallersAbove) {
4119 StatusStr = "trimmed at frame " + std::to_string(Matched) + " in " +
4120 getFuncName(OutermostMatchedFunc, 0);
4121 } else {
4122 StatusStr = "matched " + std::to_string(Matched) + " frames to " +
4123 getFuncName(OutermostMatchedFunc, 0);
4124 }
4125 }
4126
4127 // 3. The post-cloning call paths: starting from each function clone of the
4128 // outermost matched caller, follow the callee clones assigned to the
4129 // context's callsites down to the allocation clone reached, and report the
4130 // hint that clone ended up with. Paths reaching the same allocation clone
4131 // are reported once.
4132 std::string ReachesStr;
4133 if (Matched >= 1) {
4134 const ContextNode *OutermostMatchedNode = Info.Chain[Matched];
4135 const FuncTy *OutermostMatchedFunc = getCallingFunc(OutermostMatchedNode);
4136 unsigned NumOutermostMatchedFuncClones =
4137 std::max<unsigned>(1, FuncToNumClones.lookup(OutermostMatchedFunc));
4138 ReachesStr = "; reaches ";
4139 // Allocation clones (by clone number) already reported for this context.
4140 // Different function clones of the outermost matched caller often lead to
4141 // the same allocation clone; such paths are reported only once, for the
4142 // first caller clone reaching that allocation clone.
4143 SmallSet<unsigned, 4> ReportedAllocClones;
4144 bool FirstPath = true;
4145 for (unsigned OutermostMatchedFuncClone = 0;
4146 OutermostMatchedFuncClone < NumOutermostMatchedFuncClones;
4147 ++OutermostMatchedFuncClone) {
4148 // PathFuncs/PathClones[S] is the function and clone number at step S of
4149 // the path, from the outermost matched caller (S = 0) down to the
4150 // allocation function (S = Matched).
4151 std::vector<const FuncTy *> PathFuncs(Matched + 1);
4152 std::vector<unsigned> PathClones(Matched + 1);
4153 PathFuncs[0] = OutermostMatchedFunc;
4154 PathClones[0] = OutermostMatchedFuncClone;
4155 unsigned CurClone = OutermostMatchedFuncClone;
4156 for (unsigned Step = 0; Step < Matched; ++Step) {
4157 unsigned FrameIdx = Matched - Step;
4158 const ContextNode *CallNode = Info.Chain[FrameIdx];
4159 unsigned NextClone =
4160 OrigNodeAndCloneToCalleeClone.lookup({CallNode, CurClone});
4161 const FuncTy *CalleeFunc = getCallingFunc(Info.Chain[FrameIdx - 1]);
4162 PathFuncs[Step + 1] = CalleeFunc;
4163 PathClones[Step + 1] = NextClone;
4164 CurClone = NextClone;
4165 }
4166
4167 if (!ReportedAllocClones.insert(CurClone).second)
4168 continue;
4169 if (!FirstPath)
4170 ReachesStr += ", ";
4171 FirstPath = false;
4172
4173 const ContextNode *ReachedAlloc =
4174 OrigAllocAndCloneToNode.lookup({OrigAlloc, CurClone});
4175 // Report the hint actually applied to the reached allocation clone
4176 // (which accounts for allocTypeToUse and MinClonedColdBytePercent),
4177 // rather than the graph node's (possibly ambiguous) AllocTypes.
4178 std::string ReachedTypeStr;
4179 if (ReachedAlloc)
4180 ReachedTypeStr = getAllocTypeString(
4181 (uint8_t)getAllocationCallType(ReachedAlloc->Call));
4182 else if (CurClone == 0)
4183 ReachedTypeStr =
4184 getAllocTypeString((uint8_t)getAllocationCallType(OrigAlloc->Call));
4185 else
4186 ReachedTypeStr = "None";
4187
4188 // Print the path, eliding runs of uncloned (or cloned) functions in the
4189 // middle of paths longer than MemProfReportHintedSizesDetailMaxPathLen;
4190 // the ends and every transition between cloned and uncloned functions
4191 // are kept.
4192 std::string PathStr;
4193 bool SkippedPrev = false;
4194 bool PrintFullPath =
4196 for (unsigned S = 0; S <= Matched; ++S) {
4197 bool IsCloned = PathClones[S] > 0;
4198 bool PrevCloned = S > 0 ? (PathClones[S - 1] > 0) : IsCloned;
4199 bool NextCloned = S + 1 <= Matched ? (PathClones[S + 1] > 0) : IsCloned;
4200 bool Include = PrintFullPath || (S <= 1) || (S + 1 >= Matched) ||
4201 (IsCloned != PrevCloned) || (IsCloned != NextCloned);
4202 if (!Include) {
4203 if (!SkippedPrev) {
4204 PathStr += " -> ...";
4205 SkippedPrev = true;
4206 }
4207 continue;
4208 }
4209 SkippedPrev = false;
4210 if (!PathStr.empty())
4211 PathStr += " -> ";
4212 PathStr += getFuncName(PathFuncs[S], PathClones[S]);
4213 }
4214 ReachesStr += ReachedTypeStr + " via " + PathStr;
4215 }
4216 }
4217
4218 return " [alloc in " + AllocFuncStr + "; " + StatusStr + ReachesStr + "]";
4219}
4220
4221template <typename DerivedCCG, typename FuncTy, typename CallTy>
4222void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::printTotalSizes(
4223 raw_ostream &OS,
4224 function_ref<void(StringRef, StringRef, const Twine &)> EmitRemark) const {
4225 using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
4226 for (const auto Node : nodes<GraphType>(this)) {
4227 if (Node->isRemoved())
4228 continue;
4229 if (!Node->IsAllocation)
4230 continue;
4231 DenseSet<uint32_t> ContextIds = Node->getContextIds();
4232 auto AllocTypeFromCall = getAllocationCallType(Node->Call);
4233 std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
4234 std::sort(SortedIds.begin(), SortedIds.end());
4235 for (auto Id : SortedIds) {
4236 auto TypeI = ContextIdToAllocationType.find(Id);
4237 assert(TypeI != ContextIdToAllocationType.end());
4238 std::string Suffix;
4239 bool ComputedSuffix = false;
4240 auto GetSuffix = [&]() -> const std::string & {
4241 if (!ComputedSuffix) {
4243 ? formatContextReportSuffix(Node, Id)
4244 : "";
4245 ComputedSuffix = true;
4246 }
4247 return Suffix;
4248 };
4249 auto CSI = ContextIdToContextSizeInfos.find(Id);
4250 if (CSI != ContextIdToContextSizeInfos.end()) {
4251 for (auto &Info : CSI->second) {
4252 std::string Msg =
4253 "MemProf hinting: " + getAllocTypeString((uint8_t)TypeI->second) +
4254 " full allocation context " + std::to_string(Info.FullStackId) +
4255 " with total size " + std::to_string(Info.TotalSize) + " is " +
4256 getAllocTypeString(Node->AllocTypes) + " after cloning";
4257 if (allocTypeToUse(Node->AllocTypes) !=
4258 allocTypeToUse((uint8_t)AllocTypeFromCall))
4259 Msg += " marked " + getAllocTypeString((uint8_t)AllocTypeFromCall) +
4260 " due to cold byte percent";
4261 // Print the internal context id to aid debugging and visualization.
4262 Msg += " (internal context id " + std::to_string(Id) + ")";
4264 OS << Msg << GetSuffix() << "\n";
4265 if (EmitRemark)
4266 EmitRemark(DEBUG_TYPE, "MemProfReport", Msg);
4267 }
4268 } else {
4269 // This is only emitted if the context size info is not present.
4270 std::string Msg =
4271 "MemProf hinting: " + getAllocTypeString((uint8_t)TypeI->second) +
4272 " context is " + getAllocTypeString(Node->AllocTypes) +
4273 " after cloning";
4274 if (allocTypeToUse(Node->AllocTypes) !=
4275 allocTypeToUse((uint8_t)AllocTypeFromCall))
4276 Msg += " marked " + getAllocTypeString((uint8_t)AllocTypeFromCall) +
4277 " due to cold byte percent";
4278 // Print the internal context id to aid debugging and visualization.
4279 Msg += " (internal context id " + std::to_string(Id) + ")";
4281 OS << Msg << GetSuffix() << "\n";
4282 if (EmitRemark)
4283 EmitRemark(DEBUG_TYPE, "MemProfReport", Msg);
4284 }
4285 }
4286 }
4287}
4288
4289template <typename DerivedCCG, typename FuncTy, typename CallTy>
4290void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::check() const {
4291 using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
4292 for (const auto Node : nodes<GraphType>(this)) {
4293 checkNode<DerivedCCG, FuncTy, CallTy>(Node, /*CheckEdges=*/false);
4294 for (auto &Edge : Node->CallerEdges)
4296 }
4297}
4298
4299template <typename DerivedCCG, typename FuncTy, typename CallTy>
4300struct GraphTraits<const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *> {
4301 using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
4302 using NodeRef = const ContextNode<DerivedCCG, FuncTy, CallTy> *;
4303
4304 using NodePtrTy = std::unique_ptr<ContextNode<DerivedCCG, FuncTy, CallTy>>;
4305 static NodeRef getNode(const NodePtrTy &P) { return P.get(); }
4306
4309 decltype(&getNode)>;
4310
4312 return nodes_iterator(G->NodeOwner.begin(), &getNode);
4313 }
4314
4316 return nodes_iterator(G->NodeOwner.end(), &getNode);
4317 }
4318
4320 return G->NodeOwner.begin()->get();
4321 }
4322
4323 using EdgePtrTy = std::shared_ptr<ContextEdge<DerivedCCG, FuncTy, CallTy>>;
4324 static const ContextNode<DerivedCCG, FuncTy, CallTy> *
4326 return P->Callee;
4327 }
4328
4330 mapped_iterator<typename std::vector<std::shared_ptr<ContextEdge<
4331 DerivedCCG, FuncTy, CallTy>>>::const_iterator,
4332 decltype(&GetCallee)>;
4333
4335 return ChildIteratorType(N->CalleeEdges.begin(), &GetCallee);
4336 }
4337
4339 return ChildIteratorType(N->CalleeEdges.end(), &GetCallee);
4340 }
4341};
4342
4343template <typename DerivedCCG, typename FuncTy, typename CallTy>
4344struct DOTGraphTraits<const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>
4345 : public DefaultDOTGraphTraits {
4346 DOTGraphTraits(bool IsSimple = false) : DefaultDOTGraphTraits(IsSimple) {
4347 // If the user requested the full graph to be exported, but provided an
4348 // allocation id, or if the user gave a context id and requested more than
4349 // just a specific context to be exported, note that highlighting is
4350 // enabled.
4351 DoHighlight =
4352 (AllocIdForDot.getNumOccurrences() && DotGraphScope == DotScope::All) ||
4353 (ContextIdForDot.getNumOccurrences() &&
4355 }
4356
4357 using GraphType = const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *;
4359 using NodeRef = typename GTraits::NodeRef;
4360 using ChildIteratorType = typename GTraits::ChildIteratorType;
4361
4362 static std::string getNodeLabel(NodeRef Node, GraphType G) {
4363 std::string LabelString =
4364 (Twine("OrigId: ") + (Node->IsAllocation ? "Alloc" : "") +
4365 Twine(Node->OrigStackOrAllocId) + " NodeId: " + Twine(Node->NodeId))
4366 .str();
4367 LabelString += "\n";
4368 if (Node->hasCall()) {
4369 auto Func = G->NodeToCallingFunc.find(Node);
4370 assert(Func != G->NodeToCallingFunc.end());
4371 LabelString +=
4372 G->getLabel(Func->second, Node->Call.call(), Node->Call.cloneNo());
4373 for (auto &MatchingCall : Node->MatchingCalls) {
4374 LabelString += "\n";
4375 LabelString += G->getLabel(Func->second, MatchingCall.call(),
4376 MatchingCall.cloneNo());
4377 }
4378 } else {
4379 LabelString += "null call";
4380 if (Node->Recursive)
4381 LabelString += " (recursive)";
4382 else
4383 LabelString += " (external)";
4384 }
4385 return LabelString;
4386 }
4387
4389 auto ContextIds = Node->getContextIds();
4390 // If highlighting enabled, see if this node contains any of the context ids
4391 // of interest. If so, it will use a different color and a larger fontsize
4392 // (which makes the node larger as well).
4393 bool Highlight = false;
4394 if (DoHighlight) {
4395 assert(ContextIdForDot.getNumOccurrences() ||
4396 AllocIdForDot.getNumOccurrences());
4397 if (ContextIdForDot.getNumOccurrences())
4398 Highlight = ContextIds.contains(ContextIdForDot);
4399 else
4400 Highlight = set_intersects(ContextIds, G->DotAllocContextIds);
4401 }
4402 std::string AttributeString = (Twine("tooltip=\"") + getNodeId(Node) + " " +
4403 getContextIds(ContextIds) + "\"")
4404 .str();
4405 // Default fontsize is 14
4406 if (Highlight)
4407 AttributeString += ",fontsize=\"30\"";
4408 AttributeString +=
4409 (Twine(",fillcolor=\"") + getColor(Node->AllocTypes, Highlight) + "\"")
4410 .str();
4411 if (Node->CloneOf) {
4412 AttributeString += ",color=\"blue\"";
4413 AttributeString += ",style=\"filled,bold,dashed\"";
4414 } else
4415 AttributeString += ",style=\"filled\"";
4416 return AttributeString;
4417 }
4418
4419 static std::string getEdgeAttributes(NodeRef, ChildIteratorType ChildIter,
4420 GraphType G) {
4421 auto &Edge = *(ChildIter.getCurrent());
4422 // If highlighting enabled, see if this edge contains any of the context ids
4423 // of interest. If so, it will use a different color and a heavier arrow
4424 // size and weight (the larger weight makes the highlighted path
4425 // straighter).
4426 bool Highlight = false;
4427 if (DoHighlight) {
4428 assert(ContextIdForDot.getNumOccurrences() ||
4429 AllocIdForDot.getNumOccurrences());
4430 if (ContextIdForDot.getNumOccurrences())
4431 Highlight = Edge->ContextIds.contains(ContextIdForDot);
4432 else
4433 Highlight = set_intersects(Edge->ContextIds, G->DotAllocContextIds);
4434 }
4435 auto Color = getColor(Edge->AllocTypes, Highlight);
4436 std::string AttributeString =
4437 (Twine("tooltip=\"") + getContextIds(Edge->ContextIds) + "\"" +
4438 // fillcolor is the arrow head and color is the line
4439 Twine(",fillcolor=\"") + Color + "\"" + Twine(",color=\"") + Color +
4440 "\"")
4441 .str();
4442 if (Edge->IsBackedge)
4443 AttributeString += ",style=\"dotted\"";
4444 // Default penwidth and weight are both 1.
4445 if (Highlight)
4446 AttributeString += ",penwidth=\"2.0\",weight=\"2\"";
4447 return AttributeString;
4448 }
4449
4450 // Since the NodeOwners list includes nodes that are no longer connected to
4451 // the graph, skip them here.
4453 if (Node->isRemoved())
4454 return true;
4455 // If a scope smaller than the full graph was requested, see if this node
4456 // contains any of the context ids of interest.
4458 return !set_intersects(Node->getContextIds(), G->DotAllocContextIds);
4460 return !Node->getContextIds().contains(ContextIdForDot);
4461 return false;
4462 }
4463
4464private:
4465 static std::string getContextIds(const DenseSet<uint32_t> &ContextIds) {
4466 std::string IdString = "ContextIds:";
4467 if (ContextIds.size() < 100) {
4468 std::vector<uint32_t> SortedIds(ContextIds.begin(), ContextIds.end());
4469 std::sort(SortedIds.begin(), SortedIds.end());
4470 for (auto Id : SortedIds)
4471 IdString += (" " + Twine(Id)).str();
4472 } else {
4473 IdString += (" (" + Twine(ContextIds.size()) + " ids)").str();
4474 }
4475 return IdString;
4476 }
4477
4478 static std::string getColor(uint8_t AllocTypes, bool Highlight) {
4479 // If DoHighlight is not enabled, we want to use the highlight colors for
4480 // NotCold and Cold, and the non-highlight color for NotCold+Cold. This is
4481 // both compatible with the color scheme before highlighting was supported,
4482 // and for the NotCold+Cold color the non-highlight color is a bit more
4483 // readable.
4484 if (AllocTypes == (uint8_t)AllocationType::NotCold)
4485 // Color "brown1" actually looks like a lighter red.
4486 return !DoHighlight || Highlight ? "brown1" : "lightpink";
4487 if (AllocTypes == (uint8_t)AllocationType::Cold)
4488 return !DoHighlight || Highlight ? "cyan" : "lightskyblue";
4489 if (AllocTypes ==
4490 ((uint8_t)AllocationType::NotCold | (uint8_t)AllocationType::Cold))
4491 return Highlight ? "magenta" : "mediumorchid1";
4492 return "gray";
4493 }
4494
4495 static std::string getNodeId(NodeRef Node) {
4496 std::stringstream SStream;
4497 SStream << std::hex << "N0x" << (unsigned long long)Node;
4498 std::string Result = SStream.str();
4499 return Result;
4500 }
4501
4502 // True if we should highlight a specific context or allocation's contexts in
4503 // the emitted graph.
4504 static bool DoHighlight;
4505};
4506
4507template <typename DerivedCCG, typename FuncTy, typename CallTy>
4508bool DOTGraphTraits<
4509 const CallsiteContextGraph<DerivedCCG, FuncTy, CallTy> *>::DoHighlight =
4510 false;
4511
4512template <typename DerivedCCG, typename FuncTy, typename CallTy>
4513void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::exportToDot(
4514 std::string Label) const {
4515 WriteGraph(this, "", false, Label,
4516 DotFilePathPrefix + "ccg." + Label + ".dot");
4517}
4518
4519template <typename DerivedCCG, typename FuncTy, typename CallTy>
4520typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
4521CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::moveEdgeToNewCalleeClone(
4522 const std::shared_ptr<ContextEdge> &Edge,
4523 DenseSet<uint32_t> ContextIdsToMove) {
4524 ContextNode *Node = Edge->Callee;
4525 assert(NodeToCallingFunc.count(Node));
4526 ContextNode *Clone =
4527 createNewNode(Node->IsAllocation, NodeToCallingFunc[Node], Node->Call);
4528 Node->addClone(Clone);
4529 Clone->MatchingCalls = Node->MatchingCalls;
4530 moveEdgeToExistingCalleeClone(Edge, Clone, /*NewClone=*/true,
4531 ContextIdsToMove);
4532 return Clone;
4533}
4534
4535template <typename DerivedCCG, typename FuncTy, typename CallTy>
4536void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
4537 moveEdgeToExistingCalleeClone(const std::shared_ptr<ContextEdge> &Edge,
4538 ContextNode *NewCallee, bool NewClone,
4539 DenseSet<uint32_t> ContextIdsToMove) {
4540 // NewCallee and Edge's current callee must be clones of the same original
4541 // node (Edge's current callee may be the original node too).
4542 assert(NewCallee->getOrigNode() == Edge->Callee->getOrigNode());
4543
4544 bool EdgeIsRecursive = Edge->Callee == Edge->Caller;
4545
4546 ContextNode *OldCallee = Edge->Callee;
4547
4548 // We might already have an edge to the new callee from earlier cloning for a
4549 // different allocation. If one exists we will reuse it.
4550 auto ExistingEdgeToNewCallee = NewCallee->findEdgeFromCaller(Edge->Caller);
4551
4552 // Callers will pass an empty ContextIdsToMove set when they want to move the
4553 // edge. Copy in Edge's ids for simplicity.
4554 if (ContextIdsToMove.empty())
4555 ContextIdsToMove = Edge->getContextIds();
4556
4557 // If we are moving all of Edge's ids, then just move the whole Edge.
4558 // Otherwise only move the specified subset, to a new edge if needed.
4559 if (Edge->getContextIds().size() == ContextIdsToMove.size()) {
4560 // First, update the alloc types on New Callee from Edge.
4561 // Do this before we potentially clear Edge's fields below!
4562 NewCallee->AllocTypes |= Edge->AllocTypes;
4563 // Moving the whole Edge.
4564 if (ExistingEdgeToNewCallee) {
4565 // Since we already have an edge to NewCallee, simply move the ids
4566 // onto it, and remove the existing Edge.
4567 ExistingEdgeToNewCallee->getContextIds().insert_range(ContextIdsToMove);
4568 ExistingEdgeToNewCallee->AllocTypes |= Edge->AllocTypes;
4569 assert(Edge->ContextIds == ContextIdsToMove);
4570 removeEdgeFromGraph(Edge.get());
4571 } else {
4572 // Otherwise just reconnect Edge to NewCallee.
4573 Edge->Callee = NewCallee;
4574 NewCallee->CallerEdges.push_back(Edge);
4575 // Remove it from callee where it was previously connected.
4576 OldCallee->eraseCallerEdge(Edge.get());
4577 // Don't need to update Edge's context ids since we are simply
4578 // reconnecting it.
4579 }
4580 } else {
4581 // Only moving a subset of Edge's ids.
4582 // Compute the alloc type of the subset of ids being moved.
4583 auto CallerEdgeAllocType = computeAllocType(ContextIdsToMove);
4584 if (ExistingEdgeToNewCallee) {
4585 // Since we already have an edge to NewCallee, simply move the ids
4586 // onto it.
4587 ExistingEdgeToNewCallee->getContextIds().insert_range(ContextIdsToMove);
4588 ExistingEdgeToNewCallee->AllocTypes |= CallerEdgeAllocType;
4589 } else {
4590 // Otherwise, create a new edge to NewCallee for the ids being moved.
4591 auto NewEdge = std::make_shared<ContextEdge>(
4592 NewCallee, Edge->Caller, CallerEdgeAllocType, ContextIdsToMove);
4593 Edge->Caller->CalleeEdges.push_back(NewEdge);
4594 NewCallee->CallerEdges.push_back(NewEdge);
4595 }
4596 // In either case, need to update the alloc types on NewCallee, and remove
4597 // those ids and update the alloc type on the original Edge.
4598 NewCallee->AllocTypes |= CallerEdgeAllocType;
4599 set_subtract(Edge->ContextIds, ContextIdsToMove);
4600 Edge->AllocTypes = computeAllocType(Edge->ContextIds);
4601 }
4602 // Now walk the old callee node's callee edges and move Edge's context ids
4603 // over to the corresponding edge into the clone (which is created here if
4604 // this is a newly created clone).
4605 for (auto &OldCalleeEdge : OldCallee->CalleeEdges) {
4606 ContextNode *CalleeToUse = OldCalleeEdge->Callee;
4607 // If this is a direct recursion edge, use NewCallee (the clone) as the
4608 // callee as well, so that any edge updated/created here is also direct
4609 // recursive.
4610 if (CalleeToUse == OldCallee) {
4611 // If this is a recursive edge, see if we already moved a recursive edge
4612 // (which would have to have been this one) - if we were only moving a
4613 // subset of context ids it would still be on OldCallee.
4614 if (EdgeIsRecursive) {
4615 assert(OldCalleeEdge == Edge);
4616 continue;
4617 }
4618 CalleeToUse = NewCallee;
4619 }
4620 // The context ids moving to the new callee are the subset of this edge's
4621 // context ids and the context ids on the caller edge being moved.
4622 DenseSet<uint32_t> EdgeContextIdsToMove =
4623 set_intersection(OldCalleeEdge->getContextIds(), ContextIdsToMove);
4624 set_subtract(OldCalleeEdge->getContextIds(), EdgeContextIdsToMove);
4625 OldCalleeEdge->AllocTypes =
4626 computeAllocType(OldCalleeEdge->getContextIds());
4627 if (!NewClone) {
4628 // Update context ids / alloc type on corresponding edge to NewCallee.
4629 // There is a chance this may not exist if we are reusing an existing
4630 // clone, specifically during function assignment, where we would have
4631 // removed none type edges after creating the clone. If we can't find
4632 // a corresponding edge there, fall through to the cloning below.
4633 if (auto *NewCalleeEdge = NewCallee->findEdgeFromCallee(CalleeToUse)) {
4634 NewCalleeEdge->getContextIds().insert_range(EdgeContextIdsToMove);
4635 NewCalleeEdge->AllocTypes |= computeAllocType(EdgeContextIdsToMove);
4636 continue;
4637 }
4638 }
4639 auto NewEdge = std::make_shared<ContextEdge>(
4640 CalleeToUse, NewCallee, computeAllocType(EdgeContextIdsToMove),
4641 EdgeContextIdsToMove);
4642 NewCallee->CalleeEdges.push_back(NewEdge);
4643 NewEdge->Callee->CallerEdges.push_back(NewEdge);
4644 }
4645 // Recompute the node alloc type now that its callee edges have been
4646 // updated (since we will compute from those edges).
4647 OldCallee->AllocTypes = OldCallee->computeAllocType();
4648 // OldCallee alloc type should be None iff its context id set is now empty.
4649 assert((OldCallee->AllocTypes == (uint8_t)AllocationType::None) ==
4650 OldCallee->emptyContextIds());
4651 if (VerifyCCG) {
4652 checkNode<DerivedCCG, FuncTy, CallTy>(OldCallee, /*CheckEdges=*/false);
4653 checkNode<DerivedCCG, FuncTy, CallTy>(NewCallee, /*CheckEdges=*/false);
4654 for (const auto &OldCalleeEdge : OldCallee->CalleeEdges)
4655 checkNode<DerivedCCG, FuncTy, CallTy>(OldCalleeEdge->Callee,
4656 /*CheckEdges=*/false);
4657 for (const auto &NewCalleeEdge : NewCallee->CalleeEdges)
4658 checkNode<DerivedCCG, FuncTy, CallTy>(NewCalleeEdge->Callee,
4659 /*CheckEdges=*/false);
4660 }
4661}
4662
4663template <typename DerivedCCG, typename FuncTy, typename CallTy>
4664void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
4665 moveCalleeEdgeToNewCaller(const std::shared_ptr<ContextEdge> &Edge,
4666 ContextNode *NewCaller) {
4667 auto *OldCallee = Edge->Callee;
4668 auto *NewCallee = OldCallee;
4669 // If this edge was direct recursive, make any new/updated edge also direct
4670 // recursive to NewCaller.
4671 bool Recursive = Edge->Caller == Edge->Callee;
4672 if (Recursive)
4673 NewCallee = NewCaller;
4674
4675 ContextNode *OldCaller = Edge->Caller;
4676 OldCaller->eraseCalleeEdge(Edge.get());
4677
4678 // We might already have an edge to the new caller. If one exists we will
4679 // reuse it.
4680 auto ExistingEdgeToNewCaller = NewCaller->findEdgeFromCallee(NewCallee);
4681
4682 if (ExistingEdgeToNewCaller) {
4683 // Since we already have an edge to NewCaller, simply move the ids
4684 // onto it, and remove the existing Edge.
4685 ExistingEdgeToNewCaller->getContextIds().insert_range(
4686 Edge->getContextIds());
4687 ExistingEdgeToNewCaller->AllocTypes |= Edge->AllocTypes;
4688 Edge->ContextIds.clear();
4689 Edge->AllocTypes = (uint8_t)AllocationType::None;
4690 OldCallee->eraseCallerEdge(Edge.get());
4691 } else {
4692 // Otherwise just reconnect Edge to NewCaller.
4693 Edge->Caller = NewCaller;
4694 NewCaller->CalleeEdges.push_back(Edge);
4695 if (Recursive) {
4696 assert(NewCallee == NewCaller);
4697 // In the case of (direct) recursive edges, we update the callee as well
4698 // so that it becomes recursive on the new caller.
4699 Edge->Callee = NewCallee;
4700 NewCallee->CallerEdges.push_back(Edge);
4701 OldCallee->eraseCallerEdge(Edge.get());
4702 }
4703 // Don't need to update Edge's context ids since we are simply
4704 // reconnecting it.
4705 }
4706 // In either case, need to update the alloc types on New Caller.
4707 NewCaller->AllocTypes |= Edge->AllocTypes;
4708
4709 // Now walk the old caller node's caller edges and move Edge's context ids
4710 // over to the corresponding edge into the node (which is created here if
4711 // this is a newly created node). We can tell whether this is a newly created
4712 // node by seeing if it has any caller edges yet.
4713#ifndef NDEBUG
4714 bool IsNewNode = NewCaller->CallerEdges.empty();
4715#endif
4716 // If we just moved a direct recursive edge, presumably its context ids should
4717 // also flow out of OldCaller via some other non-recursive callee edge. We
4718 // don't want to remove the recursive context ids from other caller edges yet,
4719 // otherwise the context ids get into an inconsistent state on OldCaller.
4720 // We will update these context ids on the non-recursive caller edge when and
4721 // if they are updated on the non-recursive callee.
4722 if (!Recursive) {
4723 for (auto &OldCallerEdge : OldCaller->CallerEdges) {
4724 auto OldCallerCaller = OldCallerEdge->Caller;
4725 // The context ids moving to the new caller are the subset of this edge's
4726 // context ids and the context ids on the callee edge being moved.
4727 DenseSet<uint32_t> EdgeContextIdsToMove = set_intersection(
4728 OldCallerEdge->getContextIds(), Edge->getContextIds());
4729 if (OldCaller == OldCallerCaller) {
4730 OldCallerCaller = NewCaller;
4731 // Don't actually move this one. The caller will move it directly via a
4732 // call to this function with this as the Edge if it is appropriate to
4733 // move to a diff node that has a matching callee (itself).
4734 continue;
4735 }
4736 set_subtract(OldCallerEdge->getContextIds(), EdgeContextIdsToMove);
4737 OldCallerEdge->AllocTypes =
4738 computeAllocType(OldCallerEdge->getContextIds());
4739 // In this function we expect that any pre-existing node already has edges
4740 // from the same callers as the old node. That should be true in the
4741 // current use case, where we will remove None-type edges after copying
4742 // over all caller edges from the callee.
4743 auto *ExistingCallerEdge = NewCaller->findEdgeFromCaller(OldCallerCaller);
4744 // Since we would have skipped caller edges when moving a direct recursive
4745 // edge, this may not hold true when recursive handling enabled.
4746 assert(IsNewNode || ExistingCallerEdge || AllowRecursiveCallsites);
4747 if (ExistingCallerEdge) {
4748 ExistingCallerEdge->getContextIds().insert_range(EdgeContextIdsToMove);
4749 ExistingCallerEdge->AllocTypes |=
4750 computeAllocType(EdgeContextIdsToMove);
4751 continue;
4752 }
4753 auto NewEdge = std::make_shared<ContextEdge>(
4754 NewCaller, OldCallerCaller, computeAllocType(EdgeContextIdsToMove),
4755 EdgeContextIdsToMove);
4756 NewCaller->CallerEdges.push_back(NewEdge);
4757 NewEdge->Caller->CalleeEdges.push_back(NewEdge);
4758 }
4759 }
4760 // Recompute the node alloc type now that its caller edges have been
4761 // updated (since we will compute from those edges).
4762 OldCaller->AllocTypes = OldCaller->computeAllocType();
4763 // OldCaller alloc type should be None iff its context id set is now empty.
4764 assert((OldCaller->AllocTypes == (uint8_t)AllocationType::None) ==
4765 OldCaller->emptyContextIds());
4766 if (VerifyCCG) {
4767 checkNode<DerivedCCG, FuncTy, CallTy>(OldCaller, /*CheckEdges=*/false);
4768 checkNode<DerivedCCG, FuncTy, CallTy>(NewCaller, /*CheckEdges=*/false);
4769 for (const auto &OldCallerEdge : OldCaller->CallerEdges)
4770 checkNode<DerivedCCG, FuncTy, CallTy>(OldCallerEdge->Caller,
4771 /*CheckEdges=*/false);
4772 for (const auto &NewCallerEdge : NewCaller->CallerEdges)
4773 checkNode<DerivedCCG, FuncTy, CallTy>(NewCallerEdge->Caller,
4774 /*CheckEdges=*/false);
4775 }
4776}
4777
4778template <typename DerivedCCG, typename FuncTy, typename CallTy>
4779void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
4780 recursivelyRemoveNoneTypeCalleeEdges(
4781 ContextNode *Node, DenseSet<const ContextNode *> &Visited) {
4782 auto Inserted = Visited.insert(Node);
4783 if (!Inserted.second)
4784 return;
4785
4786 removeNoneTypeCalleeEdges(Node);
4787
4788 for (auto *Clone : Node->Clones)
4789 recursivelyRemoveNoneTypeCalleeEdges(Clone, Visited);
4790
4791 // The recursive call may remove some of this Node's caller edges.
4792 // Iterate over a copy and skip any that were removed.
4793 auto CallerEdges = Node->CallerEdges;
4794 for (auto &Edge : CallerEdges) {
4795 // Skip any that have been removed by an earlier recursive call.
4796 if (Edge->isRemoved()) {
4797 assert(!is_contained(Node->CallerEdges, Edge));
4798 continue;
4799 }
4800 recursivelyRemoveNoneTypeCalleeEdges(Edge->Caller, Visited);
4801 }
4802}
4803
4804// This is the standard DFS based backedge discovery algorithm.
4805template <typename DerivedCCG, typename FuncTy, typename CallTy>
4806void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::markBackedges() {
4807 // If we are cloning recursive contexts, find and mark backedges from all root
4808 // callers, using the typical DFS based backedge analysis.
4810 return;
4811 DenseSet<const ContextNode *> Visited;
4812 DenseSet<const ContextNode *> CurrentStack;
4813 for (auto &Entry : NonAllocationCallToContextNodeMap) {
4814 auto *Node = Entry.second;
4815 if (Node->isRemoved())
4816 continue;
4817 // It is a root if it doesn't have callers.
4818 if (!Node->CallerEdges.empty())
4819 continue;
4820 markBackedges(Node, Visited, CurrentStack);
4821 assert(CurrentStack.empty());
4822 }
4823}
4824
4825// Recursive helper for above markBackedges method.
4826template <typename DerivedCCG, typename FuncTy, typename CallTy>
4827void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::markBackedges(
4828 ContextNode *Node, DenseSet<const ContextNode *> &Visited,
4829 DenseSet<const ContextNode *> &CurrentStack) {
4830 auto I = Visited.insert(Node);
4831 // We should only call this for unvisited nodes.
4832 assert(I.second);
4833 (void)I;
4834 for (auto &CalleeEdge : Node->CalleeEdges) {
4835 auto *Callee = CalleeEdge->Callee;
4836 if (Visited.count(Callee)) {
4837 // Since this was already visited we need to check if it is currently on
4838 // the recursive stack in which case it is a backedge.
4839 if (CurrentStack.count(Callee))
4840 CalleeEdge->IsBackedge = true;
4841 continue;
4842 }
4843 CurrentStack.insert(Callee);
4844 markBackedges(Callee, Visited, CurrentStack);
4845 CurrentStack.erase(Callee);
4846 }
4847}
4848
4849template <typename DerivedCCG, typename FuncTy, typename CallTy>
4850void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::identifyClones() {
4851 DenseSet<const ContextNode *> Visited;
4852 for (auto &Entry : AllocationCallToContextNodeMap) {
4853 Visited.clear();
4854 identifyClones(Entry.second, Visited, Entry.second->getContextIds());
4855 }
4856 Visited.clear();
4857 for (auto &Entry : AllocationCallToContextNodeMap)
4858 recursivelyRemoveNoneTypeCalleeEdges(Entry.second, Visited);
4859 if (VerifyCCG)
4860 check();
4861}
4862
4863// helper function to check an AllocType is cold or notcold or both.
4870
4871template <typename DerivedCCG, typename FuncTy, typename CallTy>
4872void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::identifyClones(
4873 ContextNode *Node, DenseSet<const ContextNode *> &Visited,
4874 const DenseSet<uint32_t> &AllocContextIds) {
4875 if (VerifyNodes)
4876 checkNode<DerivedCCG, FuncTy, CallTy>(Node, /*CheckEdges=*/false);
4877 assert(!Node->CloneOf);
4878
4879 // If Node as a null call, then either it wasn't found in the module (regular
4880 // LTO) or summary index (ThinLTO), or there were other conditions blocking
4881 // cloning (e.g. recursion, calls multiple targets, etc).
4882 // Do this here so that we don't try to recursively clone callers below, which
4883 // isn't useful at least for this node.
4884 if (!Node->hasCall())
4885 return;
4886
4887 // No need to look at any callers if allocation type already unambiguous.
4888 if (hasSingleAllocType(Node->AllocTypes))
4889 return;
4890
4891#ifndef NDEBUG
4892 auto Insert =
4893#endif
4894 Visited.insert(Node);
4895 // We should not have visited this node yet.
4896 assert(Insert.second);
4897 // The recursive call to identifyClones may delete the current edge from the
4898 // CallerEdges vector. Make a copy and iterate on that, simpler than passing
4899 // in an iterator and having recursive call erase from it. Other edges may
4900 // also get removed during the recursion, which will have null Callee and
4901 // Caller pointers (and are deleted later), so we skip those below.
4902 {
4903 auto CallerEdges = Node->CallerEdges;
4904 for (auto &Edge : CallerEdges) {
4905 // Skip any that have been removed by an earlier recursive call.
4906 if (Edge->isRemoved()) {
4907 assert(!is_contained(Node->CallerEdges, Edge));
4908 continue;
4909 }
4910 // Defer backedges. See comments further below where these edges are
4911 // handled during the cloning of this Node.
4912 if (Edge->IsBackedge) {
4913 // We should only mark these if cloning recursive contexts, where we
4914 // need to do this deferral.
4916 continue;
4917 }
4918 // Ignore any caller we previously visited via another edge.
4919 if (!Visited.count(Edge->Caller) && !Edge->Caller->CloneOf) {
4920 identifyClones(Edge->Caller, Visited, AllocContextIds);
4921 }
4922 }
4923 }
4924
4925 // Check if we reached an unambiguous call or have have only a single caller.
4926 if (hasSingleAllocType(Node->AllocTypes) || Node->CallerEdges.size() <= 1)
4927 return;
4928
4929 // We need to clone.
4930
4931 // Try to keep the original version as alloc type NotCold. This will make
4932 // cases with indirect calls or any other situation with an unknown call to
4933 // the original function get the default behavior. We do this by sorting the
4934 // CallerEdges of the Node we will clone by alloc type.
4935 //
4936 // Give NotCold edge the lowest sort priority so those edges are at the end of
4937 // the caller edges vector, and stay on the original version (since the below
4938 // code clones greedily until it finds all remaining edges have the same type
4939 // and leaves the remaining ones on the original Node).
4940 //
4941 // We shouldn't actually have any None type edges, so the sorting priority for
4942 // that is arbitrary, and we assert in that case below.
4943 const unsigned AllocTypeCloningPriority[] = {/*None*/ 3, /*NotCold*/ 4,
4944 /*Cold*/ 1,
4945 /*NotColdCold*/ 2};
4946 llvm::stable_sort(Node->CallerEdges,
4947 [&](const std::shared_ptr<ContextEdge> &A,
4948 const std::shared_ptr<ContextEdge> &B) {
4949 // Nodes with non-empty context ids should be sorted
4950 // before those with empty context ids.
4951 if (A->ContextIds.empty())
4952 // Either B ContextIds are non-empty (in which case we
4953 // should return false because B < A), or B ContextIds
4954 // are empty, in which case they are equal, and we
4955 // should maintain the original relative ordering.
4956 return false;
4957 if (B->ContextIds.empty())
4958 return true;
4959
4960 if (A->AllocTypes == B->AllocTypes)
4961 // Use the caller node id as a deterministic
4962 // tie-breaker.
4963 return A->Caller->NodeId < B->Caller->NodeId;
4964 return AllocTypeCloningPriority[A->AllocTypes] <
4965 AllocTypeCloningPriority[B->AllocTypes];
4966 });
4967
4968 assert(Node->AllocTypes != (uint8_t)AllocationType::None);
4969
4970 DenseSet<uint32_t> RecursiveContextIds;
4972 // If we are allowing recursive callsites, but have also disabled recursive
4973 // contexts, look for context ids that show up in multiple caller edges.
4975 DenseSet<uint32_t> AllCallerContextIds;
4976 for (auto &CE : Node->CallerEdges) {
4977 // Resize to the largest set of caller context ids, since we know the
4978 // final set will be at least that large.
4979 AllCallerContextIds.reserve(CE->getContextIds().size());
4980 for (auto Id : CE->getContextIds())
4981 if (!AllCallerContextIds.insert(Id).second)
4982 RecursiveContextIds.insert(Id);
4983 }
4984 }
4985
4986 // Iterate until we find no more opportunities for disambiguating the alloc
4987 // types via cloning. In most cases this loop will terminate once the Node
4988 // has a single allocation type, in which case no more cloning is needed.
4989 // Iterate over a copy of Node's caller edges, since we may need to remove
4990 // edges in the moveEdgeTo* methods, and this simplifies the handling and
4991 // makes it less error-prone.
4992 auto CallerEdges = Node->CallerEdges;
4993 for (auto &CallerEdge : CallerEdges) {
4994 // Skip any that have been removed by an earlier recursive call.
4995 if (CallerEdge->isRemoved()) {
4996 assert(!is_contained(Node->CallerEdges, CallerEdge));
4997 continue;
4998 }
4999 assert(CallerEdge->Callee == Node);
5000
5001 // See if cloning the prior caller edge left this node with a single alloc
5002 // type or a single caller. In that case no more cloning of Node is needed.
5003 if (hasSingleAllocType(Node->AllocTypes) || Node->CallerEdges.size() <= 1)
5004 break;
5005
5006 // If the caller was not successfully matched to a call in the IR/summary,
5007 // there is no point in trying to clone for it as we can't update that call.
5008 if (!CallerEdge->Caller->hasCall())
5009 continue;
5010
5011 // Only need to process the ids along this edge pertaining to the given
5012 // allocation.
5013 auto CallerEdgeContextsForAlloc =
5014 set_intersection(CallerEdge->getContextIds(), AllocContextIds);
5015 if (!RecursiveContextIds.empty())
5016 CallerEdgeContextsForAlloc =
5017 set_difference(CallerEdgeContextsForAlloc, RecursiveContextIds);
5018 if (CallerEdgeContextsForAlloc.empty())
5019 continue;
5020
5021 auto CallerAllocTypeForAlloc = computeAllocType(CallerEdgeContextsForAlloc);
5022
5023 // Compute the node callee edge alloc types corresponding to the context ids
5024 // for this caller edge.
5025 std::vector<uint8_t> CalleeEdgeAllocTypesForCallerEdge;
5026 CalleeEdgeAllocTypesForCallerEdge.reserve(Node->CalleeEdges.size());
5027 for (auto &CalleeEdge : Node->CalleeEdges)
5028 CalleeEdgeAllocTypesForCallerEdge.push_back(intersectAllocTypes(
5029 CalleeEdge->getContextIds(), CallerEdgeContextsForAlloc));
5030
5031 // Don't clone if doing so will not disambiguate any alloc types amongst
5032 // caller edges (including the callee edges that would be cloned).
5033 // Otherwise we will simply move all edges to the clone.
5034 //
5035 // First check if by cloning we will disambiguate the caller allocation
5036 // type from node's allocation type. Query allocTypeToUse so that we don't
5037 // bother cloning to distinguish NotCold+Cold from NotCold. Note that
5038 // neither of these should be None type.
5039 //
5040 // Then check if by cloning node at least one of the callee edges will be
5041 // disambiguated by splitting out different context ids.
5042 //
5043 // However, always do the cloning if this is a backedge, in which case we
5044 // have not yet cloned along this caller edge.
5045 assert(CallerEdge->AllocTypes != (uint8_t)AllocationType::None);
5046 assert(Node->AllocTypes != (uint8_t)AllocationType::None);
5047 if (!CallerEdge->IsBackedge &&
5048 allocTypeToUse(CallerAllocTypeForAlloc) ==
5049 allocTypeToUse(Node->AllocTypes) &&
5050 allocTypesMatch<DerivedCCG, FuncTy, CallTy>(
5051 CalleeEdgeAllocTypesForCallerEdge, Node->CalleeEdges)) {
5052 continue;
5053 }
5054
5055 if (CallerEdge->IsBackedge) {
5056 // We should only mark these if cloning recursive contexts, where we
5057 // need to do this deferral.
5059 DeferredBackedges++;
5060 }
5061
5062 // If this is a backedge, we now do recursive cloning starting from its
5063 // caller since we may have moved unambiguous caller contexts to a clone
5064 // of this Node in a previous iteration of the current loop, giving more
5065 // opportunity for cloning through the backedge. Because we sorted the
5066 // caller edges earlier so that cold caller edges are first, we would have
5067 // visited and cloned this node for any unamibiguously cold non-recursive
5068 // callers before any ambiguous backedge callers. Note that we don't do this
5069 // if the caller is already cloned or visited during cloning (e.g. via a
5070 // different context path from the allocation).
5071 // TODO: Can we do better in the case where the caller was already visited?
5072 if (CallerEdge->IsBackedge && !CallerEdge->Caller->CloneOf &&
5073 !Visited.count(CallerEdge->Caller)) {
5074 const auto OrigIdCount = CallerEdge->getContextIds().size();
5075 // Now do the recursive cloning of this backedge's caller, which was
5076 // deferred earlier.
5077 identifyClones(CallerEdge->Caller, Visited, CallerEdgeContextsForAlloc);
5078 removeNoneTypeCalleeEdges(CallerEdge->Caller);
5079 // See if the recursive call to identifyClones moved the context ids to a
5080 // new edge from this node to a clone of caller, and switch to looking at
5081 // that new edge so that we clone Node for the new caller clone.
5082 bool UpdatedEdge = false;
5083 if (OrigIdCount > CallerEdge->getContextIds().size()) {
5084 for (auto E : Node->CallerEdges) {
5085 // Only interested in clones of the current edges caller.
5086 if (E->Caller->CloneOf != CallerEdge->Caller)
5087 continue;
5088 // See if this edge contains any of the context ids originally on the
5089 // current caller edge.
5090 auto CallerEdgeContextsForAllocNew =
5091 set_intersection(CallerEdgeContextsForAlloc, E->getContextIds());
5092 if (CallerEdgeContextsForAllocNew.empty())
5093 continue;
5094 // Make sure we don't pick a previously existing caller edge of this
5095 // Node, which would be processed on a different iteration of the
5096 // outer loop over the saved CallerEdges.
5097 if (llvm::is_contained(CallerEdges, E))
5098 continue;
5099 // The CallerAllocTypeForAlloc and CalleeEdgeAllocTypesForCallerEdge
5100 // are updated further below for all cases where we just invoked
5101 // identifyClones recursively.
5102 CallerEdgeContextsForAlloc.swap(CallerEdgeContextsForAllocNew);
5103 CallerEdge = E;
5104 UpdatedEdge = true;
5105 break;
5106 }
5107 }
5108 // If cloning removed this edge (and we didn't update it to a new edge
5109 // above), we're done with this edge. It's possible we moved all of the
5110 // context ids to an existing clone, in which case there's no need to do
5111 // further processing for them.
5112 if (CallerEdge->isRemoved())
5113 continue;
5114
5115 // Now we need to update the information used for the cloning decisions
5116 // further below, as we may have modified edges and their context ids.
5117
5118 // Note if we changed the CallerEdge above we would have already updated
5119 // the context ids.
5120 if (!UpdatedEdge) {
5121 CallerEdgeContextsForAlloc = set_intersection(
5122 CallerEdgeContextsForAlloc, CallerEdge->getContextIds());
5123 if (CallerEdgeContextsForAlloc.empty())
5124 continue;
5125 }
5126 // Update the other information that depends on the edges and on the now
5127 // updated CallerEdgeContextsForAlloc.
5128 CallerAllocTypeForAlloc = computeAllocType(CallerEdgeContextsForAlloc);
5129 CalleeEdgeAllocTypesForCallerEdge.clear();
5130 for (auto &CalleeEdge : Node->CalleeEdges) {
5131 CalleeEdgeAllocTypesForCallerEdge.push_back(intersectAllocTypes(
5132 CalleeEdge->getContextIds(), CallerEdgeContextsForAlloc));
5133 }
5134 }
5135
5136 // First see if we can use an existing clone. Check each clone and its
5137 // callee edges for matching alloc types.
5138 ContextNode *Clone = nullptr;
5139 for (auto *CurClone : Node->Clones) {
5140 if (allocTypeToUse(CurClone->AllocTypes) !=
5141 allocTypeToUse(CallerAllocTypeForAlloc))
5142 continue;
5143
5144 bool BothSingleAlloc = hasSingleAllocType(CurClone->AllocTypes) &&
5145 hasSingleAllocType(CallerAllocTypeForAlloc);
5146 // The above check should mean that if both have single alloc types that
5147 // they should be equal.
5148 assert(!BothSingleAlloc ||
5149 CurClone->AllocTypes == CallerAllocTypeForAlloc);
5150
5151 // If either both have a single alloc type (which are the same), or if the
5152 // clone's callee edges have the same alloc types as those for the current
5153 // allocation on Node's callee edges (CalleeEdgeAllocTypesForCallerEdge),
5154 // then we can reuse this clone.
5155 if (BothSingleAlloc || allocTypesMatchClone<DerivedCCG, FuncTy, CallTy>(
5156 CalleeEdgeAllocTypesForCallerEdge, CurClone)) {
5157 Clone = CurClone;
5158 break;
5159 }
5160 }
5161
5162 // The edge iterator is adjusted when we move the CallerEdge to the clone.
5163 if (Clone)
5164 moveEdgeToExistingCalleeClone(CallerEdge, Clone, /*NewClone=*/false,
5165 CallerEdgeContextsForAlloc);
5166 else
5167 Clone = moveEdgeToNewCalleeClone(CallerEdge, CallerEdgeContextsForAlloc);
5168
5169 // Sanity check that no alloc types on clone or its edges are None.
5170 assert(Clone->AllocTypes != (uint8_t)AllocationType::None);
5171 }
5172
5173 // We should still have some context ids on the original Node.
5174 assert(!Node->emptyContextIds());
5175
5176 // Sanity check that no alloc types on node or edges are None.
5177 assert(Node->AllocTypes != (uint8_t)AllocationType::None);
5178
5179 if (VerifyNodes)
5180 checkNode<DerivedCCG, FuncTy, CallTy>(Node, /*CheckEdges=*/false);
5181}
5182
5183void ModuleCallsiteContextGraph::updateAllocationCall(
5184 CallInfo &Call, AllocationType AllocType) {
5185 std::string AllocTypeString = getAllocTypeAttributeString(AllocType);
5186 auto *CB = cast<CallBase>(Call.call());
5187 if (isAmbiguousAllocType((uint8_t)AllocType)) {
5188 // The contexts reaching this allocation could not be disambiguated, so it
5189 // keeps (or gets) the ambiguous attribute added during matching rather than
5190 // the default not cold attribute.
5192 } else {
5194 auto A = llvm::Attribute::get(CB->getFunction()->getContext(), "memprof",
5195 AllocTypeString);
5196 CB->addFnAttr(A);
5197 }
5198 OREGetter(Call.call()->getFunction())
5199 .emit(OptimizationRemark(DEBUG_TYPE, "MemprofAttribute", Call.call())
5200 << ore::NV("AllocationCall", Call.call()) << " in clone "
5201 << ore::NV("Caller", Call.call()->getFunction())
5202 << " marked with memprof allocation attribute "
5203 << ore::NV("Attribute", AllocTypeString));
5204}
5205
5206void IndexCallsiteContextGraph::updateAllocationCall(CallInfo &Call,
5208 auto *AI = cast<AllocInfo *>(Call.call());
5209 assert(AI);
5210 assert(AI->Versions.size() > Call.cloneNo());
5211 AI->Versions[Call.cloneNo()] = (uint8_t)AllocType;
5212}
5213
5215ModuleCallsiteContextGraph::getAllocationCallType(const CallInfo &Call) const {
5216 const auto *CB = cast<CallBase>(Call.call());
5217 if (!CB->getAttributes().hasFnAttr("memprof"))
5218 return AllocationType::None;
5219 StringRef AllocTypeString =
5220 CB->getAttributes().getFnAttr("memprof").getValueAsString();
5221 if (AllocTypeString == "cold")
5222 return AllocationType::Cold;
5223 if (AllocTypeString == "ambiguous")
5224 return (AllocationType)((uint8_t)AllocationType::NotCold |
5225 (uint8_t)AllocationType::Cold);
5226 return AllocationType::NotCold;
5227}
5228
5230IndexCallsiteContextGraph::getAllocationCallType(const CallInfo &Call) const {
5231 const auto *AI = cast<AllocInfo *>(Call.call());
5232 assert(AI->Versions.size() > Call.cloneNo());
5233 return (AllocationType)AI->Versions[Call.cloneNo()];
5234}
5235
5236void ModuleCallsiteContextGraph::updateCall(CallInfo &CallerCall,
5237 FuncInfo CalleeFunc) {
5238 auto *CurF = getCalleeFunc(CallerCall.call());
5239 auto NewCalleeCloneNo = CalleeFunc.cloneNo();
5240 if (isMemProfClone(*CurF)) {
5241 // If we already assigned this callsite to call a specific non-default
5242 // clone (i.e. not the original function which is clone 0), ensure that we
5243 // aren't trying to now update it to call a different clone, which is
5244 // indicative of a bug in the graph or function assignment.
5245 auto CurCalleeCloneNo = getMemProfCloneNum(*CurF);
5246 if (CurCalleeCloneNo != NewCalleeCloneNo) {
5247 LLVM_DEBUG(dbgs() << "Mismatch in call clone assignment: was "
5248 << CurCalleeCloneNo << " now " << NewCalleeCloneNo
5249 << "\n");
5250 MismatchedCloneAssignments++;
5251 }
5252 }
5253 if (NewCalleeCloneNo > 0)
5254 cast<CallBase>(CallerCall.call())->setCalledFunction(CalleeFunc.func());
5255 OREGetter(CallerCall.call()->getFunction())
5256 .emit(OptimizationRemark(DEBUG_TYPE, "MemprofCall", CallerCall.call())
5257 << ore::NV("Call", CallerCall.call()) << " in clone "
5258 << ore::NV("Caller", CallerCall.call()->getFunction())
5259 << " assigned to call function clone "
5260 << ore::NV("Callee", CalleeFunc.func()));
5261}
5262
5263void IndexCallsiteContextGraph::updateCall(CallInfo &CallerCall,
5264 FuncInfo CalleeFunc) {
5265 auto *CI = cast<CallsiteInfo *>(CallerCall.call());
5266 assert(CI &&
5267 "Caller cannot be an allocation which should not have profiled calls");
5268 assert(CI->Clones.size() > CallerCall.cloneNo());
5269 auto NewCalleeCloneNo = CalleeFunc.cloneNo();
5270 auto &CurCalleeCloneNo = CI->Clones[CallerCall.cloneNo()];
5271 // If we already assigned this callsite to call a specific non-default
5272 // clone (i.e. not the original function which is clone 0), ensure that we
5273 // aren't trying to now update it to call a different clone, which is
5274 // indicative of a bug in the graph or function assignment.
5275 if (CurCalleeCloneNo != 0 && CurCalleeCloneNo != NewCalleeCloneNo) {
5276 LLVM_DEBUG(dbgs() << "Mismatch in call clone assignment: was "
5277 << CurCalleeCloneNo << " now " << NewCalleeCloneNo
5278 << "\n");
5279 MismatchedCloneAssignments++;
5280 }
5281 CurCalleeCloneNo = NewCalleeCloneNo;
5282}
5283
5284// Update the debug information attached to NewFunc to use the clone Name. Note
5285// this needs to be done for both any existing DISubprogram for the definition,
5286// as well as any separate declaration DISubprogram.
5288 assert(Name == NewFunc->getName());
5289 auto *SP = NewFunc->getSubprogram();
5290 if (!SP)
5291 return;
5292 auto *MDName = MDString::get(NewFunc->getParent()->getContext(), Name);
5293 SP->replaceLinkageName(MDName);
5294 DISubprogram *Decl = SP->getDeclaration();
5295 if (!Decl)
5296 return;
5297 TempDISubprogram NewDecl = Decl->clone();
5298 NewDecl->replaceLinkageName(MDName);
5299 SP->replaceDeclaration(MDNode::replaceWithUniqued(std::move(NewDecl)));
5300}
5301
5302CallsiteContextGraph<ModuleCallsiteContextGraph, Function,
5303 Instruction *>::FuncInfo
5304ModuleCallsiteContextGraph::cloneFunctionForCallsite(
5305 FuncInfo &Func, CallInfo &Call, DenseMap<CallInfo, CallInfo> &CallMap,
5306 std::vector<CallInfo> &CallsWithMetadataInFunc, unsigned CloneNo) {
5307 // Use existing LLVM facilities for cloning and obtaining Call in clone
5308 ValueToValueMapTy VMap;
5309 auto *NewFunc = CloneFunction(Func.func(), VMap);
5310 // Clones are only called directly and never have their address taken, so
5311 // strip !kcfi_type to avoid emitting KCFI type preambles/symbols on them or
5312 // making them valid KCFI indirect-call targets.
5313 NewFunc->setMetadata(LLVMContext::MD_kcfi_type, nullptr);
5314 std::string Name = getMemProfFuncName(Func.func()->getName(), CloneNo);
5315 assert(!Func.func()->getParent()->getFunction(Name));
5316 NewFunc->setName(Name);
5317 updateSubprogramLinkageName(NewFunc, Name);
5318 for (auto &Inst : CallsWithMetadataInFunc) {
5319 // This map always has the initial version in it.
5320 assert(Inst.cloneNo() == 0);
5321 CallMap[Inst] = {cast<Instruction>(VMap[Inst.call()]), CloneNo};
5322 }
5323 OREGetter(Func.func())
5324 .emit(OptimizationRemark(DEBUG_TYPE, "MemprofClone", Func.func())
5325 << "created clone " << ore::NV("NewFunction", NewFunc));
5326 return {NewFunc, CloneNo};
5327}
5328
5329CallsiteContextGraph<IndexCallsiteContextGraph, FunctionSummary,
5330 IndexCall>::FuncInfo
5331IndexCallsiteContextGraph::cloneFunctionForCallsite(
5332 FuncInfo &Func, CallInfo &Call, DenseMap<CallInfo, CallInfo> &CallMap,
5333 std::vector<CallInfo> &CallsWithMetadataInFunc, unsigned CloneNo) {
5334 // Check how many clones we have of Call (and therefore function).
5335 // The next clone number is the current size of versions array.
5336 // Confirm this matches the CloneNo provided by the caller, which is based on
5337 // the number of function clones we have.
5338 assert(CloneNo == (isa<AllocInfo *>(Call.call())
5339 ? cast<AllocInfo *>(Call.call())->Versions.size()
5340 : cast<CallsiteInfo *>(Call.call())->Clones.size()));
5341 // Walk all the instructions in this function. Create a new version for
5342 // each (by adding an entry to the Versions/Clones summary array), and copy
5343 // over the version being called for the function clone being cloned here.
5344 // Additionally, add an entry to the CallMap for the new function clone,
5345 // mapping the original call (clone 0, what is in CallsWithMetadataInFunc)
5346 // to the new call clone.
5347 for (auto &Inst : CallsWithMetadataInFunc) {
5348 // This map always has the initial version in it.
5349 assert(Inst.cloneNo() == 0);
5350 if (auto *AI = dyn_cast<AllocInfo *>(Inst.call())) {
5351 assert(AI->Versions.size() == CloneNo);
5352 // We assign the allocation type later (in updateAllocationCall), just add
5353 // an entry for it here.
5354 AI->Versions.push_back(0);
5355 } else {
5356 auto *CI = cast<CallsiteInfo *>(Inst.call());
5357 assert(CI && CI->Clones.size() == CloneNo);
5358 // We assign the clone number later (in updateCall), just add an entry for
5359 // it here.
5360 CI->Clones.push_back(0);
5361 }
5362 CallMap[Inst] = {Inst.call(), CloneNo};
5363 }
5364 return {Func.func(), CloneNo};
5365}
5366
5367// We perform cloning for each allocation node separately. However, this
5368// sometimes results in a situation where the same node calls multiple
5369// clones of the same callee, created for different allocations. This
5370// causes issues when assigning functions to these clones, as each node can
5371// in reality only call a single callee clone.
5372//
5373// To address this, before assigning functions, merge callee clone nodes as
5374// needed using a post order traversal from the allocations. We attempt to
5375// use existing clones as the merge node when legal, and to share them
5376// among callers with the same properties (callers calling the same set of
5377// callee clone nodes for the same allocations).
5378//
5379// Without this fix, in some cases incorrect function assignment will lead
5380// to calling the wrong allocation clone.
5381template <typename DerivedCCG, typename FuncTy, typename CallTy>
5382void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::mergeClones() {
5383 if (!MergeClones)
5384 return;
5385
5386 // Generate a map from context id to the associated allocation node for use
5387 // when merging clones.
5388 DenseMap<uint32_t, ContextNode *> ContextIdToAllocationNode;
5389 for (auto &Entry : AllocationCallToContextNodeMap) {
5390 auto *Node = Entry.second;
5391 for (auto Id : Node->getContextIds())
5392 ContextIdToAllocationNode[Id] = Node->getOrigNode();
5393 for (auto *Clone : Node->Clones) {
5394 for (auto Id : Clone->getContextIds())
5395 ContextIdToAllocationNode[Id] = Clone->getOrigNode();
5396 }
5397 }
5398
5399 // Post order traversal starting from allocations to ensure each callsite
5400 // calls a single clone of its callee. Callee nodes that are clones of each
5401 // other are merged (via new merge nodes if needed) to achieve this.
5402 DenseSet<const ContextNode *> Visited;
5403 for (auto &Entry : AllocationCallToContextNodeMap) {
5404 auto *Node = Entry.second;
5405
5406 mergeClones(Node, Visited, ContextIdToAllocationNode);
5407
5408 // Make a copy so the recursive post order traversal that may create new
5409 // clones doesn't mess up iteration. Note that the recursive traversal
5410 // itself does not call mergeClones on any of these nodes, which are all
5411 // (clones of) allocations.
5412 auto Clones = Node->Clones;
5413 for (auto *Clone : Clones)
5414 mergeClones(Clone, Visited, ContextIdToAllocationNode);
5415 }
5416
5417 if (DumpCCG) {
5418 dbgs() << "CCG after merging:\n";
5419 dbgs() << *this;
5420 }
5421 if (ExportToDot)
5422 exportToDot("aftermerge");
5423
5424 if (VerifyCCG) {
5425 check();
5426 }
5427}
5428
5429// Recursive helper for above mergeClones method.
5430template <typename DerivedCCG, typename FuncTy, typename CallTy>
5431void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::mergeClones(
5432 ContextNode *Node, DenseSet<const ContextNode *> &Visited,
5433 DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode) {
5434 auto Inserted = Visited.insert(Node);
5435 if (!Inserted.second)
5436 return;
5437
5438 // Iteratively perform merging on this node to handle new caller nodes created
5439 // during the recursive traversal. We could do something more elegant such as
5440 // maintain a worklist, but this is a simple approach that doesn't cause a
5441 // measureable compile time effect, as most nodes don't have many caller
5442 // edges to check.
5443 bool FoundUnvisited = true;
5444 unsigned Iters = 0;
5445 while (FoundUnvisited) {
5446 Iters++;
5447 FoundUnvisited = false;
5448 // Make a copy since the recursive call may move a caller edge to a new
5449 // callee, messing up the iterator.
5450 auto CallerEdges = Node->CallerEdges;
5451 for (auto CallerEdge : CallerEdges) {
5452 // Skip any caller edge moved onto a different callee during recursion.
5453 if (CallerEdge->Callee != Node)
5454 continue;
5455 // If we found an unvisited caller, note that we should check the caller
5456 // edges again as mergeClones may add or change caller nodes.
5457 if (DoMergeIteration && !Visited.contains(CallerEdge->Caller))
5458 FoundUnvisited = true;
5459 mergeClones(CallerEdge->Caller, Visited, ContextIdToAllocationNode);
5460 }
5461 }
5462
5463 TotalMergeInvokes++;
5464 TotalMergeIters += Iters;
5465 if (Iters > MaxMergeIters)
5466 MaxMergeIters = Iters;
5467
5468 // Merge for this node after we handle its callers.
5469 mergeNodeCalleeClones(Node, Visited, ContextIdToAllocationNode);
5470}
5471
5472template <typename DerivedCCG, typename FuncTy, typename CallTy>
5473void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::mergeNodeCalleeClones(
5474 ContextNode *Node, DenseSet<const ContextNode *> &Visited,
5475 DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode) {
5476 // Ignore Node if we moved all of its contexts to clones.
5477 if (Node->emptyContextIds())
5478 return;
5479
5480 // First identify groups of clones among Node's callee edges, by building
5481 // a map from each callee base node to the associated callee edges from Node.
5482 MapVector<ContextNode *, std::vector<std::shared_ptr<ContextEdge>>>
5483 OrigNodeToCloneEdges;
5484 for (const auto &E : Node->CalleeEdges) {
5485 auto *Callee = E->Callee;
5486 if (!Callee->CloneOf && Callee->Clones.empty())
5487 continue;
5488 ContextNode *Base = Callee->getOrigNode();
5489 OrigNodeToCloneEdges[Base].push_back(E);
5490 }
5491
5492 // Helper for callee edge sorting below. Return true if A's callee has fewer
5493 // caller edges than B, or if A is a clone and B is not, or if A's callee
5494 // node id is smaller than B's.
5495 auto CalleeCallerEdgeLessThan = [](const std::shared_ptr<ContextEdge> &A,
5496 const std::shared_ptr<ContextEdge> &B) {
5497 if (A->Callee->CallerEdges.size() != B->Callee->CallerEdges.size())
5498 return A->Callee->CallerEdges.size() < B->Callee->CallerEdges.size();
5499 if (A->Callee->CloneOf && !B->Callee->CloneOf)
5500 return true;
5501 else if (!A->Callee->CloneOf && B->Callee->CloneOf)
5502 return false;
5503 // Use the callee node id as a deterministic tie-breaker.
5504 return A->Callee->NodeId < B->Callee->NodeId;
5505 };
5506
5507 // Process each set of callee clones called by Node, performing the needed
5508 // merging.
5509 for (auto Entry : OrigNodeToCloneEdges) {
5510 // CalleeEdges is the set of edges from Node reaching callees that are
5511 // mutual clones of each other.
5512 auto &CalleeEdges = Entry.second;
5513 auto NumCalleeClones = CalleeEdges.size();
5514 // A single edge means there is no merging needed.
5515 if (NumCalleeClones == 1)
5516 continue;
5517 // Sort the CalleeEdges calling this group of clones in ascending order of
5518 // their caller edge counts, putting the original non-clone node first in
5519 // cases of a tie. This simplifies finding an existing node to use as the
5520 // merge node.
5521 llvm::stable_sort(CalleeEdges, CalleeCallerEdgeLessThan);
5522
5523 /// Find other callers of the given set of callee edges that can
5524 /// share the same callee merge node. See the comments at this method
5525 /// definition for details.
5526 DenseSet<ContextNode *> OtherCallersToShareMerge;
5527 findOtherCallersToShareMerge(Node, CalleeEdges, ContextIdToAllocationNode,
5528 OtherCallersToShareMerge);
5529
5530 // Now do the actual merging. Identify existing or create a new MergeNode
5531 // during the first iteration. Move each callee over, along with edges from
5532 // other callers we've determined above can share the same merge node.
5533 ContextNode *MergeNode = nullptr;
5534 DenseMap<ContextNode *, unsigned> CallerToMoveCount;
5535 for (auto CalleeEdge : CalleeEdges) {
5536 auto *OrigCallee = CalleeEdge->Callee;
5537 // If we don't have a MergeNode yet (only happens on the first iteration,
5538 // as a new one will be created when we go to move the first callee edge
5539 // over as needed), see if we can use this callee.
5540 if (!MergeNode) {
5541 // If there are no other callers, simply use this callee.
5542 if (CalleeEdge->Callee->CallerEdges.size() == 1) {
5543 MergeNode = OrigCallee;
5544 NonNewMergedNodes++;
5545 continue;
5546 }
5547 // Otherwise, if we have identified other caller nodes that can share
5548 // the merge node with Node, see if all of OrigCallee's callers are
5549 // going to share the same merge node. In that case we can use callee
5550 // (since all of its callers would move to the new merge node).
5551 if (!OtherCallersToShareMerge.empty()) {
5552 bool MoveAllCallerEdges = true;
5553 for (auto CalleeCallerE : OrigCallee->CallerEdges) {
5554 if (CalleeCallerE == CalleeEdge)
5555 continue;
5556 if (!OtherCallersToShareMerge.contains(CalleeCallerE->Caller)) {
5557 MoveAllCallerEdges = false;
5558 break;
5559 }
5560 }
5561 // If we are going to move all callers over, we can use this callee as
5562 // the MergeNode.
5563 if (MoveAllCallerEdges) {
5564 MergeNode = OrigCallee;
5565 NonNewMergedNodes++;
5566 continue;
5567 }
5568 }
5569 }
5570 // Move this callee edge, creating a new merge node if necessary.
5571 if (MergeNode) {
5572 assert(MergeNode != OrigCallee);
5573 moveEdgeToExistingCalleeClone(CalleeEdge, MergeNode,
5574 /*NewClone*/ false);
5575 } else {
5576 MergeNode = moveEdgeToNewCalleeClone(CalleeEdge);
5577 NewMergedNodes++;
5578 }
5579 // Now move all identified edges from other callers over to the merge node
5580 // as well.
5581 if (!OtherCallersToShareMerge.empty()) {
5582 // Make and iterate over a copy of OrigCallee's caller edges because
5583 // some of these will be moved off of the OrigCallee and that would mess
5584 // up the iteration from OrigCallee.
5585 auto OrigCalleeCallerEdges = OrigCallee->CallerEdges;
5586 for (auto &CalleeCallerE : OrigCalleeCallerEdges) {
5587 if (CalleeCallerE == CalleeEdge)
5588 continue;
5589 if (!OtherCallersToShareMerge.contains(CalleeCallerE->Caller))
5590 continue;
5591 CallerToMoveCount[CalleeCallerE->Caller]++;
5592 moveEdgeToExistingCalleeClone(CalleeCallerE, MergeNode,
5593 /*NewClone*/ false);
5594 }
5595 }
5596 removeNoneTypeCalleeEdges(OrigCallee);
5597 removeNoneTypeCalleeEdges(MergeNode);
5598 }
5599 }
5600}
5601
5602// Look for other nodes that have edges to the same set of callee
5603// clones as the current Node. Those can share the eventual merge node
5604// (reducing cloning and binary size overhead) iff:
5605// - they have edges to the same set of callee clones
5606// - each callee edge reaches a subset of the same allocations as Node's
5607// corresponding edge to the same callee clone.
5608// The second requirement is to ensure that we don't undo any of the
5609// necessary cloning to distinguish contexts with different allocation
5610// behavior.
5611// FIXME: This is somewhat conservative, as we really just need to ensure
5612// that they don't reach the same allocations as contexts on edges from Node
5613// going to any of the *other* callee clones being merged. However, that
5614// requires more tracking and checking to get right.
5615template <typename DerivedCCG, typename FuncTy, typename CallTy>
5616void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::
5617 findOtherCallersToShareMerge(
5618 ContextNode *Node,
5619 std::vector<std::shared_ptr<ContextEdge>> &CalleeEdges,
5620 DenseMap<uint32_t, ContextNode *> &ContextIdToAllocationNode,
5621 DenseSet<ContextNode *> &OtherCallersToShareMerge) {
5622 auto NumCalleeClones = CalleeEdges.size();
5623 // This map counts how many edges to the same callee clone exist for other
5624 // caller nodes of each callee clone.
5625 DenseMap<ContextNode *, unsigned> OtherCallersToSharedCalleeEdgeCount;
5626 // Counts the number of other caller nodes that have edges to all callee
5627 // clones that don't violate the allocation context checking.
5628 unsigned PossibleOtherCallerNodes = 0;
5629
5630 // We only need to look at other Caller nodes if the first callee edge has
5631 // multiple callers (recall they are sorted in ascending order above).
5632 if (CalleeEdges[0]->Callee->CallerEdges.size() < 2)
5633 return;
5634
5635 // For each callee edge:
5636 // - Collect the count of other caller nodes calling the same callees.
5637 // - Collect the alloc nodes reached by contexts on each callee edge.
5638 DenseMap<ContextEdge *, DenseSet<ContextNode *>> CalleeEdgeToAllocNodes;
5639 for (auto CalleeEdge : CalleeEdges) {
5640 assert(CalleeEdge->Callee->CallerEdges.size() > 1);
5641 // For each other caller of the same callee, increment the count of
5642 // edges reaching the same callee clone.
5643 for (auto CalleeCallerEdges : CalleeEdge->Callee->CallerEdges) {
5644 if (CalleeCallerEdges->Caller == Node) {
5645 assert(CalleeCallerEdges == CalleeEdge);
5646 continue;
5647 }
5648 OtherCallersToSharedCalleeEdgeCount[CalleeCallerEdges->Caller]++;
5649 // If this caller edge now reaches all of the same callee clones,
5650 // increment the count of candidate other caller nodes.
5651 if (OtherCallersToSharedCalleeEdgeCount[CalleeCallerEdges->Caller] ==
5652 NumCalleeClones)
5653 PossibleOtherCallerNodes++;
5654 }
5655 // Collect the alloc nodes reached by contexts on each callee edge, for
5656 // later analysis.
5657 for (auto Id : CalleeEdge->getContextIds()) {
5658 auto *Alloc = ContextIdToAllocationNode.lookup(Id);
5659 if (!Alloc) {
5660 // FIXME: unclear why this happens occasionally, presumably
5661 // imperfect graph updates possibly with recursion.
5662 MissingAllocForContextId++;
5663 continue;
5664 }
5665 CalleeEdgeToAllocNodes[CalleeEdge.get()].insert(Alloc);
5666 }
5667 }
5668
5669 // Now walk the callee edges again, and make sure that for each candidate
5670 // caller node all of its edges to the callees reach the same allocs (or
5671 // a subset) as those along the corresponding callee edge from Node.
5672 for (auto CalleeEdge : CalleeEdges) {
5673 assert(CalleeEdge->Callee->CallerEdges.size() > 1);
5674 // Stop if we do not have any (more) candidate other caller nodes.
5675 if (!PossibleOtherCallerNodes)
5676 break;
5677 auto &CurCalleeAllocNodes = CalleeEdgeToAllocNodes[CalleeEdge.get()];
5678 // Check each other caller of this callee clone.
5679 for (auto &CalleeCallerE : CalleeEdge->Callee->CallerEdges) {
5680 // Not interested in the callee edge from Node itself.
5681 if (CalleeCallerE == CalleeEdge)
5682 continue;
5683 // Skip any callers that didn't have callee edges to all the same
5684 // callee clones.
5685 if (OtherCallersToSharedCalleeEdgeCount[CalleeCallerE->Caller] !=
5686 NumCalleeClones)
5687 continue;
5688 // Make sure that each context along edge from candidate caller node
5689 // reaches an allocation also reached by this callee edge from Node.
5690 for (auto Id : CalleeCallerE->getContextIds()) {
5691 auto *Alloc = ContextIdToAllocationNode.lookup(Id);
5692 if (!Alloc)
5693 continue;
5694 // If not, simply reset the map entry to 0 so caller is ignored, and
5695 // reduce the count of candidate other caller nodes.
5696 if (!CurCalleeAllocNodes.contains(Alloc)) {
5697 OtherCallersToSharedCalleeEdgeCount[CalleeCallerE->Caller] = 0;
5698 PossibleOtherCallerNodes--;
5699 break;
5700 }
5701 }
5702 }
5703 }
5704
5705 if (!PossibleOtherCallerNodes)
5706 return;
5707
5708 // Build the set of other caller nodes that can use the same callee merge
5709 // node.
5710 for (auto &[OtherCaller, Count] : OtherCallersToSharedCalleeEdgeCount) {
5711 if (Count != NumCalleeClones)
5712 continue;
5713 OtherCallersToShareMerge.insert(OtherCaller);
5714 }
5715}
5716
5717template <typename DerivedCCG, typename FuncTy, typename CallTy>
5718const typename CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::ContextNode *
5719CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::resolveImplicitSource(
5720 const ContextNode *Node, const ImplicitSource &Src) {
5721 if (!Src.Source->emptyContextIds())
5722 return Src.Source;
5723 // Src.Source lost its contexts to another clone (e.g. its last caller was
5724 // moved to a new clone when specializing a function clone further). The
5725 // trimmed contexts stay together, so the clone carrying one of them is the
5726 // version the implicit caller requires.
5727 if (Node->hasContextId(Src.ContextId))
5728 return Node;
5729 for (const ContextNode *C : Node->Clones)
5730 if (C->hasContextId(Src.ContextId))
5731 return C;
5732 // Context ids only move between the clones of a node during function
5733 // assignment, so the id should still be on one of them. Return null, which
5734 // makes the call follow the original node, rather than an emptied clone.
5735 assert(false && "trimmed context id on no clone of its callsite node");
5736 return nullptr;
5737}
5738
5739// Contexts whose call stacks were trimmed at a callsite node's frame are on
5740// its callee edges but on none of its caller edges. When that node, or one
5741// below it on the path of such a context, is cloned for the contexts of
5742// another allocation reaching it through a caller edge, the trimmed contexts
5743// stay with the node that was cloned, and the clone gets no edge to the callee
5744// nodes the trimmed contexts flow through. The clone is therefore not a caller
5745// of any clone of such a callee node, and would otherwise be assigned a
5746// function clone of the callee without regard to it, possibly one calling the
5747// wrong version of the callee for the trimmed contexts, which are agnostic to
5748// the callers of the node they were trimmed at and apply to the clone equally.
5749//
5750// This is called once the clones of callsite Node have been assigned to
5751// function clones. For each caller Y of a clone of Node whose edge carries
5752// trimmed contexts, each clone K of Y's original node that has no edge to any
5753// clone of Node, and which some of those trimmed contexts would reach (see
5754// ImpliesContextIds), is an implicit caller of Node. For example, with a
5755// context t trimmed at the frame of Y's callsite:
5756//
5757// (callers of Y) (callers of K)
5758// | |
5759// Y K clones of the same callsite node
5760// | \
5761// t | \ K has no edge to any clone of Node
5762// v v
5763// Required (other callsites in Node's function)
5764// (clone of Node called
5765// by Y, assigned to
5766// Y's function clone)
5767//
5768// Had t not been trimmed, it would also be on an edge from K, so K should get
5769// the same version of Node's function as Y:
5770// - If K was not yet assigned a function clone, assign it Y's, so that the
5771// assignment of the callsites it does have edges to takes it into account
5772// (creating further function clones derived from it if needed).
5773// - If K was already assigned to a function clone without a clone of Node,
5774// record that the Node call in that function clone should follow Required.
5775// - Otherwise K's function clone calls a conflicting clone of Node, which is
5776// left as is (and counted).
5777template <typename DerivedCCG, typename FuncTy, typename CallTy>
5778void CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::assignImplicitCallers(
5779 ContextNode *Node, const DenseSet<uint32_t> &TrimmedContextIds,
5780 DenseMap<ContextNode *, FuncInfo> &CallsiteToCalleeFuncCloneMap,
5781 const std::map<FuncInfo, ContextNode *> &FuncCloneToCurNodeCloneMap,
5782 ImplicitSourceMap &ImplicitSources) {
5783 auto HasEdgeToNodeFamily = [&](const ContextNode *N) {
5784 return llvm::any_of(N->CalleeEdges,
5785 [&](const std::shared_ptr<ContextEdge> &E) {
5786 return E->Callee->getOrigNode() == Node;
5787 });
5788 };
5789 // Returns true if any of the trimmed context ids Ids, which are on (the
5790 // clone) Y but not on K, a clone of the same callsite node, would reach K had
5791 // they not been trimmed:
5792 // - An id terminating at Y applies to all clones of Y's original node (see
5793 // the example above assignImplicitCallers).
5794 // - An id reaching Y from a caller Z applies to K only if it would equally
5795 // apply to some caller C of K that is a clone of Z's original node, i.e.
5796 // it is not on C (if it were, C explicitly directs it to a callee other
5797 // than K) and, recursively, it applies to C the way it applies to Z (e.g.
5798 // because it terminates at Z, so it applies to all clones of Z's original
5799 // node). The id would then flow from C to K just as it flows from Z to Y,
5800 // had it not been trimmed:
5801 //
5802 // ... ... t terminates at Z or one of its callers
5803 // |
5804 // Z C clones of the same callsite node
5805 // | :
5806 // t | : (t) t is not on C or K; it applies to them
5807 // v v because it was trimmed above
5808 // Y K clones of the same callsite node
5809 // |
5810 // t v
5811 // Required
5812 //
5813 // This is checked recursively on (C, Z) with the ids not on C. If C == Z,
5814 // Z explicitly directs the ids to Y rather than K, so they don't apply to
5815 // K.
5816 //
5817 // Visited records the ids already examined for each (K, Y) pair, which
5818 // guards against cycles from recursion while still examining ids first seen
5819 // for the pair on a reconverging caller chain.
5820 auto ImpliesContextIds =
5821 [&](ContextNode *K, ContextNode *Y, const DenseSet<uint32_t> &Ids,
5822 DenseMap<std::pair<ContextNode *, ContextNode *>, DenseSet<uint32_t>>
5823 &Visited,
5824 auto &&ImpliesContextIds) -> bool {
5825 assert(K->getOrigNode() == Y->getOrigNode());
5826 DenseSet<uint32_t> &VisitedIds = Visited[{K, Y}];
5827 DenseSet<uint32_t> NewIds;
5828 for (auto Id : Ids)
5829 if (VisitedIds.insert(Id).second)
5830 NewIds.insert(Id);
5831 if (NewIds.empty())
5832 return false;
5833 // The ids found on each caller edge of Y, with that edge's caller Z, and
5834 // their union (a set rather than a count, as a recursive context can put
5835 // the same id on several caller edges).
5837 DenseSet<uint32_t> IdsOnCallerEdges;
5838 for (auto &YEdge : Y->CallerEdges) {
5839 DenseSet<uint32_t> IdsFromZ =
5840 set_intersection(NewIds, YEdge->getContextIds());
5841 if (IdsFromZ.empty())
5842 continue;
5843 set_union(IdsOnCallerEdges, IdsFromZ);
5844 IdsFromCallers.push_back({YEdge->Caller, std::move(IdsFromZ)});
5845 }
5846 // Any ids on none of Y's caller edges terminate at Y, and so apply to all
5847 // clones of Y's original node, including K.
5848 if (IdsOnCallerEdges.size() < NewIds.size())
5849 return true;
5850 // Otherwise check whether the ids reaching Y from a caller Z also apply to
5851 // K through a caller C of K that is a clone of Z's original node.
5852 for (auto &[Z, IdsFromZ] : IdsFromCallers) {
5853 for (auto &KEdge : K->CallerEdges) {
5854 ContextNode *C = KEdge->Caller;
5855 if (C == Z || C->getOrigNode() != Z->getOrigNode())
5856 continue;
5857 // Only the ids not on C can apply to C implicitly (C directs the ones
5858 // it has elsewhere); recursively check whether they do.
5859 DenseSet<uint32_t> IdsNotOnC;
5860 for (auto Id : IdsFromZ)
5861 if (!C->hasContextId(Id))
5862 IdsNotOnC.insert(Id);
5863 if (!IdsNotOnC.empty() &&
5864 ImpliesContextIds(C, Z, IdsNotOnC, Visited, ImpliesContextIds))
5865 return true;
5866 }
5867 }
5868 return false;
5869 };
5870 // Returns true if the given clones of Node are interchangeable for an
5871 // implicit caller: the same clone, allocations with the same hint, or
5872 // callsites already assigned to call the same function clone.
5873 auto SameVersion = [&](const ContextNode *A, const ContextNode *B) {
5874 if (A == B)
5875 return true;
5876 if (Node->IsAllocation)
5877 return allocTypeToUse(A->AllocTypes) == allocTypeToUse(B->AllocTypes);
5878 auto AIt = CallsiteToCalleeFuncCloneMap.find(A);
5879 auto BIt = CallsiteToCalleeFuncCloneMap.find(B);
5880 return AIt != CallsiteToCalleeFuncCloneMap.end() &&
5881 BIt != CallsiteToCalleeFuncCloneMap.end() &&
5882 AIt->second == BIt->second;
5883 };
5884 // The clones already identified as implicit callers of Node, so that each is
5885 // handled once (for the first caller edge whose trimmed contexts reach it).
5886 DenseSet<const ContextNode *> ImplicitCallers;
5887 // Handles the caller edge Edge of a clone of Node (or of Node itself): finds
5888 // the implicit callers among the clones of its caller Y's original node, and
5889 // assigns them, or records the clone of Node their calls must follow, as
5890 // described above.
5891 auto ProcessCaller = [&](const std::shared_ptr<ContextEdge> &Edge) {
5892 ContextNode *Y = Edge->Caller;
5893 ContextNode *Orig = Y->getOrigNode();
5894 if (!Y->hasCall() || Orig->Clones.empty())
5895 return;
5896 DenseSet<uint32_t> TrimmedIds =
5897 set_intersection(Edge->getContextIds(), TrimmedContextIds);
5898 if (TrimmedIds.empty())
5899 return;
5900 auto YIt = CallsiteToCalleeFuncCloneMap.find(Y);
5901 if (YIt == CallsiteToCalleeFuncCloneMap.end())
5902 return;
5903 // Copy, as recording assignments below may grow the map.
5904 const FuncInfo YFuncClone = YIt->second;
5905 // The clone of Node called by Y, which must be assigned to the same
5906 // function clone.
5907 auto RequiredIt = FuncCloneToCurNodeCloneMap.find(YFuncClone);
5908 assert(RequiredIt != FuncCloneToCurNodeCloneMap.end());
5909 const ContextNode *Required = RequiredIt->second;
5910 // Record in ImplicitSources that the Node call in function clone F must
5911 // follow the callsite clone Src. This is consulted when the calls in
5912 // function clones without their own clone of Node are collected (see
5913 // UnassignedCallClones in assignFunctions), and makes the requirement
5914 // survive if Src is later moved to another function clone derived from F
5915 // (when one of its callers conflicts at another callsite), after which the
5916 // call in F would otherwise follow the function clone F was derived from.
5917 auto RecordImplicitSource = [&](const FuncInfo &F, const ContextNode *Src) {
5918 // The calls in the original function (clone 0) are not collected there,
5919 // so nothing can be recorded for it.
5920 if (F.cloneNo() != 0)
5921 ImplicitSources.try_emplace({Node, F.cloneNo()},
5922 ImplicitSource{Src, *TrimmedIds.begin()});
5923 };
5924 // Check each clone of Y's original node (and the original itself) without
5925 // an edge to any clone of Node.
5926 for (ContextNode *K : llvm::concat<ContextNode *const>(
5927 Orig->Clones, ArrayRef<ContextNode *>(Orig))) {
5928 if (K == Y || !K->hasCall() || K->emptyContextIds() ||
5929 HasEdgeToNodeFamily(K) || ImplicitCallers.contains(K))
5930 continue;
5931 DenseMap<std::pair<ContextNode *, ContextNode *>, DenseSet<uint32_t>>
5932 Visited;
5933 if (!ImpliesContextIds(K, Y, TrimmedIds, Visited, ImpliesContextIds))
5934 continue;
5935 // K is an implicit caller of Node.
5936 ImplicitCallers.insert(K);
5937 auto KIt = CallsiteToCalleeFuncCloneMap.find(K);
5938 if (KIt == CallsiteToCalleeFuncCloneMap.end()) {
5939 // Not yet assigned: give it Y's function clone, in which the Node call
5940 // then follows Required.
5941 CallsiteToCalleeFuncCloneMap[K] = YFuncClone;
5942 RecordImplicitSource(YFuncClone, Required);
5943 NumImplicitCallersAssigned++;
5944 continue;
5945 }
5946 const FuncInfo KFuncClone = KIt->second;
5947 if (KFuncClone == YFuncClone) {
5948 // Already in Y's function clone.
5949 RecordImplicitSource(YFuncClone, Required);
5950 continue;
5951 }
5952 unsigned KCloneNo = KFuncClone.cloneNo();
5953 auto KNodeCloneIt = FuncCloneToCurNodeCloneMap.find(KFuncClone);
5954 if (KNodeCloneIt == FuncCloneToCurNodeCloneMap.end()) {
5955 // K's function clone has no clone of Node.
5956 if (KCloneNo == 0) {
5957 // The call in the original function keeps calling the original
5958 // callee function, or stays unhinted for an allocation. That
5959 // satisfies the requirement only if Required is that version.
5960 if (Node->IsAllocation) {
5961 if (allocTypeToUse(Required->AllocTypes) == AllocationType::NotCold)
5962 continue;
5963 } else if (auto It = CallsiteToCalleeFuncCloneMap.find(Required);
5964 It != CallsiteToCalleeFuncCloneMap.end() &&
5965 It->second.cloneNo() == 0)
5966 continue;
5967 } else {
5968 // Record that the Node call in K's function clone must follow
5969 // Required, unless an earlier implicit caller already recorded a
5970 // conflicting clone.
5971 auto [It, Inserted] = ImplicitSources.try_emplace(
5972 {Node, KCloneNo}, ImplicitSource{Required, *TrimmedIds.begin()});
5973 if (Inserted)
5974 NumImplicitCallersSourced++;
5975 if (SameVersion(It->second.Source, Required))
5976 continue;
5977 }
5978 } else if (SameVersion(KNodeCloneIt->second, Required)) {
5979 // K's function clone has a clone of Node interchangeable with
5980 // Required.
5981 RecordImplicitSource(KFuncClone, KNodeCloneIt->second);
5982 continue;
5983 }
5984 // K's function clone calls a conflicting clone of Node.
5985 NumImplicitCallersUnresolved++;
5986 }
5987 };
5988 // Process the caller edges of Node itself and of each of its clones: any of
5989 // them may carry trimmed contexts (a trimmed context stays with the clone
5990 // that was cloned, which may be Node or a clone made earlier).
5991 for (auto &Edge : Node->CallerEdges)
5992 ProcessCaller(Edge);
5993 for (auto *Clone : Node->Clones)
5994 for (auto &Edge : Clone->CallerEdges)
5995 ProcessCaller(Edge);
5996}
5997
5998// This method assigns cloned callsites to functions, cloning the functions as
5999// needed. The assignment is greedy and proceeds roughly as follows:
6000//
6001// For each function Func:
6002// For each call with graph Node having clones:
6003// Initialize ClonesWorklist to Node and its clones
6004// Initialize NodeCloneCount to 0
6005// While ClonesWorklist is not empty:
6006// Clone = pop front ClonesWorklist
6007// NodeCloneCount++
6008// If Func has been cloned less than NodeCloneCount times:
6009// If NodeCloneCount is 1:
6010// Assign Clone to original Func
6011// Continue
6012// Create a new function clone
6013// If other callers not assigned to call a function clone yet:
6014// Assign them to call new function clone
6015// Continue
6016// Assign any other caller calling the cloned version to new clone
6017//
6018// For each caller of Clone:
6019// If caller is assigned to call a specific function clone:
6020// If we cannot assign Clone to that function clone:
6021// Create new callsite Clone NewClone
6022// Add NewClone to ClonesWorklist
6023// Continue
6024// Assign Clone to existing caller's called function clone
6025// Else:
6026// If Clone not already assigned to a function clone:
6027// Assign to first function clone without assignment
6028// Assign caller to selected function clone
6029// For each call with graph Node having clones:
6030// If number func clones > number call's callsite Node clones:
6031// Record func CallInfo clones without Node clone in UnassignedCallClones
6032// For callsite Nodes in DFS order from allocations:
6033// If IsAllocation:
6034// Update allocation with alloc type
6035// Else:
6036// For Call, all MatchingCalls, and associated UnnassignedCallClones:
6037// Update call to call recorded callee clone
6038//
6039template <typename DerivedCCG, typename FuncTy, typename CallTy>
6040bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::assignFunctions() {
6041 bool Changed = false;
6042
6043 mergeClones();
6044
6045 // Context ids terminating at a non-allocation node which has or had callers
6046 // (i.e. is not a root), meaning their call stacks were trimmed there. Used
6047 // to identify implicit callers (see assignImplicitCallers).
6048 DenseSet<uint32_t> TrimmedContextIds;
6049 for (auto &NodePtr : NodeOwner) {
6050 ContextNode *N = NodePtr.get();
6051 if (N->IsAllocation || N->CalleeEdges.empty() ||
6052 (N->CallerEdges.empty() && N->Clones.empty() && !N->CloneOf))
6053 continue;
6054 // A context id trimmed at N is on a callee edge but on no caller edge.
6055 // Compare distinct ids rather than edge sizes: a recursive context can put
6056 // the same id on several callee and caller edges of N.
6057 DenseSet<uint32_t> CallerIds;
6058 for (auto &E : N->CallerEdges)
6059 set_union(CallerIds, E->getContextIds());
6060 for (auto &CalleeEdge : N->CalleeEdges)
6061 for (auto Id : CalleeEdge->getContextIds())
6062 if (!CallerIds.contains(Id))
6063 TrimmedContextIds.insert(Id);
6064 }
6065 NumTrimmedContextIds += TrimmedContextIds.size();
6066
6067 // Keep track of the assignment of nodes (callsites) to function clones they
6068 // call.
6069 DenseMap<ContextNode *, FuncInfo> CallsiteToCalleeFuncCloneMap;
6070
6071 // Update caller node to call function version CalleeFunc, by recording the
6072 // assignment in CallsiteToCalleeFuncCloneMap.
6073 auto RecordCalleeFuncOfCallsite = [&](ContextNode *Caller,
6074 const FuncInfo &CalleeFunc) {
6075 assert(Caller->hasCall());
6076 CallsiteToCalleeFuncCloneMap[Caller] = CalleeFunc;
6077 };
6078
6079 // Information for a single clone of this Func.
6080 struct FuncCloneInfo {
6081 // The function clone.
6082 FuncInfo FuncClone;
6083 // Remappings of each call of interest (from original uncloned call to the
6084 // corresponding cloned call in this function clone).
6085 DenseMap<CallInfo, CallInfo> CallMap;
6086 };
6087
6088 // Map to keep track of information needed to update calls in function clones
6089 // when their corresponding callsite node was not itself cloned for that
6090 // function clone. Because of call context trimming (i.e. we only keep as much
6091 // caller information as needed to distinguish hot vs cold), we may not have
6092 // caller edges coming to each callsite node from all possible function
6093 // callers. A function clone may get created for other callsites in the
6094 // function for which there are caller edges that were not trimmed. Any other
6095 // callsites in that function clone, which were not themselved cloned for
6096 // that function clone, should get updated the same way as the corresponding
6097 // callsite in the original function (which may call a clone of its callee).
6098 //
6099 // We build this map after completing function cloning for each function, so
6100 // that we can record the information from its call maps before they are
6101 // destructed. The map will be used as we update calls to update any still
6102 // unassigned call clones. Note that we may create new node clones as we clone
6103 // other functions, so later on we check which node clones were still not
6104 // created. To this end, the inner map is a map from function clone number to
6105 // the list of calls cloned for that function (can be more than one due to the
6106 // Node's MatchingCalls array).
6107 //
6108 // The alternative is creating new callsite clone nodes below as we clone the
6109 // function, but that is tricker to get right and likely more overhead.
6110 //
6111 // Each entry also records the callsite node whose callee assignment the
6112 // calls should follow (Source). By default that is the callsite clone (if
6113 // any) in the function clone the function clone was derived from, following
6114 // the chain of derivations to the original function, since a function clone
6115 // created to specialize another one should otherwise behave like it. It can
6116 // also be the callsite clone required by an implicit caller (see
6117 // assignImplicitCallers). A null Source means the original node.
6118 //
6119 // Inner map is a std::map so sorted by key (clone number), in order to get
6120 // ordered remarks in the full LTO case.
6121 struct UnassignedCallCloneInfo {
6123 const ContextNode *Source = nullptr;
6124 };
6125 DenseMap<const ContextNode *, std::map<unsigned, UnassignedCallCloneInfo>>
6126 UnassignedCallClones;
6127
6128 // Walk all functions for which we saw calls with memprof metadata, and handle
6129 // cloning for each of its calls.
6130 for (auto &[Func, CallsWithMetadata] : FuncToCallsWithMetadata) {
6131 FuncInfo OrigFunc(Func);
6132 // Map from each clone number of OrigFunc to information about that function
6133 // clone (the function clone FuncInfo and call remappings). The index into
6134 // the vector is the clone number, as function clones are created and
6135 // numbered sequentially.
6136 std::vector<FuncCloneInfo> FuncCloneInfos;
6137 // For each function clone number, the clone number of the function clone
6138 // it was derived from (created to further specialize, when a caller
6139 // already assigned to it needed a different clone of some callsite), or 0
6140 // for clones created from the original function. Used to decide which
6141 // callsite clone the calls not themselves cloned for a function clone
6142 // should follow (see UnassignedCallClones).
6143 std::vector<unsigned> FuncCloneParent;
6144 // Callsite clones required for calls in specific function clones by
6145 // implicit callers (see assignImplicitCallers).
6146 ImplicitSourceMap ImplicitSources;
6147 // Map from each original callsite node in this function to its original
6148 // call and matching calls (clone 0). The calls recorded on an original node
6149 // are updated to the calls in a function clone if that node is assigned to
6150 // a function clone, after which they can no longer be used to index the
6151 // function clone CallMaps (which are keyed by the original calls). No node
6152 // in this function has been assigned yet at this point.
6153 struct OrigCallsInfo {
6154 CallInfo Call;
6155 SmallVector<CallInfo, 0> MatchingCalls;
6156 };
6157 DenseMap<const ContextNode *, OrigCallsInfo> OrigNodeToOrigCalls;
6158 for (auto &Call : CallsWithMetadata) {
6159 ContextNode *Node = getNodeForInst(Call);
6160 if (Node && Node->Call == Call)
6161 OrigNodeToOrigCalls[Node] = {Call, Node->MatchingCalls};
6162 }
6163 for (auto &Call : CallsWithMetadata) {
6164 ContextNode *Node = getNodeForInst(Call);
6165 // Skip call if we do not have a node for it (all uses of its stack ids
6166 // were either on inlined chains or trimmed from the MIBs), or if we did
6167 // not create any clones for it.
6168 if (!Node || Node->Clones.empty())
6169 continue;
6170 assert(Node->hasCall() &&
6171 "Not having a call should have prevented cloning");
6172
6173 // Track the assignment of function clones to clones of the current
6174 // callsite Node being handled.
6175 std::map<FuncInfo, ContextNode *> FuncCloneToCurNodeCloneMap;
6176
6177 // Assign callsite version CallsiteClone to function version FuncClone,
6178 // and also assign (possibly cloned) Call to CallsiteClone.
6179 auto AssignCallsiteCloneToFuncClone = [&](const FuncInfo &FuncClone,
6180 CallInfo &Call,
6181 ContextNode *CallsiteClone,
6182 bool IsAlloc) {
6183 // Record the clone of callsite node assigned to this function clone.
6184 FuncCloneToCurNodeCloneMap[FuncClone] = CallsiteClone;
6185
6186 assert(FuncCloneInfos.size() > FuncClone.cloneNo());
6187 DenseMap<CallInfo, CallInfo> &CallMap =
6188 FuncCloneInfos[FuncClone.cloneNo()].CallMap;
6189 CallInfo CallClone(Call);
6190 if (auto It = CallMap.find(Call); It != CallMap.end())
6191 CallClone = It->second;
6192 CallsiteClone->setCall(CallClone);
6193 // Need to do the same for all matching calls. Rebuild the list of
6194 // CallsiteClone from the original matching calls (it may have been
6195 // copied from a node whose calls were already remapped), using the
6196 // same mapping as for Call.
6197 assert(OrigNodeToOrigCalls.count(Node));
6198 CallsiteClone->MatchingCalls.clear();
6199 for (const CallInfo &OrigMatchingCall :
6200 OrigNodeToOrigCalls[Node].MatchingCalls) {
6201 CallInfo CallClone(OrigMatchingCall);
6202 if (auto It = CallMap.find(OrigMatchingCall); It != CallMap.end())
6203 CallClone = It->second;
6204 CallsiteClone->MatchingCalls.push_back(CallClone);
6205 }
6206 };
6207
6208 // Invokes moveEdgeToNewCalleeClone which creates a new clone, and then
6209 // performs the necessary fixups (removing none type edges, and
6210 // importantly, propagating any function call assignment of the original
6211 // node to the new clone).
6212 auto MoveEdgeToNewCalleeCloneAndSetUp =
6213 [&](const std::shared_ptr<ContextEdge> &Edge) {
6214 ContextNode *OrigCallee = Edge->Callee;
6215 ContextNode *NewClone = moveEdgeToNewCalleeClone(Edge);
6216 removeNoneTypeCalleeEdges(NewClone);
6217 assert(NewClone->AllocTypes != (uint8_t)AllocationType::None);
6218 // If the original Callee was already assigned to call a specific
6219 // function version, make sure its new clone is assigned to call
6220 // that same function clone.
6221 if (CallsiteToCalleeFuncCloneMap.count(OrigCallee))
6222 RecordCalleeFuncOfCallsite(
6223 NewClone, CallsiteToCalleeFuncCloneMap[OrigCallee]);
6224 return NewClone;
6225 };
6226
6227 // Keep track of the clones of callsite Node that need to be assigned to
6228 // function clones. This list may be expanded in the loop body below if we
6229 // find additional cloning is required.
6230 std::deque<ContextNode *> ClonesWorklist;
6231 // Ignore original Node if we moved all of its contexts to clones.
6232 if (!Node->emptyContextIds())
6233 ClonesWorklist.push_back(Node);
6234 llvm::append_range(ClonesWorklist, Node->Clones);
6235
6236 // Now walk through all of the clones of this callsite Node that we need,
6237 // and determine the assignment to a corresponding clone of the current
6238 // function (creating new function clones as needed).
6239 unsigned NodeCloneCount = 0;
6240 while (!ClonesWorklist.empty()) {
6241 ContextNode *Clone = ClonesWorklist.front();
6242 ClonesWorklist.pop_front();
6243 NodeCloneCount++;
6244 if (VerifyNodes)
6246
6247 // Need to create a new function clone if we have more callsite clones
6248 // than existing function clones, which would have been assigned to an
6249 // earlier clone in the list (we assign callsite clones to function
6250 // clones greedily).
6251 if (FuncCloneInfos.size() < NodeCloneCount) {
6252 // If this is the first callsite copy, assign to original function.
6253 if (NodeCloneCount == 1) {
6254 // Since FuncCloneInfos is empty in this case, no clones have
6255 // been created for this function yet, and no callers should have
6256 // been assigned a function clone for this callee node yet.
6258 Clone->CallerEdges, [&](const std::shared_ptr<ContextEdge> &E) {
6259 return CallsiteToCalleeFuncCloneMap.count(E->Caller);
6260 }));
6261 // Initialize with empty call map, assign Clone to original function
6262 // and its callers, and skip to the next clone.
6263 FuncCloneInfos.push_back(
6264 {OrigFunc, DenseMap<CallInfo, CallInfo>()});
6265 FuncCloneParent.push_back(0);
6266 AssignCallsiteCloneToFuncClone(
6267 OrigFunc, Call, Clone,
6268 AllocationCallToContextNodeMap.count(Call));
6269 for (auto &CE : Clone->CallerEdges) {
6270 // Ignore any caller that does not have a recorded callsite Call.
6271 if (!CE->Caller->hasCall())
6272 continue;
6273 RecordCalleeFuncOfCallsite(CE->Caller, OrigFunc);
6274 }
6275 continue;
6276 }
6277
6278 // First locate which copy of OrigFunc to clone again. If a caller
6279 // of this callsite clone was already assigned to call a particular
6280 // function clone, we need to redirect all of those callers to the
6281 // new function clone, and update their other callees within this
6282 // function.
6283 FuncInfo PreviousAssignedFuncClone;
6284 auto EI = llvm::find_if(
6285 Clone->CallerEdges, [&](const std::shared_ptr<ContextEdge> &E) {
6286 return CallsiteToCalleeFuncCloneMap.count(E->Caller);
6287 });
6288 bool CallerAssignedToCloneOfFunc = false;
6289 if (EI != Clone->CallerEdges.end()) {
6290 const std::shared_ptr<ContextEdge> &Edge = *EI;
6291 PreviousAssignedFuncClone =
6292 CallsiteToCalleeFuncCloneMap[Edge->Caller];
6293 CallerAssignedToCloneOfFunc = true;
6294 }
6295
6296 // Clone function and save it along with the CallInfo map created
6297 // during cloning in the FuncCloneInfos.
6298 DenseMap<CallInfo, CallInfo> NewCallMap;
6299 unsigned CloneNo = FuncCloneInfos.size();
6300 assert(CloneNo > 0 && "Clone 0 is the original function, which "
6301 "should already exist in the map");
6302 FuncInfo NewFuncClone = cloneFunctionForCallsite(
6303 OrigFunc, Call, NewCallMap, CallsWithMetadata, CloneNo);
6304 FuncCloneInfos.push_back({NewFuncClone, std::move(NewCallMap)});
6305 // Record the function clone it is derived from (if any), whose
6306 // callers it will take over below.
6307 FuncCloneParent.push_back(CallerAssignedToCloneOfFunc
6308 ? PreviousAssignedFuncClone.cloneNo()
6309 : 0);
6310 FunctionClonesAnalysis++;
6311 Changed = true;
6312
6313 // If no caller callsites were already assigned to a clone of this
6314 // function, we can simply assign this clone to the new func clone
6315 // and update all callers to it, then skip to the next clone.
6316 if (!CallerAssignedToCloneOfFunc) {
6317 AssignCallsiteCloneToFuncClone(
6318 NewFuncClone, Call, Clone,
6319 AllocationCallToContextNodeMap.count(Call));
6320 for (auto &CE : Clone->CallerEdges) {
6321 // Ignore any caller that does not have a recorded callsite Call.
6322 if (!CE->Caller->hasCall())
6323 continue;
6324 RecordCalleeFuncOfCallsite(CE->Caller, NewFuncClone);
6325 }
6326 continue;
6327 }
6328
6329 // We may need to do additional node cloning in this case.
6330 // Reset the CallsiteToCalleeFuncCloneMap entry for any callers
6331 // that were previously assigned to call PreviousAssignedFuncClone,
6332 // to record that they now call NewFuncClone.
6333 // The none type edge removal may remove some of this Clone's caller
6334 // edges, if it is reached via another of its caller's callees.
6335 // Iterate over a copy and skip any that were removed.
6336 auto CallerEdges = Clone->CallerEdges;
6337 for (auto CE : CallerEdges) {
6338 // Skip any that have been removed on an earlier iteration.
6339 if (CE->isRemoved()) {
6340 assert(!is_contained(Clone->CallerEdges, CE));
6341 continue;
6342 }
6343 assert(CE);
6344 // Ignore any caller that does not have a recorded callsite Call.
6345 if (!CE->Caller->hasCall())
6346 continue;
6347
6348 if (!CallsiteToCalleeFuncCloneMap.count(CE->Caller) ||
6349 // We subsequently fall through to later handling that
6350 // will perform any additional cloning required for
6351 // callers that were calling other function clones.
6352 CallsiteToCalleeFuncCloneMap[CE->Caller] !=
6353 PreviousAssignedFuncClone)
6354 continue;
6355
6356 RecordCalleeFuncOfCallsite(CE->Caller, NewFuncClone);
6357
6358 // If we are cloning a function that was already assigned to some
6359 // callers, then essentially we are creating new callsite clones
6360 // of the other callsites in that function that are reached by those
6361 // callers. Clone the other callees of the current callsite's caller
6362 // that were already assigned to PreviousAssignedFuncClone
6363 // accordingly. This is important since we subsequently update the
6364 // calls from the nodes in the graph and their assignments to callee
6365 // functions recorded in CallsiteToCalleeFuncCloneMap.
6366 // The none type edge removal may remove some of this caller's
6367 // callee edges, if it is reached via another of its callees.
6368 // Iterate over a copy and skip any that were removed.
6369 auto CalleeEdges = CE->Caller->CalleeEdges;
6370 for (auto CalleeEdge : CalleeEdges) {
6371 // Skip any that have been removed on an earlier iteration when
6372 // cleaning up newly None type callee edges.
6373 if (CalleeEdge->isRemoved()) {
6374 assert(!is_contained(CE->Caller->CalleeEdges, CalleeEdge));
6375 continue;
6376 }
6377 assert(CalleeEdge);
6378 ContextNode *Callee = CalleeEdge->Callee;
6379 // Skip the current callsite, we are looking for other
6380 // callsites Caller calls, as well as any that does not have a
6381 // recorded callsite Call.
6382 if (Callee == Clone || !Callee->hasCall())
6383 continue;
6384 // Skip direct recursive calls. We don't need/want to clone the
6385 // caller node again, and this loop will not behave as expected if
6386 // we tried.
6387 if (Callee == CalleeEdge->Caller)
6388 continue;
6389 ContextNode *NewClone =
6390 MoveEdgeToNewCalleeCloneAndSetUp(CalleeEdge);
6391 // Moving the edge may have resulted in some none type
6392 // callee edges on the original Callee.
6393 removeNoneTypeCalleeEdges(Callee);
6394 // Update NewClone with the new Call clone of this callsite's Call
6395 // created for the new function clone created earlier.
6396 // Recall that we have already ensured when building the graph
6397 // that each caller can only call callsites within the same
6398 // function, so we are guaranteed that Callee Call is in the
6399 // current OrigFunc.
6400 // CallMap is set up as indexed by original calls at clone 0. Note
6401 // that we cannot simply use the calls recorded on Callee's
6402 // original node, as those are updated to the calls in a function
6403 // clone if the original node was itself assigned to a function
6404 // clone.
6405 assert(OrigNodeToOrigCalls.count(Callee->getOrigNode()));
6406 auto &OrigCalls = OrigNodeToOrigCalls[Callee->getOrigNode()];
6407 DenseMap<CallInfo, CallInfo> &CallMap =
6408 FuncCloneInfos[NewFuncClone.cloneNo()].CallMap;
6409 assert(CallMap.count(OrigCalls.Call));
6410 CallInfo NewCall(CallMap[OrigCalls.Call]);
6411 assert(NewCall);
6412 NewClone->setCall(NewCall);
6413 // Need to do the same for all matching calls.
6414 NewClone->MatchingCalls.clear();
6415 for (auto &OrigMatchingCall : OrigCalls.MatchingCalls) {
6416 assert(CallMap.count(OrigMatchingCall));
6417 CallInfo NewCall(CallMap[OrigMatchingCall]);
6418 assert(NewCall);
6419 NewClone->MatchingCalls.push_back(NewCall);
6420 }
6421 }
6422 }
6423 // Fall through to handling below to perform the recording of the
6424 // function for this callsite clone. This enables handling of cases
6425 // where the callers were assigned to different clones of a function.
6426 }
6427
6428 auto FindFirstAvailFuncClone = [&]() {
6429 // Find first function in FuncCloneInfos without an assigned
6430 // clone of this callsite Node. We should always have one
6431 // available at this point due to the earlier cloning when the
6432 // FuncCloneInfos size was smaller than the clone number.
6433 for (auto &CF : FuncCloneInfos) {
6434 if (!FuncCloneToCurNodeCloneMap.count(CF.FuncClone))
6435 return CF.FuncClone;
6436 }
6438 "Expected an available func clone for this callsite clone");
6439 };
6440
6441 // See if we can use existing function clone. Walk through
6442 // all caller edges to see if any have already been assigned to
6443 // a clone of this callsite's function. If we can use it, do so. If not,
6444 // because that function clone is already assigned to a different clone
6445 // of this callsite, then we need to clone again.
6446 // Basically, this checking is needed to handle the case where different
6447 // caller functions/callsites may need versions of this function
6448 // containing different mixes of callsite clones across the different
6449 // callsites within the function. If that happens, we need to create
6450 // additional function clones to handle the various combinations.
6451 //
6452 // Keep track of any new clones of this callsite created by the
6453 // following loop, as well as any existing clone that we decided to
6454 // assign this clone to.
6455 std::map<FuncInfo, ContextNode *> FuncCloneToNewCallsiteCloneMap;
6456 FuncInfo FuncCloneAssignedToCurCallsiteClone;
6457 // Iterate over a copy of Clone's caller edges, since we may need to
6458 // remove edges in the moveEdgeTo* methods, and this simplifies the
6459 // handling and makes it less error-prone.
6460 auto CloneCallerEdges = Clone->CallerEdges;
6461 for (auto &Edge : CloneCallerEdges) {
6462 // Skip removed edges (due to direct recursive edges updated when
6463 // updating callee edges when moving an edge and subsequently
6464 // removed by call to removeNoneTypeCalleeEdges on the Clone).
6465 if (Edge->isRemoved())
6466 continue;
6467 // Ignore any caller that does not have a recorded callsite Call.
6468 if (!Edge->Caller->hasCall())
6469 continue;
6470 // If this caller already assigned to call a version of OrigFunc, need
6471 // to ensure we can assign this callsite clone to that function clone.
6472 if (CallsiteToCalleeFuncCloneMap.count(Edge->Caller)) {
6473 FuncInfo FuncCloneCalledByCaller =
6474 CallsiteToCalleeFuncCloneMap[Edge->Caller];
6475 // First we need to confirm that this function clone is available
6476 // for use by this callsite node clone.
6477 //
6478 // While FuncCloneToCurNodeCloneMap is built only for this Node and
6479 // its callsite clones, one of those callsite clones X could have
6480 // been assigned to the same function clone called by Edge's caller
6481 // - if Edge's caller calls another callsite within Node's original
6482 // function, and that callsite has another caller reaching clone X.
6483 // We need to clone Node again in this case.
6484 if ((FuncCloneToCurNodeCloneMap.count(FuncCloneCalledByCaller) &&
6485 FuncCloneToCurNodeCloneMap[FuncCloneCalledByCaller] !=
6486 Clone) ||
6487 // Detect when we have multiple callers of this callsite that
6488 // have already been assigned to specific, and different, clones
6489 // of OrigFunc (due to other unrelated callsites in Func they
6490 // reach via call contexts). Is this Clone of callsite Node
6491 // assigned to a different clone of OrigFunc? If so, clone Node
6492 // again.
6493 (FuncCloneAssignedToCurCallsiteClone &&
6494 FuncCloneAssignedToCurCallsiteClone !=
6495 FuncCloneCalledByCaller)) {
6496 // We need to use a different newly created callsite clone, in
6497 // order to assign it to another new function clone on a
6498 // subsequent iteration over the Clones array (adjusted below).
6499 // Note we specifically do not reset the
6500 // CallsiteToCalleeFuncCloneMap entry for this caller, so that
6501 // when this new clone is processed later we know which version of
6502 // the function to copy (so that other callsite clones we have
6503 // assigned to that function clone are properly cloned over). See
6504 // comments in the function cloning handling earlier.
6505
6506 // Check if we already have cloned this callsite again while
6507 // walking through caller edges, for a caller calling the same
6508 // function clone. If so, we can move this edge to that new clone
6509 // rather than creating yet another new clone.
6510 if (FuncCloneToNewCallsiteCloneMap.count(
6511 FuncCloneCalledByCaller)) {
6512 ContextNode *NewClone =
6513 FuncCloneToNewCallsiteCloneMap[FuncCloneCalledByCaller];
6514 moveEdgeToExistingCalleeClone(Edge, NewClone);
6515 // Cleanup any none type edges cloned over.
6516 removeNoneTypeCalleeEdges(NewClone);
6517 } else {
6518 // Create a new callsite clone.
6519 ContextNode *NewClone = MoveEdgeToNewCalleeCloneAndSetUp(Edge);
6520 FuncCloneToNewCallsiteCloneMap[FuncCloneCalledByCaller] =
6521 NewClone;
6522 // Add to list of clones and process later.
6523 ClonesWorklist.push_back(NewClone);
6524 }
6525 // Moving the caller edge may have resulted in some none type
6526 // callee edges.
6527 removeNoneTypeCalleeEdges(Clone);
6528 // We will handle the newly created callsite clone in a subsequent
6529 // iteration over this Node's Clones.
6530 continue;
6531 }
6532
6533 // Otherwise, we can use the function clone already assigned to this
6534 // caller.
6535 if (!FuncCloneAssignedToCurCallsiteClone) {
6536 FuncCloneAssignedToCurCallsiteClone = FuncCloneCalledByCaller;
6537 // Assign Clone to FuncCloneCalledByCaller
6538 AssignCallsiteCloneToFuncClone(
6539 FuncCloneCalledByCaller, Call, Clone,
6540 AllocationCallToContextNodeMap.count(Call));
6541 } else
6542 // Don't need to do anything - callsite is already calling this
6543 // function clone.
6544 assert(FuncCloneAssignedToCurCallsiteClone ==
6545 FuncCloneCalledByCaller);
6546
6547 } else {
6548 // We have not already assigned this caller to a version of
6549 // OrigFunc. Do the assignment now.
6550
6551 // First check if we have already assigned this callsite clone to a
6552 // clone of OrigFunc for another caller during this iteration over
6553 // its caller edges.
6554 if (!FuncCloneAssignedToCurCallsiteClone) {
6555 FuncCloneAssignedToCurCallsiteClone = FindFirstAvailFuncClone();
6556 assert(FuncCloneAssignedToCurCallsiteClone);
6557 // Assign Clone to FuncCloneAssignedToCurCallsiteClone
6558 AssignCallsiteCloneToFuncClone(
6559 FuncCloneAssignedToCurCallsiteClone, Call, Clone,
6560 AllocationCallToContextNodeMap.count(Call));
6561 } else
6562 assert(FuncCloneToCurNodeCloneMap
6563 [FuncCloneAssignedToCurCallsiteClone] == Clone);
6564 // Update callers to record function version called.
6565 RecordCalleeFuncOfCallsite(Edge->Caller,
6566 FuncCloneAssignedToCurCallsiteClone);
6567 }
6568 }
6569 // If we didn't assign a function clone to this callsite clone yet, e.g.
6570 // none of its callers has a non-null call, do the assignment here.
6571 // We want to ensure that every callsite clone is assigned to some
6572 // function clone, so that the call updates below work as expected.
6573 // In particular if this is the original callsite, we want to ensure it
6574 // is assigned to the original function, otherwise the original function
6575 // will appear available for assignment to other callsite clones,
6576 // leading to unintended effects. For one, the unknown and not updated
6577 // callers will call into cloned paths leading to the wrong hints,
6578 // because they still call the original function (clone 0). Also,
6579 // because all callsites start out as being clone 0 by default, we can't
6580 // easily distinguish between callsites explicitly assigned to clone 0
6581 // vs those never assigned, which can lead to multiple updates of the
6582 // calls when invoking updateCall below, with mismatched clone values.
6583 // TODO: Add a flag to the callsite nodes or some other mechanism to
6584 // better distinguish and identify callsite clones that are not getting
6585 // assigned to function clones as expected.
6586 if (!FuncCloneAssignedToCurCallsiteClone) {
6587 FuncCloneAssignedToCurCallsiteClone = FindFirstAvailFuncClone();
6588 assert(FuncCloneAssignedToCurCallsiteClone &&
6589 "No available func clone for this callsite clone");
6590 AssignCallsiteCloneToFuncClone(
6591 FuncCloneAssignedToCurCallsiteClone, Call, Clone,
6592 /*IsAlloc=*/AllocationCallToContextNodeMap.contains(Call));
6593 }
6594 }
6595 assignImplicitCallers(Node, TrimmedContextIds,
6596 CallsiteToCalleeFuncCloneMap,
6597 FuncCloneToCurNodeCloneMap, ImplicitSources);
6598 if (VerifyCCG) {
6600 for (const auto &PE : Node->CalleeEdges)
6602 for (const auto &CE : Node->CallerEdges)
6604 for (auto *Clone : Node->Clones) {
6606 for (const auto &PE : Clone->CalleeEdges)
6608 for (const auto &CE : Clone->CallerEdges)
6610 }
6611 }
6612 }
6613
6615 FuncToNumClones[Func] = std::max<unsigned>(1, FuncCloneInfos.size());
6616 if (FuncCloneInfos.size() < 2)
6617 continue;
6618
6619 // In this case there is more than just the original function copy.
6620 // Record call clones of any callsite nodes in the function that did not
6621 // themselves get cloned for all of the function clones.
6622 for (auto &Call : CallsWithMetadata) {
6623 ContextNode *Node = getNodeForInst(Call);
6624 if (!Node || !Node->hasCall() || Node->emptyContextIds())
6625 continue;
6626 // First collect the function clones that have a clone of this callsite
6627 // node (or the node itself). They may not be sequential due to empty
6628 // clones e.g.
6629 DenseMap<unsigned, const ContextNode *> FuncCloneToNodeClone;
6630 FuncCloneToNodeClone[Node->Call.cloneNo()] = Node;
6631 // A clone left without context ids (its callers were all moved to other
6632 // clones while assigning function clones) has no edges and is not
6633 // reached by the call updates below, so the call in its function clone
6634 // keeps the original callee. If an implicit caller recorded the callsite
6635 // clone that call must follow, treat the function clone as having no
6636 // clone of this node, so that the call follows the recorded source like
6637 // any other call not cloned for its function clone.
6638 for (auto *C : Node->Clones)
6639 if (!C->emptyContextIds() ||
6640 !ImplicitSources.contains({Node, C->Call.cloneNo()}))
6641 FuncCloneToNodeClone[C->Call.cloneNo()] = C;
6642 // If Node has enough clones already to cover all function clones, we can
6643 // skip it.
6644 if (FuncCloneToNodeClone.size() >= FuncCloneInfos.size())
6645 continue;
6646 // Now check all the function clones.
6647 for (auto &FC : FuncCloneInfos) {
6648 unsigned I = FC.FuncClone.cloneNo();
6649 // Skip the first clone which got the original call.
6650 // Also skip any other clones created for this Node.
6651 if (I == 0 || FuncCloneToNodeClone.contains(I)) {
6652 continue;
6653 }
6654 // Record the call clones created for this callsite in this function
6655 // clone, and the callsite node whose callee they should follow: the
6656 // one required by an implicit caller, if any, else the clone of this
6657 // callsite node in the nearest function clone in the chain of function
6658 // clones this one was derived from (null for the original node).
6659 auto &Info = UnassignedCallClones[Node][I];
6660 auto &CallVector = Info.Calls;
6661 for (unsigned P = I;; P = FuncCloneParent[P]) {
6662 if (auto It = ImplicitSources.find({Node, P});
6663 It != ImplicitSources.end()) {
6664 Info.Source = resolveImplicitSource(Node, It->second);
6665 break;
6666 }
6667 if (auto It = FuncCloneToNodeClone.find(P);
6668 It != FuncCloneToNodeClone.end()) {
6669 Info.Source = It->second;
6670 break;
6671 }
6672 if (P == 0)
6673 break;
6674 }
6675 DenseMap<CallInfo, CallInfo> &CallMap = FC.CallMap;
6676 if (auto It = CallMap.find(Call); It != CallMap.end()) {
6677 CallInfo CallClone = It->second;
6678 CallVector.push_back(CallClone);
6679 } else {
6680 // All but the original clone (skipped earlier) should have an entry
6681 // for all calls.
6682 assert(false && "Expected to find call in CallMap");
6683 }
6684 // Need to do the same for all matching calls. Use the original
6685 // matching calls, as those on Node may have been remapped to the calls
6686 // of a function clone if Node was assigned to one.
6687 assert(OrigNodeToOrigCalls.count(Node));
6688 for (const CallInfo &OrigMatchingCall :
6689 OrigNodeToOrigCalls[Node].MatchingCalls) {
6690 if (auto It = CallMap.find(OrigMatchingCall); It != CallMap.end()) {
6691 CallInfo CallClone = It->second;
6692 CallVector.push_back(CallClone);
6693 } else {
6694 // All but the original clone (skipped earlier) should have an entry
6695 // for all calls.
6696 assert(false && "Expected to find call in CallMap");
6697 }
6698 }
6699 }
6700 }
6701 }
6702
6703 uint8_t BothTypes =
6704 (uint8_t)AllocationType::Cold | (uint8_t)AllocationType::NotCold;
6705
6706 auto UpdateCalls = [&](ContextNode *Node,
6707 DenseSet<const ContextNode *> &Visited,
6708 auto &&UpdateCalls) {
6709 auto Inserted = Visited.insert(Node);
6710 if (!Inserted.second)
6711 return;
6712
6713 for (auto *Clone : Node->Clones)
6714 UpdateCalls(Clone, Visited, UpdateCalls);
6715
6716 for (auto &Edge : Node->CallerEdges)
6717 UpdateCalls(Edge->Caller, Visited, UpdateCalls);
6718
6719 // Skip if either no call to update, or if we ended up with no context ids
6720 // (we moved all edges onto other clones).
6721 if (!Node->hasCall() || Node->emptyContextIds())
6722 return;
6723
6724 // Returns the recorded call clones in function clones which don't have a
6725 // clone of this callsite node, along with the node whose decision each
6726 // should follow (the Source recorded earlier, falling back to Node).
6727 auto GetUnassignedCallClones = [&](ContextNode *Node) {
6729 Unassigned;
6730 auto UnassignedIt = UnassignedCallClones.find(Node);
6731 if (UnassignedIt == UnassignedCallClones.end())
6732 return Unassigned;
6733 DenseSet<unsigned> NodeCallClones;
6734 for (auto *C : Node->Clones)
6735 if (!C->emptyContextIds())
6736 NodeCallClones.insert(C->Call.cloneNo());
6737 for (auto &[CloneNo, Info] : UnassignedIt->second) {
6738 // Should start at 1 as we never create an entry for original node.
6739 assert(CloneNo > 0);
6740 // If we subsequently created a clone, skip this one.
6741 if (NodeCallClones.contains(CloneNo))
6742 continue;
6743 Unassigned.push_back({Info.Source ? Info.Source : Node, Info.Calls});
6744 }
6745 return Unassigned;
6746 };
6747
6748 if (Node->IsAllocation) {
6749 auto GetAllocTypeToUse = [&](const ContextNode *Node) {
6750 auto AT = allocTypeToUse(Node->AllocTypes);
6751 // If the allocation type is still ambiguous after cloning, record
6752 // that as the allocation type (the attribute stays "ambiguous") rather
6753 // than falling back to the default not cold, unless ambiguous
6754 // attributes are disabled.
6755 if (Node->AllocTypes == BothTypes && MemProfUseAmbiguousAttributes)
6756 AT = (AllocationType)BothTypes;
6757 // If the allocation type is ambiguous, and more aggressive hinting
6758 // has been enabled via the MinClonedColdBytePercent flag, see if this
6759 // allocation should be hinted cold anyway because its fraction cold
6760 // bytes allocated is at least the given threshold.
6761 if (Node->AllocTypes == BothTypes && MinClonedColdBytePercent < 100 &&
6762 !ContextIdToContextSizeInfos.empty()) {
6763 uint64_t TotalCold = 0;
6764 uint64_t Total = 0;
6765 for (auto Id : Node->getContextIds()) {
6766 auto TypeI = ContextIdToAllocationType.find(Id);
6767 assert(TypeI != ContextIdToAllocationType.end());
6768 auto CSI = ContextIdToContextSizeInfos.find(Id);
6769 if (CSI != ContextIdToContextSizeInfos.end()) {
6770 for (auto &Info : CSI->second) {
6771 Total += Info.TotalSize;
6772 if (TypeI->second == AllocationType::Cold)
6773 TotalCold += Info.TotalSize;
6774 }
6775 }
6776 }
6777 if (TotalCold * 100 >= Total * MinClonedColdBytePercent)
6778 AT = AllocationType::Cold;
6779 }
6780 return AT;
6781 };
6782 updateAllocationCall(Node->Call, GetAllocTypeToUse(Node));
6784 OrigAllocAndCloneToNode[{Node->getOrigNode(), Node->Call.cloneNo()}] =
6785 Node;
6786 assert(Node->MatchingCalls.empty());
6787 // Update the allocation calls in function clones without a clone of this
6788 // node the same way as the node they should follow.
6789 for (auto &[Source, Calls] : GetUnassignedCallClones(Node)) {
6790 auto AT = GetAllocTypeToUse(Source);
6791 for (auto &Call : Calls) {
6792 updateAllocationCall(Call, AT);
6793 // For the report, such an allocation clone follows Source's hint.
6795 OrigAllocAndCloneToNode[{Node->getOrigNode(), Call.cloneNo()}] =
6796 Source;
6797 }
6798 }
6799 return;
6800 }
6801
6802 auto CalleeFuncIt = CallsiteToCalleeFuncCloneMap.find(Node);
6803 if (CalleeFuncIt != CallsiteToCalleeFuncCloneMap.end()) {
6804 auto &CalleeFunc = CalleeFuncIt->second;
6805 updateCall(Node->Call, CalleeFunc);
6807 OrigNodeAndCloneToCalleeClone[{
6808 Node->getOrigNode(), Node->Call.cloneNo()}] = CalleeFunc.cloneNo();
6809 // Update all the matching calls as well.
6810 for (auto &Call : Node->MatchingCalls)
6811 updateCall(Call, CalleeFunc);
6812 }
6813
6814 // Now update all calls recorded earlier that are still in function clones
6815 // which don't have a clone of this callsite node, the same way as the
6816 // node they should follow.
6817 for (auto &[Source, Calls] : GetUnassignedCallClones(Node)) {
6818 auto It = CallsiteToCalleeFuncCloneMap.find(Source);
6819 // Fall back to the original Node's callee function.
6820 if (It == CallsiteToCalleeFuncCloneMap.end())
6821 It = CalleeFuncIt;
6822 if (It == CallsiteToCalleeFuncCloneMap.end())
6823 continue;
6824 for (auto &Call : Calls) {
6825 updateCall(Call, It->second);
6827 OrigNodeAndCloneToCalleeClone[{Node->getOrigNode(), Call.cloneNo()}] =
6828 It->second.cloneNo();
6829 }
6830 }
6831 };
6832
6833 // Performs DFS traversal starting from allocation nodes to update calls to
6834 // reflect cloning decisions recorded earlier. For regular LTO this will
6835 // update the actual calls in the IR to call the appropriate function clone
6836 // (and add attributes to allocation calls), whereas for ThinLTO the decisions
6837 // are recorded in the summary entries.
6838 DenseSet<const ContextNode *> Visited;
6839 for (auto &Entry : AllocationCallToContextNodeMap)
6840 UpdateCalls(Entry.second, Visited, UpdateCalls);
6841
6842 return Changed;
6843}
6844
6845// Compute a SHA1 hash of the callsite and alloc version information of clone I
6846// in the summary, to use in detection of duplicate clones.
6848 SHA1 Hasher;
6849 // Add a record for the callsite or alloc at the given index in the summary,
6850 // with the given version. The index must be included, since two clones that
6851 // assign the same versions to different callsites or allocs are not
6852 // duplicates.
6853 auto AddRecord = [&](char Kind, uint32_t Index, uint32_t Version) {
6854 uint8_t Data[1 + 2 * sizeof(uint32_t)];
6855 Data[0] = Kind;
6856 support::endian::write32le(&Data[1], Index);
6857 support::endian::write32le(&Data[1 + sizeof(uint32_t)], Version);
6858 Hasher.update(Data);
6859 };
6860 // Update hash with any callsites that call non-default (non-zero) callee
6861 // versions.
6862 for (auto [Index, SN] : enumerate(FS->callsites())) {
6863 // In theory all callsites and allocs in this function should have the same
6864 // number of clone entries, but handle any discrepancies gracefully below
6865 // for NDEBUG builds.
6866 assert(
6867 SN.Clones.size() > I &&
6868 "Callsite summary has fewer entries than other summaries in function");
6869 if (SN.Clones.size() <= I || !SN.Clones[I])
6870 continue;
6871 AddRecord('c', Index, SN.Clones[I]);
6872 }
6873 // Update hash with any allocs that have non-default (non-None) hints.
6874 for (auto [Index, AN] : enumerate(FS->allocs())) {
6875 // In theory all callsites and allocs in this function should have the same
6876 // number of clone entries, but handle any discrepancies gracefully below
6877 // for NDEBUG builds.
6878 assert(AN.Versions.size() > I &&
6879 "Alloc summary has fewer entries than other summaries in function");
6880 if (AN.Versions.size() <= I ||
6881 (AllocationType)AN.Versions[I] == AllocationType::None)
6882 continue;
6883 AddRecord('a', Index, AN.Versions[I]);
6884 }
6885 return support::endian::read64le(Hasher.result().data());
6886}
6887
6888#ifndef NDEBUG
6889// Check that clones I and J of FS have identical callsite and alloc versions.
6890static bool clonesHaveSameVersions(const FunctionSummary *FS, unsigned I,
6891 unsigned J) {
6892 for (auto &SN : FS->callsites()) {
6893 assert(SN.Clones.size() > I && SN.Clones.size() > J &&
6894 "Callsite summary has fewer entries than other summaries in "
6895 "function");
6896 if (SN.Clones[I] != SN.Clones[J])
6897 return false;
6898 }
6899 for (auto &AN : FS->allocs()) {
6900 assert(AN.Versions.size() > I && AN.Versions.size() > J &&
6901 "Alloc summary has fewer entries than other summaries in function");
6902 if (AN.Versions[I] != AN.Versions[J])
6903 return false;
6904 }
6905 return true;
6906}
6907#endif
6908
6910 Function &F, unsigned NumClones, Module &M, OptimizationRemarkEmitter &ORE,
6912 &FuncToAliasMap,
6913 FunctionSummary *FS) {
6914 auto TakeDeclNameAndReplace = [](GlobalValue *DeclGV, GlobalValue *NewGV) {
6915 // We might have created this when adjusting callsite in another
6916 // function. It should be a declaration.
6917 assert(DeclGV->isDeclaration());
6918 NewGV->takeName(DeclGV);
6919 DeclGV->replaceAllUsesWith(NewGV);
6920 DeclGV->eraseFromParent();
6921 };
6922
6923 // Handle aliases to this function, and create analogous alias clones to the
6924 // provided clone of this function.
6925 auto CloneFuncAliases = [&](Function *NewF, unsigned I) {
6926 if (!FuncToAliasMap.count(&F))
6927 return;
6928 for (auto *A : FuncToAliasMap[&F]) {
6929 std::string AliasName = getMemProfFuncName(A->getName(), I);
6930 auto *PrevA = M.getNamedAlias(AliasName);
6931 auto *NewA = GlobalAlias::create(A->getValueType(),
6932 A->getType()->getPointerAddressSpace(),
6933 A->getLinkage(), AliasName, NewF);
6934 NewA->copyAttributesFrom(A);
6935 if (PrevA)
6936 TakeDeclNameAndReplace(PrevA, NewA);
6937 }
6938 };
6939
6940 // The first "clone" is the original copy, we should only call this if we
6941 // needed to create new clones.
6942 assert(NumClones > 1);
6944 VMaps.reserve(NumClones - 1);
6945 FunctionsClonedThinBackend++;
6946
6947 // Map of hash of callsite/alloc versions to the instantiated function clone
6948 // (possibly the original) implementing those calls and its clone number. Used
6949 // to avoid instantiating duplicate function clones.
6950 // FIXME: Ideally the thin link would not generate such duplicate clones to
6951 // start with, but right now it happens due to phase ordering in the function
6952 // assignment and possible new clones that produces. We simply make each
6953 // duplicate an alias to the matching instantiated clone recorded in the map
6954 // (except for available_externally which are made declarations as they would
6955 // be aliases in the prevailing module, and available_externally aliases are
6956 // not well supported right now).
6958
6959 // Save the hash of the original function version.
6960 HashToFunc[ComputeHash(FS, 0)] = {&F, 0};
6961
6962 for (unsigned I = 1; I < NumClones; I++) {
6963 VMaps.emplace_back(std::make_unique<ValueToValueMapTy>());
6964 std::string Name = getMemProfFuncName(F.getName(), I);
6965 auto Hash = ComputeHash(FS, I);
6966 // If this clone would duplicate a previously seen clone, don't generate the
6967 // duplicate clone body, just make an alias to satisfy any (potentially
6968 // cross-module) references.
6969 if (auto It = HashToFunc.find(Hash); It != HashToFunc.end()) {
6970 FunctionCloneDuplicatesThinBackend++;
6971 auto [Func, OrigI] = It->second;
6972 assert(clonesHaveSameVersions(FS, I, OrigI) &&
6973 "Duplicate clone hash collision");
6974 if (Func->hasAvailableExternallyLinkage()) {
6975 // Skip these as EliminateAvailableExternallyPass does not handle
6976 // available_externally aliases correctly and we end up with an
6977 // available_externally alias to a declaration. Just create a
6978 // declaration for now as we know we will have a definition in another
6979 // module.
6980 auto Decl = M.getOrInsertFunction(Name, Func->getFunctionType());
6981 ORE.emit(OptimizationRemark(DEBUG_TYPE, "MemprofClone", &F)
6982 << "created clone decl " << ore::NV("Decl", Decl.getCallee()));
6983 continue;
6984 }
6985 auto *PrevF = M.getFunction(Name);
6986 auto *Alias = GlobalAlias::create(Name, Func);
6987 if (PrevF)
6988 TakeDeclNameAndReplace(PrevF, Alias);
6989 ORE.emit(OptimizationRemark(DEBUG_TYPE, "MemprofClone", &F)
6990 << "created clone alias " << ore::NV("Alias", Alias));
6991
6992 // Now handle aliases to this function, and clone those as well.
6993 CloneFuncAliases(Func, I);
6994 continue;
6995 }
6996 auto *NewF = CloneFunction(&F, *VMaps.back());
6997 HashToFunc[Hash] = {NewF, I};
6998 FunctionClonesThinBackend++;
6999 // Strip !kcfi_type (see comment in cloneFunctionForCallsite).
7000 NewF->setMetadata(LLVMContext::MD_kcfi_type, nullptr);
7001 // Strip memprof and callsite metadata from clone as they are no longer
7002 // needed.
7003 for (auto &BB : *NewF) {
7004 for (auto &Inst : BB) {
7005 Inst.setMetadata(LLVMContext::MD_memprof, nullptr);
7006 Inst.setMetadata(LLVMContext::MD_callsite, nullptr);
7007 }
7008 }
7009 auto *PrevF = M.getFunction(Name);
7010 if (PrevF)
7011 TakeDeclNameAndReplace(PrevF, NewF);
7012 else
7013 NewF->setName(Name);
7014 updateSubprogramLinkageName(NewF, Name);
7015 ORE.emit(OptimizationRemark(DEBUG_TYPE, "MemprofClone", &F)
7016 << "created clone " << ore::NV("NewFunction", NewF));
7017
7018 // Now handle aliases to this function, and clone those as well.
7019 CloneFuncAliases(NewF, I);
7020 }
7021 return VMaps;
7022}
7023
7024// If F is an imported copy of an alias (materialized as a Function by
7025// replaceAliasWithAliasee during FunctionImport while preserving the aliasee's
7026// !guid metadata for body-level passes), return the original alias GUID
7027// recorded in its thinlto_alias_guid metadata.
7028static std::optional<GlobalValue::GUID>
7030 if (auto *MD = F.getMetadata("thinlto_alias_guid"))
7031 return mdconst::extract<ConstantInt>(MD->getOperand(0))->getZExtValue();
7032 return std::nullopt;
7033}
7034
7035// Locate the summary for GV. This is complicated by the fact that it might
7036// have been internalized, promoted, or imported as a copy of an alias.
7038 const ModuleSummaryIndex *ImportSummary,
7039 const Function *CallingFunc = nullptr) {
7040 if (const auto *F = dyn_cast<Function>(&GV))
7041 if (auto AliasGUID = getImportedAliasGUID(*F)) {
7042 ValueInfo AliasVI = ImportSummary->getValueInfo(*AliasGUID);
7043 assert(AliasVI && "Expected ValueInfo for imported alias GUID");
7044 return AliasVI;
7045 }
7046
7047 // FIXME: Ideally we would retain the original GUID in some fashion on the
7048 // function (e.g. as metadata), but for now do our best to locate the
7049 // summary without that information.
7050 ValueInfo TheFnVI = ImportSummary->getValueInfo(GV.getGUID());
7051 if (!TheFnVI)
7052 // See if GV was internalized, by checking index directly with
7053 // original name (this avoids the name adjustment done by getGUID() for
7054 // internal symbols).
7055 TheFnVI = ImportSummary->getValueInfo(
7057 if (TheFnVI)
7058 return TheFnVI;
7059 // Now query with the original name before any promotion was performed.
7060 StringRef OrigName =
7062 // When this pass is enabled, we always add thinlto_src_file provenance
7063 // metadata to imported function definitions, which allows us to recreate the
7064 // original internal symbol's GUID.
7065 const auto *GO = dyn_cast<GlobalObject>(&GV);
7066 MDNode *SrcFileMD = GO ? GO->getMetadata("thinlto_src_file") : nullptr;
7067 // If this is a call to an imported/promoted local for which we didn't import
7068 // the definition, the metadata will not exist on the declaration. However,
7069 // since we are doing this early, before any inlining in the LTO backend, we
7070 // can simply look at the metadata on the calling function which must have
7071 // been from the same module if GV was an internal symbol originally.
7072 if (!SrcFileMD && GV.isDeclaration()) {
7073 // We would only call this for a declaration for a direct callsite, in which
7074 // case the caller would have provided the calling function pointer.
7075 assert(CallingFunc);
7076 SrcFileMD = CallingFunc->getMetadata("thinlto_src_file");
7077 // If this is a promoted local (OrigName != GV.getName()), since this is a
7078 // declaration, it must be imported from a different module and therefore we
7079 // should always find the metadata on its calling function. Any call to a
7080 // promoted local that came from this module should still be a definition.
7081 assert(SrcFileMD || OrigName == GV.getName());
7082 }
7083 StringRef SrcFile = M.getSourceFileName();
7084 if (SrcFileMD)
7085 SrcFile = dyn_cast<MDString>(SrcFileMD->getOperand(0))->getString();
7086 std::string OrigId = GlobalValue::getGlobalIdentifier(
7087 OrigName, GlobalValue::InternalLinkage, SrcFile);
7088 TheFnVI = ImportSummary->getValueInfo(
7090 // Internal func in original module may have gotten a numbered suffix if we
7091 // imported an external function with the same name. This happens
7092 // automatically during IR linking for naming conflicts. It would have to
7093 // still be internal in that case (otherwise it would have been renamed on
7094 // promotion in which case we wouldn't have a naming conflict).
7095 if (!TheFnVI && OrigName == GV.getName() && GV.hasLocalLinkage() &&
7096 GV.getName().contains('.')) {
7097 OrigName = GV.getName().rsplit('.').first;
7099 OrigName, GlobalValue::InternalLinkage, SrcFile);
7100 TheFnVI = ImportSummary->getValueInfo(
7102 }
7103 // The only way we may not have a VI is if this is a declaration created for
7104 // an imported reference. For distributed ThinLTO we may not have a VI for
7105 // such declarations in the distributed summary.
7106 assert(TheFnVI || GV.isDeclaration());
7107 return TheFnVI;
7108}
7109
7110// Find the CallsiteInfo record synthesized by the thin link for profiled target
7111// CandidateGUID of an indirect tail call, in MapTailCallCalleeVIToCallsite
7112// (which maps each synthesized record's callee to the record). Returns null if
7113// there is none. OrigNameGUIDToTailCallCallsite caches a map from the original
7114// name GUID of each record's callee to the record, and is built on first use
7115// (along with AliasVIToGV, which caches alias ValueInfos in M).
7117 uint64_t CandidateGUID,
7118 const DenseMap<ValueInfo, CallsiteInfo> &MapTailCallCalleeVIToCallsite,
7120 &OrigNameGUIDToTailCallCallsite,
7121 std::optional<DenseMap<ValueInfo, const GlobalAlias *>> &AliasVIToGV,
7122 const Module &M, const ModuleSummaryIndex *ImportSummary,
7123 InstrProfSymtab &Symtab) {
7124 auto End = MapTailCallCalleeVIToCallsite.end();
7125 // The callee of the synthesized record is normally the VP metadata GUID, so
7126 // check that first.
7127 if (ValueInfo VI = ImportSummary->getValueInfo(CandidateGUID)) {
7128 auto It = MapTailCallCalleeVIToCallsite.find(VI);
7129 if (It != End)
7130 return &It->second;
7131 }
7132 // For local targets the VP metadata may contain the GUID of the original
7133 // name without the source file prefix, which the thin link translates to the
7134 // actual GUID of the local function (see updateValueInfoForIndirectCalls).
7135 if (auto LocalGUID = ImportSummary->getGUIDFromOriginalID(CandidateGUID)) {
7136 if (ValueInfo LocalVI = ImportSummary->getValueInfo(LocalGUID)) {
7137 auto It = MapTailCallCalleeVIToCallsite.find(LocalVI);
7138 if (It != End)
7139 return &It->second;
7140 }
7141 }
7142 // The index for a distributed ThinLTO backend does not record the original
7143 // names of local functions, in which case the above lookup via
7144 // getGUIDFromOriginalID fails. Instead, compute the original name GUID of
7145 // each record's callee from its function or alias in this module, using its
7146 // name before any promotion.
7147 if (!OrigNameGUIDToTailCallCallsite) {
7148 // Construct the (initially empty) map so that it is marked as built, even
7149 // if no entries are added below, to avoid rebuilding it on later calls.
7150 OrigNameGUIDToTailCallCallsite.emplace();
7151 for (const auto &[CalleeVI, Callsite] : MapTailCallCalleeVIToCallsite) {
7152 const GlobalValue *Callee = Symtab.getFunction(CalleeVI.getGUID());
7153 if (!Callee) {
7154 // InstrProfSymtab only indexes Functions, not GlobalAliases (imported
7155 // alias copies are materialized as Functions with thinlto_alias_guid
7156 // metadata and are found via Symtab above). Lazily build a module-wide
7157 // map for same-module function GlobalAliases once per module so we can
7158 // recover the alias's pre-promotion name.
7159 if (!AliasVIToGV) {
7160 AliasVIToGV.emplace();
7161 for (const GlobalAlias &A : M.aliases()) {
7162 if (isMemProfClone(A) ||
7163 !isa_and_nonnull<Function>(A.getAliaseeObject()))
7164 continue;
7165 if (ValueInfo VI = findValueInfoForGV(A, M, ImportSummary))
7166 AliasVIToGV->try_emplace(VI, &A);
7167 }
7168 }
7169 Callee = AliasVIToGV->lookup(CalleeVI);
7170 if (!Callee)
7171 continue;
7172 }
7173 StringRef OrigName =
7175 auto [It, Inserted] = OrigNameGUIDToTailCallCallsite->try_emplace(
7176 GlobalValue::getGUIDAssumingExternalLinkage(OrigName), &Callsite);
7177 // If multiple callees have the same original name, we can't tell which
7178 // corresponds to the profiled target.
7179 if (!Inserted && It->second != &Callsite)
7180 It->second = nullptr;
7181 }
7182 }
7183 return OrigNameGUIDToTailCallCallsite->lookup(CandidateGUID);
7184}
7185
7186bool MemProfContextDisambiguation::initializeIndirectCallPromotionInfo(
7187 Module &M) {
7188 ICallAnalysis = std::make_unique<ICallPromotionAnalysis>();
7189 Symtab = std::make_unique<InstrProfSymtab>();
7190 // Don't add canonical names, to avoid multiple functions to the symtab
7191 // when they both have the same root name with "." suffixes stripped.
7192 // If we pick the wrong one then this could lead to incorrect ICP and calling
7193 // a memprof clone that we don't actually create (resulting in linker unsats).
7194 // What this means is that the GUID of the function *must* match that in the
7195 // VP metadata to allow promotion. In practice this should not be a
7196 // limitation, since local functions have a GUID from before they were
7197 // promoted and global function names shouldn't need any special handling
7198 // (they should not get the ".llvm.*" suffix that the canonicalization
7199 // handling is attempting to strip).
7200 // Key imported copies of aliases by the alias's GUID rather than the
7201 // aliasee's GUID retained in their !guid metadata, since we rely on the
7202 // looked up function's name (to compute its original name GUID and the names
7203 // of its clones), and so that they don't shadow the aliasee's own function
7204 // when both are in the module. See AliaseeGUIDToImportedAlias for how
7205 // performICP handles profiled targets that are the aliasee of an imported
7206 // alias copy when the aliasee itself is not in the module.
7207 AliaseeGUIDToImportedAlias.reset();
7208 if (Error E = Symtab->create(M, /*InLTO=*/true, /*AddCanonical=*/false,
7209 /*KeyImportedAliasesByAliasGUID=*/true)) {
7210 std::string SymtabFailure = toString(std::move(E));
7211 M.getContext().emitError("Failed to create symtab: " + SymtabFailure);
7212 return false;
7213 }
7214 return true;
7215}
7216
7217#ifndef NDEBUG
7218// Sanity check that the MIB stack ids match between the summary and
7219// instruction metadata.
7221 const AllocInfo &AllocNode, const MDNode *MemProfMD,
7222 const CallStack<MDNode, MDNode::op_iterator> &CallsiteContext,
7223 const ModuleSummaryIndex *ImportSummary) {
7224 auto MIBIter = AllocNode.MIBs.begin();
7225 for (auto &MDOp : MemProfMD->operands()) {
7226 assert(MIBIter != AllocNode.MIBs.end());
7227 // The combined index may have been written without the MIB contexts (see
7228 // -combined-index-memprof-context, which defaults to false in NDEBUG
7229 // builds), e.g. if the thin link was performed by a release build while
7230 // this backend has assertions enabled. Since the MIB context in the summary
7231 // can never be empty after trimming the allocation's callsite context, an
7232 // empty list means the context was not serialized, and there is nothing to
7233 // check.
7234 if (MIBIter->StackIdIndices.empty()) {
7235 MIBIter++;
7236 continue;
7237 }
7238 auto StackIdIndexIter = MIBIter->StackIdIndices.begin();
7239 auto *MIBMD = cast<const MDNode>(MDOp);
7240 MDNode *StackMDNode = getMIBStackNode(MIBMD);
7241 assert(StackMDNode);
7242 CallStack<MDNode, MDNode::op_iterator> StackContext(StackMDNode);
7243 auto ContextIterBegin =
7244 StackContext.beginAfterSharedPrefix(CallsiteContext);
7245 // Skip the checking on the first iteration.
7246 uint64_t LastStackContextId =
7247 (ContextIterBegin != StackContext.end() && *ContextIterBegin == 0) ? 1
7248 : 0;
7249 for (auto ContextIter = ContextIterBegin; ContextIter != StackContext.end();
7250 ++ContextIter) {
7251 // If this is a direct recursion, simply skip the duplicate
7252 // entries, to be consistent with how the summary ids were
7253 // generated during ModuleSummaryAnalysis.
7254 if (LastStackContextId == *ContextIter)
7255 continue;
7256 LastStackContextId = *ContextIter;
7257 assert(StackIdIndexIter != MIBIter->StackIdIndices.end());
7258 assert(ImportSummary->getStackIdAtIndex(*StackIdIndexIter) ==
7259 *ContextIter);
7260 StackIdIndexIter++;
7261 }
7262 MIBIter++;
7263 }
7264}
7265#endif
7266
7267bool MemProfContextDisambiguation::applyImport(Module &M) {
7268 assert(ImportSummary);
7269 bool Changed = false;
7270
7271 // We also need to clone any aliases that reference cloned functions, because
7272 // the modified callsites may invoke via the alias. Keep track of the aliases
7273 // for each function.
7274 std::map<const Function *, SmallPtrSet<const GlobalAlias *, 1>>
7275 FuncToAliasMap;
7276 for (auto &A : M.aliases()) {
7277 auto *Aliasee = A.getAliaseeObject();
7278 if (auto *F = dyn_cast<Function>(Aliasee))
7279 FuncToAliasMap[F].insert(&A);
7280 }
7281
7282 if (!initializeIndirectCallPromotionInfo(M))
7283 return false;
7284
7285 // Lazily populated map from alias ValueInfo to the corresponding same-module
7286 // GlobalAlias, used by findIndirectTailCallCallsite when a local alias target
7287 // is not in Symtab.
7288 std::optional<DenseMap<ValueInfo, const GlobalAlias *>> AliasVIToGV;
7289
7290 for (auto &F : M) {
7291 if (F.isDeclaration() || isMemProfClone(F))
7292 continue;
7293
7294 OptimizationRemarkEmitter ORE(&F);
7295
7297 bool ClonesCreated = false;
7298 unsigned NumClonesCreated = 0;
7299 auto CloneFuncIfNeeded = [&](unsigned NumClones, FunctionSummary *FS) {
7300 // We should at least have version 0 which is the original copy.
7301 assert(NumClones > 0);
7302 // If only one copy needed use original.
7303 if (NumClones == 1)
7304 return;
7305 // If we already performed cloning of this function, confirm that the
7306 // requested number of clones matches (the thin link should ensure the
7307 // number of clones for each constituent callsite is consistent within
7308 // each function), before returning.
7309 if (ClonesCreated) {
7310 assert(NumClonesCreated == NumClones);
7311 return;
7312 }
7313 VMaps = createFunctionClones(F, NumClones, M, ORE, FuncToAliasMap, FS);
7314 // The first "clone" is the original copy, which doesn't have a VMap.
7315 assert(VMaps.size() == NumClones - 1);
7316 Changed = true;
7317 ClonesCreated = true;
7318 NumClonesCreated = NumClones;
7319 };
7320
7321 auto CloneCallsite = [&](const CallsiteInfo &StackNode, CallBase *CB,
7322 Function *CalledFunction, FunctionSummary *FS) {
7323 // Perform cloning if not yet done.
7324 CloneFuncIfNeeded(/*NumClones=*/StackNode.Clones.size(), FS);
7325
7326 assert(!isMemProfClone(*CalledFunction));
7327
7328 // Because we update the cloned calls by calling setCalledOperand (see
7329 // comment below), out of an abundance of caution make sure the called
7330 // function was actually the called operand (or its aliasee). We also
7331 // strip pointer casts when looking for calls (to match behavior during
7332 // summary generation), however, with opaque pointers in theory this
7333 // should not be an issue. Note we still clone the current function
7334 // (containing this call) above, as that could be needed for its callers.
7335 auto *GA = dyn_cast_or_null<GlobalAlias>(CB->getCalledOperand());
7336 if (CalledFunction != CB->getCalledOperand() &&
7337 (!GA || CalledFunction != GA->getAliaseeObject())) {
7338 SkippedCallsCloning++;
7339 return;
7340 }
7341 // Update the calls per the summary info.
7342 // Save orig name since it gets updated in the first iteration
7343 // below.
7344 auto CalleeOrigName = CalledFunction->getName();
7345 for (unsigned J = 0; J < StackNode.Clones.size(); J++) {
7346 // If the VMap is empty, this clone was a duplicate of another and was
7347 // created as an alias or a declaration.
7348 if (J > 0 && VMaps[J - 1]->empty())
7349 continue;
7350 // Do nothing if this version calls the original version of its
7351 // callee.
7352 if (!StackNode.Clones[J])
7353 continue;
7354 auto NewF = M.getOrInsertFunction(
7355 getMemProfFuncName(CalleeOrigName, StackNode.Clones[J]),
7356 CalledFunction->getFunctionType());
7357 CallBase *CBClone;
7358 // Copy 0 is the original function.
7359 if (!J)
7360 CBClone = CB;
7361 else
7362 CBClone = cast<CallBase>((*VMaps[J - 1])[CB]);
7363 // Set the called operand directly instead of calling setCalledFunction,
7364 // as the latter mutates the function type on the call. In rare cases
7365 // we may have a slightly different type on a callee function
7366 // declaration due to it being imported from a different module with
7367 // incomplete types. We really just want to change the name of the
7368 // function to the clone, and not make any type changes.
7369 CBClone->setCalledOperand(NewF.getCallee());
7370 ORE.emit(OptimizationRemark(DEBUG_TYPE, "MemprofCall", CBClone)
7371 << ore::NV("Call", CBClone) << " in clone "
7372 << ore::NV("Caller", CBClone->getFunction())
7373 << " assigned to call function clone "
7374 << ore::NV("Callee", NewF.getCallee()));
7375 }
7376 };
7377
7378 // Locate the summary for F.
7379 ValueInfo TheFnVI = findValueInfoForGV(F, M, ImportSummary);
7380 // If not found, this could be an imported local (see comment in
7381 // findValueInfoForGV). Skip for now as it will be cloned in its original
7382 // module (where it would have been promoted to global scope so should
7383 // satisfy any reference in this module).
7384 if (!TheFnVI)
7385 continue;
7386
7387 auto *GVSummary =
7388 ImportSummary->findSummaryInModule(TheFnVI, M.getModuleIdentifier());
7389 if (!GVSummary) {
7390 // Must have been imported, use the summary which matches the definition。
7391 // (might be multiple if this was a linkonce_odr).
7392 auto SrcModuleMD = F.getMetadata("thinlto_src_module");
7393 assert(SrcModuleMD &&
7394 "enable-import-metadata is needed to emit thinlto_src_module");
7395 StringRef SrcModule =
7396 dyn_cast<MDString>(SrcModuleMD->getOperand(0))->getString();
7397 for (auto &GVS : TheFnVI.getSummaryList()) {
7398 if (GVS->modulePath() == SrcModule) {
7399 GVSummary = GVS.get();
7400 break;
7401 }
7402 }
7403 assert(GVSummary && GVSummary->modulePath() == SrcModule);
7404 }
7405
7406 // If this was an imported alias skip it as we won't have the function
7407 // summary, and it should be cloned in the original module.
7408 if (isa<AliasSummary>(GVSummary))
7409 continue;
7410
7411 auto *FS = cast<FunctionSummary>(GVSummary->getBaseObject());
7412
7413 if (FS->allocs().empty() && FS->callsites().empty())
7414 continue;
7415
7416 auto SI = FS->callsites().begin();
7417 auto AI = FS->allocs().begin();
7418
7419 // To handle callsite infos synthesized for tail calls which have missing
7420 // frames in the profiled context, map callee VI to the synthesized callsite
7421 // info.
7422 DenseMap<ValueInfo, CallsiteInfo> MapTailCallCalleeVIToCallsite;
7423 // Iterate the callsites for this function in reverse, since we place all
7424 // those synthesized for tail calls at the end.
7425 for (auto CallsiteIt = FS->callsites().rbegin();
7426 CallsiteIt != FS->callsites().rend(); CallsiteIt++) {
7427 auto &Callsite = *CallsiteIt;
7428 // Stop as soon as we see a non-synthesized callsite info (see comment
7429 // above loop). All the entries added for discovered tail calls have empty
7430 // stack ids.
7431 if (!Callsite.StackIdIndices.empty())
7432 break;
7433 MapTailCallCalleeVIToCallsite.insert({Callsite.Callee, Callsite});
7434 }
7435 // Maps the original name GUID of each synthesized callsite info's callee to
7436 // the callsite info, built on first use by findIndirectTailCallCallsite.
7437 std::optional<DenseMap<uint64_t, const CallsiteInfo *>>
7438 OrigNameGUIDToTailCallCallsite;
7439
7440 // Keeps track of needed ICP for the function.
7441 SmallVector<ICallAnalysisData> ICallAnalysisInfo;
7442
7443 // Assume for now that the instructions are in the exact same order
7444 // as when the summary was created, but confirm this is correct by
7445 // matching the stack ids.
7446 for (auto &BB : F) {
7447 for (auto &I : BB) {
7448 auto *CB = dyn_cast<CallBase>(&I);
7449 // Same handling as when creating module summary.
7450 if (!mayHaveMemprofSummary(CB))
7451 continue;
7452
7453 auto *CalledValue = CB->getCalledOperand();
7454 auto *CalledFunction = CB->getCalledFunction();
7455 if (CalledValue && !CalledFunction) {
7456 CalledValue = CalledValue->stripPointerCasts();
7457 // Stripping pointer casts can reveal a called function.
7458 CalledFunction = dyn_cast<Function>(CalledValue);
7459 }
7460 // Check if this is an alias to a function. If so, get the
7461 // called aliasee for the checks below.
7462 if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
7463 assert(!CalledFunction &&
7464 "Expected null called function in callsite for alias");
7465 CalledFunction = dyn_cast<Function>(GA->getAliaseeObject());
7466 }
7467
7468 CallStack<MDNode, MDNode::op_iterator> CallsiteContext(
7469 I.getMetadata(LLVMContext::MD_callsite));
7470 auto *MemProfMD = I.getMetadata(LLVMContext::MD_memprof);
7471
7472 // Include allocs that were already assigned a memprof function
7473 // attribute in the statistics. Only do this for those that do not have
7474 // memprof metadata, since we add an "ambiguous" memprof attribute by
7475 // default.
7476 if (CB->getAttributes().hasFnAttr("memprof") && !MemProfMD) {
7477 CB->getAttributes().getFnAttr("memprof").getValueAsString() == "cold"
7478 ? AllocTypeColdThinBackend++
7479 : AllocTypeNotColdThinBackend++;
7480 OrigAllocsThinBackend++;
7481 AllocVersionsThinBackend++;
7482 if (!MaxAllocVersionsThinBackend)
7483 MaxAllocVersionsThinBackend = 1;
7484 continue;
7485 }
7486
7487 if (MemProfMD) {
7488 // Consult the next alloc node.
7489 assert(AI != FS->allocs().end());
7490 auto &AllocNode = *(AI++);
7491
7492#ifndef NDEBUG
7493 checkAllocContextIds(AllocNode, MemProfMD, CallsiteContext,
7494 ImportSummary);
7495#endif
7496
7497 // Perform cloning if not yet done.
7498 CloneFuncIfNeeded(/*NumClones=*/AllocNode.Versions.size(), FS);
7499
7500 OrigAllocsThinBackend++;
7501 AllocVersionsThinBackend += AllocNode.Versions.size();
7502 if (MaxAllocVersionsThinBackend < AllocNode.Versions.size())
7503 MaxAllocVersionsThinBackend = AllocNode.Versions.size();
7504
7505 // If there is only one version that means we didn't end up
7506 // considering this function for cloning. Count that as unclonable
7507 // unless the single version is cold due to the
7508 // MinClonedColdBytePercent heuristic. The alloc may still be none
7509 // type (not reached during whole program analysis), in which case
7510 // there is nothing to apply; otherwise it got the type the whole
7511 // program analysis resolved it to (not cold, or both types when the
7512 // contexts could not be disambiguated), which is applied below like
7513 // for cloned functions so that the attribute matches the regular
7514 // LTO result.
7515 if (AllocNode.Versions.size() == 1 &&
7516 (AllocationType)AllocNode.Versions[0] != AllocationType::Cold)
7517 UnclonableAllocsThinBackend++;
7518
7519 // Update the allocation types per the summary info.
7520 for (unsigned J = 0; J < AllocNode.Versions.size(); J++) {
7521 // If the VMap is empty, this clone was a duplicate of another and
7522 // was created as an alias or a declaration.
7523 if (J > 0 && VMaps[J - 1]->empty())
7524 continue;
7525 // Ignore any that didn't get an assigned allocation type.
7526 if (AllocNode.Versions[J] == (uint8_t)AllocationType::None)
7527 continue;
7528 AllocationType AllocTy = (AllocationType)AllocNode.Versions[J];
7529 // The whole program analysis only records both types when
7530 // ambiguous attributes are enabled; if the backend runs without
7531 // them, fall back to the not cold default.
7532 if (isAmbiguousAllocType((uint8_t)AllocTy) &&
7534 AllocTy = AllocationType::NotCold;
7535 const bool IsAmbiguous = isAmbiguousAllocType((uint8_t)AllocTy);
7536 if (AllocTy == AllocationType::Cold)
7537 AllocTypeColdThinBackend++;
7538 else if (IsAmbiguous)
7539 AllocTypeAmbiguousThinBackend++;
7540 else
7541 AllocTypeNotColdThinBackend++;
7542 std::string AllocTypeString = getAllocTypeAttributeString(AllocTy);
7543 CallBase *CBClone;
7544 // Copy 0 is the original function.
7545 if (!J)
7546 CBClone = CB;
7547 else
7548 // Since VMaps are only created for new clones, we index with
7549 // clone J-1 (J==0 is the original clone and does not have a VMaps
7550 // entry).
7551 CBClone = cast<CallBase>((*VMaps[J - 1])[CB]);
7552 if (IsAmbiguous) {
7553 // The contexts could not be disambiguated: keep (or add) the
7554 // ambiguous attribute from matching instead of a specific type.
7555 addAmbiguousAttribute(CBClone);
7556 } else {
7558 CBClone->addFnAttr(llvm::Attribute::get(F.getContext(), "memprof",
7559 AllocTypeString));
7560 }
7561 ORE.emit(OptimizationRemark(DEBUG_TYPE, "MemprofAttribute", CBClone)
7562 << ore::NV("AllocationCall", CBClone) << " in clone "
7563 << ore::NV("Caller", CBClone->getFunction())
7564 << " marked with memprof allocation attribute "
7565 << ore::NV("Attribute", AllocTypeString));
7566 }
7567 } else if (!CallsiteContext.empty()) {
7568 if (!CalledFunction) {
7569#ifndef NDEBUG
7570 // We should have skipped inline assembly calls.
7571 auto *CI = dyn_cast<CallInst>(CB);
7572 assert(!CI || !CI->isInlineAsm());
7573#endif
7574 // We should have skipped direct calls via a Constant.
7575 assert(CalledValue && !isa<Constant>(CalledValue));
7576
7577 // This is an indirect call, see if we have profile information and
7578 // whether any clones were recorded for the profiled targets (that
7579 // we synthesized CallsiteInfo summary records for when building the
7580 // index). These records are the next ones in the function's
7581 // callsites, one per profiled target.
7582 auto NumClones = recordICPInfo(
7583 CB,
7584 [&](const InstrProfValueData &Candidate) {
7585 assert(SI != FS->callsites().end());
7586#ifndef NDEBUG
7587 auto CalleeValueInfo =
7588 ImportSummary->getValueInfo(Candidate.Value);
7589 auto LocalGUID =
7590 ImportSummary->getGUIDFromOriginalID(Candidate.Value);
7591#endif
7592 // We might not have a ValueInfo if this is a distributed
7593 // ThinLTO backend and decided not to import that function.
7594 // The thin link may also have updated the callee from the
7595 // original name GUID recorded in the VP metadata to the GUID
7596 // of the local target function (see
7597 // updateValueInfoForIndirectCalls).
7598 assert(!CalleeValueInfo || SI->Callee == CalleeValueInfo ||
7599 (SI->Callee && SI->Callee.getGUID() == LocalGUID));
7600 return &*(SI++);
7601 },
7602 ICallAnalysisInfo);
7603
7604 // Perform cloning if not yet done. This is done here in case
7605 // we don't need to do ICP, but might need to clone this
7606 // function as it is the target of other cloned calls.
7607 if (NumClones)
7608 CloneFuncIfNeeded(NumClones, FS);
7609 }
7610
7611 else {
7612 // Consult the next callsite node.
7613 assert(SI != FS->callsites().end());
7614 auto &StackNode = *(SI++);
7615
7616#ifndef NDEBUG
7617 // Sanity check that the stack ids match between the summary and
7618 // instruction metadata.
7619 auto StackIdIndexIter = StackNode.StackIdIndices.begin();
7620 for (auto StackId : CallsiteContext) {
7621 assert(StackIdIndexIter != StackNode.StackIdIndices.end());
7622 assert(ImportSummary->getStackIdAtIndex(*StackIdIndexIter) ==
7623 StackId);
7624 StackIdIndexIter++;
7625 }
7626#endif
7627
7628 CloneCallsite(StackNode, CB, CalledFunction, FS);
7629 }
7630 } else if (CB->isTailCall() && CalledFunction) {
7631 // Locate the synthesized callsite info for the callee VI, if any was
7632 // created, and use that for cloning.
7633 ValueInfo CalleeVI = findValueInfoForGV(
7634 *cast<GlobalValue>(CalledValue), M, ImportSummary, &F);
7635 if (CalleeVI && MapTailCallCalleeVIToCallsite.count(CalleeVI)) {
7636 auto Callsite = MapTailCallCalleeVIToCallsite.find(CalleeVI);
7637 assert(Callsite != MapTailCallCalleeVIToCallsite.end());
7638 CloneCallsite(Callsite->second, CB, CalledFunction, FS);
7639 }
7640 } else if (CB->isTailCall() && !CalledFunction &&
7641 !MapTailCallCalleeVIToCallsite.empty()) {
7642 // An indirect tail call, which the thin link may have identified as
7643 // being on profiled contexts with a missing frame, via the profiled
7644 // targets in its VP metadata. If so, the callsite infos synthesized
7645 // for those targets are used to perform ICP so that each clone of
7646 // this function calls the assigned clone of the target.
7647 auto PrevNumICallAnalysisInfo = ICallAnalysisInfo.size();
7648 auto NumClones = recordICPInfo(
7649 CB,
7650 [&](const InstrProfValueData &Candidate) {
7652 Candidate.Value, MapTailCallCalleeVIToCallsite,
7653 OrigNameGUIDToTailCallCallsite, AliasVIToGV, M,
7654 ImportSummary, *Symtab);
7655 },
7656 ICallAnalysisInfo);
7657 if (ICallAnalysisInfo.size() > PrevNumICallAnalysisInfo)
7658 IndirectTailCallsRecordedForICPThinBackend++;
7659 if (NumClones)
7660 CloneFuncIfNeeded(NumClones, FS);
7661 }
7662 }
7663 }
7664
7665 // Now do any promotion required for cloning.
7666 performICP(M, VMaps, ICallAnalysisInfo, ORE);
7667 }
7668
7669 // We skip some of the functions and instructions above, so remove all the
7670 // metadata in a single sweep here.
7671 for (auto &F : M) {
7672 // We can skip memprof clones because createFunctionClones already strips
7673 // the metadata from the newly created clones.
7674 if (F.isDeclaration() || isMemProfClone(F))
7675 continue;
7676 for (auto &BB : F) {
7677 for (auto &I : BB) {
7678 if (!isa<CallBase>(I))
7679 continue;
7680 I.setMetadata(LLVMContext::MD_memprof, nullptr);
7681 I.setMetadata(LLVMContext::MD_callsite, nullptr);
7682 }
7683 }
7684 }
7685
7686 return Changed;
7687}
7688
7689unsigned MemProfContextDisambiguation::recordICPInfo(
7690 CallBase *CB,
7691 function_ref<const CallsiteInfo *(const InstrProfValueData &)> GetCallsite,
7692 SmallVector<ICallAnalysisData> &ICallAnalysisInfo) {
7693 // First see if we have profile information for this indirect call.
7694 uint32_t NumCandidates;
7695 uint64_t TotalCount;
7696 auto CandidateProfileData =
7697 ICallAnalysis->getPromotionCandidatesForInstruction(
7698 CB, TotalCount, NumCandidates,
7700 if (CandidateProfileData.empty())
7701 return 0;
7702
7703 // Iterate through all of the candidate profiled targets along with the
7704 // CallsiteInfo summary records synthesized for them, and see if any are
7705 // cloned and/or refer to clones.
7706 bool ICPNeeded = false;
7707 unsigned NumClones = 0;
7709 for (const auto &Candidate : CandidateProfileData) {
7710 const CallsiteInfo *Callsite = GetCallsite(Candidate);
7711 Callsites.push_back(Callsite);
7712 if (!Callsite)
7713 continue;
7714 // See if any of the clones of the indirect callsite for this
7715 // profiled target should call a cloned version of the profiled
7716 // target. We only need to do the ICP here if so.
7717 ICPNeeded |= llvm::any_of(Callsite->Clones,
7718 [](unsigned CloneNo) { return CloneNo != 0; });
7719 // Every callsite in the same function should have been cloned the same
7720 // number of times.
7721 assert(!NumClones || NumClones == Callsite->Clones.size());
7722 NumClones = Callsite->Clones.size();
7723 }
7724 if (!ICPNeeded)
7725 return NumClones;
7726 // Save information for ICP, which is performed later to avoid messing up the
7727 // current function traversal.
7728 ICallAnalysisInfo.push_back({CB, CandidateProfileData.vec(), NumCandidates,
7729 TotalCount, NumClones, std::move(Callsites)});
7730 return NumClones;
7731}
7732
7733void MemProfContextDisambiguation::performICP(
7734 Module &M, ArrayRef<std::unique_ptr<ValueToValueMapTy>> VMaps,
7735 ArrayRef<ICallAnalysisData> ICallAnalysisInfo,
7736 OptimizationRemarkEmitter &ORE) {
7737 // Now do any promotion required for cloning. Specifically, for each
7738 // recorded ICP candidate (which was only recorded because one clone of that
7739 // candidate should call a cloned target), we perform ICP (speculative
7740 // devirtualization) for each clone of the callsite, and update its callee
7741 // to the appropriate clone. Note that the ICP compares against the original
7742 // version of the target, which is what is in the vtable.
7743 for (auto &Info : ICallAnalysisInfo) {
7744 auto *CB = Info.CB;
7745 auto TotalCount = Info.TotalCount;
7746 unsigned NumClones = Info.NumClones;
7747 SmallVector<InstrProfValueData, 8> RemainingCandidates;
7748
7749 assert(Info.CandidateProfileData.size() == Info.Callsites.size());
7750 for (unsigned I = 0; I < Info.CandidateProfileData.size(); I++) {
7751 auto &Candidate = Info.CandidateProfileData[I];
7752 const CallsiteInfo *StackNode = Info.Callsites[I];
7753 // All calls in the same function must have the same number of clones.
7754 assert(!StackNode || NumClones == StackNode->Clones.size());
7755
7756 // See if the target is in the module. If it wasn't imported, it is
7757 // possible that this profile could have been collected on a different
7758 // target (or version of the code), and we need to be conservative
7759 // (similar to what is done in the ICP pass).
7760 Function *TargetFunction = Symtab->getFunction(Candidate.Value);
7761 // If not found, try the GUID of the callee recorded in the summary. For
7762 // local targets the VP metadata may contain the GUID of the original
7763 // name without the source file prefix (e.g. if it was synthesized from
7764 // the MemProf profile's callee GUIDs or from a SamplePGO profile), which
7765 // the thin link translates to the actual GUID of the local function
7766 // (see updateValueInfoForIndirectCalls). The latter is recorded in the
7767 // function's !guid metadata, which is preserved across promotion and
7768 // importing, and is added to the Symtab. If there is no summary record,
7769 // the target was not cloned, so we conservatively only promote it if
7770 // found directly via the VP metadata GUID.
7771 if (!TargetFunction && StackNode && StackNode->Callee &&
7772 StackNode->Callee.getGUID() != Candidate.Value)
7773 TargetFunction = Symtab->getFunction(StackNode->Callee.getGUID());
7774 // The target may be the aliasee of an alias imported as a copy of the
7775 // aliasee, without the aliasee itself being imported (e.g. an IRPGO value
7776 // profile records the aliasee for calls through the alias, as the alias
7777 // has no profile data of its own). The Symtab keys such a copy by the
7778 // alias GUID (see initializeIndirectCallPromotionInfo), so look for one
7779 // with the target's GUID as its aliasee GUID. Promoting to the copy is
7780 // correct as it has the same address, and its clones are aliases of the
7781 // aliasee's clones in the defining module.
7782 if (!TargetFunction) {
7783 if (!AliaseeGUIDToImportedAlias) {
7784 AliaseeGUIDToImportedAlias.emplace();
7785 for (Function &Fn : M) {
7786 if (isMemProfClone(Fn) || !getImportedAliasGUID(Fn))
7787 continue;
7788 if (auto AliaseeGUID = Fn.getGUIDIfAssigned())
7789 AliaseeGUIDToImportedAlias->try_emplace(*AliaseeGUID, &Fn);
7790 }
7791 }
7792 TargetFunction = AliaseeGUIDToImportedAlias->lookup(Candidate.Value);
7793 if (!TargetFunction && StackNode && StackNode->Callee &&
7794 StackNode->Callee.getGUID() != Candidate.Value)
7795 TargetFunction =
7796 AliaseeGUIDToImportedAlias->lookup(StackNode->Callee.getGUID());
7797 }
7798 if (TargetFunction == nullptr ||
7799 // Any ThinLTO global dead symbol removal should have already
7800 // occurred, so it should be safe to promote when the target is a
7801 // declaration.
7802 // TODO: Remove internal option once more fully tested.
7804 TargetFunction->isDeclaration())) {
7805 ORE.emit([&]() {
7806 return OptimizationRemarkMissed(DEBUG_TYPE, "UnableToFindTarget", CB)
7807 << "Memprof cannot promote indirect call: target with md5sum "
7808 << ore::NV("target md5sum", Candidate.Value) << " not found";
7809 });
7810 // FIXME: See if we can use the new declaration importing support to
7811 // at least get the declarations imported for this case. Hot indirect
7812 // targets should have been imported normally, however.
7813 RemainingCandidates.push_back(Candidate);
7814 continue;
7815 }
7816
7817 // Check if legal to promote
7818 const char *Reason = nullptr;
7819 if (!isLegalToPromote(*CB, TargetFunction, &Reason)) {
7820 ORE.emit([&]() {
7821 return OptimizationRemarkMissed(DEBUG_TYPE, "UnableToPromote", CB)
7822 << "Memprof cannot promote indirect call to "
7823 << ore::NV("TargetFunction", TargetFunction)
7824 << " with count of " << ore::NV("TotalCount", TotalCount)
7825 << ": " << Reason;
7826 });
7827 RemainingCandidates.push_back(Candidate);
7828 continue;
7829 }
7830
7831 assert(!isMemProfClone(*TargetFunction));
7832
7833 // Handle each call clone, applying ICP so that each clone directly
7834 // calls the specified callee clone, guarded by the appropriate ICP
7835 // check.
7836 CallBase *CBClone = CB;
7837 for (unsigned J = 0; J < NumClones; J++) {
7838 // If the VMap is empty, this clone was a duplicate of another and was
7839 // created as an alias or a declaration.
7840 if (J > 0 && VMaps[J - 1]->empty())
7841 continue;
7842 // Copy 0 is the original function.
7843 if (J > 0)
7844 CBClone = cast<CallBase>((*VMaps[J - 1])[CB]);
7845 // We do the promotion using the original name, so that the comparison
7846 // is against the name in the vtable. Then just below, change the new
7847 // direct call to call the cloned function.
7848 auto &DirectCall =
7849 pgo::promoteIndirectCall(*CBClone, TargetFunction, Candidate.Count,
7850 TotalCount, isSamplePGO, &ORE);
7851 auto *TargetToUse = TargetFunction;
7852 // Call original if this version calls the original version of its
7853 // callee (always the case if there is no summary record).
7854 if (StackNode && StackNode->Clones[J]) {
7855 TargetToUse =
7856 cast<Function>(M.getOrInsertFunction(
7857 getMemProfFuncName(TargetFunction->getName(),
7858 StackNode->Clones[J]),
7859 TargetFunction->getFunctionType())
7860 .getCallee());
7861 }
7862 DirectCall.setCalledFunction(TargetToUse);
7863 // During matching we generate synthetic VP metadata for indirect calls
7864 // not already having any, from the memprof profile's callee GUIDs. If
7865 // we subsequently promote and inline those callees, we currently lose
7866 // the ability to generate this synthetic VP metadata. Optionally apply
7867 // a noinline attribute to promoted direct calls, where the threshold is
7868 // set to capture synthetic VP metadata targets which get a count of 1.
7870 Candidate.Count < MemProfICPNoInlineThreshold)
7871 DirectCall.setIsNoInline();
7872 ORE.emit(OptimizationRemark(DEBUG_TYPE, "MemprofCall", CBClone)
7873 << ore::NV("Call", CBClone) << " in clone "
7874 << ore::NV("Caller", CBClone->getFunction())
7875 << " promoted and assigned to call function clone "
7876 << ore::NV("Callee", TargetToUse));
7877 }
7878
7879 // Update TotalCount (all clones should get same count above)
7880 TotalCount -= Candidate.Count;
7881 }
7882 // Adjust the MD.prof metadata for all clones, now that we have the new
7883 // TotalCount and the remaining candidates.
7884 CallBase *CBClone = CB;
7885 for (unsigned J = 0; J < NumClones; J++) {
7886 // If the VMap is empty, this clone was a duplicate of another and was
7887 // created as an alias or a declaration.
7888 if (J > 0 && VMaps[J - 1]->empty())
7889 continue;
7890 // Copy 0 is the original function.
7891 if (J > 0)
7892 CBClone = cast<CallBase>((*VMaps[J - 1])[CB]);
7893 // First delete the old one.
7894 CBClone->setMetadata(LLVMContext::MD_prof, nullptr);
7895 // If all promoted, we don't need the MD.prof metadata.
7896 // Otherwise we need update with the un-promoted records back.
7897 if (TotalCount != 0)
7898 annotateValueSite(M, *CBClone, RemainingCandidates, TotalCount,
7899 IPVK_IndirectCallTarget, Info.NumCandidates);
7900 }
7901 }
7902}
7903
7904template <typename DerivedCCG, typename FuncTy, typename CallTy>
7905bool CallsiteContextGraph<DerivedCCG, FuncTy, CallTy>::process(
7906 function_ref<void(StringRef, StringRef, const Twine &)> EmitRemark,
7907 bool AllowExtraAnalysis) {
7908 if (DumpCCG) {
7909 dbgs() << "CCG before cloning:\n";
7910 dbgs() << *this;
7911 }
7912 if (ExportToDot)
7913 exportToDot("postbuild");
7914
7915 if (VerifyCCG) {
7916 check();
7917 }
7918
7919 // The detail suffix needs the pre-cloning state of each context; only
7920 // capture it (extra memory and time) when it will be printed.
7922 capturePreCloningContextInfo();
7923
7924 identifyClones();
7925
7926 if (VerifyCCG) {
7927 check();
7928 }
7929
7930 if (DumpCCG) {
7931 dbgs() << "CCG after cloning:\n";
7932 dbgs() << *this;
7933 }
7934 if (ExportToDot)
7935 exportToDot("cloned");
7936
7937 bool Changed = assignFunctions();
7938
7939 if (DumpCCG) {
7940 dbgs() << "CCG after assigning function clones:\n";
7941 dbgs() << *this;
7942 }
7943 if (ExportToDot)
7944 exportToDot("clonefuncassign");
7945
7946 // AllowExtraAnalysis is set when the hinted size messages were enabled via
7947 // remarks (-pass-remarks=memprof-context-disambiguation).
7948 // TODO: Consider also supporting the detail suffix for the remark output.
7949 if (MemProfReportHintedSizes || AllowExtraAnalysis) {
7950 // The referenced-function set can be large; build it only now, when it is
7951 // used by the detail suffix, rather than keeping it across cloning.
7953 populateFunctionsWithCallers();
7954 printTotalSizes(errs(), EmitRemark);
7955 }
7956
7957 return Changed;
7958}
7959
7960bool MemProfContextDisambiguation::processModule(
7961 Module &M,
7962 llvm::function_ref<OptimizationRemarkEmitter &(Function *)> OREGetter) {
7963
7964 // If we have an import summary, then the cloning decisions were made during
7965 // the thin link on the index. Apply them and return.
7966 if (ImportSummary)
7967 return applyImport(M);
7968
7969 // TODO: If/when other types of memprof cloning are enabled beyond just for
7970 // hot and cold, we will need to change this to individually control the
7971 // AllocationType passed to addStackNodesForMIB during CCG construction.
7972 // Note that we specifically check this after applying imports above, so that
7973 // the option isn't needed to be passed to distributed ThinLTO backend
7974 // clang processes, which won't necessarily have visibility into the linker
7975 // dependences. Instead the information is communicated from the LTO link to
7976 // the backends via the combined summary index.
7977 if (!SupportsHotColdNew)
7978 return false;
7979
7980 ModuleCallsiteContextGraph CCG(M, OREGetter);
7981 // TODO: Set up remarks for regular LTO. We need to decide what function to
7982 // use in the callback.
7983 return CCG.process();
7984}
7985
7987 const ModuleSummaryIndex *Summary, bool isSamplePGO)
7988 : ImportSummary(Summary), isSamplePGO(isSamplePGO) {
7989 // Check the dot graph printing options once here, to make sure we have valid
7990 // and expected combinations.
7991 if (DotGraphScope == DotScope::Alloc && !AllocIdForDot.getNumOccurrences())
7993 "-memprof-dot-scope=alloc requires -memprof-dot-alloc-id");
7995 !ContextIdForDot.getNumOccurrences())
7997 "-memprof-dot-scope=context requires -memprof-dot-context-id");
7998 if (DotGraphScope == DotScope::All && AllocIdForDot.getNumOccurrences() &&
7999 ContextIdForDot.getNumOccurrences())
8001 "-memprof-dot-scope=all can't have both -memprof-dot-alloc-id and "
8002 "-memprof-dot-context-id");
8003 if (ImportSummary) {
8004 // The MemProfImportSummary should only be used for testing ThinLTO
8005 // distributed backend handling via opt, in which case we don't have a
8006 // summary from the pass pipeline.
8008 return;
8009 }
8010 if (MemProfImportSummary.empty())
8011 return;
8012
8013 auto ReadSummaryFile =
8015 if (!ReadSummaryFile) {
8016 logAllUnhandledErrors(ReadSummaryFile.takeError(), errs(),
8017 "Error loading file '" + MemProfImportSummary +
8018 "': ");
8019 return;
8020 }
8021 auto ImportSummaryForTestingOrErr = getModuleSummaryIndex(**ReadSummaryFile);
8022 if (!ImportSummaryForTestingOrErr) {
8023 logAllUnhandledErrors(ImportSummaryForTestingOrErr.takeError(), errs(),
8024 "Error parsing file '" + MemProfImportSummary +
8025 "': ");
8026 return;
8027 }
8028 ImportSummaryForTesting = std::move(*ImportSummaryForTestingOrErr);
8029 ImportSummary = ImportSummaryForTesting.get();
8030}
8031
8034 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
8035 auto OREGetter = [&](Function *F) -> OptimizationRemarkEmitter & {
8036 return FAM.getResult<OptimizationRemarkEmitterAnalysis>(*F);
8037 };
8038 if (!processModule(M, OREGetter))
8039 return PreservedAnalyses::all();
8040 return PreservedAnalyses::none();
8041}
8042
8043// After the thin link cloning decisions have been recorded in the summary
8044// callsite records, mark the summary indirect call edges to any callee that a
8045// callsite was assigned to call a (non-original) clone of as requiring a
8046// declaration import.
8047// See the comments for the MemProfImportClonedIndirectTargets option.
8049 ModuleSummaryIndex &Index,
8051 isPrevailing) {
8052 // Map from callee to the corresponding call edge in the current function
8053 // summary, so that we don't need to walk the function's call edges for each
8054 // callsite assigned to call a cloned callee. Note that there is a single call
8055 // edge per callee. The map is built lazily, the first time we encounter such
8056 // a callsite in a given function summary, since most functions have none
8057 // (the map is cleared before processing each function summary). The
8058 // pointers into the call edge vector remain valid since it isn't modified
8059 // while the map is in use.
8061 for (auto &I : Index) {
8062 auto VI = Index.getValueInfo(I);
8063 for (auto &S : VI.getSummaryList()) {
8064 // Only the prevailing copies were added to the CCG (see the
8065 // IndexCallsiteContextGraph constructor).
8066 if (!GlobalValue::isLocalLinkage(S->linkage()) &&
8067 !isPrevailing(VI.getGUID(), S.get()))
8068 continue;
8069 auto *FS = dyn_cast<FunctionSummary>(S.get());
8070 if (!FS || FS->callsites().empty())
8071 continue;
8072 CalleeToEdge.clear();
8073 for (auto &CI : FS->callsites()) {
8074 if (!CI.Callee || llvm::all_of(CI.Clones, [](unsigned CloneNo) {
8075 return CloneNo == 0;
8076 }))
8077 continue;
8078 // Note that we mark the edge even if the callee is defined in the
8079 // same module as the caller, since the caller may be imported into
8080 // other modules. The function importer will skip importing the callee
8081 // into any module that already defines it.
8082 NumClonedCalleeCallsites++;
8083 // Lazily build the map on the first such callsite in this function.
8084 if (CalleeToEdge.empty())
8085 for (auto &[CalleeVI, Edge] : FS->mutableCalls())
8086 CalleeToEdge[CalleeVI] = &Edge;
8087 auto It = CalleeToEdge.find(CI.Callee);
8088 if (It == CalleeToEdge.end())
8089 continue;
8090 CalleeInfo *Edge = It->second;
8091 // Direct calls don't require importing the callee. Note that the
8092 // edge is marked indirect if any call to the callee in this function
8093 // is indirect, in which case we can't distinguish whether this
8094 // callsite is direct and conservatively treat it as indirect.
8095 if (!Edge->hasIndirectCall())
8096 continue;
8097 NumClonedCalleeIndirectCallsites++;
8098 if (!Edge->requiresDeclImport()) {
8099 Edge->setRequiresDeclImport(true);
8100 NumClonedCalleeEdgesMarkedRequiresDeclImport++;
8101 }
8102 }
8103 }
8104 }
8105}
8106
8108 ModuleSummaryIndex &Index,
8110 isPrevailing,
8111 LLVMContext &Ctx,
8112 function_ref<void(StringRef, StringRef, const Twine &)> EmitRemark) {
8113 // TODO: If/when other types of memprof cloning are enabled beyond just for
8114 // hot and cold, we will need to change this to individually control the
8115 // AllocationType passed to addStackNodesForMIB during CCG construction.
8116 // The index was set from the option, so these should be in sync.
8117 assert(Index.withSupportsHotColdNew() == SupportsHotColdNew);
8118 if (!SupportsHotColdNew)
8119 return;
8120
8121 bool AllowExtraAnalysis =
8123
8124 bool Changed = false;
8125 {
8126 IndexCallsiteContextGraph CCG(Index, isPrevailing);
8127 Changed = CCG.process(EmitRemark, AllowExtraAnalysis);
8128 }
8129 // Note this must be done after the CCG is destroyed, which is when the
8130 // callsite records synthesized for missing tail call frames are added to the
8131 // function summaries. Importing declarations is only useful if MemProf ICP
8132 // is allowed to promote to a declaration of the target.
8135 updateClonedCalleeEdgesForImport(Index, isPrevailing);
8136}
8137
8138// Strips MemProf attributes and metadata. Can be invoked by the pass pipeline
8139// when we don't have an index that has recorded that we are linking with
8140// allocation libraries containing the necessary APIs for downstream
8141// transformations.
8143 // The profile matcher applies hotness attributes directly for allocations,
8144 // and those will cause us to generate calls to the hot/cold interfaces
8145 // unconditionally. If supports-hot-cold-new was not enabled in the LTO
8146 // link then assume we don't want these calls (e.g. not linking with
8147 // the appropriate library, or otherwise trying to disable this behavior).
8148 bool Changed = false;
8149 for (auto &F : M) {
8150 for (auto &BB : F) {
8151 for (auto &I : BB) {
8152 auto *CI = dyn_cast<CallBase>(&I);
8153 if (!CI)
8154 continue;
8155 if (CI->hasFnAttr("memprof")) {
8156 CI->removeFnAttr("memprof");
8157 Changed = true;
8158 }
8159 if (!CI->hasMetadata(LLVMContext::MD_callsite)) {
8160 assert(!CI->hasMetadata(LLVMContext::MD_memprof));
8161 continue;
8162 }
8163 // Strip off all memprof metadata as it is no longer needed.
8164 // Importantly, this avoids the addition of new memprof attributes
8165 // after inlining propagation.
8166 CI->setMetadata(LLVMContext::MD_memprof, nullptr);
8167 CI->setMetadata(LLVMContext::MD_callsite, nullptr);
8168 Changed = true;
8169 }
8170 }
8171 }
8172 if (!Changed)
8173 return PreservedAnalyses::all();
8174 return PreservedAnalyses::none();
8175}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
aarch64 promote const
uint64_t uint64_t
Unify divergent function exit nodes
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
#define DEBUG_TYPE
Module.h This file contains the declarations for the Module class.
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
#define G(x, y, z)
Definition MD5.cpp:55
Machine Check Debug Module
This file implements a map that provides insertion order iteration.
static cl::opt< unsigned > TailCallSearchDepth("memprof-tail-call-search-depth", cl::init(5), cl::Hidden, cl::desc("Max depth to recursively search for missing " "frames through tail calls."))
uint64_t ComputeHash(const FunctionSummary *FS, unsigned I)
static cl::opt< DotScope > DotGraphScope("memprof-dot-scope", cl::desc("Scope of graph to export to dot"), cl::Hidden, cl::init(DotScope::All), cl::values(clEnumValN(DotScope::All, "all", "Export full callsite graph"), clEnumValN(DotScope::Alloc, "alloc", "Export only nodes with contexts feeding given " "-memprof-dot-alloc-id"), clEnumValN(DotScope::Context, "context", "Export only nodes with given -memprof-dot-context-id")))
static cl::opt< bool > DoMergeIteration("memprof-merge-iteration", cl::init(true), cl::Hidden, cl::desc("Iteratively apply merging on a node to catch new callers"))
static cl::opt< unsigned > AllocIdForDot("memprof-dot-alloc-id", cl::init(0), cl::Hidden, cl::desc("Id of alloc to export if -memprof-dot-scope=alloc " "or to highlight if -memprof-dot-scope=all"))
static bool clonesHaveSameVersions(const FunctionSummary *FS, unsigned I, unsigned J)
static std::optional< GlobalValue::GUID > getImportedAliasGUID(const Function &F)
static void updateClonedCalleeEdgesForImport(ModuleSummaryIndex &Index, llvm::function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing)
static cl::opt< unsigned > ContextIdForDot("memprof-dot-context-id", cl::init(0), cl::Hidden, cl::desc("Id of context to export if -memprof-dot-scope=context or to " "highlight otherwise"))
static cl::opt< bool > ExportToDot("memprof-export-to-dot", cl::init(false), cl::Hidden, cl::desc("Export graph to dot files."))
static ValueInfo findValueInfoForGV(const GlobalValue &GV, const Module &M, const ModuleSummaryIndex *ImportSummary, const Function *CallingFunc=nullptr)
static void checkEdge(const std::shared_ptr< ContextEdge< DerivedCCG, FuncTy, CallTy > > &Edge)
static cl::opt< bool > AllowRecursiveCallsites("memprof-allow-recursive-callsites", cl::init(true), cl::Hidden, cl::desc("Allow cloning of callsites involved in recursive cycles"))
bool checkColdOrNotCold(uint8_t AllocType)
static cl::opt< bool > CloneRecursiveContexts("memprof-clone-recursive-contexts", cl::init(true), cl::Hidden, cl::desc("Allow cloning of contexts through recursive cycles"))
static std::string getAllocTypeString(uint8_t AllocTypes)
static cl::opt< bool > IndirectTailCallSearch("memprof-indirect-tail-call-search", cl::init(true), cl::Hidden, cl::desc("Search for missing frames through indirect tail calls to their " "profiled targets (ThinLTO)."))
bool DOTGraphTraits< constCallsiteContextGraph< DerivedCCG, FuncTy, CallTy > * >::DoHighlight
static cl::opt< bool > AssignTrimmedInlinedContexts("memprof-assign-trimmed-inlined-contexts", cl::init(true), cl::Hidden, cl::desc("Assign contexts trimmed within a call's inlined stack id " "sequence to that call"))
static unsigned getMemProfCloneNum(const Function &F)
static cl::opt< unsigned > MemProfICPNoInlineThreshold("memprof-icp-noinline-threshold", cl::init(0), cl::Hidden, cl::desc("Minimum absolute count for promoted target to be inlinable"))
static SmallVector< std::unique_ptr< ValueToValueMapTy >, 4 > createFunctionClones(Function &F, unsigned NumClones, Module &M, OptimizationRemarkEmitter &ORE, std::map< const Function *, SmallPtrSet< const GlobalAlias *, 1 > > &FuncToAliasMap, FunctionSummary *FS)
static cl::opt< bool > VerifyCCG("memprof-verify-ccg", cl::init(false), cl::Hidden, cl::desc("Perform verification checks on CallingContextGraph."))
static void checkNode(const ContextNode< DerivedCCG, FuncTy, CallTy > *Node, bool CheckEdges=true)
static cl::opt< bool > MergeClones("memprof-merge-clones", cl::init(true), cl::Hidden, cl::desc("Merge clones before assigning functions"))
static std::string getMemProfFuncName(Twine Base, unsigned CloneNo)
static cl::opt< std::string > MemProfImportSummary("memprof-import-summary", cl::desc("Import summary to use for testing the ThinLTO backend via opt"), cl::Hidden)
static const CallsiteInfo * findIndirectTailCallCallsite(uint64_t CandidateGUID, const DenseMap< ValueInfo, CallsiteInfo > &MapTailCallCalleeVIToCallsite, std::optional< DenseMap< uint64_t, const CallsiteInfo * > > &OrigNameGUIDToTailCallCallsite, std::optional< DenseMap< ValueInfo, const GlobalAlias * > > &AliasVIToGV, const Module &M, const ModuleSummaryIndex *ImportSummary, InstrProfSymtab &Symtab)
static void updateSubprogramLinkageName(Function *NewFunc, StringRef Name)
static cl::opt< bool > AllowRecursiveContexts("memprof-allow-recursive-contexts", cl::init(true), cl::Hidden, cl::desc("Allow cloning of contexts having recursive cycles"))
static cl::opt< std::string > DotFilePathPrefix("memprof-dot-file-path-prefix", cl::init(""), cl::Hidden, cl::value_desc("filename"), cl::desc("Specify the path prefix of the MemProf dot files."))
static cl::opt< bool > VerifyNodes("memprof-verify-nodes", cl::init(false), cl::Hidden, cl::desc("Perform frequent verification checks on nodes."))
static void checkAllocContextIds(const AllocInfo &AllocNode, const MDNode *MemProfMD, const CallStack< MDNode, MDNode::op_iterator > &CallsiteContext, const ModuleSummaryIndex *ImportSummary)
static cl::opt< bool > DumpCCG("memprof-dump-ccg", cl::init(false), cl::Hidden, cl::desc("Dump CallingContextGraph to stdout after each stage."))
AllocType
This is the interface to build a ModuleSummaryIndex for a module.
ModuleSummaryIndex.h This file contains the declarations the classes that hold the module index and s...
#define P(N)
if(auto Err=PB.parsePassPipeline(MPM, Passes)) return wrap(std MPM run * Mod
FunctionAnalysisManager FAM
if(PassOpts->AAPipeline)
const char * Msg
std::pair< BasicBlock *, BasicBlock * > Edge
This file defines generic set operations that may be used on set's of different types,...
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
This file contains some functions that are useful when dealing with strings.
#define LLVM_DEBUG(...)
Definition Debug.h:119
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
ValueInfo getAliaseeVI() const
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
ArrayRef< T > drop_back(size_t N=1) const
Drop the last N elements of the array.
Definition ArrayRef.h:200
bool empty() const
Check if the array is empty.
Definition ArrayRef.h:136
static LLVM_ABI Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
void addFnAttr(Attribute::AttrKind Kind)
Adds the attribute to the function.
void setCalledOperand(Value *V)
Subprogram description. Uses SubclassData1.
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Definition DenseMap.h:757
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition DenseMap.h:762
bool empty() const
Definition DenseMap.h:716
iterator find(const_arg_type_t< KeyT > Val)
Definition DenseMap.h:766
ValueT & at(const_arg_type_t< KeyT > Val)
Return the entry for the specified key, or abort if no such entry exists.
Definition DenseMap.h:811
void reserve(size_type NumEntries)
Grow the DenseMap so that it can contain at least NumEntries items before resizing again.
Definition DenseMap.h:721
iterator end()
Definition DenseMap.h:686
unsigned size() const
Definition DenseMap.h:717
bool erase(const KeyT &Val)
Definition DenseMap.h:930
ValueT lookup(const_arg_type_t< KeyT > Val) const
Return the entry for the specified key, or a default constructed value if no such entry exists.
Definition DenseMap.h:793
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:827
Implements a dense probed hash-table based set.
Definition DenseSet.h:283
Function summary information to aid decisions and implementation of importing.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:212
DISubprogram * getSubprogram() const
Get the attached subprogram.
const Function & getFunction() const
Definition Function.h:167
static LLVM_ABI GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
Definition Globals.cpp:692
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set a particular kind of metadata attachment.
Function and variable summary information to aid decisions and implementation of importing.
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
Definition Globals.cpp:80
static bool isLocalLinkage(LinkageTypes Linkage)
LLVM_ABI bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition Globals.cpp:408
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
bool hasLocalLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
LLVM_ABI void eraseFromParent()
This method unlinks 'this' from the containing module and deletes it.
Definition Globals.cpp:158
LLVM_ABI GUID getGUID() const
Return a 64-bit global unique ID for this value.
Definition Globals.cpp:103
static LLVM_ABI std::string getGlobalIdentifier(StringRef Name, GlobalValue::LinkageTypes Linkage, StringRef FileName)
Return the modified name for a global value suitable to be used as the key for a global lookup (e....
Definition Globals.cpp:234
bool isWeakForLinker() const
@ InternalLinkage
Rename collisions when linking (static functions).
Definition GlobalValue.h:60
A symbol table used for function [IR]PGO name look-up with keys (such as pointers,...
Definition InstrProf.h:500
Function * getFunction(uint64_t FuncMD5Hash) const
Return function from the name's md5 hash. Return nullptr if not found.
Definition InstrProf.h:800
LLVM_ABI const Function * getFunction() const
Return the function this instruction belongs to.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
Metadata node.
Definition Metadata.h:1087
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1443
ArrayRef< MDOperand > operands() const
Definition Metadata.h:1441
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1449
LLVM_ABI TempMDNode clone() const
Create a (temporary) clone of this.
Definition Metadata.cpp:668
static std::enable_if_t< std::is_base_of< MDNode, T >::value, T * > replaceWithUniqued(std::unique_ptr< T, TempMDNodeDeleter > N)
Replace a temporary node with a uniqued one.
Definition Metadata.h:1318
static LLVM_ABI MDString * get(LLVMContext &Context, StringRef Str)
Definition Metadata.cpp:595
This class implements a map that also provides access to all stored values in a deterministic order.
Definition MapVector.h:39
size_type count(const KeyT &Key) const
Definition MapVector.h:153
LLVM_ABI MemProfContextDisambiguation(const ModuleSummaryIndex *Summary=nullptr, bool isSamplePGO=false)
LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFile(const Twine &Filename, bool IsText=false, bool RequiresNullTerminator=true, bool IsVolatile=false, std::optional< Align > Alignment=std::nullopt)
Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful,...
Class to hold module path string table and global value map, and encapsulate methods for operating on...
static StringRef getOriginalNameBeforePromote(StringRef Name)
Helper to obtain the unpromoted name for a global value (or the original name if not promoted).
ValueInfo getValueInfo(const GlobalValueSummaryMapTy::value_type &R) const
Return a ValueInfo for the index value_type (convenient when iterating index).
uint64_t getStackIdAtIndex(unsigned Index) const
GlobalValue::GUID getGUIDFromOriginalID(GlobalValue::GUID OriginalID) const
Return the GUID for OriginalId in the OidGuidMap.
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
LLVMContext & getContext() const
Get the global data context.
Definition Module.h:332
A NodeSet contains a set of SUnit DAG nodes with additional information that assigns a priority to th...
unsigned size() const
bool insert(SUnit *SU)
The optimization diagnostic interface.
bool allowExtraAnalysis(StringRef PassName) const
Whether we allow for extra compile-time budget to perform more analysis to produce fewer false positi...
LLVM_ABI void emit(DiagnosticInfoOptimizationBase &OptDiag)
Output the remark via the diagnostic handler and to the optimization record file.
Diagnostic information for applied optimization remarks.
A discriminated union of two or more pointer types, with the discriminator in the low bits of the poi...
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition Analysis.h:115
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
A class that wrap the SHA1 algorithm.
Definition SHA1.h:27
LLVM_ABI void update(ArrayRef< uint8_t > Data)
Digest more data.
Definition SHA1.cpp:208
LLVM_ABI std::array< uint8_t, 20 > result()
Return the current raw 160-bits SHA1 for the digested data since the last call to init().
Definition SHA1.cpp:288
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition SmallSet.h:184
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
bool contains(StringRef Other) const
Return true if the given string is a substring of *this, and false otherwise.
Definition StringRef.h:438
std::pair< StringRef, StringRef > rsplit(StringRef Separator) const
Split into two substrings around the last occurrence of a separator string.
Definition StringRef.h:761
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
LLVM_ABI void print(raw_ostream &O, bool IsForDebug=false) const
Implement operator<< on Value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:548
LLVMContext & getContext() const
All values hold a context through their type.
Definition Value.h:260
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
Definition Value.cpp:314
std::pair< iterator, bool > insert(const ValueT &V)
Definition DenseSet.h:210
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition DenseSet.h:183
bool erase(const ValueT &V)
Definition DenseSet.h:98
void insert_range(Range &&R)
Definition DenseSet.h:236
size_type size() const
Definition DenseSet.h:85
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition DenseSet.h:188
void reserve(size_t Size)
Grow the DenseSet so that it can contain at least NumEntries items before resizing again.
Definition DenseSet.h:94
An efficient, type-erasing, non-owning reference to a callable.
Helper class to iterate through stack ids in both metadata (memprof MIB and callsite) and the corresp...
CallStackIterator beginAfterSharedPrefix(const CallStack &Other)
CallStackIterator end() const
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
CallInst * Call
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Entry
Definition COFF.h:862
@ CE
Windows NT (Windows on ARM)
Definition MCAsmInfo.h:51
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
Definition Metadata.h:708
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract(Y &&MD)
Extract a Value from Metadata.
Definition Metadata.h:680
LLVM_ABI AllocationType getMIBAllocType(const MDNode *MIB)
Returns the allocation type from an MIB metadata node.
LLVM_ABI bool metadataMayIncludeContextSizeInfo()
Whether the alloc memprof metadata may include context size info for some MIBs (but possibly not all)...
LLVM_ABI bool hasSingleAllocType(uint8_t AllocTypes)
True if the AllocTypes bitmask contains just a single type.
LLVM_ABI bool isAmbiguousAllocType(uint8_t AllocTypes)
True if the AllocTypes bitmask contains both NotCold and Cold, i.e.
LLVM_ABI std::string getAllocTypeAttributeString(AllocationType Type)
Returns the string to use in attributes with the given type.
LLVM_ABI MDNode * getMIBStackNode(const MDNode *MIB)
Returns the stack node from an MIB metadata node.
LLVM_ABI void removeAnyExistingAmbiguousAttribute(CallBase *CB)
Removes any existing "ambiguous" memprof attribute.
LLVM_ABI void addAmbiguousAttribute(CallBase *CB)
Adds an "ambiguous" memprof attribute to call with a matched allocation profile but that we haven't y...
DiagnosticInfoOptimizationBase::Argument NV
LLVM_ABI CallBase & promoteIndirectCall(CallBase &CB, Function *F, uint64_t Count, uint64_t TotalCount, bool AttachProfToDirectCall, OptimizationRemarkEmitter *ORE)
uint32_t NodeId
Definition RDFGraph.h:262
NodeAddr< NodeBase * > Node
Definition RDFGraph.h:381
NodeAddr< FuncNode * > Func
Definition RDFGraph.h:393
bool empty() const
Definition BasicBlock.h:101
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
uint64_t read64le(const void *P)
Definition Endian.h:415
void write32le(void *P, uint32_t V)
Definition Endian.h:455
This is an optimization pass for GlobalISel generic memory operations.
cl::opt< unsigned > MinClonedColdBytePercent("memprof-cloning-cold-threshold", cl::init(100), cl::Hidden, cl::desc("Min percent of cold bytes to hint alloc cold during cloning"))
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
LLVM_ABI void logAllUnhandledErrors(Error E, raw_ostream &OS, Twine ErrorBanner={})
Log all errors (if any) in E to OS.
Definition Error.cpp:61
void stable_sort(R &&Range)
Definition STLExtras.h:2132
cl::opt< bool > MemProfReportHintedSizes("memprof-report-hinted-sizes", cl::init(false), cl::Hidden, cl::desc("Report total allocation sizes of hinted allocations"))
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
LLVM_ABI bool isLegalToPromote(const CallBase &CB, Function *Callee, const char **FailureReason=nullptr)
Return true if the given indirect call site can be made to call Callee.
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr, unsigned DynamicVGPRBlockSize=0)
cl::opt< bool > MemProfUseAmbiguousAttributes
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2575
void set_intersect(S1Ty &S1, const S2Ty &S2)
set_intersect(A, B) - Compute A := A ^ B Identical to set_intersection, except that it works on set<>...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
LLVM_ABI bool mayHaveMemprofSummary(const CallBase *CB)
Returns true if the instruction could have memprof metadata, used to ensure consistency between summa...
constexpr from_range_t from_range
static cl::opt< bool > MemProfRequireDefinitionForPromotion("memprof-require-definition-for-promotion", cl::init(false), cl::Hidden, cl::desc("Require target function definition when promoting indirect calls"))
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
cl::opt< unsigned > MemProfTopNImportant("memprof-top-n-important", cl::init(10), cl::Hidden, cl::desc("Number of largest cold contexts to consider important"))
bool set_is_subset(const S1Ty &S1, const S2Ty &S2)
set_is_subset(A, B) - Return true iff A in B
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition STLExtras.h:2229
static cl::opt< bool > MemProfImportClonedIndirectTargets("memprof-import-cloned-indirect-targets", cl::init(true), cl::Hidden, cl::desc("Import at least a declaration of the indirect call targets of " "callsites assigned to call a cloned callee (needed for MemProf " "ICP in the ThinLTO backend)"))
void set_subtract(S1Ty &S1, const S2Ty &S2)
set_subtract(A, B) - Compute A := A - B
InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy
Provide the FunctionAnalysisManager to Module proxy.
raw_ostream & WriteGraph(raw_ostream &O, const GraphType &G, bool ShortNames=false, const Twine &Title="")
bool isa_and_nonnull(const Y &Val)
Definition Casting.h:676
bool set_intersects(const S1Ty &S1, const S2Ty &S2)
set_intersects(A, B) - Return true iff A ^ B is non empty
detail::concat_range< ValueT, RangeTs... > concat(RangeTs &&...Ranges)
Returns a concatenated range across two or more ranges.
Definition STLExtras.h:1167
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
LLVM_ABI Expected< std::unique_ptr< ModuleSummaryIndex > > getModuleSummaryIndex(MemoryBufferRef Buffer)
Parse the specified bitcode buffer, returning the module summary index.
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
LLVM_ABI void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
static cl::opt< bool > MemProfReportHintedSizesDetail("memprof-report-hinted-sizes-detail", cl::init(false), cl::Hidden, cl::desc("Append post-cloning call path and matching details to each " "-memprof-report-hinted-sizes line"))
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1769
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
bool set_union(S1Ty &S1, const S2Ty &S2)
set_union(A, B) - Compute A := A u B, return whether A changed.
cl::opt< bool > SupportsHotColdNew
Indicate we are linking with an allocator that supports hot/cold operator new interfaces.
static cl::opt< unsigned > MemProfReportHintedSizesDetailMaxPathLen("memprof-report-hinted-sizes-detail-max-path-len", cl::init(7), cl::Hidden, cl::desc("Maximum number of functions printed in full for a post-cloning " "call path in the -memprof-report-hinted-sizes-detail suffix; " "longer paths are elided in the middle"))
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
S1Ty set_intersection(const S1Ty &S1, const S2Ty &S2)
set_intersection(A, B) - Return A ^ B
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
LLVM_ABI unsigned getMaxSummaryIndirectEdges(const LLVMContext &C)
Return -module-summary-max-indirect-edges.
cl::opt< bool > EnableMemProfContextDisambiguation
Enable MemProf context disambiguation for thin link.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
Definition InstrProf.h:145
S1Ty set_difference(const S1Ty &S1, const S2Ty &S2)
set_difference(A, B) - Return A - B
bool isMemProfClone(const GlobalValue &GV)
Returns true if GV is a MemProf context-disambiguated clone.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Expected< T > errorOrToExpected(ErrorOr< T > &&EO)
Convert an ErrorOr<T> to an Expected<T>.
Definition Error.h:1261
ArrayRef(const T &OneElt) -> ArrayRef< T >
std::string toString(const APInt &I, unsigned Radix, bool Signed, bool formatAsCLiteral=false, bool UpperCase=true, bool InsertSeparators=false)
ValueMap< const Value *, WeakTrackingVH > ValueToValueMapTy
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1933
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
Definition InstrProf.h:147
constexpr StringLiteral MemProfCloneSuffix
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
LLVM_ABI Function * CloneFunction(Function *F, ValueToValueMapTy &VMap, ClonedCodeInfo *CodeInfo=nullptr)
Return a copy of the specified function and add it to that function's module.
AnalysisManager< Module > ModuleAnalysisManager
Convenience typedef for the Module analysis manager.
Definition MIRParser.h:39
cl::opt< bool > MemProfFixupImportant("memprof-fixup-important", cl::init(true), cl::Hidden, cl::desc("Enables edge fixup for important contexts"))
#define N
static std::string getEdgeAttributes(NodeRef, ChildIteratorType ChildIter, GraphType G)
static const ContextNode< DerivedCCG, FuncTy, CallTy > * GetCallee(const EdgePtrTy &P)
std::unique_ptr< ContextNode< DerivedCCG, FuncTy, CallTy > > NodePtrTy
mapped_iterator< typename std::vector< std::shared_ptr< ContextEdge< DerivedCCG, FuncTy, CallTy > > >::const_iterator, decltype(&GetCallee)> ChildIteratorType
mapped_iterator< typename std::vector< NodePtrTy >::const_iterator, decltype(&getNode)> nodes_iterator
std::shared_ptr< ContextEdge< DerivedCCG, FuncTy, CallTy > > EdgePtrTy
Summary of memprof metadata on allocations.
std::vector< MIBInfo > MIBs
Class to accumulate and hold information about a callee.
Summary of memprof callsite metadata.
SmallVector< unsigned > StackIdIndices
SmallVector< unsigned > Clones
DefaultDOTGraphTraits(bool simple=false)
An information struct used to provide DenseMap with the various necessary components for a given valu...
typename GraphType::UnknownGraphTypeError NodeRef
Definition GraphTraits.h:94
Struct that holds a reference to a particular GUID in a global value summary.
ArrayRef< std::unique_ptr< GlobalValueSummary > > getSummaryList() const
GlobalValue::GUID getGUID() const
PointerUnion< CallsiteInfo *, AllocInfo * > SimpleType
static SimpleType getSimplifiedValue(IndexCall &Val)
const PointerUnion< CallsiteInfo *, AllocInfo * > SimpleType
static SimpleType getSimplifiedValue(const IndexCall &Val)
Define a template that can be specialized by smart pointers to reflect the fact that they are automat...
Definition Casting.h:34