LLVM 24.0.0git
ModuleSummaryAnalysis.cpp
Go to the documentation of this file.
1//===- ModuleSummaryAnalysis.cpp - Module summary index builder -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass builds a ModuleSummaryIndex object for the module, to be written
10// to bitcode or LLVM assembly.
11//
12//===----------------------------------------------------------------------===//
13
15#include "AnalysisOptions.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/DenseSet.h"
18#include "llvm/ADT/MapVector.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SetVector.h"
23#include "llvm/ADT/StringRef.h"
33#include "llvm/IR/Attributes.h"
34#include "llvm/IR/BasicBlock.h"
35#include "llvm/IR/Constant.h"
36#include "llvm/IR/Constants.h"
37#include "llvm/IR/CycleInfo.h"
38#include "llvm/IR/Dominators.h"
39#include "llvm/IR/Function.h"
40#include "llvm/IR/GlobalAlias.h"
41#include "llvm/IR/GlobalValue.h"
45#include "llvm/IR/LLVMContext.h"
46#include "llvm/IR/Metadata.h"
47#include "llvm/IR/Module.h"
49#include "llvm/IR/Use.h"
50#include "llvm/IR/User.h"
54#include "llvm/Pass.h"
58#include <cassert>
59#include <cstdint>
60#include <vector>
61
62using namespace llvm;
63using namespace llvm::memprof;
64
65#define DEBUG_TYPE "module-summary-analysis"
66
68 return getAnalysisOptions(C).module_summary_max_indirect_edges;
69}
70
71// Walk through the operands of a given User via worklist iteration and populate
72// the set of GlobalValue references encountered. Invoked either on an
73// Instruction or a GlobalVariable (which walks its initializer).
74// Return true if any of the operands contains blockaddress. This is important
75// to know when computing summary for global var, because if global variable
76// references basic block address we can't import it separately from function
77// containing that basic block. For simplicity we currently don't import such
78// global vars at all. When importing function we aren't interested if any
79// instruction in it takes an address of any basic block, because instruction
80// can only take an address of basic block located in the same function.
81// Set `RefLocalLinkageIFunc` to true if the analyzed value references a
82// local-linkage ifunc.
83static bool
84findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
87 bool &RefLocalLinkageIFunc) {
88 bool HasBlockAddress = false;
90 if (Visited.insert(CurUser).second)
91 Worklist.push_back(CurUser);
92
93 while (!Worklist.empty()) {
94 const User *U = Worklist.pop_back_val();
95 const auto *CB = dyn_cast<CallBase>(U);
96
97 for (const auto &OI : U->operands()) {
98 const User *Operand = dyn_cast<User>(OI);
99 if (!Operand)
100 continue;
101 if (isa<BlockAddress>(Operand)) {
102 HasBlockAddress = true;
103 continue;
104 }
105 if (auto *GV = dyn_cast<GlobalValue>(Operand)) {
106 // We have a reference to a global value. This should be added to
107 // the reference set unless it is a callee. Callees are handled
108 // specially by WriteFunction and are added to a separate list.
109 if (!(CB && CB->isCallee(&OI))) {
110 // If an ifunc has local linkage, do not add it into ref edges, and
111 // sets `RefLocalLinkageIFunc` to true. The referencer is not eligible
112 // for import. An ifunc doesn't have summary and ThinLTO cannot
113 // promote it; importing the referencer may cause linkage errors.
114 if (auto *GI = dyn_cast_if_present<GlobalIFunc>(GV);
115 GI && GI->hasLocalLinkage()) {
116 RefLocalLinkageIFunc = true;
117 continue;
118 }
119 RefEdges.insert(Index.getOrInsertValueInfo(GV));
120 }
121 continue;
122 }
123 if (Visited.insert(Operand).second)
124 Worklist.push_back(Operand);
125 }
126 }
127
128 const Instruction *I = dyn_cast<Instruction>(CurUser);
129 if (I) {
130 uint64_t TotalCount = 0;
131 // MaxNumVTableAnnotations is the maximum number of vtables annotated on
132 // the instruction.
133 auto ValueDataArray = getValueProfDataFromInst(
134 *I, IPVK_VTableTarget, getMaxNumVTableAnnotations(I->getContext()),
135 TotalCount);
136
137 for (const auto &V : ValueDataArray)
138 RefEdges.insert(Index.getOrInsertValueInfo(/* VTableGUID = */
139 V.Value));
140 }
141 return HasBlockAddress;
142}
143
144/// Collect globals referenced via !implicit.ref metadata on a function
145/// and add them as reference edges in the module summary. This ensures
146/// ThinLTO liveness analysis treats them as live when the function is
147/// live, and imports them alongside the function during cross-module
148/// importing.
150 ModuleSummaryIndex &Index, const Function &F,
152 if (!F.hasMetadata(LLVMContext::MD_implicit_ref))
153 return;
155 F.getMetadata(LLVMContext::MD_implicit_ref, MDs);
156 for (MDNode *MD : MDs) {
157 for (const MDOperand &Op : MD->operands()) {
158 if (auto *VAM = dyn_cast_or_null<ValueAsMetadata>(Op.get()))
159 if (auto *GV = dyn_cast<GlobalValue>(VAM->getValue()))
160 RefEdges.insert(Index.getOrInsertValueInfo(GV));
161 }
162 }
163}
164
166 ProfileSummaryInfo *PSI) {
167 if (!PSI)
169 if (PSI->isHotCount(ProfileCount))
171 if (PSI->isColdCount(ProfileCount))
174}
175
176static bool isNonRenamableLocal(const GlobalValue &GV) {
177 return GV.hasSection() && GV.hasLocalLinkage();
178}
179
180/// Determine whether this call has all constant integer arguments (excluding
181/// "this") and summarize it to VCalls or ConstVCalls as appropriate.
182static void addVCallToSet(
184 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
185 &VCalls,
187 std::vector<FunctionSummary::ConstVCall>> &ConstVCalls) {
188 std::vector<uint64_t> Args;
189 // Start from the second argument to skip the "this" pointer.
190 for (auto &Arg : drop_begin(Call.CB.args())) {
191 auto *CI = dyn_cast<ConstantInt>(Arg);
192 if (!CI || CI->getBitWidth() > 64) {
193 VCalls.insert({Guid, Call.Offset});
194 return;
195 }
196 Args.push_back(CI->getZExtValue());
197 }
198 ConstVCalls.insert({{Guid, Call.Offset}, std::move(Args)});
199}
200
201/// If this intrinsic call requires that we add information to the function
202/// summary, do so via the non-constant reference arguments.
204 const CallInst *CI,
205 SetVector<GlobalValue::GUID, std::vector<GlobalValue::GUID>> &TypeTests,
206 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
207 &TypeTestAssumeVCalls,
208 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
209 &TypeCheckedLoadVCalls,
211 std::vector<FunctionSummary::ConstVCall>>
212 &TypeTestAssumeConstVCalls,
214 std::vector<FunctionSummary::ConstVCall>>
215 &TypeCheckedLoadConstVCalls,
216 DominatorTree &DT) {
217 switch (CI->getCalledFunction()->getIntrinsicID()) {
218 case Intrinsic::type_test:
219 case Intrinsic::public_type_test: {
220 auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(1));
221 auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
222 if (!TypeId)
223 break;
226
227 // Produce a summary from type.test intrinsics. We only summarize type.test
228 // intrinsics that are used other than by an llvm.assume intrinsic.
229 // Intrinsics that are assumed are relevant only to the devirtualization
230 // pass, not the type test lowering pass.
231 bool HasNonAssumeUses = llvm::any_of(CI->uses(), [](const Use &CIU) {
232 return !isa<AssumeInst>(CIU.getUser());
233 });
234 if (HasNonAssumeUses)
235 TypeTests.insert(Guid);
236
239 findDevirtualizableCallsForTypeTest(DevirtCalls, Assumes, CI, DT);
240 for (auto &Call : DevirtCalls)
241 addVCallToSet(Call, Guid, TypeTestAssumeVCalls,
242 TypeTestAssumeConstVCalls);
243
244 break;
245 }
246
247 case Intrinsic::type_checked_load_relative:
248 case Intrinsic::type_checked_load: {
249 auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(2));
250 auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
251 if (!TypeId)
252 break;
255
259 bool HasNonCallUses = false;
260 findDevirtualizableCallsForTypeCheckedLoad(DevirtCalls, LoadedPtrs, Preds,
261 HasNonCallUses, CI, DT);
262 // Any non-call uses of the result of llvm.type.checked.load will
263 // prevent us from optimizing away the llvm.type.test.
264 if (HasNonCallUses)
265 TypeTests.insert(Guid);
266 for (auto &Call : DevirtCalls)
267 addVCallToSet(Call, Guid, TypeCheckedLoadVCalls,
268 TypeCheckedLoadConstVCalls);
269
270 break;
271 }
272 default:
273 break;
274 }
275}
276
277static bool isNonVolatileLoad(const Instruction *I) {
278 if (const auto *LI = dyn_cast<LoadInst>(I))
279 return !LI->isVolatile();
280
281 return false;
282}
283
284static bool isNonVolatileStore(const Instruction *I) {
285 if (const auto *SI = dyn_cast<StoreInst>(I))
286 return !SI->isVolatile();
287
288 return false;
289}
290
291// Returns true if the function definition must be unreachable.
292//
293// Note if this helper function returns true, `F` is guaranteed
294// to be unreachable; if it returns false, `F` might still
295// be unreachable but not covered by this helper function.
297 // A function must be unreachable if its entry block ends with an
298 // 'unreachable'.
299 assert(!F.isDeclaration());
300 return isa<UnreachableInst>(F.getEntryBlock().getTerminator());
301}
302
304 const AnalysisOptions &Opts, ModuleSummaryIndex &Index, const Module &M,
306 DominatorTree &DT, bool HasLocalsInUsedOrAsm,
307 DenseSet<GlobalValue::GUID> &CantBePromoted, bool IsThinLTO,
308 std::function<const StackSafetyInfo *(const Function &F)> GetSSICallback) {
309 // Summary not currently supported for anonymous functions, they should
310 // have been named.
311 assert(F.hasName());
312
313 unsigned NumInsts = 0;
314 // Map from callee ValueId to profile count. Used to accumulate profile
315 // counts for all static calls to a given callee.
318 CallGraphEdges;
320 StoreRefEdges;
323 TypeTestAssumeVCalls, TypeCheckedLoadVCalls;
325 std::vector<FunctionSummary::ConstVCall>>
326 TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls;
327 ICallPromotionAnalysis ICallAnalysis;
329
330 // Add personality function, prefix data and prologue data to function's ref
331 // list.
332 bool HasLocalIFuncCallOrRef = false;
333 findRefEdges(Index, &F, RefEdges, Visited, HasLocalIFuncCallOrRef);
334 findImplicitRefEdges(Index, F, RefEdges);
335
336 std::vector<const Instruction *> NonVolatileLoads;
337 std::vector<const Instruction *> NonVolatileStores;
338
339 std::vector<CallsiteInfo> Callsites;
340 std::vector<AllocInfo> Allocs;
341
342#ifndef NDEBUG
343 DenseSet<const CallBase *> CallsThatMayHaveMemprofSummary;
344#endif
345
346 bool HasInlineAsmMaybeReferencingInternal = false;
347 bool HasIndirBranchToBlockAddress = false;
348 bool HasUnknownCall = false;
349 bool MayThrow = false;
350 for (const BasicBlock &BB : F) {
351 // We don't allow inlining of function with indirect branch to blockaddress.
352 // If the blockaddress escapes the function, e.g., via a global variable,
353 // inlining may lead to an invalid cross-function reference. So we shouldn't
354 // import such function either.
355 if (BB.hasAddressTaken()) {
356 for (User *U : BlockAddress::get(const_cast<BasicBlock *>(&BB))->users())
357 if (!isa<CallBrInst>(*U)) {
358 HasIndirBranchToBlockAddress = true;
359 break;
360 }
361 }
362
363 for (const Instruction &I : BB) {
365 continue;
366 ++NumInsts;
367
368 // Regular LTO module doesn't participate in ThinLTO import,
369 // so no reference from it can be read/writeonly, since this
370 // would require importing variable as local copy
371 if (IsThinLTO) {
372 if (isNonVolatileLoad(&I)) {
373 // Postpone processing of non-volatile load instructions
374 // See comments below
375 Visited.insert(&I);
376 NonVolatileLoads.push_back(&I);
377 continue;
378 } else if (isNonVolatileStore(&I)) {
379 Visited.insert(&I);
380 NonVolatileStores.push_back(&I);
381 // All references from second operand of store (destination address)
382 // can be considered write-only if they're not referenced by any
383 // non-store instruction. References from first operand of store
384 // (stored value) can't be treated either as read- or as write-only
385 // so we add them to RefEdges as we do with all other instructions
386 // except non-volatile load.
387 Value *Stored = I.getOperand(0);
388 if (auto *GV = dyn_cast<GlobalValue>(Stored))
389 // findRefEdges will try to examine GV operands, so instead
390 // of calling it we should add GV to RefEdges directly.
391 RefEdges.insert(Index.getOrInsertValueInfo(GV));
392 else if (auto *U = dyn_cast<User>(Stored))
393 findRefEdges(Index, U, RefEdges, Visited, HasLocalIFuncCallOrRef);
394 continue;
395 }
396 }
397 findRefEdges(Index, &I, RefEdges, Visited, HasLocalIFuncCallOrRef);
398 const auto *CB = dyn_cast<CallBase>(&I);
399 if (!CB) {
400 if (I.mayThrow())
401 MayThrow = true;
402 continue;
403 }
404
405 const auto *CI = dyn_cast<CallInst>(&I);
406 // Since we don't know exactly which local values are referenced in inline
407 // assembly, conservatively mark the function as possibly referencing
408 // a local value from inline assembly to ensure we don't export a
409 // reference (which would require renaming and promotion of the
410 // referenced value).
411 if (HasLocalsInUsedOrAsm && CI && CI->isInlineAsm())
412 HasInlineAsmMaybeReferencingInternal = true;
413
414 // Compute this once per indirect call.
415 uint32_t NumCandidates = 0;
416 uint64_t TotalCount = 0;
417 MutableArrayRef<InstrProfValueData> CandidateProfileData;
418
419 auto *CalledValue = CB->getCalledOperand();
420 auto *CalledFunction = CB->getCalledFunction();
421 if (CalledValue && !CalledFunction) {
422 CalledValue = CalledValue->stripPointerCasts();
423 // Stripping pointer casts can reveal a called function.
424 CalledFunction = dyn_cast<Function>(CalledValue);
425 }
426 // Check if this is an alias to a function. If so, get the
427 // called aliasee for the checks below.
428 if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
429 assert(!CalledFunction && "Expected null called function in callsite for alias");
430 CalledFunction = dyn_cast<Function>(GA->getAliaseeObject());
431 }
432 // Check if this is a direct call to a known function or a known
433 // intrinsic, or an indirect call with profile data.
434 if (CalledFunction) {
435 if (CI && CalledFunction->isIntrinsic()) {
437 CI, TypeTests, TypeTestAssumeVCalls, TypeCheckedLoadVCalls,
438 TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls, DT);
439 continue;
440 }
441 // We should have named any anonymous globals
442 assert(CalledFunction->hasName());
443 auto ScaledCount = PSI->getProfileCount(*CB, BFI);
444 auto Hotness = ScaledCount ? getHotness(*ScaledCount, PSI)
446 if (Opts.force_summary_edges_cold != ForceSummaryHotness::None)
448
449 // Use the original CalledValue, in case it was an alias. We want
450 // to record the call edge to the alias in that case. Eventually
451 // an alias summary will be created to associate the alias and
452 // aliasee.
453 auto &ValueInfo = CallGraphEdges[Index.getOrInsertValueInfo(
454 cast<GlobalValue>(CalledValue))];
455 ValueInfo.updateHotness(Hotness);
456 if (CB->isTailCall())
457 ValueInfo.setHasTailCall(true);
458 } else {
459 HasUnknownCall = true;
460 // If F is imported, a local linkage ifunc (e.g. target_clones on a
461 // static function) called by F will be cloned. Since summaries don't
462 // track ifunc, we do not know implementation functions referenced by
463 // the ifunc resolver need to be promoted in the exporter, and we will
464 // get linker errors due to cloned declarations for implementation
465 // functions. As a simple fix, just mark F as not eligible for import.
466 // Non-local ifunc is not cloned and does not have the issue.
467 if (auto *GI = dyn_cast_if_present<GlobalIFunc>(CalledValue))
468 if (GI->hasLocalLinkage())
469 HasLocalIFuncCallOrRef = true;
470 // Skip inline assembly calls.
471 if (CI && CI->isInlineAsm())
472 continue;
473 // Skip direct calls.
474 if (!CalledValue || isa<Constant>(CalledValue))
475 continue;
476
477 // Check if the instruction has a callees metadata. If so, add callees
478 // to CallGraphEdges to reflect the references from the metadata, and
479 // to enable importing for subsequent indirect call promotion and
480 // inlining.
481 if (auto *MD = I.getMetadata(LLVMContext::MD_callees)) {
482 for (const auto &Op : MD->operands()) {
484 if (Callee)
485 CallGraphEdges[Index.getOrInsertValueInfo(Callee)]
486 .setHasIndirectCall(true);
487 }
488 }
489
490 CandidateProfileData =
492 &I, TotalCount, NumCandidates,
493 Opts.module_summary_max_indirect_edges);
494 for (const auto &Candidate : CandidateProfileData) {
495 auto &Edge =
496 CallGraphEdges[Index.getOrInsertValueInfo(Candidate.Value)];
497 Edge.updateHotness(getHotness(Candidate.Count, PSI));
498 Edge.setHasIndirectCall(true);
499 // Record that the profiled target was reached via a tail call, so
500 // that MemProf context disambiguation can identify frames missing
501 // from profiled contexts due to indirect tail calls (see
502 // findProfiledCalleeThroughTailCalls).
503 if (CB->isTailCall())
504 Edge.setHasTailCall(true);
505 }
506 }
507
508 // Summarize memprof related metadata. This is only needed for ThinLTO.
509 if (!IsThinLTO)
510 continue;
511
512 // Skip indirect calls if we haven't enabled memprof ICP.
513 if (!CalledFunction && !Opts.enable_memprof_indirect_call_support)
514 continue;
515
516 // Ensure we keep this analysis in sync with the handling in the ThinLTO
517 // backend (see MemProfContextDisambiguation::applyImport). Save this call
518 // so that we can skip it in checking the reverse case later.
520#ifndef NDEBUG
521 CallsThatMayHaveMemprofSummary.insert(CB);
522#endif
523
524 // Compute the list of stack ids first (so we can trim them from the stack
525 // ids on any MIBs).
527 I.getMetadata(LLVMContext::MD_callsite));
528 auto *MemProfMD = I.getMetadata(LLVMContext::MD_memprof);
529 if (MemProfMD) {
530 std::vector<MIBInfo> MIBs;
531 std::vector<std::vector<ContextTotalSize>> ContextSizeInfos;
532 bool HasNonZeroContextSizeInfos = false;
533 for (auto &MDOp : MemProfMD->operands()) {
534 auto *MIBMD = cast<const MDNode>(MDOp);
537 SmallVector<unsigned> StackIdIndices;
539 // Collapse out any on the allocation call (inlining).
540 for (auto ContextIter =
541 StackContext.beginAfterSharedPrefix(InstCallsite);
542 ContextIter != StackContext.end(); ++ContextIter) {
543 unsigned StackIdIdx = Index.addOrGetStackIdIndex(*ContextIter);
544 // If this is a direct recursion, simply skip the duplicate
545 // entries. If this is mutual recursion, handling is left to
546 // the LTO link analysis client.
547 if (StackIdIndices.empty() || StackIdIndices.back() != StackIdIdx)
548 StackIdIndices.push_back(StackIdIdx);
549 }
550 // If we have context size information, collect it for inclusion in
551 // the summary.
552 assert(MIBMD->getNumOperands() > 2 ||
554 if (MIBMD->getNumOperands() > 2) {
555 std::vector<ContextTotalSize> ContextSizes;
556 for (unsigned I = 2; I < MIBMD->getNumOperands(); I++) {
557 MDNode *ContextSizePair = dyn_cast<MDNode>(MIBMD->getOperand(I));
558 assert(ContextSizePair->getNumOperands() == 2);
560 ContextSizePair->getOperand(0))
561 ->getZExtValue();
563 ContextSizePair->getOperand(1))
564 ->getZExtValue();
565 ContextSizes.push_back({FullStackId, TS});
566 }
567 // Flag that we need to keep the ContextSizeInfos array for this
568 // alloc as it now contains non-zero context info sizes.
569 HasNonZeroContextSizeInfos = true;
570 ContextSizeInfos.push_back(std::move(ContextSizes));
571 } else {
572 // The ContextSizeInfos must be in the same relative position as the
573 // associated MIB. In some cases we only include a ContextSizeInfo
574 // for a subset of MIBs in an allocation. To handle that, eagerly
575 // fill any MIB entries that don't have context size info metadata
576 // with a pair of 0s. Later on we will only use this array if it
577 // ends up containing any non-zero entries (see where we set
578 // HasNonZeroContextSizeInfos above).
579 ContextSizeInfos.push_back({{0, 0}});
580 }
581 MIBs.push_back(
582 MIBInfo(getMIBAllocType(MIBMD), std::move(StackIdIndices)));
583 }
584 Allocs.push_back(AllocInfo(std::move(MIBs)));
585 assert(HasNonZeroContextSizeInfos ||
587 // We eagerly build the ContextSizeInfos array, but it will be filled
588 // with sub arrays of pairs of 0s if no MIBs on this alloc actually
589 // contained context size info metadata. Only save it if any MIBs had
590 // any such metadata.
591 if (HasNonZeroContextSizeInfos) {
592 assert(Allocs.back().MIBs.size() == ContextSizeInfos.size());
593 Allocs.back().ContextSizeInfos = std::move(ContextSizeInfos);
594 }
595 } else if (!InstCallsite.empty()) {
596 SmallVector<unsigned> StackIdIndices;
597 for (auto StackId : InstCallsite)
598 StackIdIndices.push_back(Index.addOrGetStackIdIndex(StackId));
599 if (CalledFunction) {
600 // Use the original CalledValue, in case it was an alias. We want
601 // to record the call edge to the alias in that case. Eventually
602 // an alias summary will be created to associate the alias and
603 // aliasee.
604 auto CalleeValueInfo =
605 Index.getOrInsertValueInfo(cast<GlobalValue>(CalledValue));
606 Callsites.push_back({CalleeValueInfo, StackIdIndices});
607 } else {
608 assert(Opts.enable_memprof_indirect_call_support);
609 // For indirect callsites, create multiple Callsites, one per target.
610 // This enables having a different set of clone versions per target,
611 // and we will apply the cloning decisions while speculatively
612 // devirtualizing in the ThinLTO backends.
613 for (const auto &Candidate : CandidateProfileData) {
614 auto CalleeValueInfo = Index.getOrInsertValueInfo(Candidate.Value);
615 Callsites.push_back({CalleeValueInfo, StackIdIndices});
616 }
617 }
618 }
619 }
620 }
621
622 if (PSI->hasPartialSampleProfile() &&
623 Opts.scale_partial_sample_profile_working_set_size)
624 Index.addBlockCount(F.size());
625
627 if (IsThinLTO) {
628 auto AddRefEdges =
629 [&](const std::vector<const Instruction *> &Instrs,
632 for (const auto *I : Instrs) {
633 Cache.erase(I);
634 findRefEdges(Index, I, Edges, Cache, HasLocalIFuncCallOrRef);
635 }
636 };
637
638 // By now we processed all instructions in a function, except
639 // non-volatile loads and non-volatile value stores. Let's find
640 // ref edges for both of instruction sets
641 AddRefEdges(NonVolatileLoads, LoadRefEdges, Visited);
642 // We can add some values to the Visited set when processing load
643 // instructions which are also used by stores in NonVolatileStores.
644 // For example this can happen if we have following code:
645 //
646 // store %Derived* @foo, %Derived** bitcast (%Base** @bar to %Derived**)
647 // %42 = load %Derived*, %Derived** bitcast (%Base** @bar to %Derived**)
648 //
649 // After processing loads we'll add bitcast to the Visited set, and if
650 // we use the same set while processing stores, we'll never see store
651 // to @bar and @bar will be mistakenly treated as readonly.
653 AddRefEdges(NonVolatileStores, StoreRefEdges, StoreCache);
654
655 // If both load and store instruction reference the same variable
656 // we won't be able to optimize it. Add all such reference edges
657 // to RefEdges set.
658 for (const auto &VI : StoreRefEdges)
659 if (LoadRefEdges.remove(VI))
660 RefEdges.insert(VI);
661
662 unsigned RefCnt = RefEdges.size();
663 // All new reference edges inserted in two loops below are either
664 // read or write only. They will be grouped in the end of RefEdges
665 // vector, so we can use a single integer value to identify them.
666 RefEdges.insert_range(LoadRefEdges);
667
668 unsigned FirstWORef = RefEdges.size();
669 RefEdges.insert_range(StoreRefEdges);
670
671 Refs = RefEdges.takeVector();
672 for (; RefCnt < FirstWORef; ++RefCnt)
673 Refs[RefCnt].setReadOnly();
674
675 for (; RefCnt < Refs.size(); ++RefCnt)
676 Refs[RefCnt].setWriteOnly();
677 } else {
678 Refs = RefEdges.takeVector();
679 }
680 // Explicit add hot edges to enforce importing for designated GUIDs for
681 // sample PGO, to enable the same inlines as the profiled optimized binary.
682 for (auto &I : F.getImportGUIDs())
683 CallGraphEdges[Index.getOrInsertValueInfo(I)].updateHotness(
684 Opts.force_summary_edges_cold == ForceSummaryHotness::All
687
688#ifndef NDEBUG
689 // Make sure that all calls we decided could not have memprof summaries get a
690 // false value for mayHaveMemprofSummary, to ensure that this handling remains
691 // in sync with the ThinLTO backend handling.
692 if (IsThinLTO) {
693 for (const BasicBlock &BB : F) {
694 for (const Instruction &I : BB) {
695 const auto *CB = dyn_cast<CallBase>(&I);
696 if (!CB)
697 continue;
698 // We already checked these above.
699 if (CallsThatMayHaveMemprofSummary.count(CB))
700 continue;
702 }
703 }
704 }
705#endif
706
707 bool NonRenamableLocal = isNonRenamableLocal(F);
708 bool NotEligibleForImport =
709 NonRenamableLocal || HasInlineAsmMaybeReferencingInternal ||
710 HasIndirBranchToBlockAddress || HasLocalIFuncCallOrRef;
712 F.getLinkage(), F.getVisibility(), NotEligibleForImport,
713 /* Live = */ false, F.isDSOLocal(), F.canBeOmittedFromSymbolTable(),
715 /* NoRenameOnPromotion = */ false);
717 F.doesNotAccessMemory(), F.onlyReadsMemory() && !F.doesNotAccessMemory(),
718 F.hasFnAttribute(Attribute::NoRecurse), F.returnDoesNotAlias(),
719 // FIXME: refactor this to use the same code that inliner is using.
720 // Don't try to import functions with noinline attribute.
721 F.isNoInline(), F.hasFnAttribute(Attribute::AlwaysInline),
722 F.hasFnAttribute(Attribute::NoUnwind), MayThrow, HasUnknownCall,
724 std::vector<FunctionSummary::ParamAccess> ParamAccesses;
725 if (auto *SSI = GetSSICallback(F))
726 ParamAccesses = SSI->getParamAccesses(Index);
727 auto FuncSummary = std::make_unique<FunctionSummary>(
728 Flags, NumInsts, FunFlags, std::move(Refs), CallGraphEdges.takeVector(),
729 TypeTests.takeVector(), TypeTestAssumeVCalls.takeVector(),
730 TypeCheckedLoadVCalls.takeVector(),
731 TypeTestAssumeConstVCalls.takeVector(),
732 TypeCheckedLoadConstVCalls.takeVector(), std::move(ParamAccesses),
733 std::move(Callsites), std::move(Allocs));
734 if (NonRenamableLocal)
735 CantBePromoted.insert(F.getGUID());
736 Index.addGlobalValueSummary(F, std::move(FuncSummary));
737}
738
739/// Find function pointers referenced within the given vtable initializer
740/// (or subset of an initializer) \p I. The starting offset of \p I within
741/// the vtable initializer is \p StartingOffset. Any discovered function
742/// pointers are added to \p VTableFuncs along with their cumulative offset
743/// within the initializer.
744static void findFuncPointers(const Constant *I, uint64_t StartingOffset,
745 const Module &M, ModuleSummaryIndex &Index,
746 VTableFuncList &VTableFuncs,
747 const GlobalVariable &OrigGV) {
748 // First check if this is a function pointer.
749 if (I->getType()->isPointerTy()) {
750 auto C = I->stripPointerCasts();
751 auto A = dyn_cast<GlobalAlias>(C);
752 if (isa<Function>(C) || (A && isa<Function>(A->getAliasee()))) {
753 auto GV = dyn_cast<GlobalValue>(C);
754 assert(GV);
755 // We can disregard __cxa_pure_virtual as a possible call target, as
756 // calls to pure virtuals are UB.
757 if (GV && GV->getName() != "__cxa_pure_virtual")
758 VTableFuncs.push_back({Index.getOrInsertValueInfo(GV), StartingOffset});
759 return;
760 }
761 }
762
763 // Walk through the elements in the constant struct or array and recursively
764 // look for virtual function pointers.
765 const DataLayout &DL = M.getDataLayout();
766 if (auto *C = dyn_cast<ConstantStruct>(I)) {
767 StructType *STy = C->getType();
768 assert(STy);
769 const StructLayout *SL = DL.getStructLayout(C->getType());
770
771 for (auto EI : llvm::enumerate(STy->elements())) {
772 auto Offset = SL->getElementOffset(EI.index());
773 unsigned Op = SL->getElementContainingOffset(Offset);
774 findFuncPointers(cast<Constant>(I->getOperand(Op)),
775 StartingOffset + Offset, M, Index, VTableFuncs, OrigGV);
776 }
777 } else if (auto *C = dyn_cast<ConstantArray>(I)) {
778 ArrayType *ATy = C->getType();
779 Type *EltTy = ATy->getElementType();
780 uint64_t EltSize = DL.getTypeAllocSize(EltTy);
781 for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) {
782 findFuncPointers(cast<Constant>(I->getOperand(i)),
783 StartingOffset + i * EltSize, M, Index, VTableFuncs,
784 OrigGV);
785 }
786 } else if (const auto *CE = dyn_cast<ConstantExpr>(I)) {
787 // For relative vtables, the next sub-component should be a trunc.
788 if (CE->getOpcode() != Instruction::Trunc ||
789 !(CE = dyn_cast<ConstantExpr>(CE->getOperand(0))))
790 return;
791
792 // If this constant can be reduced to the offset between a function and a
793 // global, then we know this is a valid virtual function if the RHS is the
794 // original vtable we're scanning through.
795 if (CE->getOpcode() == Instruction::Sub) {
797 APSInt LHSOffset, RHSOffset;
798 if (IsConstantOffsetFromGlobal(CE->getOperand(0), LHS, LHSOffset, DL) &&
799 IsConstantOffsetFromGlobal(CE->getOperand(1), RHS, RHSOffset, DL) &&
800 RHS == &OrigGV &&
801
802 // For relative vtables, this component should point to the callable
803 // function without any offsets.
804 LHSOffset == 0 &&
805
806 // Also, the RHS should always point to somewhere within the vtable.
807 RHSOffset <=
808 static_cast<uint64_t>(DL.getTypeAllocSize(OrigGV.getInitializer()->getType()))) {
809 findFuncPointers(LHS, StartingOffset, M, Index, VTableFuncs, OrigGV);
810 }
811 }
812 }
813}
814
815// Identify the function pointers referenced by vtable definition \p V.
817 const GlobalVariable &V, const Module &M,
818 VTableFuncList &VTableFuncs) {
819 if (!V.isConstant())
820 return;
821
822 findFuncPointers(V.getInitializer(), /*StartingOffset=*/0, M, Index,
823 VTableFuncs, V);
824
825#ifndef NDEBUG
826 // Validate that the VTableFuncs list is ordered by offset.
827 uint64_t PrevOffset = 0;
828 for (auto &P : VTableFuncs) {
829 // The findVFuncPointers traversal should have encountered the
830 // functions in offset order. We need to use ">=" since PrevOffset
831 // starts at 0.
832 assert(P.VTableOffset >= PrevOffset);
833 PrevOffset = P.VTableOffset;
834 }
835#endif
836}
837
838/// Record vtable definition \p V for each type metadata it references.
839static void
841 const GlobalVariable &V,
843 for (MDNode *Type : Types) {
844 auto TypeID = Type->getOperand(1).get();
845
848 cast<ConstantAsMetadata>(Type->getOperand(0))->getValue())
849 ->getZExtValue();
850
851 if (auto *TypeId = dyn_cast<MDString>(TypeID))
852 Index.getOrInsertTypeIdCompatibleVtableSummary(TypeId->getString())
853 .push_back({Offset, Index.getOrInsertValueInfo(&V)});
854 }
855}
856
858 const GlobalVariable &V,
859 DenseSet<GlobalValue::GUID> &CantBePromoted,
860 const Module &M,
864 bool RefLocalIFunc = false;
865 bool HasBlockAddress =
866 findRefEdges(Index, &V, RefEdges, Visited, RefLocalIFunc);
867 const bool NotEligibleForImport = (HasBlockAddress || RefLocalIFunc);
868 bool NonRenamableLocal = isNonRenamableLocal(V);
870 V.getLinkage(), V.getVisibility(), NonRenamableLocal,
871 /* Live = */ false, V.isDSOLocal(), V.canBeOmittedFromSymbolTable(),
872 GlobalValueSummary::Definition, /* NoRenameOnPromotion = */ false);
873
874 VTableFuncList VTableFuncs;
875 // If splitting is not enabled, then we compute the summary information
876 // necessary for index-based whole program devirtualization.
877 if (!Index.enableSplitLTOUnit()) {
878 Types.clear();
879 V.getMetadata(LLVMContext::MD_type, Types);
880 if (!Types.empty()) {
881 // Identify the function pointers referenced by this vtable definition.
882 computeVTableFuncs(Index, V, M, VTableFuncs);
883
884 // Record this vtable definition for each type metadata it references.
886 }
887 }
888
889 // Don't mark variables we won't be able to internalize as read/write-only.
890 bool CanBeInternalized =
891 !V.hasComdat() && !V.hasAppendingLinkage() && !V.isInterposable() &&
892 !V.hasAvailableExternallyLinkage() && !V.hasDLLExportStorageClass();
893 bool Constant = V.isConstant();
894 GlobalVarSummary::GVarFlags VarFlags(CanBeInternalized,
895 Constant ? false : CanBeInternalized,
896 Constant, V.getVCallVisibility());
897 auto GVarSummary = std::make_unique<GlobalVarSummary>(Flags, VarFlags,
898 RefEdges.takeVector());
899 if (NonRenamableLocal)
900 CantBePromoted.insert(V.getGUID());
901 if (NotEligibleForImport)
902 GVarSummary->setNotEligibleToImport();
903 if (!VTableFuncs.empty())
904 GVarSummary->setVTableFuncs(VTableFuncs);
905 Index.addGlobalValueSummary(V, std::move(GVarSummary));
906}
907
909 DenseSet<GlobalValue::GUID> &CantBePromoted) {
910 // Skip summary for indirect function aliases as summary for aliasee will not
911 // be emitted.
912 const GlobalObject *Aliasee = A.getAliaseeObject();
913 if (isa<GlobalIFunc>(Aliasee))
914 return;
915 bool NonRenamableLocal = isNonRenamableLocal(A);
917 A.getLinkage(), A.getVisibility(), NonRenamableLocal,
918 /* Live = */ false, A.isDSOLocal(), A.canBeOmittedFromSymbolTable(),
919 GlobalValueSummary::Definition, /* NoRenameOnPromotion = */ false);
920 auto AS = std::make_unique<AliasSummary>(Flags);
921 auto AliaseeVI = Index.getValueInfo(Aliasee->getGUID());
922 assert(AliaseeVI && "Alias expects aliasee summary to be available");
923 assert(AliaseeVI.getSummaryList().size() == 1 &&
924 "Expected a single entry per aliasee in per-module index");
925 AS->setAliasee(AliaseeVI, AliaseeVI.getSummaryList()[0].get());
926 if (NonRenamableLocal)
927 CantBePromoted.insert(A.getGUID());
928 Index.addGlobalValueSummary(A, std::move(AS));
929}
930
931// Set LiveRoot flag on entries matching the given value name.
932static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name) {
933 if (ValueInfo VI =
934 Index.getValueInfo(GlobalValue::getGUIDAssumingExternalLinkage(Name)))
935 for (const auto &Summary : VI.getSummaryList())
936 Summary->setLive(true);
937}
938
940 const Module &M,
941 std::function<BlockFrequencyInfo *(const Function &F)> GetBFICallback,
943 std::function<const StackSafetyInfo *(const Function &F)> GetSSICallback) {
944 assert(PSI);
945 const AnalysisOptions &Opts = getAnalysisOptions(M);
946 bool EnableSplitLTOUnit = false;
947 bool UnifiedLTO = false;
949 M.getModuleFlag("EnableSplitLTOUnit")))
950 EnableSplitLTOUnit = MD->getZExtValue();
951 if (auto *MD =
952 mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("UnifiedLTO")))
953 UnifiedLTO = MD->getZExtValue();
954 ModuleSummaryIndex Index(/*HaveGVs=*/true, EnableSplitLTOUnit, UnifiedLTO);
955
956 // Identify the local values in the llvm.used and llvm.compiler.used sets,
957 // which should not be exported as they would then require renaming and
958 // promotion, but we may have opaque uses e.g. in inline asm. We collect them
959 // here because we use this information to mark functions containing inline
960 // assembly calls as not importable.
963 // First collect those in the llvm.used set.
964 collectUsedGlobalVariables(M, Used, /*CompilerUsed=*/false);
965 // Next collect those in the llvm.compiler.used set.
966 collectUsedGlobalVariables(M, Used, /*CompilerUsed=*/true);
967 DenseSet<GlobalValue::GUID> CantBePromoted;
968 for (auto *V : Used) {
969 if (V->hasLocalLinkage()) {
970 LocalsUsed.insert(V);
971 CantBePromoted.insert(V->getGUID());
972 }
973 }
974
975 bool HasLocalInlineAsmSymbol = false;
976 if (!M.getModuleInlineAsm().empty()) {
977 // Collect the local values defined by module level asm, and set up
978 // summaries for these symbols so that they can be marked as NoRename,
979 // to prevent export of any use of them in regular IR that would require
980 // renaming within the module level asm. Note we don't need to create a
981 // summary for weak or global defs, as they don't need to be flagged as
982 // NoRename, and defs in module level asm can't be imported anyway.
983 // Also, any values used but not defined within module level asm should
984 // be listed on the llvm.used or llvm.compiler.used global and marked as
985 // referenced from there.
987 M, [&](StringRef Name, object::BasicSymbolRef::Flags Flags) {
988 // Symbols not marked as Weak or Global are local definitions.
991 return;
992 HasLocalInlineAsmSymbol = true;
993 GlobalValue *GV = M.getNamedValue(Name);
994 if (!GV)
995 return;
996 assert(GV->isDeclaration() && "Def in module asm already has definition");
999 /* NotEligibleToImport = */ true,
1000 /* Live = */ true,
1001 /* Local */ GV->isDSOLocal(), GV->canBeOmittedFromSymbolTable(),
1003 /* NoRenameOnPromotion = */ false);
1004 CantBePromoted.insert(GV->getGUID());
1005 // Create the appropriate summary type.
1006 if (Function *F = dyn_cast<Function>(GV)) {
1007 std::unique_ptr<FunctionSummary> Summary =
1008 std::make_unique<FunctionSummary>(
1009 GVFlags, /*InstCount=*/0,
1011 F->hasFnAttribute(Attribute::ReadNone),
1012 F->hasFnAttribute(Attribute::ReadOnly),
1013 F->hasFnAttribute(Attribute::NoRecurse),
1014 F->returnDoesNotAlias(),
1015 /* NoInline = */ false,
1016 F->hasFnAttribute(Attribute::AlwaysInline),
1017 F->hasFnAttribute(Attribute::NoUnwind),
1018 /* MayThrow */ true,
1019 /* HasUnknownCall */ true,
1020 /* MustBeUnreachable */ false},
1030 Index.addGlobalValueSummary(*GV, std::move(Summary));
1031 } else {
1032 std::unique_ptr<GlobalVarSummary> Summary =
1033 std::make_unique<GlobalVarSummary>(
1034 GVFlags,
1036 false, false, cast<GlobalVariable>(GV)->isConstant(),
1039 Index.addGlobalValueSummary(*GV, std::move(Summary));
1040 }
1041 });
1042 }
1043
1044 bool IsThinLTO = true;
1045 if (auto *MD =
1046 mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("ThinLTO")))
1047 IsThinLTO = MD->getZExtValue();
1048
1049 // Compute summaries for all functions defined in module, and save in the
1050 // index.
1051 for (const auto &F : M) {
1052 if (F.isDeclaration())
1053 continue;
1054
1055 DominatorTree DT(const_cast<Function &>(F));
1056 BlockFrequencyInfo *BFI = nullptr;
1057 std::unique_ptr<BlockFrequencyInfo> BFIPtr;
1058 if (F.hasProfileData()) {
1059 if (GetBFICallback) {
1060 BFI = GetBFICallback(F);
1061 } else {
1062 CycleInfo CI;
1063 CI.compute(const_cast<Function &>(F));
1064 BranchProbabilityInfo BPI{F, CI};
1065 BFIPtr = std::make_unique<BlockFrequencyInfo>(F, BPI, CI);
1066 BFI = BFIPtr.get();
1067 }
1068 }
1069
1070 computeFunctionSummary(Opts, Index, M, F, BFI, PSI, DT,
1071 !LocalsUsed.empty() || HasLocalInlineAsmSymbol,
1072 CantBePromoted, IsThinLTO, GetSSICallback);
1073 }
1074
1075 // Compute summaries for all variables defined in module, and save in the
1076 // index.
1078 for (const GlobalVariable &G : M.globals()) {
1079 if (G.isDeclaration())
1080 continue;
1081 computeVariableSummary(Index, G, CantBePromoted, M, Types);
1082 }
1083
1084 // Compute summaries for all aliases defined in module, and save in the
1085 // index.
1086 for (const GlobalAlias &A : M.aliases())
1087 computeAliasSummary(Index, A, CantBePromoted);
1088
1089 // Iterate through ifuncs, set their resolvers all alive.
1090 for (const GlobalIFunc &I : M.ifuncs()) {
1091 I.applyAlongResolverPath([&Index](const GlobalValue &GV) {
1092 Index.getGlobalValueSummary(GV)->setLive(true);
1093 });
1094 }
1095
1096 for (auto *V : LocalsUsed) {
1097 auto *Summary = Index.getGlobalValueSummary(*V);
1098 assert(Summary && "Missing summary for global value");
1099 Summary->setNotEligibleToImport();
1100 }
1101
1102 // The linker doesn't know about these LLVM produced values, so we need
1103 // to flag them as live in the index to ensure index-based dead value
1104 // analysis treats them as live roots of the analysis.
1105 setLiveRoot(Index, "llvm.used");
1106 setLiveRoot(Index, "llvm.compiler.used");
1107 setLiveRoot(Index, "llvm.global_ctors");
1108 setLiveRoot(Index, "llvm.global_dtors");
1109 setLiveRoot(Index, "llvm.global.annotations");
1110
1111 for (auto &GlobalList : Index) {
1112 // Ignore entries for references that are undefined in the current module.
1113 if (GlobalList.second.getSummaryList().empty())
1114 continue;
1115
1116 assert(GlobalList.second.getSummaryList().size() == 1 &&
1117 "Expected module's index to have one summary per GUID");
1118 auto &Summary = GlobalList.second.getSummaryList()[0];
1119 if (!IsThinLTO) {
1120 Summary->setNotEligibleToImport();
1121 continue;
1122 }
1123
1124 bool AllRefsCanBeExternallyReferenced =
1125 llvm::all_of(Summary->refs(), [&](const ValueInfo &VI) {
1126 return !CantBePromoted.count(VI.getGUID());
1127 });
1128 if (!AllRefsCanBeExternallyReferenced) {
1129 Summary->setNotEligibleToImport();
1130 continue;
1131 }
1132
1133 if (auto *FuncSummary = dyn_cast<FunctionSummary>(Summary.get())) {
1134 bool AllCallsCanBeExternallyReferenced = llvm::all_of(
1135 FuncSummary->calls(), [&](const FunctionSummary::EdgeTy &Edge) {
1136 return !CantBePromoted.count(Edge.first.getGUID());
1137 });
1138 if (!AllCallsCanBeExternallyReferenced)
1139 Summary->setNotEligibleToImport();
1140 }
1141 }
1142
1143 if (!Opts.module_summary_dot_file.empty()) {
1144 std::error_code EC;
1145 raw_fd_ostream OSDot(Opts.module_summary_dot_file, EC,
1147 if (EC)
1148 report_fatal_error(Twine("Failed to open dot file ") +
1149 Opts.module_summary_dot_file + ": " + EC.message() +
1150 "\n");
1151 Index.exportToDot(OSDot, {});
1152 }
1153
1154 return Index;
1155}
1156
1157AnalysisKey ModuleSummaryIndexAnalysis::Key;
1158AnalysisKey ImmutableModuleSummaryIndexAnalysis::Key;
1159
1163 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
1164 bool NeedSSI = needsParamAccessSummary(M);
1166 M,
1167 [&FAM](const Function &F) {
1168 return &FAM.getResult<BlockFrequencyAnalysis>(
1169 *const_cast<Function *>(&F));
1170 },
1171 &PSI,
1172 [&FAM, NeedSSI](const Function &F) -> const StackSafetyInfo * {
1173 return NeedSSI ? &FAM.getResult<StackSafetyAnalysis>(
1174 const_cast<Function &>(F))
1175 : nullptr;
1176 });
1177}
1178
1180
1182 "Module Summary Analysis", false, true)
1187 "Module Summary Analysis", false, true)
1188
1192
1195
1197 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
1198 bool NeedSSI = needsParamAccessSummary(M);
1199 Index.emplace(buildModuleSummaryIndex(
1200 M,
1201 [this](const Function &F) {
1203 *const_cast<Function *>(&F))
1204 .getBFI());
1205 },
1206 PSI,
1207 [&](const Function &F) -> const StackSafetyInfo * {
1209 const_cast<Function &>(F))
1210 .getResult()
1211 : nullptr;
1212 }));
1213 return false;
1214}
1215
1217 Index.reset();
1218 return false;
1219}
1220
1227
1229
1233
1238
1243
1245 "Module summary info", false, true)
1246
1248 if (!CB)
1249 return false;
1250 auto *CI = dyn_cast<CallInst>(CB);
1251 auto *CalledValue = CB->getCalledOperand();
1252 auto *CalledFunction = CB->getCalledFunction();
1253 if (CalledValue && !CalledFunction) {
1254 CalledValue = CalledValue->stripPointerCasts();
1255 // Stripping pointer casts can reveal a called function.
1256 CalledFunction = dyn_cast<Function>(CalledValue);
1257 }
1258 // Check if this is an alias to a function. If so, get the
1259 // called aliasee for the checks below.
1260 if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
1261 assert(!CalledFunction &&
1262 "Expected null called function in callsite for alias");
1263 CalledFunction = dyn_cast<Function>(GA->getAliaseeObject());
1264 }
1265 // Check if this is a direct call to a known function or a known
1266 // intrinsic, or an indirect call with profile data.
1267 if (CalledFunction) {
1268 if (CI && CalledFunction->isIntrinsic())
1269 return false;
1270 } else {
1271 // Skip indirect calls if we haven't enabled memprof ICP.
1272 if (!getAnalysisOptions(CB->getContext())
1273 .enable_memprof_indirect_call_support)
1274 return false;
1275 // Skip inline assembly calls.
1276 if (CI && CI->isInlineAsm())
1277 return false;
1278 // Skip direct calls via Constant.
1279 if (!CalledValue || isa<Constant>(CalledValue))
1280 return false;
1281 return true;
1282 }
1283 return true;
1284}
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
aarch64 promote const
uint64_t uint64_t
static bool isConstant(const MachineInstr &MI)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file declares the LLVM IR specialization of the GenericCycle templates.
This file defines the DenseSet and SmallDenseSet classes.
Module.h This file contains the declarations for the Module class.
This defines the Use class.
iv users
Definition IVUsers.cpp:48
Interface to identify indirect call promotion candidates.
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
#define G(x, y, z)
Definition MD5.cpp:55
This file implements a map that provides insertion order iteration.
This file contains the declarations for metadata subclasses.
Type::TypeID TypeID
static void addVCallToSet(DevirtCallSite Call, GlobalValue::GUID Guid, SetVector< FunctionSummary::VFuncId, std::vector< FunctionSummary::VFuncId > > &VCalls, SetVector< FunctionSummary::ConstVCall, std::vector< FunctionSummary::ConstVCall > > &ConstVCalls)
Determine whether this call has all constant integer arguments (excluding "this") and summarize it to...
static void computeVTableFuncs(ModuleSummaryIndex &Index, const GlobalVariable &V, const Module &M, VTableFuncList &VTableFuncs)
static void computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A, DenseSet< GlobalValue::GUID > &CantBePromoted)
static void findFuncPointers(const Constant *I, uint64_t StartingOffset, const Module &M, ModuleSummaryIndex &Index, VTableFuncList &VTableFuncs, const GlobalVariable &OrigGV)
Find function pointers referenced within the given vtable initializer (or subset of an initializer) I...
static void computeVariableSummary(ModuleSummaryIndex &Index, const GlobalVariable &V, DenseSet< GlobalValue::GUID > &CantBePromoted, const Module &M, SmallVectorImpl< MDNode * > &Types)
static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name)
static CalleeInfo::HotnessType getHotness(uint64_t ProfileCount, ProfileSummaryInfo *PSI)
static bool isNonVolatileLoad(const Instruction *I)
static void findImplicitRefEdges(ModuleSummaryIndex &Index, const Function &F, SetVector< ValueInfo, SmallVector< ValueInfo, 0 > > &RefEdges)
Collect globals referenced via !implicit.ref metadata on a function and add them as reference edges i...
static bool isNonRenamableLocal(const GlobalValue &GV)
static void computeFunctionSummary(const AnalysisOptions &Opts, ModuleSummaryIndex &Index, const Module &M, const Function &F, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, DominatorTree &DT, bool HasLocalsInUsedOrAsm, DenseSet< GlobalValue::GUID > &CantBePromoted, bool IsThinLTO, std::function< const StackSafetyInfo *(const Function &F)> GetSSICallback)
static bool mustBeUnreachableFunction(const Function &F)
static bool isNonVolatileStore(const Instruction *I)
static bool findRefEdges(ModuleSummaryIndex &Index, const User *CurUser, SetVector< ValueInfo, SmallVector< ValueInfo, 0 > > &RefEdges, SmallPtrSet< const User *, 8 > &Visited, bool &RefLocalLinkageIFunc)
static void addIntrinsicToSummary(const CallInst *CI, SetVector< GlobalValue::GUID, std::vector< GlobalValue::GUID > > &TypeTests, SetVector< FunctionSummary::VFuncId, std::vector< FunctionSummary::VFuncId > > &TypeTestAssumeVCalls, SetVector< FunctionSummary::VFuncId, std::vector< FunctionSummary::VFuncId > > &TypeCheckedLoadVCalls, SetVector< FunctionSummary::ConstVCall, std::vector< FunctionSummary::ConstVCall > > &TypeTestAssumeConstVCalls, SetVector< FunctionSummary::ConstVCall, std::vector< FunctionSummary::ConstVCall > > &TypeCheckedLoadConstVCalls, DominatorTree &DT)
If this intrinsic call requires that we add information to the function summary, do so via the non-co...
static void recordTypeIdCompatibleVtableReferences(ModuleSummaryIndex &Index, const GlobalVariable &V, SmallVectorImpl< MDNode * > &Types)
Record vtable definition V for each type metadata it references.
This is the interface to build a ModuleSummaryIndex for a module.
ModuleSummaryIndex.h This file contains the declarations the classes that hold the module index and s...
#define P(N)
FunctionAnalysisManager FAM
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition PassSupport.h:56
This file contains some templates that are useful if you are working with the STL at all.
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
Value * RHS
Value * LHS
An arbitrary precision integer that knows its signedness.
Definition APSInt.h:24
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesAll()
Set by analyses that do not transform their input at all.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
LLVM Basic Block Representation.
Definition BasicBlock.h:62
static LLVM_ABI BlockAddress * get(Function *F, BasicBlock *BB)
Return a BlockAddress for the specified function and basic block.
Analysis pass which computes BlockFrequencyInfo.
Legacy analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Analysis providing branch probability information.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Value * getArgOperand(unsigned i) const
This class represents a function call, abstracting a target machine's calling convention.
This is an important base class in LLVM.
Definition Constant.h:44
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
Implements a dense probed hash-table based set.
Definition DenseSet.h:283
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition Dominators.h:122
std::pair< ValueInfo, CalleeInfo > EdgeTy
<CalleeValueInfo, CalleeInfo> call edge pair.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition Function.h:247
void compute(FunctionT &F)
Compute the cycle info for a function.
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
Definition Globals.cpp:80
bool isDSOLocal() const
LLVM_ABI bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition Globals.cpp:408
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
bool hasLocalLinkage() const
LLVM_ABI GUID getGUID() const
Return a 64-bit global unique ID for this value.
Definition Globals.cpp:103
@ DefaultVisibility
The GV is visible.
Definition GlobalValue.h:68
bool hasSection() const
LLVM_ABI bool canBeOmittedFromSymbolTable() const
True if GV can be left out of the object symbol table.
Definition Globals.cpp:546
@ InternalLinkage
Rename collisions when linking (static functions).
Definition GlobalValue.h:60
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
LLVM_ABI MutableArrayRef< InstrProfValueData > getPromotionCandidatesForInstruction(const Instruction *I, uint64_t &TotalCount, uint32_t &NumCandidates, unsigned MaxNumValueData=0)
Returns reference to array of InstrProfValueData for the given instruction I.
Legacy wrapper pass to provide the ModuleSummaryIndex object.
ImmutableModuleSummaryIndexWrapperPass(const ModuleSummaryIndex *Index=nullptr)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
ImmutablePass class - This class is used to provide information that does not need to be run.
Definition Pass.h:283
ImmutablePass(char &pid)
Definition Pass.h:285
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
Metadata node.
Definition Metadata.h:1081
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1437
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1443
Tracking metadata reference owned by Metadata.
Definition Metadata.h:902
This class implements a map that also provides access to all stored values in a deterministic order.
Definition MapVector.h:39
VectorType takeVector()
Clear the MapVector and return the underlying vector.
Definition MapVector.h:51
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition Pass.h:253
ModulePass(char &pid)
Definition Pass.h:255
LLVM_ABI Result run(Module &M, ModuleAnalysisManager &AM)
Legacy wrapper pass to provide the ModuleSummaryIndex object.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
bool doFinalization(Module &M) override
doFinalization - Virtual method overriden by subclasses to do any necessary clean up after all passes...
bool runOnModule(Module &M) override
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
Class to hold module path string table and global value map, and encapsulate methods for operating on...
static LLVM_ABI void CollectAsmSymbols(const Module &M, function_ref< void(StringRef, object::BasicSymbolRef::Flags)> AsmSymbol)
Parse inline ASM and collect the symbols that are defined or referenced in the current module.
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:294
AnalysisType & getAnalysis() const
getAnalysis<AnalysisType>() - This function is used by subclasses to get to the analysis information ...
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
Analysis providing profile information.
LLVM_ABI bool isColdCount(uint64_t C) const
Returns true if count C is considered cold.
LLVM_ABI bool hasPartialSampleProfile() const
Returns true if module M has partial-profile sample profile.
LLVM_ABI bool isHotCount(uint64_t C) const
Returns true if count C is considered hot.
LLVM_ABI std::optional< uint64_t > getProfileCount(const CallBase &CallInst, BlockFrequencyInfo *BFI) const
Returns the profile count for CallInst.
A vector that has set insertion semantics.
Definition SetVector.h:57
bool remove(const value_type &X)
Remove an item from the set vector.
Definition SetVector.h:187
size_type size() const
Determine the number of elements in the SetVector.
Definition SetVector.h:103
void insert_range(Range &&R)
Definition SetVector.h:182
Vector takeVector()
Clear the SetVector and return the underlying vector.
Definition SetVector.h:94
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition SetVector.h:157
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackSafetyInfo wrapper for the new pass manager.
StackSafetyInfo wrapper for the legacy pass manager.
Interface to access stack safety analysis results for single function.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
Definition DataLayout.h:746
LLVM_ABI unsigned getElementContainingOffset(uint64_t FixedOffset) const
Given a valid byte offset into the structure, returns the structure index that contains it.
TypeSize getElementOffset(unsigned Idx) const
Definition DataLayout.h:777
Class to represent struct types.
ArrayRef< Type * > elements() const
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
iterator_range< use_iterator > uses()
Definition Value.h:382
std::pair< iterator, bool > insert(const ValueT &V)
Definition DenseSet.h:210
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition DenseSet.h:188
Helper class to iterate through stack ids in both metadata (memprof MIB and callsite) and the corresp...
CallStackIterator beginAfterSharedPrefix(const CallStack &Other)
CallStackIterator end() const
A raw_ostream that writes to a file descriptor.
CallInst * Call
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > extract_or_null(Y &&MD)
Extract a Value from Metadata, allowing null.
Definition Metadata.h:694
std::enable_if_t< detail::IsValidPointer< X, Y >::value, X * > dyn_extract(Y &&MD)
Extract a Value from Metadata, if any.
Definition Metadata.h:707
LLVM_ABI bool metadataIncludesAllContextSizeInfo()
Whether the alloc memeprof metadata will include context size info for all MIBs.
LLVM_ABI AllocationType getMIBAllocType(const MDNode *MIB)
Returns the allocation type from an MIB metadata node.
LLVM_ABI MDNode * getMIBStackNode(const MDNode *MIB)
Returns the stack node from an MIB metadata node.
@ OF_Text
The file should be opened in text mode on platforms like z/OS that make this distinction.
Definition FileSystem.h:793
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
@ Offset
Definition DWP.cpp:577
std::vector< VirtFuncOffset > VTableFuncList
List of functions referenced by a particular vtable definition.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2570
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
LLVM_ABI bool mayHaveMemprofSummary(const CallBase *CB)
Returns true if the instruction could have memprof metadata, used to ensure consistency between summa...
LLVM_ABI bool IsConstantOffsetFromGlobal(Constant *C, GlobalValue *&GV, APInt &Offset, const DataLayout &DL, DSOLocalEquivalent **DSOEquiv=nullptr)
If this constant is a constant offset from a global, return the global and the constant.
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
Definition Casting.h:732
LLVM_ABI bool needsParamAccessSummary(const Module &M)
InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy
Provide the FunctionAnalysisManager to Module proxy.
LLVM_ABI ModuleSummaryIndex buildModuleSummaryIndex(const Module &M, std::function< BlockFrequencyInfo *(const Function &F)> GetBFICallback, ProfileSummaryInfo *PSI, std::function< const StackSafetyInfo *(const Function &F)> GetSSICallback=[](const Function &F) -> const StackSafetyInfo *{ return nullptr;})
Direct function to compute a ModuleSummaryIndex from a given module.
const AnalysisOptions & getAnalysisOptions(const LLVMContext &C)
auto dyn_cast_or_null(const Y &Val)
Definition Casting.h:753
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
LLVM_ABI void findDevirtualizableCallsForTypeCheckedLoad(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< Instruction * > &LoadedPtrs, SmallVectorImpl< Instruction * > &Preds, bool &HasNonCallUses, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.checked.load, find all devirtualizable call sites based on t...
LLVM_ABI SmallVector< InstrProfValueData, 4 > getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst and returns them if Inst is annotated with value profile dat...
LLVM_ABI ModulePass * createModuleSummaryIndexWrapperPass()
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ABI unsigned getMaxSummaryIndirectEdges(const LLVMContext &C)
Return -module-summary-max-indirect-edges.
DWARFExpression::Operation Op
LLVM_ABI ImmutablePass * createImmutableModuleSummaryIndexWrapperPass(const ModuleSummaryIndex *Index)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI unsigned getMaxNumVTableAnnotations(const LLVMContext &C)
Return -icp-max-num-vtables.
LLVM_ABI void findDevirtualizableCallsForTypeTest(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< CallInst * > &Assumes, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.test, find all devirtualizable call sites based on the call ...
AnalysisManager< Module > ModuleAnalysisManager
Convenience typedef for the Module analysis manager.
Definition MIRParser.h:39
LLVM_ABI GlobalVariable * collectUsedGlobalVariables(const Module &M, SmallVectorImpl< GlobalValue * > &Vec, bool CompilerUsed)
Given "llvm.used" or "llvm.compiler.used" as a global name, collect the initializer elements of that ...
Definition Module.cpp:991
Summary of memprof metadata on allocations.
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition Analysis.h:29
A call site that could be devirtualized.
A specification for a virtual function call with all constant integer arguments.
Flags specific to function summaries.
An "identifier" for a virtual function.
Group flags (Linkage, NotEligibleToImport, etc.) as a bitfield.
Summary of a single MIB in a memprof metadata on allocations.
Struct that holds a reference to a particular GUID in a global value summary.