LLVM 20.0.0git
ModuleSummaryAnalysis.cpp
Go to the documentation of this file.
1//===- ModuleSummaryAnalysis.cpp - Module summary index builder -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass builds a ModuleSummaryIndex object for the module, to be written
10// to bitcode or LLVM assembly.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/DenseSet.h"
17#include "llvm/ADT/MapVector.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/SetVector.h"
22#include "llvm/ADT/StringRef.h"
32#include "llvm/IR/Attributes.h"
33#include "llvm/IR/BasicBlock.h"
34#include "llvm/IR/Constant.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/Dominators.h"
37#include "llvm/IR/Function.h"
38#include "llvm/IR/GlobalAlias.h"
39#include "llvm/IR/GlobalValue.h"
43#include "llvm/IR/Metadata.h"
44#include "llvm/IR/Module.h"
46#include "llvm/IR/Use.h"
47#include "llvm/IR/User.h"
51#include "llvm/Pass.h"
55#include <cassert>
56#include <cstdint>
57#include <vector>
58
59using namespace llvm;
60using namespace llvm::memprof;
61
62#define DEBUG_TYPE "module-summary-analysis"
63
64// Option to force edges cold which will block importing when the
65// -import-cold-multiplier is set to 0. Useful for debugging.
66namespace llvm {
69} // namespace llvm
70
72 "force-summary-edges-cold", cl::Hidden, cl::location(ForceSummaryEdgesCold),
73 cl::desc("Force all edges in the function summary to cold"),
76 "all-non-critical", "All non-critical edges."),
77 clEnumValN(FunctionSummary::FSHT_All, "all", "All edges.")));
78
80 "module-summary-dot-file", cl::Hidden, cl::value_desc("filename"),
81 cl::desc("File to emit dot graph of new summary into"));
82
84 "enable-memprof-indirect-call-support", cl::init(false), cl::Hidden,
86 "Enable MemProf support for summarizing and cloning indirect calls"));
87
89
91
93
94// Walk through the operands of a given User via worklist iteration and populate
95// the set of GlobalValue references encountered. Invoked either on an
96// Instruction or a GlobalVariable (which walks its initializer).
97// Return true if any of the operands contains blockaddress. This is important
98// to know when computing summary for global var, because if global variable
99// references basic block address we can't import it separately from function
100// containing that basic block. For simplicity we currently don't import such
101// global vars at all. When importing function we aren't interested if any
102// instruction in it takes an address of any basic block, because instruction
103// can only take an address of basic block located in the same function.
104// Set `RefLocalLinkageIFunc` to true if the analyzed value references a
105// local-linkage ifunc.
106static bool
107findRefEdges(ModuleSummaryIndex &Index, const User *CurUser,
110 bool &RefLocalLinkageIFunc) {
111 bool HasBlockAddress = false;
113 if (Visited.insert(CurUser).second)
114 Worklist.push_back(CurUser);
115
116 while (!Worklist.empty()) {
117 const User *U = Worklist.pop_back_val();
118 const auto *CB = dyn_cast<CallBase>(U);
119
120 for (const auto &OI : U->operands()) {
121 const User *Operand = dyn_cast<User>(OI);
122 if (!Operand)
123 continue;
124 if (isa<BlockAddress>(Operand)) {
125 HasBlockAddress = true;
126 continue;
127 }
128 if (auto *GV = dyn_cast<GlobalValue>(Operand)) {
129 // We have a reference to a global value. This should be added to
130 // the reference set unless it is a callee. Callees are handled
131 // specially by WriteFunction and are added to a separate list.
132 if (!(CB && CB->isCallee(&OI))) {
133 // If an ifunc has local linkage, do not add it into ref edges, and
134 // sets `RefLocalLinkageIFunc` to true. The referencer is not eligible
135 // for import. An ifunc doesn't have summary and ThinLTO cannot
136 // promote it; importing the referencer may cause linkage errors.
137 if (auto *GI = dyn_cast_if_present<GlobalIFunc>(GV);
138 GI && GI->hasLocalLinkage()) {
139 RefLocalLinkageIFunc = true;
140 continue;
141 }
142 RefEdges.insert(Index.getOrInsertValueInfo(GV));
143 }
144 continue;
145 }
146 if (Visited.insert(Operand).second)
147 Worklist.push_back(Operand);
148 }
149 }
150
151 const Instruction *I = dyn_cast<Instruction>(CurUser);
152 if (I) {
153 uint64_t TotalCount = 0;
154 // MaxNumVTableAnnotations is the maximum number of vtables annotated on
155 // the instruction.
156 auto ValueDataArray = getValueProfDataFromInst(
157 *I, IPVK_VTableTarget, MaxNumVTableAnnotations, TotalCount);
158
159 for (const auto &V : ValueDataArray)
160 RefEdges.insert(Index.getOrInsertValueInfo(/* VTableGUID = */
161 V.Value));
162 }
163 return HasBlockAddress;
164}
165
167 ProfileSummaryInfo *PSI) {
168 if (!PSI)
169 return CalleeInfo::HotnessType::Unknown;
170 if (PSI->isHotCount(ProfileCount))
171 return CalleeInfo::HotnessType::Hot;
172 if (PSI->isColdCount(ProfileCount))
173 return CalleeInfo::HotnessType::Cold;
174 return CalleeInfo::HotnessType::None;
175}
176
177static bool isNonRenamableLocal(const GlobalValue &GV) {
178 return GV.hasSection() && GV.hasLocalLinkage();
179}
180
181/// Determine whether this call has all constant integer arguments (excluding
182/// "this") and summarize it to VCalls or ConstVCalls as appropriate.
183static void addVCallToSet(
185 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
186 &VCalls,
188 std::vector<FunctionSummary::ConstVCall>> &ConstVCalls) {
189 std::vector<uint64_t> Args;
190 // Start from the second argument to skip the "this" pointer.
191 for (auto &Arg : drop_begin(Call.CB.args())) {
192 auto *CI = dyn_cast<ConstantInt>(Arg);
193 if (!CI || CI->getBitWidth() > 64) {
194 VCalls.insert({Guid, Call.Offset});
195 return;
196 }
197 Args.push_back(CI->getZExtValue());
198 }
199 ConstVCalls.insert({{Guid, Call.Offset}, std::move(Args)});
200}
201
202/// If this intrinsic call requires that we add information to the function
203/// summary, do so via the non-constant reference arguments.
205 const CallInst *CI,
206 SetVector<GlobalValue::GUID, std::vector<GlobalValue::GUID>> &TypeTests,
207 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
208 &TypeTestAssumeVCalls,
209 SetVector<FunctionSummary::VFuncId, std::vector<FunctionSummary::VFuncId>>
210 &TypeCheckedLoadVCalls,
212 std::vector<FunctionSummary::ConstVCall>>
213 &TypeTestAssumeConstVCalls,
215 std::vector<FunctionSummary::ConstVCall>>
216 &TypeCheckedLoadConstVCalls,
217 DominatorTree &DT) {
218 switch (CI->getCalledFunction()->getIntrinsicID()) {
219 case Intrinsic::type_test:
220 case Intrinsic::public_type_test: {
221 auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(1));
222 auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
223 if (!TypeId)
224 break;
225 GlobalValue::GUID Guid = GlobalValue::getGUID(TypeId->getString());
226
227 // Produce a summary from type.test intrinsics. We only summarize type.test
228 // intrinsics that are used other than by an llvm.assume intrinsic.
229 // Intrinsics that are assumed are relevant only to the devirtualization
230 // pass, not the type test lowering pass.
231 bool HasNonAssumeUses = llvm::any_of(CI->uses(), [](const Use &CIU) {
232 return !isa<AssumeInst>(CIU.getUser());
233 });
234 if (HasNonAssumeUses)
235 TypeTests.insert(Guid);
236
239 findDevirtualizableCallsForTypeTest(DevirtCalls, Assumes, CI, DT);
240 for (auto &Call : DevirtCalls)
241 addVCallToSet(Call, Guid, TypeTestAssumeVCalls,
242 TypeTestAssumeConstVCalls);
243
244 break;
245 }
246
247 case Intrinsic::type_checked_load_relative:
248 case Intrinsic::type_checked_load: {
249 auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(2));
250 auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
251 if (!TypeId)
252 break;
253 GlobalValue::GUID Guid = GlobalValue::getGUID(TypeId->getString());
254
258 bool HasNonCallUses = false;
259 findDevirtualizableCallsForTypeCheckedLoad(DevirtCalls, LoadedPtrs, Preds,
260 HasNonCallUses, CI, DT);
261 // Any non-call uses of the result of llvm.type.checked.load will
262 // prevent us from optimizing away the llvm.type.test.
263 if (HasNonCallUses)
264 TypeTests.insert(Guid);
265 for (auto &Call : DevirtCalls)
266 addVCallToSet(Call, Guid, TypeCheckedLoadVCalls,
267 TypeCheckedLoadConstVCalls);
268
269 break;
270 }
271 default:
272 break;
273 }
274}
275
276static bool isNonVolatileLoad(const Instruction *I) {
277 if (const auto *LI = dyn_cast<LoadInst>(I))
278 return !LI->isVolatile();
279
280 return false;
281}
282
283static bool isNonVolatileStore(const Instruction *I) {
284 if (const auto *SI = dyn_cast<StoreInst>(I))
285 return !SI->isVolatile();
286
287 return false;
288}
289
290// Returns true if the function definition must be unreachable.
291//
292// Note if this helper function returns true, `F` is guaranteed
293// to be unreachable; if it returns false, `F` might still
294// be unreachable but not covered by this helper function.
296 // A function must be unreachable if its entry block ends with an
297 // 'unreachable'.
298 assert(!F.isDeclaration());
299 return isa<UnreachableInst>(F.getEntryBlock().getTerminator());
300}
301
303 ModuleSummaryIndex &Index, const Module &M, const Function &F,
305 bool HasLocalsInUsedOrAsm, DenseSet<GlobalValue::GUID> &CantBePromoted,
306 bool IsThinLTO,
307 std::function<const StackSafetyInfo *(const Function &F)> GetSSICallback) {
308 // Summary not currently supported for anonymous functions, they should
309 // have been named.
310 assert(F.hasName());
311
312 unsigned NumInsts = 0;
313 // Map from callee ValueId to profile count. Used to accumulate profile
314 // counts for all static calls to a given callee.
317 CallGraphEdges;
319 StoreRefEdges;
322 TypeTestAssumeVCalls, TypeCheckedLoadVCalls;
324 std::vector<FunctionSummary::ConstVCall>>
325 TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls;
326 ICallPromotionAnalysis ICallAnalysis;
328
329 // Add personality function, prefix data and prologue data to function's ref
330 // list.
331 bool HasLocalIFuncCallOrRef = false;
332 findRefEdges(Index, &F, RefEdges, Visited, HasLocalIFuncCallOrRef);
333 std::vector<const Instruction *> NonVolatileLoads;
334 std::vector<const Instruction *> NonVolatileStores;
335
336 std::vector<CallsiteInfo> Callsites;
337 std::vector<AllocInfo> Allocs;
338
339#ifndef NDEBUG
340 DenseSet<const CallBase *> CallsThatMayHaveMemprofSummary;
341#endif
342
343 bool HasInlineAsmMaybeReferencingInternal = false;
344 bool HasIndirBranchToBlockAddress = false;
345 bool HasUnknownCall = false;
346 bool MayThrow = false;
347 for (const BasicBlock &BB : F) {
348 // We don't allow inlining of function with indirect branch to blockaddress.
349 // If the blockaddress escapes the function, e.g., via a global variable,
350 // inlining may lead to an invalid cross-function reference. So we shouldn't
351 // import such function either.
352 if (BB.hasAddressTaken()) {
353 for (User *U : BlockAddress::get(const_cast<BasicBlock *>(&BB))->users())
354 if (!isa<CallBrInst>(*U)) {
355 HasIndirBranchToBlockAddress = true;
356 break;
357 }
358 }
359
360 for (const Instruction &I : BB) {
361 if (I.isDebugOrPseudoInst())
362 continue;
363 ++NumInsts;
364
365 // Regular LTO module doesn't participate in ThinLTO import,
366 // so no reference from it can be read/writeonly, since this
367 // would require importing variable as local copy
368 if (IsThinLTO) {
369 if (isNonVolatileLoad(&I)) {
370 // Postpone processing of non-volatile load instructions
371 // See comments below
372 Visited.insert(&I);
373 NonVolatileLoads.push_back(&I);
374 continue;
375 } else if (isNonVolatileStore(&I)) {
376 Visited.insert(&I);
377 NonVolatileStores.push_back(&I);
378 // All references from second operand of store (destination address)
379 // can be considered write-only if they're not referenced by any
380 // non-store instruction. References from first operand of store
381 // (stored value) can't be treated either as read- or as write-only
382 // so we add them to RefEdges as we do with all other instructions
383 // except non-volatile load.
384 Value *Stored = I.getOperand(0);
385 if (auto *GV = dyn_cast<GlobalValue>(Stored))
386 // findRefEdges will try to examine GV operands, so instead
387 // of calling it we should add GV to RefEdges directly.
388 RefEdges.insert(Index.getOrInsertValueInfo(GV));
389 else if (auto *U = dyn_cast<User>(Stored))
390 findRefEdges(Index, U, RefEdges, Visited, HasLocalIFuncCallOrRef);
391 continue;
392 }
393 }
394 findRefEdges(Index, &I, RefEdges, Visited, HasLocalIFuncCallOrRef);
395 const auto *CB = dyn_cast<CallBase>(&I);
396 if (!CB) {
397 if (I.mayThrow())
398 MayThrow = true;
399 continue;
400 }
401
402 const auto *CI = dyn_cast<CallInst>(&I);
403 // Since we don't know exactly which local values are referenced in inline
404 // assembly, conservatively mark the function as possibly referencing
405 // a local value from inline assembly to ensure we don't export a
406 // reference (which would require renaming and promotion of the
407 // referenced value).
408 if (HasLocalsInUsedOrAsm && CI && CI->isInlineAsm())
409 HasInlineAsmMaybeReferencingInternal = true;
410
411 // Compute this once per indirect call.
412 uint32_t NumCandidates = 0;
413 uint64_t TotalCount = 0;
414 MutableArrayRef<InstrProfValueData> CandidateProfileData;
415
416 auto *CalledValue = CB->getCalledOperand();
417 auto *CalledFunction = CB->getCalledFunction();
418 if (CalledValue && !CalledFunction) {
419 CalledValue = CalledValue->stripPointerCasts();
420 // Stripping pointer casts can reveal a called function.
421 CalledFunction = dyn_cast<Function>(CalledValue);
422 }
423 // Check if this is an alias to a function. If so, get the
424 // called aliasee for the checks below.
425 if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
426 assert(!CalledFunction && "Expected null called function in callsite for alias");
427 CalledFunction = dyn_cast<Function>(GA->getAliaseeObject());
428 }
429 // Check if this is a direct call to a known function or a known
430 // intrinsic, or an indirect call with profile data.
431 if (CalledFunction) {
432 if (CI && CalledFunction->isIntrinsic()) {
434 CI, TypeTests, TypeTestAssumeVCalls, TypeCheckedLoadVCalls,
435 TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls, DT);
436 continue;
437 }
438 // We should have named any anonymous globals
439 assert(CalledFunction->hasName());
440 auto ScaledCount = PSI->getProfileCount(*CB, BFI);
441 auto Hotness = ScaledCount ? getHotness(*ScaledCount, PSI)
442 : CalleeInfo::HotnessType::Unknown;
444 Hotness = CalleeInfo::HotnessType::Cold;
445
446 // Use the original CalledValue, in case it was an alias. We want
447 // to record the call edge to the alias in that case. Eventually
448 // an alias summary will be created to associate the alias and
449 // aliasee.
450 auto &ValueInfo = CallGraphEdges[Index.getOrInsertValueInfo(
451 cast<GlobalValue>(CalledValue))];
452 ValueInfo.updateHotness(Hotness);
453 if (CB->isTailCall())
454 ValueInfo.setHasTailCall(true);
455 // Add the relative block frequency to CalleeInfo if there is no profile
456 // information.
457 if (BFI != nullptr && Hotness == CalleeInfo::HotnessType::Unknown) {
458 uint64_t BBFreq = BFI->getBlockFreq(&BB).getFrequency();
459 uint64_t EntryFreq = BFI->getEntryFreq().getFrequency();
460 ValueInfo.updateRelBlockFreq(BBFreq, EntryFreq);
461 }
462 } else {
463 HasUnknownCall = true;
464 // If F is imported, a local linkage ifunc (e.g. target_clones on a
465 // static function) called by F will be cloned. Since summaries don't
466 // track ifunc, we do not know implementation functions referenced by
467 // the ifunc resolver need to be promoted in the exporter, and we will
468 // get linker errors due to cloned declarations for implementation
469 // functions. As a simple fix, just mark F as not eligible for import.
470 // Non-local ifunc is not cloned and does not have the issue.
471 if (auto *GI = dyn_cast_if_present<GlobalIFunc>(CalledValue))
472 if (GI->hasLocalLinkage())
473 HasLocalIFuncCallOrRef = true;
474 // Skip inline assembly calls.
475 if (CI && CI->isInlineAsm())
476 continue;
477 // Skip direct calls.
478 if (!CalledValue || isa<Constant>(CalledValue))
479 continue;
480
481 // Check if the instruction has a callees metadata. If so, add callees
482 // to CallGraphEdges to reflect the references from the metadata, and
483 // to enable importing for subsequent indirect call promotion and
484 // inlining.
485 if (auto *MD = I.getMetadata(LLVMContext::MD_callees)) {
486 for (const auto &Op : MD->operands()) {
487 Function *Callee = mdconst::extract_or_null<Function>(Op);
488 if (Callee)
489 CallGraphEdges[Index.getOrInsertValueInfo(Callee)];
490 }
491 }
492
493 CandidateProfileData =
494 ICallAnalysis.getPromotionCandidatesForInstruction(&I, TotalCount,
495 NumCandidates);
496 for (const auto &Candidate : CandidateProfileData)
497 CallGraphEdges[Index.getOrInsertValueInfo(Candidate.Value)]
498 .updateHotness(getHotness(Candidate.Count, PSI));
499 }
500
501 // Summarize memprof related metadata. This is only needed for ThinLTO.
502 if (!IsThinLTO)
503 continue;
504
505 // Skip indirect calls if we haven't enabled memprof ICP.
506 if (!CalledFunction && !EnableMemProfIndirectCallSupport)
507 continue;
508
509 // Ensure we keep this analysis in sync with the handling in the ThinLTO
510 // backend (see MemProfContextDisambiguation::applyImport). Save this call
511 // so that we can skip it in checking the reverse case later.
513#ifndef NDEBUG
514 CallsThatMayHaveMemprofSummary.insert(CB);
515#endif
516
517 // Compute the list of stack ids first (so we can trim them from the stack
518 // ids on any MIBs).
520 I.getMetadata(LLVMContext::MD_callsite));
521 auto *MemProfMD = I.getMetadata(LLVMContext::MD_memprof);
522 if (MemProfMD) {
523 std::vector<MIBInfo> MIBs;
524 std::vector<uint64_t> TotalSizes;
525 std::vector<std::vector<ContextTotalSize>> ContextSizeInfos;
526 for (auto &MDOp : MemProfMD->operands()) {
527 auto *MIBMD = cast<const MDNode>(MDOp);
530 SmallVector<unsigned> StackIdIndices;
532 // Collapse out any on the allocation call (inlining).
533 for (auto ContextIter =
534 StackContext.beginAfterSharedPrefix(InstCallsite);
535 ContextIter != StackContext.end(); ++ContextIter) {
536 unsigned StackIdIdx = Index.addOrGetStackIdIndex(*ContextIter);
537 // If this is a direct recursion, simply skip the duplicate
538 // entries. If this is mutual recursion, handling is left to
539 // the LTO link analysis client.
540 if (StackIdIndices.empty() || StackIdIndices.back() != StackIdIdx)
541 StackIdIndices.push_back(StackIdIdx);
542 }
543 // If we have context size information, collect it for inclusion in
544 // the summary.
545 assert(MIBMD->getNumOperands() > 2 || !MemProfReportHintedSizes);
546 if (MIBMD->getNumOperands() > 2) {
547 std::vector<ContextTotalSize> ContextSizes;
548 for (unsigned I = 2; I < MIBMD->getNumOperands(); I++) {
549 MDNode *ContextSizePair = dyn_cast<MDNode>(MIBMD->getOperand(I));
550 assert(ContextSizePair->getNumOperands() == 2);
551 uint64_t FullStackId = mdconst::dyn_extract<ConstantInt>(
552 ContextSizePair->getOperand(0))
553 ->getZExtValue();
554 uint64_t TS = mdconst::dyn_extract<ConstantInt>(
555 ContextSizePair->getOperand(1))
556 ->getZExtValue();
557 ContextSizes.push_back({FullStackId, TS});
558 }
559 ContextSizeInfos.push_back(std::move(ContextSizes));
560 }
561 MIBs.push_back(
562 MIBInfo(getMIBAllocType(MIBMD), std::move(StackIdIndices)));
563 }
564 Allocs.push_back(AllocInfo(std::move(MIBs)));
565 assert(!ContextSizeInfos.empty() || !MemProfReportHintedSizes);
566 if (!ContextSizeInfos.empty()) {
567 assert(Allocs.back().MIBs.size() == ContextSizeInfos.size());
568 Allocs.back().ContextSizeInfos = std::move(ContextSizeInfos);
569 }
570 } else if (!InstCallsite.empty()) {
571 SmallVector<unsigned> StackIdIndices;
572 for (auto StackId : InstCallsite)
573 StackIdIndices.push_back(Index.addOrGetStackIdIndex(StackId));
574 if (CalledFunction) {
575 // Use the original CalledValue, in case it was an alias. We want
576 // to record the call edge to the alias in that case. Eventually
577 // an alias summary will be created to associate the alias and
578 // aliasee.
579 auto CalleeValueInfo =
580 Index.getOrInsertValueInfo(cast<GlobalValue>(CalledValue));
581 Callsites.push_back({CalleeValueInfo, StackIdIndices});
582 } else {
584 // For indirect callsites, create multiple Callsites, one per target.
585 // This enables having a different set of clone versions per target,
586 // and we will apply the cloning decisions while speculatively
587 // devirtualizing in the ThinLTO backends.
588 for (const auto &Candidate : CandidateProfileData) {
589 auto CalleeValueInfo = Index.getOrInsertValueInfo(Candidate.Value);
590 Callsites.push_back({CalleeValueInfo, StackIdIndices});
591 }
592 }
593 }
594 }
595 }
596
598 Index.addBlockCount(F.size());
599
601 if (IsThinLTO) {
602 auto AddRefEdges =
603 [&](const std::vector<const Instruction *> &Instrs,
606 for (const auto *I : Instrs) {
607 Cache.erase(I);
608 findRefEdges(Index, I, Edges, Cache, HasLocalIFuncCallOrRef);
609 }
610 };
611
612 // By now we processed all instructions in a function, except
613 // non-volatile loads and non-volatile value stores. Let's find
614 // ref edges for both of instruction sets
615 AddRefEdges(NonVolatileLoads, LoadRefEdges, Visited);
616 // We can add some values to the Visited set when processing load
617 // instructions which are also used by stores in NonVolatileStores.
618 // For example this can happen if we have following code:
619 //
620 // store %Derived* @foo, %Derived** bitcast (%Base** @bar to %Derived**)
621 // %42 = load %Derived*, %Derived** bitcast (%Base** @bar to %Derived**)
622 //
623 // After processing loads we'll add bitcast to the Visited set, and if
624 // we use the same set while processing stores, we'll never see store
625 // to @bar and @bar will be mistakenly treated as readonly.
627 AddRefEdges(NonVolatileStores, StoreRefEdges, StoreCache);
628
629 // If both load and store instruction reference the same variable
630 // we won't be able to optimize it. Add all such reference edges
631 // to RefEdges set.
632 for (const auto &VI : StoreRefEdges)
633 if (LoadRefEdges.remove(VI))
634 RefEdges.insert(VI);
635
636 unsigned RefCnt = RefEdges.size();
637 // All new reference edges inserted in two loops below are either
638 // read or write only. They will be grouped in the end of RefEdges
639 // vector, so we can use a single integer value to identify them.
640 for (const auto &VI : LoadRefEdges)
641 RefEdges.insert(VI);
642
643 unsigned FirstWORef = RefEdges.size();
644 for (const auto &VI : StoreRefEdges)
645 RefEdges.insert(VI);
646
647 Refs = RefEdges.takeVector();
648 for (; RefCnt < FirstWORef; ++RefCnt)
649 Refs[RefCnt].setReadOnly();
650
651 for (; RefCnt < Refs.size(); ++RefCnt)
652 Refs[RefCnt].setWriteOnly();
653 } else {
654 Refs = RefEdges.takeVector();
655 }
656 // Explicit add hot edges to enforce importing for designated GUIDs for
657 // sample PGO, to enable the same inlines as the profiled optimized binary.
658 for (auto &I : F.getImportGUIDs())
659 CallGraphEdges[Index.getOrInsertValueInfo(I)].updateHotness(
661 ? CalleeInfo::HotnessType::Cold
662 : CalleeInfo::HotnessType::Critical);
663
664#ifndef NDEBUG
665 // Make sure that all calls we decided could not have memprof summaries get a
666 // false value for mayHaveMemprofSummary, to ensure that this handling remains
667 // in sync with the ThinLTO backend handling.
668 if (IsThinLTO) {
669 for (const BasicBlock &BB : F) {
670 for (const Instruction &I : BB) {
671 const auto *CB = dyn_cast<CallBase>(&I);
672 if (!CB)
673 continue;
674 // We already checked these above.
675 if (CallsThatMayHaveMemprofSummary.count(CB))
676 continue;
678 }
679 }
680 }
681#endif
682
683 bool NonRenamableLocal = isNonRenamableLocal(F);
684 bool NotEligibleForImport =
685 NonRenamableLocal || HasInlineAsmMaybeReferencingInternal ||
686 HasIndirBranchToBlockAddress || HasLocalIFuncCallOrRef;
688 F.getLinkage(), F.getVisibility(), NotEligibleForImport,
689 /* Live = */ false, F.isDSOLocal(), F.canBeOmittedFromSymbolTable(),
690 GlobalValueSummary::ImportKind::Definition);
692 F.doesNotAccessMemory(), F.onlyReadsMemory() && !F.doesNotAccessMemory(),
693 F.hasFnAttribute(Attribute::NoRecurse), F.returnDoesNotAlias(),
694 // FIXME: refactor this to use the same code that inliner is using.
695 // Don't try to import functions with noinline attribute.
696 F.getAttributes().hasFnAttr(Attribute::NoInline),
697 F.hasFnAttribute(Attribute::AlwaysInline),
698 F.hasFnAttribute(Attribute::NoUnwind), MayThrow, HasUnknownCall,
700 std::vector<FunctionSummary::ParamAccess> ParamAccesses;
701 if (auto *SSI = GetSSICallback(F))
702 ParamAccesses = SSI->getParamAccesses(Index);
703 auto FuncSummary = std::make_unique<FunctionSummary>(
704 Flags, NumInsts, FunFlags, std::move(Refs), CallGraphEdges.takeVector(),
705 TypeTests.takeVector(), TypeTestAssumeVCalls.takeVector(),
706 TypeCheckedLoadVCalls.takeVector(),
707 TypeTestAssumeConstVCalls.takeVector(),
708 TypeCheckedLoadConstVCalls.takeVector(), std::move(ParamAccesses),
709 std::move(Callsites), std::move(Allocs));
710 if (NonRenamableLocal)
711 CantBePromoted.insert(F.getGUID());
712 Index.addGlobalValueSummary(F, std::move(FuncSummary));
713}
714
715/// Find function pointers referenced within the given vtable initializer
716/// (or subset of an initializer) \p I. The starting offset of \p I within
717/// the vtable initializer is \p StartingOffset. Any discovered function
718/// pointers are added to \p VTableFuncs along with their cumulative offset
719/// within the initializer.
720static void findFuncPointers(const Constant *I, uint64_t StartingOffset,
721 const Module &M, ModuleSummaryIndex &Index,
722 VTableFuncList &VTableFuncs,
723 const GlobalVariable &OrigGV) {
724 // First check if this is a function pointer.
725 if (I->getType()->isPointerTy()) {
726 auto C = I->stripPointerCasts();
727 auto A = dyn_cast<GlobalAlias>(C);
728 if (isa<Function>(C) || (A && isa<Function>(A->getAliasee()))) {
729 auto GV = dyn_cast<GlobalValue>(C);
730 assert(GV);
731 // We can disregard __cxa_pure_virtual as a possible call target, as
732 // calls to pure virtuals are UB.
733 if (GV && GV->getName() != "__cxa_pure_virtual")
734 VTableFuncs.push_back({Index.getOrInsertValueInfo(GV), StartingOffset});
735 return;
736 }
737 }
738
739 // Walk through the elements in the constant struct or array and recursively
740 // look for virtual function pointers.
741 const DataLayout &DL = M.getDataLayout();
742 if (auto *C = dyn_cast<ConstantStruct>(I)) {
743 StructType *STy = dyn_cast<StructType>(C->getType());
744 assert(STy);
745 const StructLayout *SL = DL.getStructLayout(C->getType());
746
747 for (auto EI : llvm::enumerate(STy->elements())) {
748 auto Offset = SL->getElementOffset(EI.index());
749 unsigned Op = SL->getElementContainingOffset(Offset);
750 findFuncPointers(cast<Constant>(I->getOperand(Op)),
751 StartingOffset + Offset, M, Index, VTableFuncs, OrigGV);
752 }
753 } else if (auto *C = dyn_cast<ConstantArray>(I)) {
754 ArrayType *ATy = C->getType();
755 Type *EltTy = ATy->getElementType();
756 uint64_t EltSize = DL.getTypeAllocSize(EltTy);
757 for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) {
758 findFuncPointers(cast<Constant>(I->getOperand(i)),
759 StartingOffset + i * EltSize, M, Index, VTableFuncs,
760 OrigGV);
761 }
762 } else if (const auto *CE = dyn_cast<ConstantExpr>(I)) {
763 // For relative vtables, the next sub-component should be a trunc.
764 if (CE->getOpcode() != Instruction::Trunc ||
765 !(CE = dyn_cast<ConstantExpr>(CE->getOperand(0))))
766 return;
767
768 // If this constant can be reduced to the offset between a function and a
769 // global, then we know this is a valid virtual function if the RHS is the
770 // original vtable we're scanning through.
771 if (CE->getOpcode() == Instruction::Sub) {
773 APSInt LHSOffset, RHSOffset;
774 if (IsConstantOffsetFromGlobal(CE->getOperand(0), LHS, LHSOffset, DL) &&
775 IsConstantOffsetFromGlobal(CE->getOperand(1), RHS, RHSOffset, DL) &&
776 RHS == &OrigGV &&
777
778 // For relative vtables, this component should point to the callable
779 // function without any offsets.
780 LHSOffset == 0 &&
781
782 // Also, the RHS should always point to somewhere within the vtable.
783 RHSOffset <=
784 static_cast<uint64_t>(DL.getTypeAllocSize(OrigGV.getInitializer()->getType()))) {
785 findFuncPointers(LHS, StartingOffset, M, Index, VTableFuncs, OrigGV);
786 }
787 }
788 }
789}
790
791// Identify the function pointers referenced by vtable definition \p V.
793 const GlobalVariable &V, const Module &M,
794 VTableFuncList &VTableFuncs) {
795 if (!V.isConstant())
796 return;
797
798 findFuncPointers(V.getInitializer(), /*StartingOffset=*/0, M, Index,
799 VTableFuncs, V);
800
801#ifndef NDEBUG
802 // Validate that the VTableFuncs list is ordered by offset.
803 uint64_t PrevOffset = 0;
804 for (auto &P : VTableFuncs) {
805 // The findVFuncPointers traversal should have encountered the
806 // functions in offset order. We need to use ">=" since PrevOffset
807 // starts at 0.
808 assert(P.VTableOffset >= PrevOffset);
809 PrevOffset = P.VTableOffset;
810 }
811#endif
812}
813
814/// Record vtable definition \p V for each type metadata it references.
815static void
817 const GlobalVariable &V,
819 for (MDNode *Type : Types) {
820 auto TypeID = Type->getOperand(1).get();
821
823 cast<ConstantInt>(
824 cast<ConstantAsMetadata>(Type->getOperand(0))->getValue())
825 ->getZExtValue();
826
827 if (auto *TypeId = dyn_cast<MDString>(TypeID))
828 Index.getOrInsertTypeIdCompatibleVtableSummary(TypeId->getString())
829 .push_back({Offset, Index.getOrInsertValueInfo(&V)});
830 }
831}
832
834 const GlobalVariable &V,
835 DenseSet<GlobalValue::GUID> &CantBePromoted,
836 const Module &M,
840 bool RefLocalIFunc = false;
841 bool HasBlockAddress =
842 findRefEdges(Index, &V, RefEdges, Visited, RefLocalIFunc);
843 const bool NotEligibleForImport = (HasBlockAddress || RefLocalIFunc);
844 bool NonRenamableLocal = isNonRenamableLocal(V);
846 V.getLinkage(), V.getVisibility(), NonRenamableLocal,
847 /* Live = */ false, V.isDSOLocal(), V.canBeOmittedFromSymbolTable(),
849
850 VTableFuncList VTableFuncs;
851 // If splitting is not enabled, then we compute the summary information
852 // necessary for index-based whole program devirtualization.
853 if (!Index.enableSplitLTOUnit()) {
854 Types.clear();
855 V.getMetadata(LLVMContext::MD_type, Types);
856 if (!Types.empty()) {
857 // Identify the function pointers referenced by this vtable definition.
858 computeVTableFuncs(Index, V, M, VTableFuncs);
859
860 // Record this vtable definition for each type metadata it references.
862 }
863 }
864
865 // Don't mark variables we won't be able to internalize as read/write-only.
866 bool CanBeInternalized =
867 !V.hasComdat() && !V.hasAppendingLinkage() && !V.isInterposable() &&
868 !V.hasAvailableExternallyLinkage() && !V.hasDLLExportStorageClass();
869 bool Constant = V.isConstant();
870 GlobalVarSummary::GVarFlags VarFlags(CanBeInternalized,
871 Constant ? false : CanBeInternalized,
872 Constant, V.getVCallVisibility());
873 auto GVarSummary = std::make_unique<GlobalVarSummary>(Flags, VarFlags,
874 RefEdges.takeVector());
875 if (NonRenamableLocal)
876 CantBePromoted.insert(V.getGUID());
877 if (NotEligibleForImport)
878 GVarSummary->setNotEligibleToImport();
879 if (!VTableFuncs.empty())
880 GVarSummary->setVTableFuncs(VTableFuncs);
881 Index.addGlobalValueSummary(V, std::move(GVarSummary));
882}
883
885 DenseSet<GlobalValue::GUID> &CantBePromoted) {
886 // Skip summary for indirect function aliases as summary for aliasee will not
887 // be emitted.
888 const GlobalObject *Aliasee = A.getAliaseeObject();
889 if (isa<GlobalIFunc>(Aliasee))
890 return;
891 bool NonRenamableLocal = isNonRenamableLocal(A);
893 A.getLinkage(), A.getVisibility(), NonRenamableLocal,
894 /* Live = */ false, A.isDSOLocal(), A.canBeOmittedFromSymbolTable(),
896 auto AS = std::make_unique<AliasSummary>(Flags);
897 auto AliaseeVI = Index.getValueInfo(Aliasee->getGUID());
898 assert(AliaseeVI && "Alias expects aliasee summary to be available");
899 assert(AliaseeVI.getSummaryList().size() == 1 &&
900 "Expected a single entry per aliasee in per-module index");
901 AS->setAliasee(AliaseeVI, AliaseeVI.getSummaryList()[0].get());
902 if (NonRenamableLocal)
903 CantBePromoted.insert(A.getGUID());
904 Index.addGlobalValueSummary(A, std::move(AS));
905}
906
907// Set LiveRoot flag on entries matching the given value name.
909 if (ValueInfo VI = Index.getValueInfo(GlobalValue::getGUID(Name)))
910 for (const auto &Summary : VI.getSummaryList())
911 Summary->setLive(true);
912}
913
915 const Module &M,
916 std::function<BlockFrequencyInfo *(const Function &F)> GetBFICallback,
918 std::function<const StackSafetyInfo *(const Function &F)> GetSSICallback) {
919 assert(PSI);
920 bool EnableSplitLTOUnit = false;
921 bool UnifiedLTO = false;
922 if (auto *MD = mdconst::extract_or_null<ConstantInt>(
923 M.getModuleFlag("EnableSplitLTOUnit")))
924 EnableSplitLTOUnit = MD->getZExtValue();
925 if (auto *MD =
926 mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("UnifiedLTO")))
927 UnifiedLTO = MD->getZExtValue();
928 ModuleSummaryIndex Index(/*HaveGVs=*/true, EnableSplitLTOUnit, UnifiedLTO);
929
930 // Identify the local values in the llvm.used and llvm.compiler.used sets,
931 // which should not be exported as they would then require renaming and
932 // promotion, but we may have opaque uses e.g. in inline asm. We collect them
933 // here because we use this information to mark functions containing inline
934 // assembly calls as not importable.
937 // First collect those in the llvm.used set.
938 collectUsedGlobalVariables(M, Used, /*CompilerUsed=*/false);
939 // Next collect those in the llvm.compiler.used set.
940 collectUsedGlobalVariables(M, Used, /*CompilerUsed=*/true);
941 DenseSet<GlobalValue::GUID> CantBePromoted;
942 for (auto *V : Used) {
943 if (V->hasLocalLinkage()) {
944 LocalsUsed.insert(V);
945 CantBePromoted.insert(V->getGUID());
946 }
947 }
948
949 bool HasLocalInlineAsmSymbol = false;
950 if (!M.getModuleInlineAsm().empty()) {
951 // Collect the local values defined by module level asm, and set up
952 // summaries for these symbols so that they can be marked as NoRename,
953 // to prevent export of any use of them in regular IR that would require
954 // renaming within the module level asm. Note we don't need to create a
955 // summary for weak or global defs, as they don't need to be flagged as
956 // NoRename, and defs in module level asm can't be imported anyway.
957 // Also, any values used but not defined within module level asm should
958 // be listed on the llvm.used or llvm.compiler.used global and marked as
959 // referenced from there.
962 // Symbols not marked as Weak or Global are local definitions.
965 return;
966 HasLocalInlineAsmSymbol = true;
967 GlobalValue *GV = M.getNamedValue(Name);
968 if (!GV)
969 return;
970 assert(GV->isDeclaration() && "Def in module asm already has definition");
973 /* NotEligibleToImport = */ true,
974 /* Live = */ true,
975 /* Local */ GV->isDSOLocal(), GV->canBeOmittedFromSymbolTable(),
977 CantBePromoted.insert(GV->getGUID());
978 // Create the appropriate summary type.
979 if (Function *F = dyn_cast<Function>(GV)) {
980 std::unique_ptr<FunctionSummary> Summary =
981 std::make_unique<FunctionSummary>(
982 GVFlags, /*InstCount=*/0,
984 F->hasFnAttribute(Attribute::ReadNone),
985 F->hasFnAttribute(Attribute::ReadOnly),
986 F->hasFnAttribute(Attribute::NoRecurse),
987 F->returnDoesNotAlias(),
988 /* NoInline = */ false,
989 F->hasFnAttribute(Attribute::AlwaysInline),
990 F->hasFnAttribute(Attribute::NoUnwind),
991 /* MayThrow */ true,
992 /* HasUnknownCall */ true,
993 /* MustBeUnreachable */ false},
1003 Index.addGlobalValueSummary(*GV, std::move(Summary));
1004 } else {
1005 std::unique_ptr<GlobalVarSummary> Summary =
1006 std::make_unique<GlobalVarSummary>(
1007 GVFlags,
1009 false, false, cast<GlobalVariable>(GV)->isConstant(),
1012 Index.addGlobalValueSummary(*GV, std::move(Summary));
1013 }
1014 });
1015 }
1016
1017 bool IsThinLTO = true;
1018 if (auto *MD =
1019 mdconst::extract_or_null<ConstantInt>(M.getModuleFlag("ThinLTO")))
1020 IsThinLTO = MD->getZExtValue();
1021
1022 // Compute summaries for all functions defined in module, and save in the
1023 // index.
1024 for (const auto &F : M) {
1025 if (F.isDeclaration())
1026 continue;
1027
1028 DominatorTree DT(const_cast<Function &>(F));
1029 BlockFrequencyInfo *BFI = nullptr;
1030 std::unique_ptr<BlockFrequencyInfo> BFIPtr;
1031 if (GetBFICallback)
1032 BFI = GetBFICallback(F);
1033 else if (F.hasProfileData()) {
1034 LoopInfo LI{DT};
1035 BranchProbabilityInfo BPI{F, LI};
1036 BFIPtr = std::make_unique<BlockFrequencyInfo>(F, BPI, LI);
1037 BFI = BFIPtr.get();
1038 }
1039
1040 computeFunctionSummary(Index, M, F, BFI, PSI, DT,
1041 !LocalsUsed.empty() || HasLocalInlineAsmSymbol,
1042 CantBePromoted, IsThinLTO, GetSSICallback);
1043 }
1044
1045 // Compute summaries for all variables defined in module, and save in the
1046 // index.
1048 for (const GlobalVariable &G : M.globals()) {
1049 if (G.isDeclaration())
1050 continue;
1051 computeVariableSummary(Index, G, CantBePromoted, M, Types);
1052 }
1053
1054 // Compute summaries for all aliases defined in module, and save in the
1055 // index.
1056 for (const GlobalAlias &A : M.aliases())
1057 computeAliasSummary(Index, A, CantBePromoted);
1058
1059 // Iterate through ifuncs, set their resolvers all alive.
1060 for (const GlobalIFunc &I : M.ifuncs()) {
1061 I.applyAlongResolverPath([&Index](const GlobalValue &GV) {
1062 Index.getGlobalValueSummary(GV)->setLive(true);
1063 });
1064 }
1065
1066 for (auto *V : LocalsUsed) {
1067 auto *Summary = Index.getGlobalValueSummary(*V);
1068 assert(Summary && "Missing summary for global value");
1069 Summary->setNotEligibleToImport();
1070 }
1071
1072 // The linker doesn't know about these LLVM produced values, so we need
1073 // to flag them as live in the index to ensure index-based dead value
1074 // analysis treats them as live roots of the analysis.
1075 setLiveRoot(Index, "llvm.used");
1076 setLiveRoot(Index, "llvm.compiler.used");
1077 setLiveRoot(Index, "llvm.global_ctors");
1078 setLiveRoot(Index, "llvm.global_dtors");
1079 setLiveRoot(Index, "llvm.global.annotations");
1080
1081 for (auto &GlobalList : Index) {
1082 // Ignore entries for references that are undefined in the current module.
1083 if (GlobalList.second.SummaryList.empty())
1084 continue;
1085
1086 assert(GlobalList.second.SummaryList.size() == 1 &&
1087 "Expected module's index to have one summary per GUID");
1088 auto &Summary = GlobalList.second.SummaryList[0];
1089 if (!IsThinLTO) {
1090 Summary->setNotEligibleToImport();
1091 continue;
1092 }
1093
1094 bool AllRefsCanBeExternallyReferenced =
1095 llvm::all_of(Summary->refs(), [&](const ValueInfo &VI) {
1096 return !CantBePromoted.count(VI.getGUID());
1097 });
1098 if (!AllRefsCanBeExternallyReferenced) {
1099 Summary->setNotEligibleToImport();
1100 continue;
1101 }
1102
1103 if (auto *FuncSummary = dyn_cast<FunctionSummary>(Summary.get())) {
1104 bool AllCallsCanBeExternallyReferenced = llvm::all_of(
1105 FuncSummary->calls(), [&](const FunctionSummary::EdgeTy &Edge) {
1106 return !CantBePromoted.count(Edge.first.getGUID());
1107 });
1108 if (!AllCallsCanBeExternallyReferenced)
1109 Summary->setNotEligibleToImport();
1110 }
1111 }
1112
1113 if (!ModuleSummaryDotFile.empty()) {
1114 std::error_code EC;
1115 raw_fd_ostream OSDot(ModuleSummaryDotFile, EC, sys::fs::OpenFlags::OF_Text);
1116 if (EC)
1117 report_fatal_error(Twine("Failed to open dot file ") +
1118 ModuleSummaryDotFile + ": " + EC.message() + "\n");
1119 Index.exportToDot(OSDot, {});
1120 }
1121
1122 return Index;
1123}
1124
1125AnalysisKey ModuleSummaryIndexAnalysis::Key;
1126
1130 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
1131 bool NeedSSI = needsParamAccessSummary(M);
1133 M,
1134 [&FAM](const Function &F) {
1136 *const_cast<Function *>(&F));
1137 },
1138 &PSI,
1139 [&FAM, NeedSSI](const Function &F) -> const StackSafetyInfo * {
1140 return NeedSSI ? &FAM.getResult<StackSafetyAnalysis>(
1141 const_cast<Function &>(F))
1142 : nullptr;
1143 });
1144}
1145
1147
1149 "Module Summary Analysis", false, true)
1155
1157 return new ModuleSummaryIndexWrapperPass();
1158}
1159
1161 : ModulePass(ID) {
1163}
1164
1166 auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
1167 bool NeedSSI = needsParamAccessSummary(M);
1168 Index.emplace(buildModuleSummaryIndex(
1169 M,
1170 [this](const Function &F) {
1171 return &(this->getAnalysis<BlockFrequencyInfoWrapperPass>(
1172 *const_cast<Function *>(&F))
1173 .getBFI());
1174 },
1175 PSI,
1176 [&](const Function &F) -> const StackSafetyInfo * {
1177 return NeedSSI ? &getAnalysis<StackSafetyInfoWrapperPass>(
1178 const_cast<Function &>(F))
1179 .getResult()
1180 : nullptr;
1181 }));
1182 return false;
1183}
1184
1186 Index.reset();
1187 return false;
1188}
1189
1191 AU.setPreservesAll();
1195}
1196
1198
1200 const ModuleSummaryIndex *Index)
1201 : ImmutablePass(ID), Index(Index) {
1204}
1205
1207 AnalysisUsage &AU) const {
1208 AU.setPreservesAll();
1209}
1210
1212 const ModuleSummaryIndex *Index) {
1214}
1215
1217 "Module summary info", false, true)
1218
1220 if (!CB)
1221 return false;
1222 if (CB->isDebugOrPseudoInst())
1223 return false;
1224 auto *CI = dyn_cast<CallInst>(CB);
1225 auto *CalledValue = CB->getCalledOperand();
1226 auto *CalledFunction = CB->getCalledFunction();
1227 if (CalledValue && !CalledFunction) {
1228 CalledValue = CalledValue->stripPointerCasts();
1229 // Stripping pointer casts can reveal a called function.
1230 CalledFunction = dyn_cast<Function>(CalledValue);
1231 }
1232 // Check if this is an alias to a function. If so, get the
1233 // called aliasee for the checks below.
1234 if (auto *GA = dyn_cast<GlobalAlias>(CalledValue)) {
1235 assert(!CalledFunction &&
1236 "Expected null called function in callsite for alias");
1237 CalledFunction = dyn_cast<Function>(GA->getAliaseeObject());
1238 }
1239 // Check if this is a direct call to a known function or a known
1240 // intrinsic, or an indirect call with profile data.
1241 if (CalledFunction) {
1242 if (CI && CalledFunction->isIntrinsic())
1243 return false;
1244 } else {
1245 // Skip indirect calls if we haven't enabled memprof ICP.
1247 return false;
1248 // Skip inline assembly calls.
1249 if (CI && CI->isInlineAsm())
1250 return false;
1251 // Skip direct calls via Constant.
1252 if (!CalledValue || isa<Constant>(CalledValue))
1253 return false;
1254 return true;
1255 }
1256 return true;
1257}
aarch64 promote const
static bool isConstant(const MachineInstr &MI)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the simple types necessary to represent the attributes associated with functions a...
basic Basic Alias true
block Block Frequency Analysis
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:686
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file defines the DenseSet and SmallDenseSet classes.
std::string Name
Module.h This file contains the declarations for the Module class.
This defines the Use class.
iv users
Definition: IVUsers.cpp:48
Interface to identify indirect call promotion candidates.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
This file implements a map that provides insertion order iteration.
This file contains the declarations for metadata subclasses.
static void addVCallToSet(DevirtCallSite Call, GlobalValue::GUID Guid, SetVector< FunctionSummary::VFuncId, std::vector< FunctionSummary::VFuncId > > &VCalls, SetVector< FunctionSummary::ConstVCall, std::vector< FunctionSummary::ConstVCall > > &ConstVCalls)
Determine whether this call has all constant integer arguments (excluding "this") and summarize it to...
cl::opt< unsigned > MaxNumVTableAnnotations
static void computeVTableFuncs(ModuleSummaryIndex &Index, const GlobalVariable &V, const Module &M, VTableFuncList &VTableFuncs)
static void computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A, DenseSet< GlobalValue::GUID > &CantBePromoted)
static void findFuncPointers(const Constant *I, uint64_t StartingOffset, const Module &M, ModuleSummaryIndex &Index, VTableFuncList &VTableFuncs, const GlobalVariable &OrigGV)
Find function pointers referenced within the given vtable initializer (or subset of an initializer) I...
static void computeVariableSummary(ModuleSummaryIndex &Index, const GlobalVariable &V, DenseSet< GlobalValue::GUID > &CantBePromoted, const Module &M, SmallVectorImpl< MDNode * > &Types)
static void setLiveRoot(ModuleSummaryIndex &Index, StringRef Name)
static CalleeInfo::HotnessType getHotness(uint64_t ProfileCount, ProfileSummaryInfo *PSI)
static cl::opt< bool > EnableMemProfIndirectCallSupport("enable-memprof-indirect-call-support", cl::init(false), cl::Hidden, cl::desc("Enable MemProf support for summarizing and cloning indirect calls"))
static bool isNonVolatileLoad(const Instruction *I)
cl::opt< bool > ScalePartialSampleProfileWorkingSetSize
cl::opt< bool > MemProfReportHintedSizes
static bool isNonRenamableLocal(const GlobalValue &GV)
static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, const Function &F, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, DominatorTree &DT, bool HasLocalsInUsedOrAsm, DenseSet< GlobalValue::GUID > &CantBePromoted, bool IsThinLTO, std::function< const StackSafetyInfo *(const Function &F)> GetSSICallback)
static cl::opt< FunctionSummary::ForceSummaryHotnessType, true > FSEC("force-summary-edges-cold", cl::Hidden, cl::location(ForceSummaryEdgesCold), cl::desc("Force all edges in the function summary to cold"), cl::values(clEnumValN(FunctionSummary::FSHT_None, "none", "None."), clEnumValN(FunctionSummary::FSHT_AllNonCritical, "all-non-critical", "All non-critical edges."), clEnumValN(FunctionSummary::FSHT_All, "all", "All edges.")))
static bool mustBeUnreachableFunction(const Function &F)
static bool isNonVolatileStore(const Instruction *I)
module summary analysis
static cl::opt< std::string > ModuleSummaryDotFile("module-summary-dot-file", cl::Hidden, cl::value_desc("filename"), cl::desc("File to emit dot graph of new summary into"))
static bool findRefEdges(ModuleSummaryIndex &Index, const User *CurUser, SetVector< ValueInfo, SmallVector< ValueInfo, 0 > > &RefEdges, SmallPtrSet< const User *, 8 > &Visited, bool &RefLocalLinkageIFunc)
static void addIntrinsicToSummary(const CallInst *CI, SetVector< GlobalValue::GUID, std::vector< GlobalValue::GUID > > &TypeTests, SetVector< FunctionSummary::VFuncId, std::vector< FunctionSummary::VFuncId > > &TypeTestAssumeVCalls, SetVector< FunctionSummary::VFuncId, std::vector< FunctionSummary::VFuncId > > &TypeCheckedLoadVCalls, SetVector< FunctionSummary::ConstVCall, std::vector< FunctionSummary::ConstVCall > > &TypeTestAssumeConstVCalls, SetVector< FunctionSummary::ConstVCall, std::vector< FunctionSummary::ConstVCall > > &TypeCheckedLoadConstVCalls, DominatorTree &DT)
If this intrinsic call requires that we add information to the function summary, do so via the non-co...
static void recordTypeIdCompatibleVtableReferences(ModuleSummaryIndex &Index, const GlobalVariable &V, SmallVectorImpl< MDNode * > &Types)
Record vtable definition V for each type metadata it references.
This is the interface to build a ModuleSummaryIndex for a module.
ModuleSummaryIndex.h This file contains the declarations the classes that hold the module index and s...
#define P(N)
FunctionAnalysisManager FAM
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:57
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
Value * RHS
Value * LHS
An arbitrary precision integer that knows its signedness.
Definition: APSInt.h:23
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:253
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:410
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesAll()
Set by analyses that do not transform their input at all.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
LLVM Basic Block Representation.
Definition: BasicBlock.h:61
static BlockAddress * get(Function *F, BasicBlock *BB)
Return a BlockAddress for the specified function and basic block.
Definition: Constants.cpp:1897
Analysis pass which computes BlockFrequencyInfo.
Legacy analysis pass which computes BlockFrequencyInfo.
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Analysis providing branch probability information.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1120
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1349
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1294
This class represents a function call, abstracting a target machine's calling convention.
This is an important base class in LLVM.
Definition: Constant.h:42
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:63
Implements a dense probed hash-table based set.
Definition: DenseSet.h:278
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:162
std::pair< ValueInfo, CalleeInfo > EdgeTy
<CalleeValueInfo, CalleeInfo> call edge pair.
ForceSummaryHotnessType
Types for -force-summary-edges-cold debugging option.
Class to represent profile counts.
Definition: Function.h:292
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:251
bool isDSOLocal() const
Definition: GlobalValue.h:305
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:296
bool hasLocalLinkage() const
Definition: GlobalValue.h:528
static GUID getGUID(StringRef GlobalName)
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: Globals.cpp:75
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:595
@ DefaultVisibility
The GV is visible.
Definition: GlobalValue.h:67
bool hasSection() const
Definition: GlobalValue.h:290
bool canBeOmittedFromSymbolTable() const
True if GV can be left out of the object symbol table.
Definition: Globals.cpp:425
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:59
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
MutableArrayRef< InstrProfValueData > getPromotionCandidatesForInstruction(const Instruction *I, uint64_t &TotalCount, uint32_t &NumCandidates)
Returns reference to array of InstrProfValueData for the given instruction I.
Legacy wrapper pass to provide the ModuleSummaryIndex object.
ImmutableModuleSummaryIndexWrapperPass(const ModuleSummaryIndex *Index=nullptr)
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
ImmutablePass class - This class is used to provide information that does not need to be run.
Definition: Pass.h:281
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:567
Metadata node.
Definition: Metadata.h:1069
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1430
unsigned getNumOperands() const
Return number of MDNode operands.
Definition: Metadata.h:1436
This class implements a map that also provides access to all stored values in a deterministic order.
Definition: MapVector.h:36
VectorType takeVector()
Clear the MapVector and return the underlying vector.
Definition: MapVector.h:55
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:251
Result run(Module &M, ModuleAnalysisManager &AM)
Legacy wrapper pass to provide the ModuleSummaryIndex object.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
bool doFinalization(Module &M) override
doFinalization - Virtual method overriden by subclasses to do any necessary clean up after all passes...
bool runOnModule(Module &M) override
runOnModule - Virtual method overriden by subclasses to process the module being operated on.
Class to hold module path string table and global value map, and encapsulate methods for operating on...
static void CollectAsmSymbols(const Module &M, function_ref< void(StringRef, object::BasicSymbolRef::Flags)> AsmSymbol)
Parse inline ASM and collect the symbols that are defined or referenced in the current module.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:310
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
An analysis pass based on the new PM to deliver ProfileSummaryInfo.
An analysis pass based on legacy pass manager to deliver ProfileSummaryInfo.
Analysis providing profile information.
std::optional< uint64_t > getProfileCount(const CallBase &CallInst, BlockFrequencyInfo *BFI, bool AllowSynthetic=false) const
Returns the profile count for CallInst.
bool isColdCount(uint64_t C) const
Returns true if count C is considered cold.
bool hasPartialSampleProfile() const
Returns true if module M has partial-profile sample profile.
bool isHotCount(uint64_t C) const
Returns true if count C is considered hot.
A vector that has set insertion semantics.
Definition: SetVector.h:57
bool remove(const value_type &X)
Remove an item from the set vector.
Definition: SetVector.h:188
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:98
Vector takeVector()
Clear the SetVector and return the underlying vector.
Definition: SetVector.h:87
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:384
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:519
bool empty() const
Definition: SmallVector.h:81
size_t size() const
Definition: SmallVector.h:78
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:573
iterator erase(const_iterator CI)
Definition: SmallVector.h:737
void push_back(const T &Elt)
Definition: SmallVector.h:413
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1196
StackSafetyInfo wrapper for the new pass manager.
StackSafetyInfo wrapper for the legacy pass manager.
Interface to access stack safety analysis results for single function.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:51
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
Definition: DataLayout.h:567
unsigned getElementContainingOffset(uint64_t FixedOffset) const
Given a valid byte offset into the structure, returns the structure index that contains it.
Definition: DataLayout.cpp:92
TypeSize getElementOffset(unsigned Idx) const
Definition: DataLayout.h:596
Class to represent struct types.
Definition: DerivedTypes.h:218
ArrayRef< Type * > elements() const
Definition: DerivedTypes.h:357
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
TypeID
Definitions of all of the base types for the Type system.
Definition: Type.h:54
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
iterator_range< use_iterator > uses()
Definition: Value.h:376
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:213
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:95
Helper class to iterate through stack ids in both metadata (memprof MIB and callsite) and the corresp...
CallStackIterator end() const
CallStackIterator beginAfterSharedPrefix(CallStack &Other)
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:460
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
Definition: CommandLine.h:711
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
LocationClass< Ty > location(Ty &L)
Definition: CommandLine.h:463
AllocationType getMIBAllocType(const MDNode *MIB)
Returns the allocation type from an MIB metadata node.
MDNode * getMIBStackNode(const MDNode *MIB)
Returns the stack node from an MIB metadata node.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:329
@ Offset
Definition: DWP.cpp:480
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1739
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition: STLExtras.h:2448
bool IsConstantOffsetFromGlobal(Constant *C, GlobalValue *&GV, APInt &Offset, const DataLayout &DL, DSOLocalEquivalent **DSOEquiv=nullptr)
If this constant is a constant offset from a global, return the global and the constant.
FunctionSummary::ForceSummaryHotnessType ForceSummaryEdgesCold
bool needsParamAccessSummary(const Module &M)
ModuleSummaryIndex buildModuleSummaryIndex(const Module &M, std::function< BlockFrequencyInfo *(const Function &F)> GetBFICallback, ProfileSummaryInfo *PSI, std::function< const StackSafetyInfo *(const Function &F)> GetSSICallback=[](const Function &F) -> const StackSafetyInfo *{ return nullptr;})
Direct function to compute a ModuleSummaryIndex from a given module.
void initializeModuleSummaryIndexWrapperPassPass(PassRegistry &)
std::vector< VirtFuncOffset > VTableFuncList
List of functions referenced by a particular vtable definition.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1746
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
void findDevirtualizableCallsForTypeCheckedLoad(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< Instruction * > &LoadedPtrs, SmallVectorImpl< Instruction * > &Preds, bool &HasNonCallUses, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.checked.load, find all devirtualizable call sites based on t...
SmallVector< InstrProfValueData, 4 > getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst and returns them if Inst is annotated with value profile dat...
Definition: InstrProf.cpp:1369
ModulePass * createModuleSummaryIndexWrapperPass()
ImmutablePass * createImmutableModuleSummaryIndexWrapperPass(const ModuleSummaryIndex *Index)
void initializeImmutableModuleSummaryIndexWrapperPassPass(PassRegistry &)
bool mayHaveMemprofSummary(const CallBase *CB)
Returns true if the instruction could have memprof metadata, used to ensure consistency between summa...
void findDevirtualizableCallsForTypeTest(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< CallInst * > &Assumes, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.test, find all devirtualizable call sites based on the call ...
GlobalVariable * collectUsedGlobalVariables(const Module &M, SmallVectorImpl< GlobalValue * > &Vec, bool CompilerUsed)
Given "llvm.used" or "llvm.compiler.used" as a global name, collect the initializer elements of that ...
Definition: Module.cpp:865
Summary of memprof metadata on allocations.
A special type used by analysis passes to provide an address that identifies that particular analysis...
Definition: Analysis.h:28
A call site that could be devirtualized.
A specification for a virtual function call with all constant integer arguments.
Flags specific to function summaries.
An "identifier" for a virtual function.
Group flags (Linkage, NotEligibleToImport, etc.) as a bitfield.
Summary of a single MIB in a memprof metadata on allocations.
Struct that holds a reference to a particular GUID in a global value summary.