50#include <system_error>
56#define DEBUG_TYPE "function-import"
59 "Number of functions thin link decided to import");
61 "Number of hot functions thin link decided to import");
63 "Number of critical functions thin link decided to import");
65 "Number of global variables thin link decided to import");
66STATISTIC(NumImportedFunctions,
"Number of functions imported in backend");
68 "Number of global variables imported in backend");
69STATISTIC(NumImportedModules,
"Number of modules imported from");
70STATISTIC(NumDeadSymbols,
"Number of dead stripped symbols in index");
71STATISTIC(NumLiveSymbols,
"Number of live symbols in index");
76 cl::desc(
"Only import functions with less than N instructions"));
80 cl::desc(
"Only import first N functions if N>=0 (default -1)"));
84 cl::desc(
"Import functions with noinline attribute"));
89 cl::desc(
"As we import functions, multiply the "
90 "`import-instr-limit` threshold by this factor "
91 "before processing newly imported functions"));
96 cl::desc(
"As we import functions called from hot callsite, multiply the "
97 "`import-instr-limit` threshold by this factor "
98 "before processing newly imported functions"));
102 cl::desc(
"Multiply the `import-instr-limit` threshold for hot callsites"));
108 "Multiply the `import-instr-limit` threshold for critical callsites"));
113 cl::desc(
"Multiply the `import-instr-limit` threshold for cold callsites"));
116 cl::desc(
"Print imported functions"));
120 cl::desc(
"Print information for functions rejected for importing"));
127 cl::desc(
"Enable import metadata like 'thinlto_src_module'"));
133 cl::desc(
"The summary file to use for function importing."));
139 cl::desc(
"Import all external functions in index."));
142static std::unique_ptr<Module>
loadFile(
const std::string &FileName,
148 std::unique_ptr<Module> Result =
152 Err.print(
"function-import",
errs());
165 ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList,
169 [&
Index, CalleeSummaryList,
170 CallerModulePath](
const std::unique_ptr<GlobalValueSummary> &SummaryPtr)
173 auto *GVSummary = SummaryPtr.get();
174 if (!
Index.isGlobalValueLive(GVSummary))
175 return {FunctionImporter::ImportFailureReason::NotLive, GVSummary};
178 return {FunctionImporter::ImportFailureReason::InterposableLinkage,
181 auto *Summary = dyn_cast<FunctionSummary>(GVSummary->getBaseObject());
189 return {FunctionImporter::ImportFailureReason::GlobalVar, GVSummary};
202 CalleeSummaryList.size() > 1 &&
203 Summary->modulePath() != CallerModulePath)
205 FunctionImporter::ImportFailureReason::LocalLinkageNotInModule,
210 if (Summary->notEligibleToImport())
211 return {FunctionImporter::ImportFailureReason::NotEligible,
214 return {FunctionImporter::ImportFailureReason::None, GVSummary};
231 ArrayRef<std::unique_ptr<GlobalValueSummary>> CalleeSummaryList,
232 unsigned Threshold,
StringRef CallerModulePath,
234 auto QualifiedCandidates =
236 for (
auto QualifiedValue : QualifiedCandidates) {
237 Reason = QualifiedValue.first;
238 if (Reason != FunctionImporter::ImportFailureReason::None)
241 cast<FunctionSummary>(QualifiedValue.second->getBaseObject());
243 if ((Summary->instCount() > Threshold) && !Summary->fflags().AlwaysInline &&
245 Reason = FunctionImporter::ImportFailureReason::TooLarge;
251 Reason = FunctionImporter::ImportFailureReason::NoInline;
276 bool shouldImportGlobal(
const ValueInfo &VI) {
277 const auto &GVS = DefinedGVSummaries.
find(VI.getGUID());
278 if (GVS == DefinedGVSummaries.
end())
290 if (VI.getSummaryList().size() > 1 &&
292 !IsPrevailing(VI.getGUID(), GVS->second))
301 for (
const auto &VI : Summary.refs()) {
302 if (!shouldImportGlobal(VI)) {
304 dbgs() <<
"Ref ignored! Target already in destination module.\n");
316 auto LocalNotInModule =
319 RefSummary->modulePath() != Summary.modulePath();
322 for (
const auto &RefSummary : VI.getSummaryList()) {
323 const auto *GVS = dyn_cast<GlobalVarSummary>(RefSummary.get());
329 if (!GVS || !
Index.canImportGlobalVar(GVS,
true) ||
330 LocalNotInModule(GVS))
332 auto ILI = ImportList[RefSummary->modulePath()].
insert(VI.getGUID());
337 NumImportedGlobalVarsThinLink++;
342 (*ExportLists)[RefSummary->modulePath()].insert(VI);
346 if (!
Index.isWriteOnly(GVS))
360 :
Index(
Index), DefinedGVSummaries(DefinedGVSummaries),
361 IsPrevailing(IsPrevailing), ImportList(ImportList),
362 ExportLists(ExportLists) {}
366 onImportingSummaryImpl(Summary, Worklist);
367 while (!Worklist.
empty())
368 onImportingSummaryImpl(*Worklist.
pop_back_val(), Worklist);
385 : IsPrevailing(IsPrevailing),
Index(
Index), ExportLists(ExportLists) {}
398 case FunctionImporter::ImportFailureReason::None:
400 case FunctionImporter::ImportFailureReason::GlobalVar:
402 case FunctionImporter::ImportFailureReason::NotLive:
404 case FunctionImporter::ImportFailureReason::TooLarge:
406 case FunctionImporter::ImportFailureReason::InterposableLinkage:
407 return "InterposableLinkage";
408 case FunctionImporter::ImportFailureReason::LocalLinkageNotInModule:
409 return "LocalLinkageNotInModule";
410 case FunctionImporter::ImportFailureReason::NotEligible:
411 return "NotEligible";
412 case FunctionImporter::ImportFailureReason::NoInline:
423 const unsigned Threshold,
const GVSummaryMapTy &DefinedGVSummaries,
431 static int ImportCount = 0;
432 for (
const auto &Edge : Summary.calls()) {
434 LLVM_DEBUG(
dbgs() <<
" edge -> " << VI <<
" Threshold:" << Threshold
443 if (DefinedGVSummaries.
count(VI.getGUID())) {
447 LLVM_DEBUG(
dbgs() <<
"ignored! Target already in destination module.\n");
452 if (Hotness == CalleeInfo::HotnessType::Hot)
454 if (Hotness == CalleeInfo::HotnessType::Cold)
456 if (Hotness == CalleeInfo::HotnessType::Critical)
461 const auto NewThreshold =
462 Threshold * GetBonusMultiplier(Edge.second.getHotness());
464 auto IT = ImportThresholds.
insert(std::make_pair(
465 VI.getGUID(), std::make_tuple(NewThreshold,
nullptr,
nullptr)));
466 bool PreviouslyVisited = !
IT.second;
467 auto &ProcessedThreshold = std::get<0>(
IT.first->second);
468 auto &CalleeSummary = std::get<1>(
IT.first->second);
469 auto &FailureInfo = std::get<2>(
IT.first->second);
472 Edge.second.getHotness() == CalleeInfo::HotnessType::Hot;
473 bool IsCriticalCallsite =
474 Edge.second.getHotness() == CalleeInfo::HotnessType::Critical;
478 assert(PreviouslyVisited);
483 if (NewThreshold <= ProcessedThreshold) {
485 dbgs() <<
"ignored! Target was already imported with Threshold "
486 << ProcessedThreshold <<
"\n");
490 ProcessedThreshold = NewThreshold;
491 ResolvedCalleeSummary = cast<FunctionSummary>(CalleeSummary);
495 if (PreviouslyVisited && NewThreshold <= ProcessedThreshold) {
497 dbgs() <<
"ignored! Target was already rejected with Threshold "
498 << ProcessedThreshold <<
"\n");
501 "Expected FailureInfo for previously rejected candidate");
502 FailureInfo->Attempts++;
509 Summary.modulePath(), Reason);
510 if (!CalleeSummary) {
514 if (PreviouslyVisited) {
515 ProcessedThreshold = NewThreshold;
518 "Expected FailureInfo for previously rejected candidate");
519 FailureInfo->Reason = Reason;
520 FailureInfo->Attempts++;
521 FailureInfo->MaxHotness =
522 std::max(FailureInfo->MaxHotness, Edge.second.getHotness());
526 "Expected no FailureInfo for newly rejected candidate");
527 FailureInfo = std::make_unique<FunctionImporter::ImportFailureInfo>(
528 VI, Edge.second.getHotness(), Reason, 1);
531 std::string Msg = std::string(
"Failed to import function ") +
532 VI.name().str() +
" due to " +
534 auto Error = make_error<StringError>(
537 "Error importing module: ");
541 <<
"ignored! No qualifying callee with summary found.\n");
547 CalleeSummary = CalleeSummary->getBaseObject();
548 ResolvedCalleeSummary = cast<FunctionSummary>(CalleeSummary);
551 (ResolvedCalleeSummary->
instCount() <= NewThreshold)) &&
552 "selectCallee() didn't honor the threshold");
554 auto ExportModulePath = ResolvedCalleeSummary->
modulePath();
555 auto ILI = ImportList[ExportModulePath].
insert(VI.getGUID());
558 bool PreviouslyImported = !ILI.second;
559 if (!PreviouslyImported) {
560 NumImportedFunctionsThinLink++;
562 NumImportedHotFunctionsThinLink++;
563 if (IsCriticalCallsite)
564 NumImportedCriticalFunctionsThinLink++;
571 (*ExportLists)[ExportModulePath].insert(VI);
574 auto GetAdjustedThreshold = [](
unsigned Threshold,
bool IsHotCallsite) {
583 const auto AdjThreshold = GetAdjustedThreshold(Threshold, IsHotCallsite);
588 Worklist.
emplace_back(ResolvedCalleeSummary, AdjThreshold);
604 for (
const auto &GVSummary : DefinedGVSummaries) {
608 auto VI =
Index.getValueInfo(GVSummary.first);
610 if (!
Index.isGlobalValueLive(GVSummary.second)) {
615 dyn_cast<FunctionSummary>(GVSummary.second->getBaseObject());
621 DefinedGVSummaries, IsPrevailing, Worklist, GVI,
622 ImportList, ExportLists, ImportThresholds);
626 while (!Worklist.
empty()) {
628 auto *Summary = std::get<0>(GVInfo);
629 auto Threshold = std::get<1>(GVInfo);
631 if (
auto *FS = dyn_cast<FunctionSummary>(Summary))
633 IsPrevailing, Worklist, GVI, ImportList,
634 ExportLists, ImportThresholds);
640 dbgs() <<
"Missed imports into module " << ModName <<
"\n";
641 for (
auto &
I : ImportThresholds) {
642 auto &ProcessedThreshold = std::get<0>(
I.second);
643 auto &CalleeSummary = std::get<1>(
I.second);
644 auto &FailureInfo = std::get<2>(
I.second);
649 if (!FailureInfo->VI.getSummaryList().empty())
650 FS = dyn_cast<FunctionSummary>(
651 FailureInfo->VI.getSummaryList()[0]->getBaseObject());
652 dbgs() << FailureInfo->VI
654 <<
", Threshold = " << ProcessedThreshold
655 <<
", Size = " << (FS ? (int)FS->instCount() : -1)
657 <<
", Attempts = " << FailureInfo->Attempts <<
"\n";
664 auto SL = VI.getSummaryList();
672 if (
const auto &VI =
Index.getValueInfo(
G))
696 for (
auto &ImportPerModule : ImportLists)
697 for (
auto &ExportPerModule : ImportPerModule.second)
698 FlattenedImports.
insert(ExportPerModule.second.begin(),
699 ExportPerModule.second.end());
707 auto IsReadOrWriteOnlyVarNeedingImporting = [&](
StringRef ModulePath,
709 auto *GVS = dyn_cast_or_null<GlobalVarSummary>(
710 Index.findSummaryInModule(VI, ModulePath));
711 return GVS && (
Index.isReadOnly(GVS) ||
Index.isWriteOnly(GVS)) &&
717 for (
auto &ExportPerModule : ExportLists)
718 for (
auto &VI : ExportPerModule.second)
719 if (!FlattenedImports.
count(VI.getGUID()) &&
720 IsReadOrWriteOnlyVarNeedingImporting(ExportPerModule.first, VI))
737 for (
const auto &DefinedGVSummaries : ModuleToDefinedGVSummaries) {
738 auto &ImportList = ImportLists[DefinedGVSummaries.first];
740 << DefinedGVSummaries.first <<
"'\n");
742 DefinedGVSummaries.first, ImportList);
750 for (
auto &ELI : ExportLists) {
752 const auto &DefinedGVSummaries =
753 ModuleToDefinedGVSummaries.
lookup(ELI.first);
754 for (
auto &EI : ELI.second) {
761 auto DS = DefinedGVSummaries.
find(EI.getGUID());
765 auto *S = DS->getSecond();
766 S = S->getBaseObject();
767 if (
auto *GVS = dyn_cast<GlobalVarSummary>(S)) {
772 if (!
Index.isWriteOnly(GVS))
773 for (
const auto &VI : GVS->refs())
776 auto *FS = cast<FunctionSummary>(S);
777 for (
const auto &Edge : FS->calls())
778 NewExports.
insert(Edge.first);
779 for (
const auto &
Ref : FS->refs())
787 for (
auto EI = NewExports.
begin(); EI != NewExports.
end();) {
788 if (!DefinedGVSummaries.
count(EI->getGUID()))
789 NewExports.
erase(EI++);
793 ELI.second.insert(NewExports.
begin(), NewExports.
end());
800 for (
auto &ModuleImports : ImportLists) {
801 auto ModName = ModuleImports.first;
802 auto &Exports = ExportLists[ModName];
805 << Exports.size() - NumGVS <<
" functions and " << NumGVS
806 <<
" vars. Imports from " << ModuleImports.second.size()
808 for (
auto &Src : ModuleImports.second) {
809 auto SrcModName = Src.first;
812 <<
" functions imported from " << SrcModName <<
"\n");
814 <<
" global vars imported from " << SrcModName <<
"\n");
824 LLVM_DEBUG(
dbgs() <<
"* Module " << ModulePath <<
" imports from "
825 << ImportList.
size() <<
" modules.\n");
826 for (
auto &Src : ImportList) {
827 auto SrcModName = Src.first;
830 <<
" functions imported from " << SrcModName <<
"\n");
831 LLVM_DEBUG(
dbgs() <<
" - " << NumGVSPerMod <<
" vars imported from "
832 << SrcModName <<
"\n");
854 Index.collectDefinedFunctionsForModule(ModulePath, FunctionSummaryMap);
857 LLVM_DEBUG(
dbgs() <<
"Computing import for Module '" << ModulePath <<
"'\n");
874 for (
const auto &GlobalList :
Index) {
876 if (GlobalList.second.SummaryList.empty())
879 auto GUID = GlobalList.first;
880 assert(GlobalList.second.SummaryList.size() == 1 &&
881 "Expected individual combined index to have one summary per GUID");
882 auto &Summary = GlobalList.second.SummaryList[0];
885 if (Summary->modulePath() == ModulePath)
888 ImportList[Summary->modulePath()].
insert(GUID);
901 for (
auto &EI : FS->mutableCalls()) {
902 if (!EI.first.getSummaryList().empty())
904 auto GUID =
Index.getGUIDFromOriginalID(EI.first.getGUID());
908 auto VI =
Index.getValueInfo(GUID);
911 [&](
const std::unique_ptr<GlobalValueSummary> &SummaryPtr) {
921 return SummaryPtr->getSummaryKind() ==
922 GlobalValueSummary::GlobalVarKind;
930 for (
const auto &Entry :
Index) {
931 for (
const auto &S : Entry.second.SummaryList) {
932 if (
auto *FS = dyn_cast<FunctionSummary>(S.get()))
945 GUIDPreservedSymbols.
empty()) {
950 unsigned LiveSymbols = 0;
953 for (
auto GUID : GUIDPreservedSymbols) {
957 for (
const auto &S : VI.getSummaryList())
962 for (
const auto &Entry :
Index) {
963 auto VI =
Index.getValueInfo(Entry);
964 for (
const auto &S : Entry.second.SummaryList) {
965 if (
auto *FS = dyn_cast<FunctionSummary>(S.get()))
977 auto visit = [&](
ValueInfo VI,
bool IsAliasee) {
988 [](
const std::unique_ptr<llvm::GlobalValueSummary> &S) {
998 if (isPrevailing(VI.getGUID()) == PrevailingType::No) {
999 bool KeepAliveLinkage =
false;
1000 bool Interposable =
false;
1001 for (
const auto &S : VI.getSummaryList()) {
1005 KeepAliveLinkage =
true;
1007 Interposable =
true;
1011 if (!KeepAliveLinkage)
1016 "Interposable and available_externally/linkonce_odr/weak_odr "
1021 for (
const auto &S : VI.getSummaryList())
1027 while (!Worklist.
empty()) {
1029 for (
const auto &Summary : VI.getSummaryList()) {
1030 if (
auto *AS = dyn_cast<AliasSummary>(Summary.get())) {
1034 visit(AS->getAliaseeVI(),
true);
1037 for (
auto Ref : Summary->refs())
1039 if (
auto *FS = dyn_cast<FunctionSummary>(Summary.get()))
1040 for (
auto Call : FS->calls())
1041 visit(Call.first,
false);
1044 Index.setWithGlobalValueDeadStripping();
1046 unsigned DeadSymbols =
Index.size() - LiveSymbols;
1047 LLVM_DEBUG(
dbgs() << LiveSymbols <<
" symbols Live, and " << DeadSymbols
1048 <<
" symbols Dead \n");
1049 NumDeadSymbols += DeadSymbols;
1050 NumLiveSymbols += LiveSymbols;
1058 bool ImportEnabled) {
1062 Index.propagateAttributes(GUIDPreservedSymbols);
1071 std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) {
1073 ModuleToSummariesForIndex[std::string(ModulePath)] =
1074 ModuleToDefinedGVSummaries.
lookup(ModulePath);
1076 for (
const auto &ILI : ImportList) {
1077 auto &SummariesForIndex = ModuleToSummariesForIndex[std::string(ILI.first)];
1078 const auto &DefinedGVSummaries =
1079 ModuleToDefinedGVSummaries.
lookup(ILI.first);
1080 for (
const auto &GI : ILI.second) {
1081 const auto &DS = DefinedGVSummaries.
find(GI);
1082 assert(DS != DefinedGVSummaries.
end() &&
1083 "Expected a defined summary for imported global value");
1084 SummariesForIndex[GI] = DS->second;
1092 const std::map<std::string, GVSummaryMapTy> &ModuleToSummariesForIndex) {
1097 for (
const auto &ILI : ModuleToSummariesForIndex)
1101 if (ILI.first != ModulePath)
1102 ImportsOS << ILI.first <<
"\n";
1103 return std::error_code();
1109 if (
Function *
F = dyn_cast<Function>(&GV)) {
1112 F->setComdat(
nullptr);
1114 V->setInitializer(
nullptr);
1117 V->setComdat(
nullptr);
1145 auto FinalizeInModule = [&](
GlobalValue &GV,
bool Propagate =
false) {
1147 const auto &GS = DefinedGlobals.
find(GV.
getGUID());
1148 if (GS == DefinedGlobals.
end())
1153 if (
Function *
F = dyn_cast<Function>(&GV)) {
1155 if (FS->fflags().ReadNone && !
F->doesNotAccessMemory())
1156 F->setDoesNotAccessMemory();
1158 if (FS->fflags().ReadOnly && !
F->onlyReadsMemory())
1159 F->setOnlyReadsMemory();
1161 if (FS->fflags().NoRecurse && !
F->doesNotRecurse())
1162 F->setDoesNotRecurse();
1164 if (FS->fflags().NoUnwind && !
F->doesNotThrow())
1165 F->setDoesNotThrow();
1169 auto NewLinkage = GS->second->linkage();
1208 GS->second->canAutoHide()) {
1214 <<
"` from " << GV.
getLinkage() <<
" to " << NewLinkage
1221 auto *GO = dyn_cast_or_null<GlobalObject>(&GV);
1222 if (GO && GO->isDeclarationForLinker() && GO->hasComdat()) {
1223 if (GO->getComdat()->getName() == GO->getName())
1224 NonPrevailingComdats.
insert(GO->getComdat());
1225 GO->setComdat(
nullptr);
1230 for (
auto &GV : TheModule)
1232 for (
auto &GV : TheModule.
globals())
1233 FinalizeInModule(GV);
1234 for (
auto &GV : TheModule.
aliases())
1235 FinalizeInModule(GV);
1240 if (NonPrevailingComdats.
empty())
1243 if (
auto *
C = GO.getComdat();
C && NonPrevailingComdats.
count(
C)) {
1244 GO.setComdat(
nullptr);
1255 for (
auto &GA : TheModule.
aliases()) {
1256 if (GA.hasAvailableExternallyLinkage())
1259 assert(Obj &&
"aliasee without an base object is unimplemented");
1273 auto MustPreserveGV = [&](
const GlobalValue &GV) ->
bool {
1277 if (isa<GlobalIFunc>(&GV) ||
1278 (isa<GlobalAlias>(&GV) &&
1279 isa<GlobalIFunc>(cast<GlobalAlias>(&GV)->getAliaseeObject())))
1283 auto GS = DefinedGlobals.
find(GV.getGUID());
1284 if (GS == DefinedGlobals.
end()) {
1296 if (GS == DefinedGlobals.
end()) {
1333 for (
auto &GV : M.globals())
1336 if (!GV.isDeclaration() && GV.hasAttribute(
"thinlto-internalize")) {
1348 unsigned ImportedCount = 0, ImportedGVCount = 0;
1352 std::set<StringRef> ModuleNameOrderedList;
1353 for (
const auto &FunctionsToImportPerModule : ImportList) {
1354 ModuleNameOrderedList.insert(FunctionsToImportPerModule.first);
1356 for (
const auto &
Name : ModuleNameOrderedList) {
1358 const auto &FunctionsToImportPerModule = ImportList.
find(
Name);
1359 assert(FunctionsToImportPerModule != ImportList.
end());
1361 if (!SrcModuleOrErr)
1363 std::unique_ptr<Module> SrcModule = std::move(*SrcModuleOrErr);
1365 "Context mismatch");
1369 if (
Error Err = SrcModule->materializeMetadata())
1370 return std::move(Err);
1372 auto &ImportGUIDs = FunctionsToImportPerModule->second;
1378 auto GUID =
F.getGUID();
1379 auto Import = ImportGUIDs.count(GUID);
1381 << GUID <<
" " <<
F.getName() <<
" from "
1382 << SrcModule->getSourceFileName() <<
"\n");
1384 if (
Error Err =
F.materialize())
1385 return std::move(Err);
1389 "thinlto_src_module",
1391 {MDString::get(DestModule.getContext(),
1392 SrcModule->getSourceFileName())}));
1400 auto GUID = GV.getGUID();
1401 auto Import = ImportGUIDs.count(GUID);
1403 << GUID <<
" " << GV.getName() <<
" from "
1404 << SrcModule->getSourceFileName() <<
"\n");
1406 if (
Error Err = GV.materialize())
1407 return std::move(Err);
1408 ImportedGVCount += GlobalsToImport.
insert(&GV);
1412 if (!GA.hasName() || isa<GlobalIFunc>(GA.getAliaseeObject()))
1414 auto GUID = GA.getGUID();
1415 auto Import = ImportGUIDs.count(GUID);
1417 << GUID <<
" " << GA.getName() <<
" from "
1418 << SrcModule->getSourceFileName() <<
"\n");
1420 if (
Error Err = GA.materialize())
1421 return std::move(Err);
1425 return std::move(Err);
1429 << SrcModule->getSourceFileName() <<
"\n");
1433 "thinlto_src_module",
1435 {MDString::get(DestModule.getContext(),
1436 SrcModule->getSourceFileName())}));
1438 GlobalsToImport.
insert(Fn);
1450 SrcModule->setPartialSampleProfileRatio(
Index);
1458 for (
const auto *GV : GlobalsToImport)
1460 <<
" from " << SrcModule->getSourceFileName() <<
"\n";
1463 if (
Error Err = Mover.
move(std::move(SrcModule),
1467 Twine(
"Function Import: link error: ") +
1470 ImportedCount += GlobalsToImport.
size();
1471 NumImportedModules++;
1476 NumImportedFunctions += (ImportedCount - ImportedGVCount);
1477 NumImportedGlobalVars += ImportedGVCount;
1479 LLVM_DEBUG(
dbgs() <<
"Imported " << ImportedCount - ImportedGVCount
1480 <<
" functions for Module "
1483 <<
" global variables for Module "
1485 return ImportedCount;
1495 if (!IndexPtrOrErr) {
1500 std::unique_ptr<ModuleSummaryIndex>
Index = std::move(*IndexPtrOrErr);
1509 *
Index, ImportList);
1512 isPrevailing, *
Index, ImportList);
1519 for (
auto &S :
I.second.SummaryList) {
1529 errs() <<
"Error renaming module\n";
1534 auto ModuleLoader = [&M](
StringRef Identifier) {
1535 return loadFile(std::string(Identifier), M.getContext());
1544 "Error importing module: ");
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate any type of IT block"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow complex IT blocks")))
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static auto qualifyCalleeCandidates(const ModuleSummaryIndex &Index, ArrayRef< std::unique_ptr< GlobalValueSummary > > CalleeSummaryList, StringRef CallerModulePath)
Given a list of possible callee implementation for a call site, qualify the legality of importing eac...
static cl::opt< float > ImportColdMultiplier("import-cold-multiplier", cl::init(0), cl::Hidden, cl::value_desc("N"), cl::desc("Multiply the `import-instr-limit` threshold for cold callsites"))
static cl::opt< float > ImportHotMultiplier("import-hot-multiplier", cl::init(10.0), cl::Hidden, cl::value_desc("x"), cl::desc("Multiply the `import-instr-limit` threshold for hot callsites"))
static const GlobalValueSummary * selectCallee(const ModuleSummaryIndex &Index, ArrayRef< std::unique_ptr< GlobalValueSummary > > CalleeSummaryList, unsigned Threshold, StringRef CallerModulePath, FunctionImporter::ImportFailureReason &Reason)
Given a list of possible callee implementation for a call site, select one that fits the Threshold.
static bool checkVariableImport(const ModuleSummaryIndex &Index, DenseMap< StringRef, FunctionImporter::ImportMapTy > &ImportLists, DenseMap< StringRef, FunctionImporter::ExportSetTy > &ExportLists)
static bool doImportingForModuleForTest(Module &M, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing)
static cl::opt< bool > EnableImportMetadata("enable-import-metadata", cl::init(false), cl::Hidden, cl::desc("Enable import metadata like 'thinlto_src_module'"))
static cl::opt< float > ImportHotInstrFactor("import-hot-evolution-factor", cl::init(1.0), cl::Hidden, cl::value_desc("x"), cl::desc("As we import functions called from hot callsite, multiply the " "`import-instr-limit` threshold by this factor " "before processing newly imported functions"))
static void computeImportForFunction(const FunctionSummary &Summary, const ModuleSummaryIndex &Index, const unsigned Threshold, const GVSummaryMapTy &DefinedGVSummaries, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, SmallVectorImpl< EdgeInfo > &Worklist, GlobalsImporter &GVImporter, FunctionImporter::ImportMapTy &ImportList, DenseMap< StringRef, FunctionImporter::ExportSetTy > *ExportLists, FunctionImporter::ImportThresholdsTy &ImportThresholds)
Compute the list of functions to import for a given caller.
static cl::opt< float > ImportCriticalMultiplier("import-critical-multiplier", cl::init(100.0), cl::Hidden, cl::value_desc("x"), cl::desc("Multiply the `import-instr-limit` threshold for critical callsites"))
static cl::opt< int > ImportCutoff("import-cutoff", cl::init(-1), cl::Hidden, cl::value_desc("N"), cl::desc("Only import first N functions if N>=0 (default -1)"))
static const char * getFailureName(FunctionImporter::ImportFailureReason Reason)
static cl::opt< float > ImportInstrFactor("import-instr-evolution-factor", cl::init(0.7), cl::Hidden, cl::value_desc("x"), cl::desc("As we import functions, multiply the " "`import-instr-limit` threshold by this factor " "before processing newly imported functions"))
static void internalizeGVsAfterImport(Module &M)
static cl::opt< bool > PrintImports("print-imports", cl::init(false), cl::Hidden, cl::desc("Print imported functions"))
void updateValueInfoForIndirectCalls(ModuleSummaryIndex &Index, FunctionSummary *FS)
static std::unique_ptr< Module > loadFile(const std::string &FileName, LLVMContext &Context)
static cl::opt< bool > ForceImportAll("force-import-all", cl::init(false), cl::Hidden, cl::desc("Import functions with noinline attribute"))
static bool isGlobalVarSummary(const ModuleSummaryIndex &Index, ValueInfo VI)
static cl::opt< unsigned > ImportInstrLimit("import-instr-limit", cl::init(100), cl::Hidden, cl::value_desc("N"), cl::desc("Only import functions with less than N instructions"))
Limit on instruction count of imported functions.
static cl::opt< bool > ComputeDead("compute-dead", cl::init(true), cl::Hidden, cl::desc("Compute dead symbols"))
static cl::opt< std::string > SummaryFile("summary-file", cl::desc("The summary file to use for function importing."))
Summary file to use for function importing when using -function-import from the command line.
static cl::opt< bool > ImportAllIndex("import-all-index", cl::desc("Import all external functions in index."))
Used when testing importing from distributed indexes via opt.
static cl::opt< bool > PrintImportFailures("print-import-failures", cl::init(false), cl::Hidden, cl::desc("Print information for functions rejected for importing"))
static Function * replaceAliasWithAliasee(Module *SrcModule, GlobalAlias *GA)
Make alias a clone of its aliasee.
static void dumpImportListForModule(const ModuleSummaryIndex &Index, StringRef ModulePath, FunctionImporter::ImportMapTy &ImportList)
static void ComputeCrossModuleImportForModuleFromIndexForTest(StringRef ModulePath, const ModuleSummaryIndex &Index, FunctionImporter::ImportMapTy &ImportList)
Mark all external summaries in Index for import into the given module.
static unsigned numGlobalVarSummaries(const ModuleSummaryIndex &Index, T &Cont)
static void ComputeCrossModuleImportForModuleForTest(StringRef ModulePath, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, const ModuleSummaryIndex &Index, FunctionImporter::ImportMapTy &ImportList)
Compute all the imports for the given module using the Index.
static cl::opt< std::string > OutputFilename("o", cl::desc("Output filename"), cl::value_desc("filename"), cl::init("-"))
static cl::opt< bool > PropagateAttrs("propagate-attrs", cl::init(true), cl::Hidden, cl::desc("Propagate attributes in index"))
ModuleSummaryIndex.h This file contains the declarations the classes that hold the module index and s...
Module.h This file contains the declarations for the Module class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Import globals referenced by a function or other globals that are being imported, if importing such g...
void onImportingSummary(const GlobalValueSummary &Summary)
GlobalsImporter(const ModuleSummaryIndex &Index, const GVSummaryMapTy &DefinedGVSummaries, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> IsPrevailing, FunctionImporter::ImportMapTy &ImportList, DenseMap< StringRef, FunctionImporter::ExportSetTy > *ExportLists)
Determine the list of imports and exports for each module.
ModuleImportsManager(function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> IsPrevailing, const ModuleSummaryIndex &Index, DenseMap< StringRef, FunctionImporter::ExportSetTy > *ExportLists=nullptr)
void computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries, StringRef ModName, FunctionImporter::ImportMapTy &ImportList)
Given the list of globals defined in a module, compute the list of imports as well as the list of "ex...
A container for analyses that lazily runs them and caches their results.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
iterator find(const_arg_type_t< KeyT > Val)
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
Lightweight error class with error context and mandatory checking.
Tagged union holding either a T or a Error.
Error takeError()
Take ownership of the stored error.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
The function importer is automatically importing function from other modules based on the provided su...
Expected< bool > importFunctions(Module &M, const ImportMapTy &ImportList)
Import functions in Module M based on the supplied import list.
ImportFailureReason
The different reasons selectCallee will chose not to import a candidate.
Function summary information to aid decisions and implementation of importing.
unsigned instCount() const
Get the instruction count recorded for this function.
FFlags fflags() const
Get function summary flags.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
const GlobalObject * getAliaseeObject() const
Function and variable summary information to aid decisions and implementation of importing.
StringRef modulePath() const
Get the path to the module containing this function.
VisibilityTypes getVisibility() const
bool isImplicitDSOLocal() const
static bool isLocalLinkage(LinkageTypes Linkage)
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
LinkageTypes getLinkage() const
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
ThreadLocalMode getThreadLocalMode() const
static bool isAvailableExternallyLinkage(LinkageTypes Linkage)
void setLinkage(LinkageTypes LT)
unsigned getAddressSpace() const
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Module * getParent()
Get the module that this global value is contained inside of...
const GlobalObject * getAliaseeObject() const
void setDSOLocal(bool Local)
PointerType * getType() const
Global values are always pointers.
@ DefaultVisibility
The GV is visible.
@ HiddenVisibility
The GV is hidden.
static GUID getGUID(StringRef GlobalName)
Return a 64-bit global unique ID constructed from global value name (i.e.
void setVisibility(VisibilityTypes V)
static bool isInterposableLinkage(LinkageTypes Linkage)
Whether the definition of this global may be replaced by something non-equivalent at link time.
Error materialize()
Make sure this GlobalValue is fully read.
bool canBeOmittedFromSymbolTable() const
True if GV can be left out of the object symbol table.
bool hasAvailableExternallyLinkage() const
std::string getGlobalIdentifier() const
Return the modified name for this global value suitable to be used as the key for a global lookup (e....
@ InternalLinkage
Rename collisions when linking (static functions).
@ WeakODRLinkage
Same, but only replaced by something equivalent.
@ ExternalLinkage
Externally visible function.
@ AvailableExternallyLinkage
Available for inspection, not emission.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Type * getValueType() const
Error move(std::unique_ptr< Module > Src, ArrayRef< GlobalValue * > ValuesToLink, LazyCallback AddLazyFor, bool IsPerformingImport)
Move in the provide values in ValuesToLink from Src.
This is an important class for using LLVM in a threaded context.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Class to hold module path string table and global value map, and encapsulate methods for operating on...
static StringRef getOriginalNameBeforePromote(StringRef Name)
Helper to obtain the unpromoted name for a global value (or the original name if not promoted).
A Module instance is used to store all the information related to an LLVM module.
LLVMContext & getContext() const
Get the global data context.
const std::string & getSourceFileName() const
Get the module's original source file name.
iterator_range< alias_iterator > aliases()
iterator_range< global_iterator > globals()
const std::string & getModuleIdentifier() const
Get the module identifier which is, essentially, the name of the module.
iterator_range< global_object_iterator > global_objects()
unsigned getAddressSpace() const
Return the address space of the Pointer type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Instances of this class encapsulate one diagnostic report, allowing printing to a raw_ostream as a ca...
A vector that has set insertion semantics.
ArrayRef< value_type > getArrayRef() const
size_type size() const
Determine the number of elements in the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
bool isFunctionTy() const
True if this is an instance of FunctionType.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
StringRef getName() const
Return a constant reference to the value's name.
void takeName(Value *V)
Transfer the name from V to this value.
std::pair< iterator, bool > insert(const ValueT &V)
bool erase(const ValueT &V)
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
initializer< Ty > init(const Ty &Val)
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
This is an optimization pass for GlobalISel generic memory operations.
void logAllUnhandledErrors(Error E, raw_ostream &OS, Twine ErrorBanner={})
Log all errors (if any) in E to OS.
void ComputeCrossModuleImport(const ModuleSummaryIndex &Index, const DenseMap< StringRef, GVSummaryMapTy > &ModuleToDefinedGVSummaries, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, DenseMap< StringRef, FunctionImporter::ImportMapTy > &ImportLists, DenseMap< StringRef, FunctionImporter::ExportSetTy > &ExportLists)
Compute all the imports and exports for every module in the Index.
bool internalizeModule(Module &TheModule, std::function< bool(const GlobalValue &)> MustPreserveGV)
Helper function to internalize functions and variables in a Module.
std::error_code make_error_code(BitcodeError E)
const char * getHotnessName(CalleeInfo::HotnessType HT)
std::error_code EmitImportsFiles(StringRef ModulePath, StringRef OutputFilename, const std::map< std::string, GVSummaryMapTy > &ModuleToSummariesForIndex)
Emit into OutputFilename the files module ModulePath will import from.
@ Import
Import information from summary.
bool convertToDeclaration(GlobalValue &GV)
Converts value GV to declaration, or replaces with a declaration if it is an alias.
Error createStringError(std::error_code EC, char const *Fmt, const Ts &... Vals)
Create formatted StringError object.
auto map_range(ContainerTy &&C, FuncTy F)
bool renameModuleForThinLTO(Module &M, const ModuleSummaryIndex &Index, bool ClearDSOLocalOnDeclarations, SetVector< GlobalValue * > *GlobalsToImport=nullptr)
Perform in-place global value handling on the given Module for exported local functions renamed and p...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
void computeDeadSymbolsAndUpdateIndirectCalls(ModuleSummaryIndex &Index, const DenseSet< GlobalValue::GUID > &GUIDPreservedSymbols, function_ref< PrevailingType(GlobalValue::GUID)> isPrevailing)
Compute all the symbols that are "dead": i.e these that can't be reached in the graph from any of the...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
void gatherImportedSummariesForModule(StringRef ModulePath, const DenseMap< StringRef, GVSummaryMapTy > &ModuleToDefinedGVSummaries, const FunctionImporter::ImportMapTy &ImportList, std::map< std::string, GVSummaryMapTy > &ModuleToSummariesForIndex)
Compute the set of summaries needed for a ThinLTO backend compilation of ModulePath.
void updateIndirectCalls(ModuleSummaryIndex &Index)
Update call edges for indirect calls to local functions added from SamplePGO when needed.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Ref
The access may reference the value stored in memory.
std::unique_ptr< Module > getLazyIRFileModule(StringRef Filename, SMDiagnostic &Err, LLVMContext &Context, bool ShouldLazyLoadMetadata=false)
If the given file holds a bitcode image, return a Module for it which does lazy deserialization of fu...
void thinLTOInternalizeModule(Module &TheModule, const GVSummaryMapTy &DefinedGlobals)
Internalize TheModule based on the information recorded in the summaries during global summary-based ...
PrevailingType
PrevailingType enum used as a return type of callback passed to computeDeadSymbolsAndUpdateIndirectCa...
bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
Function * CloneFunction(Function *F, ValueToValueMapTy &VMap, ClonedCodeInfo *CodeInfo=nullptr)
Return a copy of the specified function and add it to that function's module.
void computeDeadSymbolsWithConstProp(ModuleSummaryIndex &Index, const DenseSet< GlobalValue::GUID > &GUIDPreservedSymbols, function_ref< PrevailingType(GlobalValue::GUID)> isPrevailing, bool ImportEnabled)
Compute dead symbols and run constant propagation in combined index after that.
Expected< std::unique_ptr< ModuleSummaryIndex > > getModuleSummaryIndexForFile(StringRef Path, bool IgnoreEmptyThinLTOIndexFile=false)
Parse the module summary index out of an IR file and return the module summary index object if found,...
void thinLTOFinalizeInModule(Module &TheModule, const GVSummaryMapTy &DefinedGlobals, bool PropagateAttrs)
Based on the information recorded in the summaries during global summary-based analysis:
Struct that holds a reference to a particular GUID in a global value summary.