LLVM 19.0.0git
LTO.cpp
Go to the documentation of this file.
1//===-LTO.cpp - LLVM Link Time Optimizer ----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements functions and classes used to support LTO.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/LTO/LTO.h"
14#include "llvm/ADT/ScopeExit.h"
15#include "llvm/ADT/SmallSet.h"
16#include "llvm/ADT/Statistic.h"
25#include "llvm/Config/llvm-config.h"
26#include "llvm/IR/AutoUpgrade.h"
28#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/Mangler.h"
32#include "llvm/IR/Metadata.h"
33#include "llvm/LTO/LTOBackend.h"
35#include "llvm/Linker/IRMover.h"
39#include "llvm/Support/Error.h"
43#include "llvm/Support/Path.h"
44#include "llvm/Support/SHA1.h"
50#include "llvm/Support/VCSRevision.h"
53#include "llvm/Transforms/IPO.h"
58
59#include <optional>
60#include <set>
61
62using namespace llvm;
63using namespace lto;
64using namespace object;
65
66#define DEBUG_TYPE "lto"
67
68static cl::opt<bool>
69 DumpThinCGSCCs("dump-thin-cg-sccs", cl::init(false), cl::Hidden,
70 cl::desc("Dump the SCCs in the ThinLTO index's callgraph"));
71
72namespace llvm {
73/// Enable global value internalization in LTO.
75 "enable-lto-internalization", cl::init(true), cl::Hidden,
76 cl::desc("Enable global value internalization in LTO"));
77
78/// Indicate we are linking with an allocator that supports hot/cold operator
79/// new interfaces.
81
82/// Enable MemProf context disambiguation for thin link.
84} // namespace llvm
85
86// Computes a unique hash for the Module considering the current list of
87// export/import and other global analysis results.
88// The hash is produced in \p Key.
90 SmallString<40> &Key, const Config &Conf, const ModuleSummaryIndex &Index,
91 StringRef ModuleID, const FunctionImporter::ImportMapTy &ImportList,
92 const FunctionImporter::ExportSetTy &ExportList,
93 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
94 const GVSummaryMapTy &DefinedGlobals,
95 const std::set<GlobalValue::GUID> &CfiFunctionDefs,
96 const std::set<GlobalValue::GUID> &CfiFunctionDecls) {
97 // Compute the unique hash for this entry.
98 // This is based on the current compiler version, the module itself, the
99 // export list, the hash for every single module in the import list, the
100 // list of ResolvedODR for the module, and the list of preserved symbols.
101 SHA1 Hasher;
102
103 // Start with the compiler revision
104 Hasher.update(LLVM_VERSION_STRING);
105#ifdef LLVM_REVISION
106 Hasher.update(LLVM_REVISION);
107#endif
108
109 // Include the parts of the LTO configuration that affect code generation.
110 auto AddString = [&](StringRef Str) {
111 Hasher.update(Str);
112 Hasher.update(ArrayRef<uint8_t>{0});
113 };
114 auto AddUnsigned = [&](unsigned I) {
115 uint8_t Data[4];
117 Hasher.update(ArrayRef<uint8_t>{Data, 4});
118 };
119 auto AddUint64 = [&](uint64_t I) {
120 uint8_t Data[8];
122 Hasher.update(ArrayRef<uint8_t>{Data, 8});
123 };
124 AddString(Conf.CPU);
125 // FIXME: Hash more of Options. For now all clients initialize Options from
126 // command-line flags (which is unsupported in production), but may set
127 // X86RelaxRelocations. The clang driver can also pass FunctionSections,
128 // DataSections and DebuggerTuning via command line flags.
129 AddUnsigned(Conf.Options.MCOptions.X86RelaxRelocations);
130 AddUnsigned(Conf.Options.FunctionSections);
131 AddUnsigned(Conf.Options.DataSections);
132 AddUnsigned((unsigned)Conf.Options.DebuggerTuning);
133 for (auto &A : Conf.MAttrs)
134 AddString(A);
135 if (Conf.RelocModel)
136 AddUnsigned(*Conf.RelocModel);
137 else
138 AddUnsigned(-1);
139 if (Conf.CodeModel)
140 AddUnsigned(*Conf.CodeModel);
141 else
142 AddUnsigned(-1);
143 for (const auto &S : Conf.MllvmArgs)
144 AddString(S);
145 AddUnsigned(static_cast<int>(Conf.CGOptLevel));
146 AddUnsigned(static_cast<int>(Conf.CGFileType));
147 AddUnsigned(Conf.OptLevel);
148 AddUnsigned(Conf.Freestanding);
149 AddString(Conf.OptPipeline);
150 AddString(Conf.AAPipeline);
151 AddString(Conf.OverrideTriple);
152 AddString(Conf.DefaultTriple);
153 AddString(Conf.DwoDir);
154
155 // Include the hash for the current module
156 auto ModHash = Index.getModuleHash(ModuleID);
157 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
158
159 std::vector<uint64_t> ExportsGUID;
160 ExportsGUID.reserve(ExportList.size());
161 for (const auto &VI : ExportList) {
162 auto GUID = VI.getGUID();
163 ExportsGUID.push_back(GUID);
164 }
165
166 // Sort the export list elements GUIDs.
167 llvm::sort(ExportsGUID);
168 for (uint64_t GUID : ExportsGUID) {
169 // The export list can impact the internalization, be conservative here
170 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&GUID, sizeof(GUID)));
171 }
172
173 // Include the hash for every module we import functions from. The set of
174 // imported symbols for each module may affect code generation and is
175 // sensitive to link order, so include that as well.
176 using ImportMapIteratorTy = FunctionImporter::ImportMapTy::const_iterator;
177 struct ImportModule {
178 ImportMapIteratorTy ModIt;
179 const ModuleSummaryIndex::ModuleInfo *ModInfo;
180
181 StringRef getIdentifier() const { return ModIt->getFirst(); }
182 const FunctionImporter::FunctionsToImportTy &getFunctions() const {
183 return ModIt->second;
184 }
185
186 const ModuleHash &getHash() const { return ModInfo->second; }
187 };
188
189 std::vector<ImportModule> ImportModulesVector;
190 ImportModulesVector.reserve(ImportList.size());
191
192 for (ImportMapIteratorTy It = ImportList.begin(); It != ImportList.end();
193 ++It) {
194 ImportModulesVector.push_back({It, Index.getModule(It->getFirst())});
195 }
196 // Order using module hash, to be both independent of module name and
197 // module order.
198 llvm::sort(ImportModulesVector,
199 [](const ImportModule &Lhs, const ImportModule &Rhs) -> bool {
200 return Lhs.getHash() < Rhs.getHash();
201 });
202 for (const ImportModule &Entry : ImportModulesVector) {
203 auto ModHash = Entry.getHash();
204 Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
205
206 AddUint64(Entry.getFunctions().size());
207 for (auto &Fn : Entry.getFunctions())
208 AddUint64(Fn);
209 }
210
211 // Include the hash for the resolved ODR.
212 for (auto &Entry : ResolvedODR) {
213 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.first,
214 sizeof(GlobalValue::GUID)));
215 Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.second,
217 }
218
219 // Members of CfiFunctionDefs and CfiFunctionDecls that are referenced or
220 // defined in this module.
221 std::set<GlobalValue::GUID> UsedCfiDefs;
222 std::set<GlobalValue::GUID> UsedCfiDecls;
223
224 // Typeids used in this module.
225 std::set<GlobalValue::GUID> UsedTypeIds;
226
227 auto AddUsedCfiGlobal = [&](GlobalValue::GUID ValueGUID) {
228 if (CfiFunctionDefs.count(ValueGUID))
229 UsedCfiDefs.insert(ValueGUID);
230 if (CfiFunctionDecls.count(ValueGUID))
231 UsedCfiDecls.insert(ValueGUID);
232 };
233
234 auto AddUsedThings = [&](GlobalValueSummary *GS) {
235 if (!GS) return;
236 AddUnsigned(GS->getVisibility());
237 AddUnsigned(GS->isLive());
238 AddUnsigned(GS->canAutoHide());
239 for (const ValueInfo &VI : GS->refs()) {
240 AddUnsigned(VI.isDSOLocal(Index.withDSOLocalPropagation()));
241 AddUsedCfiGlobal(VI.getGUID());
242 }
243 if (auto *GVS = dyn_cast<GlobalVarSummary>(GS)) {
244 AddUnsigned(GVS->maybeReadOnly());
245 AddUnsigned(GVS->maybeWriteOnly());
246 }
247 if (auto *FS = dyn_cast<FunctionSummary>(GS)) {
248 for (auto &TT : FS->type_tests())
249 UsedTypeIds.insert(TT);
250 for (auto &TT : FS->type_test_assume_vcalls())
251 UsedTypeIds.insert(TT.GUID);
252 for (auto &TT : FS->type_checked_load_vcalls())
253 UsedTypeIds.insert(TT.GUID);
254 for (auto &TT : FS->type_test_assume_const_vcalls())
255 UsedTypeIds.insert(TT.VFunc.GUID);
256 for (auto &TT : FS->type_checked_load_const_vcalls())
257 UsedTypeIds.insert(TT.VFunc.GUID);
258 for (auto &ET : FS->calls()) {
259 AddUnsigned(ET.first.isDSOLocal(Index.withDSOLocalPropagation()));
260 AddUsedCfiGlobal(ET.first.getGUID());
261 }
262 }
263 };
264
265 // Include the hash for the linkage type to reflect internalization and weak
266 // resolution, and collect any used type identifier resolutions.
267 for (auto &GS : DefinedGlobals) {
268 GlobalValue::LinkageTypes Linkage = GS.second->linkage();
269 Hasher.update(
270 ArrayRef<uint8_t>((const uint8_t *)&Linkage, sizeof(Linkage)));
271 AddUsedCfiGlobal(GS.first);
272 AddUsedThings(GS.second);
273 }
274
275 // Imported functions may introduce new uses of type identifier resolutions,
276 // so we need to collect their used resolutions as well.
277 for (const ImportModule &ImpM : ImportModulesVector)
278 for (auto &ImpF : ImpM.getFunctions()) {
280 Index.findSummaryInModule(ImpF, ImpM.getIdentifier());
281 AddUsedThings(S);
282 // If this is an alias, we also care about any types/etc. that the aliasee
283 // may reference.
284 if (auto *AS = dyn_cast_or_null<AliasSummary>(S))
285 AddUsedThings(AS->getBaseObject());
286 }
287
288 auto AddTypeIdSummary = [&](StringRef TId, const TypeIdSummary &S) {
289 AddString(TId);
290
291 AddUnsigned(S.TTRes.TheKind);
292 AddUnsigned(S.TTRes.SizeM1BitWidth);
293
294 AddUint64(S.TTRes.AlignLog2);
295 AddUint64(S.TTRes.SizeM1);
296 AddUint64(S.TTRes.BitMask);
297 AddUint64(S.TTRes.InlineBits);
298
299 AddUint64(S.WPDRes.size());
300 for (auto &WPD : S.WPDRes) {
301 AddUnsigned(WPD.first);
302 AddUnsigned(WPD.second.TheKind);
303 AddString(WPD.second.SingleImplName);
304
305 AddUint64(WPD.second.ResByArg.size());
306 for (auto &ByArg : WPD.second.ResByArg) {
307 AddUint64(ByArg.first.size());
308 for (uint64_t Arg : ByArg.first)
309 AddUint64(Arg);
310 AddUnsigned(ByArg.second.TheKind);
311 AddUint64(ByArg.second.Info);
312 AddUnsigned(ByArg.second.Byte);
313 AddUnsigned(ByArg.second.Bit);
314 }
315 }
316 };
317
318 // Include the hash for all type identifiers used by this module.
319 for (GlobalValue::GUID TId : UsedTypeIds) {
320 auto TidIter = Index.typeIds().equal_range(TId);
321 for (auto It = TidIter.first; It != TidIter.second; ++It)
322 AddTypeIdSummary(It->second.first, It->second.second);
323 }
324
325 AddUnsigned(UsedCfiDefs.size());
326 for (auto &V : UsedCfiDefs)
327 AddUint64(V);
328
329 AddUnsigned(UsedCfiDecls.size());
330 for (auto &V : UsedCfiDecls)
331 AddUint64(V);
332
333 if (!Conf.SampleProfile.empty()) {
334 auto FileOrErr = MemoryBuffer::getFile(Conf.SampleProfile);
335 if (FileOrErr) {
336 Hasher.update(FileOrErr.get()->getBuffer());
337
338 if (!Conf.ProfileRemapping.empty()) {
339 FileOrErr = MemoryBuffer::getFile(Conf.ProfileRemapping);
340 if (FileOrErr)
341 Hasher.update(FileOrErr.get()->getBuffer());
342 }
343 }
344 }
345
346 Key = toHex(Hasher.result());
347}
348
350 const Config &C, ValueInfo VI,
351 DenseSet<GlobalValueSummary *> &GlobalInvolvedWithAlias,
353 isPrevailing,
355 recordNewLinkage,
356 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
358 C.VisibilityScheme == Config::ELF ? VI.getELFVisibility()
360 for (auto &S : VI.getSummaryList()) {
361 GlobalValue::LinkageTypes OriginalLinkage = S->linkage();
362 // Ignore local and appending linkage values since the linker
363 // doesn't resolve them.
364 if (GlobalValue::isLocalLinkage(OriginalLinkage) ||
366 continue;
367 // We need to emit only one of these. The prevailing module will keep it,
368 // but turned into a weak, while the others will drop it when possible.
369 // This is both a compile-time optimization and a correctness
370 // transformation. This is necessary for correctness when we have exported
371 // a reference - we need to convert the linkonce to weak to
372 // ensure a copy is kept to satisfy the exported reference.
373 // FIXME: We may want to split the compile time and correctness
374 // aspects into separate routines.
375 if (isPrevailing(VI.getGUID(), S.get())) {
376 if (GlobalValue::isLinkOnceLinkage(OriginalLinkage)) {
377 S->setLinkage(GlobalValue::getWeakLinkage(
378 GlobalValue::isLinkOnceODRLinkage(OriginalLinkage)));
379 // The kept copy is eligible for auto-hiding (hidden visibility) if all
380 // copies were (i.e. they were all linkonce_odr global unnamed addr).
381 // If any copy is not (e.g. it was originally weak_odr), then the symbol
382 // must remain externally available (e.g. a weak_odr from an explicitly
383 // instantiated template). Additionally, if it is in the
384 // GUIDPreservedSymbols set, that means that it is visibile outside
385 // the summary (e.g. in a native object or a bitcode file without
386 // summary), and in that case we cannot hide it as it isn't possible to
387 // check all copies.
388 S->setCanAutoHide(VI.canAutoHide() &&
389 !GUIDPreservedSymbols.count(VI.getGUID()));
390 }
391 if (C.VisibilityScheme == Config::FromPrevailing)
392 Visibility = S->getVisibility();
393 }
394 // Alias and aliasee can't be turned into available_externally.
395 else if (!isa<AliasSummary>(S.get()) &&
396 !GlobalInvolvedWithAlias.count(S.get()))
398
399 // For ELF, set visibility to the computed visibility from summaries. We
400 // don't track visibility from declarations so this may be more relaxed than
401 // the most constraining one.
402 if (C.VisibilityScheme == Config::ELF)
403 S->setVisibility(Visibility);
404
405 if (S->linkage() != OriginalLinkage)
406 recordNewLinkage(S->modulePath(), VI.getGUID(), S->linkage());
407 }
408
409 if (C.VisibilityScheme == Config::FromPrevailing) {
410 for (auto &S : VI.getSummaryList()) {
411 GlobalValue::LinkageTypes OriginalLinkage = S->linkage();
412 if (GlobalValue::isLocalLinkage(OriginalLinkage) ||
414 continue;
415 S->setVisibility(Visibility);
416 }
417 }
418}
419
420/// Resolve linkage for prevailing symbols in the \p Index.
421//
422// We'd like to drop these functions if they are no longer referenced in the
423// current module. However there is a chance that another module is still
424// referencing them because of the import. We make sure we always emit at least
425// one copy.
429 isPrevailing,
431 recordNewLinkage,
432 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
433 // We won't optimize the globals that are referenced by an alias for now
434 // Ideally we should turn the alias into a global and duplicate the definition
435 // when needed.
436 DenseSet<GlobalValueSummary *> GlobalInvolvedWithAlias;
437 for (auto &I : Index)
438 for (auto &S : I.second.SummaryList)
439 if (auto AS = dyn_cast<AliasSummary>(S.get()))
440 GlobalInvolvedWithAlias.insert(&AS->getAliasee());
441
442 for (auto &I : Index)
443 thinLTOResolvePrevailingGUID(C, Index.getValueInfo(I),
444 GlobalInvolvedWithAlias, isPrevailing,
445 recordNewLinkage, GUIDPreservedSymbols);
446}
447
449 ValueInfo VI, function_ref<bool(StringRef, ValueInfo)> isExported,
451 isPrevailing) {
452 auto ExternallyVisibleCopies =
453 llvm::count_if(VI.getSummaryList(),
454 [](const std::unique_ptr<GlobalValueSummary> &Summary) {
455 return !GlobalValue::isLocalLinkage(Summary->linkage());
456 });
457
458 for (auto &S : VI.getSummaryList()) {
459 // First see if we need to promote an internal value because it is not
460 // exported.
461 if (isExported(S->modulePath(), VI)) {
462 if (GlobalValue::isLocalLinkage(S->linkage()))
463 S->setLinkage(GlobalValue::ExternalLinkage);
464 continue;
465 }
466
467 // Otherwise, see if we can internalize.
469 continue;
470
471 // Non-exported values with external linkage can be internalized.
472 if (GlobalValue::isExternalLinkage(S->linkage())) {
473 S->setLinkage(GlobalValue::InternalLinkage);
474 continue;
475 }
476
477 // Non-exported function and variable definitions with a weak-for-linker
478 // linkage can be internalized in certain cases. The minimum legality
479 // requirements would be that they are not address taken to ensure that we
480 // don't break pointer equality checks, and that variables are either read-
481 // or write-only. For functions, this is the case if either all copies are
482 // [local_]unnamed_addr, or we can propagate reference edge attributes
483 // (which is how this is guaranteed for variables, when analyzing whether
484 // they are read or write-only).
485 //
486 // However, we only get to this code for weak-for-linkage values in one of
487 // two cases:
488 // 1) The prevailing copy is not in IR (it is in native code).
489 // 2) The prevailing copy in IR is not exported from its module.
490 // Additionally, at least for the new LTO API, case 2 will only happen if
491 // there is exactly one definition of the value (i.e. in exactly one
492 // module), as duplicate defs are result in the value being marked exported.
493 // Likely, users of the legacy LTO API are similar, however, currently there
494 // are llvm-lto based tests of the legacy LTO API that do not mark
495 // duplicate linkonce_odr copies as exported via the tool, so we need
496 // to handle that case below by checking the number of copies.
497 //
498 // Generally, we only want to internalize a weak-for-linker value in case
499 // 2, because in case 1 we cannot see how the value is used to know if it
500 // is read or write-only. We also don't want to bloat the binary with
501 // multiple internalized copies of non-prevailing linkonce/weak functions.
502 // Note if we don't internalize, we will convert non-prevailing copies to
503 // available_externally anyway, so that we drop them after inlining. The
504 // only reason to internalize such a function is if we indeed have a single
505 // copy, because internalizing it won't increase binary size, and enables
506 // use of inliner heuristics that are more aggressive in the face of a
507 // single call to a static (local). For variables, internalizing a read or
508 // write only variable can enable more aggressive optimization. However, we
509 // already perform this elsewhere in the ThinLTO backend handling for
510 // read or write-only variables (processGlobalForThinLTO).
511 //
512 // Therefore, only internalize linkonce/weak if there is a single copy, that
513 // is prevailing in this IR module. We can do so aggressively, without
514 // requiring the address to be insignificant, or that a variable be read or
515 // write-only.
516 if (!GlobalValue::isWeakForLinker(S->linkage()) ||
518 continue;
519
520 if (isPrevailing(VI.getGUID(), S.get()) && ExternallyVisibleCopies == 1)
521 S->setLinkage(GlobalValue::InternalLinkage);
522 }
523}
524
525// Update the linkages in the given \p Index to mark exported values
526// as external and non-exported values as internal.
529 function_ref<bool(StringRef, ValueInfo)> isExported,
531 isPrevailing) {
532 for (auto &I : Index)
533 thinLTOInternalizeAndPromoteGUID(Index.getValueInfo(I), isExported,
534 isPrevailing);
535}
536
537// Requires a destructor for std::vector<InputModule>.
538InputFile::~InputFile() = default;
539
541 std::unique_ptr<InputFile> File(new InputFile);
542
543 Expected<IRSymtabFile> FOrErr = readIRSymtab(Object);
544 if (!FOrErr)
545 return FOrErr.takeError();
546
547 File->TargetTriple = FOrErr->TheReader.getTargetTriple();
548 File->SourceFileName = FOrErr->TheReader.getSourceFileName();
549 File->COFFLinkerOpts = FOrErr->TheReader.getCOFFLinkerOpts();
550 File->DependentLibraries = FOrErr->TheReader.getDependentLibraries();
551 File->ComdatTable = FOrErr->TheReader.getComdatTable();
552
553 for (unsigned I = 0; I != FOrErr->Mods.size(); ++I) {
554 size_t Begin = File->Symbols.size();
555 for (const irsymtab::Reader::SymbolRef &Sym :
556 FOrErr->TheReader.module_symbols(I))
557 // Skip symbols that are irrelevant to LTO. Note that this condition needs
558 // to match the one in Skip() in LTO::addRegularLTO().
559 if (Sym.isGlobal() && !Sym.isFormatSpecific())
560 File->Symbols.push_back(Sym);
561 File->ModuleSymIndices.push_back({Begin, File->Symbols.size()});
562 }
563
564 File->Mods = FOrErr->Mods;
565 File->Strtab = std::move(FOrErr->Strtab);
566 return std::move(File);
567}
568
570 return Mods[0].getModuleIdentifier();
571}
572
574 assert(Mods.size() == 1 && "Expect only one bitcode module");
575 return Mods[0];
576}
577
578LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel,
579 const Config &Conf)
580 : ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel),
581 Ctx(Conf), CombinedModule(std::make_unique<Module>("ld-temp.o", Ctx)),
582 Mover(std::make_unique<IRMover>(*CombinedModule)) {}
583
584LTO::ThinLTOState::ThinLTOState(ThinBackend Backend)
585 : Backend(Backend), CombinedIndex(/*HaveGVs*/ false) {
586 if (!Backend)
587 this->Backend =
589}
590
592 unsigned ParallelCodeGenParallelismLevel, LTOKind LTOMode)
593 : Conf(std::move(Conf)),
594 RegularLTO(ParallelCodeGenParallelismLevel, this->Conf),
595 ThinLTO(std::move(Backend)),
596 GlobalResolutions(std::make_optional<StringMap<GlobalResolution>>()),
597 LTOMode(LTOMode) {}
598
599// Requires a destructor for MapVector<BitcodeModule>.
600LTO::~LTO() = default;
601
602// Add the symbols in the given module to the GlobalResolutions map, and resolve
603// their partitions.
604void LTO::addModuleToGlobalRes(ArrayRef<InputFile::Symbol> Syms,
606 unsigned Partition, bool InSummary) {
607 auto *ResI = Res.begin();
608 auto *ResE = Res.end();
609 (void)ResE;
610 const Triple TT(RegularLTO.CombinedModule->getTargetTriple());
611 for (const InputFile::Symbol &Sym : Syms) {
612 assert(ResI != ResE);
613 SymbolResolution Res = *ResI++;
614
615 auto &GlobalRes = (*GlobalResolutions)[Sym.getName()];
616 GlobalRes.UnnamedAddr &= Sym.isUnnamedAddr();
617 if (Res.Prevailing) {
618 assert(!GlobalRes.Prevailing &&
619 "Multiple prevailing defs are not allowed");
620 GlobalRes.Prevailing = true;
621 GlobalRes.IRName = std::string(Sym.getIRName());
622 } else if (!GlobalRes.Prevailing && GlobalRes.IRName.empty()) {
623 // Sometimes it can be two copies of symbol in a module and prevailing
624 // symbol can have no IR name. That might happen if symbol is defined in
625 // module level inline asm block. In case we have multiple modules with
626 // the same symbol we want to use IR name of the prevailing symbol.
627 // Otherwise, if we haven't seen a prevailing symbol, set the name so that
628 // we can later use it to check if there is any prevailing copy in IR.
629 GlobalRes.IRName = std::string(Sym.getIRName());
630 }
631
632 // In rare occasion, the symbol used to initialize GlobalRes has a different
633 // IRName from the inspected Symbol. This can happen on macOS + iOS, when a
634 // symbol is referenced through its mangled name, say @"\01_symbol" while
635 // the IRName is @symbol (the prefix underscore comes from MachO mangling).
636 // In that case, we have the same actual Symbol that can get two different
637 // GUID, leading to some invalid internalization. Workaround this by marking
638 // the GlobalRes external.
639
640 // FIXME: instead of this check, it would be desirable to compute GUIDs
641 // based on mangled name, but this requires an access to the Target Triple
642 // and would be relatively invasive on the codebase.
643 if (GlobalRes.IRName != Sym.getIRName()) {
644 GlobalRes.Partition = GlobalResolution::External;
645 GlobalRes.VisibleOutsideSummary = true;
646 }
647
648 // Set the partition to external if we know it is re-defined by the linker
649 // with -defsym or -wrap options, used elsewhere, e.g. it is visible to a
650 // regular object, is referenced from llvm.compiler.used/llvm.used, or was
651 // already recorded as being referenced from a different partition.
652 if (Res.LinkerRedefined || Res.VisibleToRegularObj || Sym.isUsed() ||
653 (GlobalRes.Partition != GlobalResolution::Unknown &&
654 GlobalRes.Partition != Partition)) {
655 GlobalRes.Partition = GlobalResolution::External;
656 } else
657 // First recorded reference, save the current partition.
658 GlobalRes.Partition = Partition;
659
660 // Flag as visible outside of summary if visible from a regular object or
661 // from a module that does not have a summary.
662 GlobalRes.VisibleOutsideSummary |=
663 (Res.VisibleToRegularObj || Sym.isUsed() || !InSummary);
664
665 GlobalRes.ExportDynamic |= Res.ExportDynamic;
666 }
667}
668
671 StringRef Path = Input->getName();
672 OS << Path << '\n';
673 auto ResI = Res.begin();
674 for (const InputFile::Symbol &Sym : Input->symbols()) {
675 assert(ResI != Res.end());
676 SymbolResolution Res = *ResI++;
677
678 OS << "-r=" << Path << ',' << Sym.getName() << ',';
679 if (Res.Prevailing)
680 OS << 'p';
682 OS << 'l';
683 if (Res.VisibleToRegularObj)
684 OS << 'x';
685 if (Res.LinkerRedefined)
686 OS << 'r';
687 OS << '\n';
688 }
689 OS.flush();
690 assert(ResI == Res.end());
691}
692
693Error LTO::add(std::unique_ptr<InputFile> Input,
695 assert(!CalledGetMaxTasks);
696
697 if (Conf.ResolutionFile)
698 writeToResolutionFile(*Conf.ResolutionFile, Input.get(), Res);
699
700 if (RegularLTO.CombinedModule->getTargetTriple().empty()) {
701 RegularLTO.CombinedModule->setTargetTriple(Input->getTargetTriple());
702 if (Triple(Input->getTargetTriple()).isOSBinFormatELF())
704 }
705
706 const SymbolResolution *ResI = Res.begin();
707 for (unsigned I = 0; I != Input->Mods.size(); ++I)
708 if (Error Err = addModule(*Input, I, ResI, Res.end()))
709 return Err;
710
711 assert(ResI == Res.end());
712 return Error::success();
713}
714
715Error LTO::addModule(InputFile &Input, unsigned ModI,
716 const SymbolResolution *&ResI,
717 const SymbolResolution *ResE) {
718 Expected<BitcodeLTOInfo> LTOInfo = Input.Mods[ModI].getLTOInfo();
719 if (!LTOInfo)
720 return LTOInfo.takeError();
721
722 if (EnableSplitLTOUnit) {
723 // If only some modules were split, flag this in the index so that
724 // we can skip or error on optimizations that need consistently split
725 // modules (whole program devirt and lower type tests).
726 if (*EnableSplitLTOUnit != LTOInfo->EnableSplitLTOUnit)
727 ThinLTO.CombinedIndex.setPartiallySplitLTOUnits();
728 } else
729 EnableSplitLTOUnit = LTOInfo->EnableSplitLTOUnit;
730
731 BitcodeModule BM = Input.Mods[ModI];
732
733 if ((LTOMode == LTOK_UnifiedRegular || LTOMode == LTOK_UnifiedThin) &&
734 !LTOInfo->UnifiedLTO)
735 return make_error<StringError>(
736 "unified LTO compilation must use "
737 "compatible bitcode modules (use -funified-lto)",
739
740 if (LTOInfo->UnifiedLTO && LTOMode == LTOK_Default)
741 LTOMode = LTOK_UnifiedThin;
742
743 bool IsThinLTO = LTOInfo->IsThinLTO && (LTOMode != LTOK_UnifiedRegular);
744
745 auto ModSyms = Input.module_symbols(ModI);
746 addModuleToGlobalRes(ModSyms, {ResI, ResE},
747 IsThinLTO ? ThinLTO.ModuleMap.size() + 1 : 0,
748 LTOInfo->HasSummary);
749
750 if (IsThinLTO)
751 return addThinLTO(BM, ModSyms, ResI, ResE);
752
753 RegularLTO.EmptyCombinedModule = false;
755 addRegularLTO(BM, ModSyms, ResI, ResE);
756 if (!ModOrErr)
757 return ModOrErr.takeError();
758
759 if (!LTOInfo->HasSummary)
760 return linkRegularLTO(std::move(*ModOrErr), /*LivenessFromIndex=*/false);
761
762 // Regular LTO module summaries are added to a dummy module that represents
763 // the combined regular LTO module.
764 if (Error Err = BM.readSummary(ThinLTO.CombinedIndex, ""))
765 return Err;
766 RegularLTO.ModsWithSummaries.push_back(std::move(*ModOrErr));
767 return Error::success();
768}
769
770// Checks whether the given global value is in a non-prevailing comdat
771// (comdat containing values the linker indicated were not prevailing,
772// which we then dropped to available_externally), and if so, removes
773// it from the comdat. This is called for all global values to ensure the
774// comdat is empty rather than leaving an incomplete comdat. It is needed for
775// regular LTO modules, in case we are in a mixed-LTO mode (both regular
776// and thin LTO modules) compilation. Since the regular LTO module will be
777// linked first in the final native link, we want to make sure the linker
778// doesn't select any of these incomplete comdats that would be left
779// in the regular LTO module without this cleanup.
780static void
782 std::set<const Comdat *> &NonPrevailingComdats) {
783 Comdat *C = GV.getComdat();
784 if (!C)
785 return;
786
787 if (!NonPrevailingComdats.count(C))
788 return;
789
790 // Additionally need to drop all global values from the comdat to
791 // available_externally, to satisfy the COMDAT requirement that all members
792 // are discarded as a unit. The non-local linkage global values avoid
793 // duplicate definition linker errors.
795
796 if (auto GO = dyn_cast<GlobalObject>(&GV))
797 GO->setComdat(nullptr);
798}
799
800// Add a regular LTO object to the link.
801// The resulting module needs to be linked into the combined LTO module with
802// linkRegularLTO.
804LTO::addRegularLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
805 const SymbolResolution *&ResI,
806 const SymbolResolution *ResE) {
807 RegularLTOState::AddedModule Mod;
809 BM.getLazyModule(RegularLTO.Ctx, /*ShouldLazyLoadMetadata*/ true,
810 /*IsImporting*/ false);
811 if (!MOrErr)
812 return MOrErr.takeError();
813 Module &M = **MOrErr;
814 Mod.M = std::move(*MOrErr);
815
816 if (Error Err = M.materializeMetadata())
817 return std::move(Err);
818
819 // If cfi.functions is present and we are in regular LTO mode, LowerTypeTests
820 // will rename local functions in the merged module as "<function name>.1".
821 // This causes linking errors, since other parts of the module expect the
822 // original function name.
823 if (LTOMode == LTOK_UnifiedRegular)
824 if (NamedMDNode *CfiFunctionsMD = M.getNamedMetadata("cfi.functions"))
825 M.eraseNamedMetadata(CfiFunctionsMD);
826
828
829 ModuleSymbolTable SymTab;
830 SymTab.addModule(&M);
831
832 for (GlobalVariable &GV : M.globals())
833 if (GV.hasAppendingLinkage())
834 Mod.Keep.push_back(&GV);
835
836 DenseSet<GlobalObject *> AliasedGlobals;
837 for (auto &GA : M.aliases())
838 if (GlobalObject *GO = GA.getAliaseeObject())
839 AliasedGlobals.insert(GO);
840
841 // In this function we need IR GlobalValues matching the symbols in Syms
842 // (which is not backed by a module), so we need to enumerate them in the same
843 // order. The symbol enumeration order of a ModuleSymbolTable intentionally
844 // matches the order of an irsymtab, but when we read the irsymtab in
845 // InputFile::create we omit some symbols that are irrelevant to LTO. The
846 // Skip() function skips the same symbols from the module as InputFile does
847 // from the symbol table.
848 auto MsymI = SymTab.symbols().begin(), MsymE = SymTab.symbols().end();
849 auto Skip = [&]() {
850 while (MsymI != MsymE) {
851 auto Flags = SymTab.getSymbolFlags(*MsymI);
852 if ((Flags & object::BasicSymbolRef::SF_Global) &&
854 return;
855 ++MsymI;
856 }
857 };
858 Skip();
859
860 std::set<const Comdat *> NonPrevailingComdats;
861 SmallSet<StringRef, 2> NonPrevailingAsmSymbols;
862 for (const InputFile::Symbol &Sym : Syms) {
863 assert(ResI != ResE);
864 SymbolResolution Res = *ResI++;
865
866 assert(MsymI != MsymE);
867 ModuleSymbolTable::Symbol Msym = *MsymI++;
868 Skip();
869
870 if (GlobalValue *GV = dyn_cast_if_present<GlobalValue *>(Msym)) {
871 if (Res.Prevailing) {
872 if (Sym.isUndefined())
873 continue;
874 Mod.Keep.push_back(GV);
875 // For symbols re-defined with linker -wrap and -defsym options,
876 // set the linkage to weak to inhibit IPO. The linkage will be
877 // restored by the linker.
878 if (Res.LinkerRedefined)
879 GV->setLinkage(GlobalValue::WeakAnyLinkage);
880
881 GlobalValue::LinkageTypes OriginalLinkage = GV->getLinkage();
882 if (GlobalValue::isLinkOnceLinkage(OriginalLinkage))
883 GV->setLinkage(GlobalValue::getWeakLinkage(
884 GlobalValue::isLinkOnceODRLinkage(OriginalLinkage)));
885 } else if (isa<GlobalObject>(GV) &&
886 (GV->hasLinkOnceODRLinkage() || GV->hasWeakODRLinkage() ||
887 GV->hasAvailableExternallyLinkage()) &&
888 !AliasedGlobals.count(cast<GlobalObject>(GV))) {
889 // Any of the above three types of linkage indicates that the
890 // chosen prevailing symbol will have the same semantics as this copy of
891 // the symbol, so we may be able to link it with available_externally
892 // linkage. We will decide later whether to do that when we link this
893 // module (in linkRegularLTO), based on whether it is undefined.
894 Mod.Keep.push_back(GV);
896 if (GV->hasComdat())
897 NonPrevailingComdats.insert(GV->getComdat());
898 cast<GlobalObject>(GV)->setComdat(nullptr);
899 }
900
901 // Set the 'local' flag based on the linker resolution for this symbol.
903 GV->setDSOLocal(true);
904 if (GV->hasDLLImportStorageClass())
905 GV->setDLLStorageClass(GlobalValue::DLLStorageClassTypes::
906 DefaultStorageClass);
907 }
908 } else if (auto *AS =
909 dyn_cast_if_present<ModuleSymbolTable::AsmSymbol *>(Msym)) {
910 // Collect non-prevailing symbols.
911 if (!Res.Prevailing)
912 NonPrevailingAsmSymbols.insert(AS->first);
913 } else {
914 llvm_unreachable("unknown symbol type");
915 }
916
917 // Common resolution: collect the maximum size/alignment over all commons.
918 // We also record if we see an instance of a common as prevailing, so that
919 // if none is prevailing we can ignore it later.
920 if (Sym.isCommon()) {
921 // FIXME: We should figure out what to do about commons defined by asm.
922 // For now they aren't reported correctly by ModuleSymbolTable.
923 auto &CommonRes = RegularLTO.Commons[std::string(Sym.getIRName())];
924 CommonRes.Size = std::max(CommonRes.Size, Sym.getCommonSize());
925 if (uint32_t SymAlignValue = Sym.getCommonAlignment()) {
926 CommonRes.Alignment =
927 std::max(Align(SymAlignValue), CommonRes.Alignment);
928 }
929 CommonRes.Prevailing |= Res.Prevailing;
930 }
931 }
932
933 if (!M.getComdatSymbolTable().empty())
934 for (GlobalValue &GV : M.global_values())
935 handleNonPrevailingComdat(GV, NonPrevailingComdats);
936
937 // Prepend ".lto_discard <sym>, <sym>*" directive to each module inline asm
938 // block.
939 if (!M.getModuleInlineAsm().empty()) {
940 std::string NewIA = ".lto_discard";
941 if (!NonPrevailingAsmSymbols.empty()) {
942 // Don't dicard a symbol if there is a live .symver for it.
944 M, [&](StringRef Name, StringRef Alias) {
945 if (!NonPrevailingAsmSymbols.count(Alias))
946 NonPrevailingAsmSymbols.erase(Name);
947 });
948 NewIA += " " + llvm::join(NonPrevailingAsmSymbols, ", ");
949 }
950 NewIA += "\n";
951 M.setModuleInlineAsm(NewIA + M.getModuleInlineAsm());
952 }
953
954 assert(MsymI == MsymE);
955 return std::move(Mod);
956}
957
958Error LTO::linkRegularLTO(RegularLTOState::AddedModule Mod,
959 bool LivenessFromIndex) {
960 std::vector<GlobalValue *> Keep;
961 for (GlobalValue *GV : Mod.Keep) {
962 if (LivenessFromIndex && !ThinLTO.CombinedIndex.isGUIDLive(GV->getGUID())) {
963 if (Function *F = dyn_cast<Function>(GV)) {
964 if (DiagnosticOutputFile) {
965 if (Error Err = F->materialize())
966 return Err;
967 OptimizationRemarkEmitter ORE(F, nullptr);
968 ORE.emit(OptimizationRemark(DEBUG_TYPE, "deadfunction", F)
969 << ore::NV("Function", F)
970 << " not added to the combined module ");
971 }
972 }
973 continue;
974 }
975
977 Keep.push_back(GV);
978 continue;
979 }
980
981 // Only link available_externally definitions if we don't already have a
982 // definition.
983 GlobalValue *CombinedGV =
984 RegularLTO.CombinedModule->getNamedValue(GV->getName());
985 if (CombinedGV && !CombinedGV->isDeclaration())
986 continue;
987
988 Keep.push_back(GV);
989 }
990
991 return RegularLTO.Mover->move(std::move(Mod.M), Keep, nullptr,
992 /* IsPerformingImport */ false);
993}
994
995// Add a ThinLTO module to the link.
996Error LTO::addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
997 const SymbolResolution *&ResI,
998 const SymbolResolution *ResE) {
999 const SymbolResolution *ResITmp = ResI;
1000 for (const InputFile::Symbol &Sym : Syms) {
1001 assert(ResITmp != ResE);
1002 SymbolResolution Res = *ResITmp++;
1003
1004 if (!Sym.getIRName().empty()) {
1006 Sym.getIRName(), GlobalValue::ExternalLinkage, ""));
1007 if (Res.Prevailing)
1008 ThinLTO.PrevailingModuleForGUID[GUID] = BM.getModuleIdentifier();
1009 }
1010 }
1011
1012 if (Error Err =
1013 BM.readSummary(ThinLTO.CombinedIndex, BM.getModuleIdentifier(),
1014 [&](GlobalValue::GUID GUID) {
1015 return ThinLTO.PrevailingModuleForGUID[GUID] ==
1016 BM.getModuleIdentifier();
1017 }))
1018 return Err;
1019 LLVM_DEBUG(dbgs() << "Module " << BM.getModuleIdentifier() << "\n");
1020
1021 for (const InputFile::Symbol &Sym : Syms) {
1022 assert(ResI != ResE);
1023 SymbolResolution Res = *ResI++;
1024
1025 if (!Sym.getIRName().empty()) {
1027 Sym.getIRName(), GlobalValue::ExternalLinkage, ""));
1028 if (Res.Prevailing) {
1029 assert(ThinLTO.PrevailingModuleForGUID[GUID] ==
1030 BM.getModuleIdentifier());
1031
1032 // For linker redefined symbols (via --wrap or --defsym) we want to
1033 // switch the linkage to `weak` to prevent IPOs from happening.
1034 // Find the summary in the module for this very GV and record the new
1035 // linkage so that we can switch it when we import the GV.
1036 if (Res.LinkerRedefined)
1037 if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(
1038 GUID, BM.getModuleIdentifier()))
1039 S->setLinkage(GlobalValue::WeakAnyLinkage);
1040 }
1041
1042 // If the linker resolved the symbol to a local definition then mark it
1043 // as local in the summary for the module we are adding.
1045 if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(
1046 GUID, BM.getModuleIdentifier())) {
1047 S->setDSOLocal(true);
1048 }
1049 }
1050 }
1051 }
1052
1053 if (!ThinLTO.ModuleMap.insert({BM.getModuleIdentifier(), BM}).second)
1054 return make_error<StringError>(
1055 "Expected at most one ThinLTO module per bitcode file",
1057
1058 if (!Conf.ThinLTOModulesToCompile.empty()) {
1059 if (!ThinLTO.ModulesToCompile)
1060 ThinLTO.ModulesToCompile = ModuleMapType();
1061 // This is a fuzzy name matching where only modules with name containing the
1062 // specified switch values are going to be compiled.
1063 for (const std::string &Name : Conf.ThinLTOModulesToCompile) {
1064 if (BM.getModuleIdentifier().contains(Name)) {
1065 ThinLTO.ModulesToCompile->insert({BM.getModuleIdentifier(), BM});
1066 llvm::errs() << "[ThinLTO] Selecting " << BM.getModuleIdentifier()
1067 << " to compile\n";
1068 }
1069 }
1070 }
1071
1072 return Error::success();
1073}
1074
1075unsigned LTO::getMaxTasks() const {
1076 CalledGetMaxTasks = true;
1077 auto ModuleCount = ThinLTO.ModulesToCompile ? ThinLTO.ModulesToCompile->size()
1078 : ThinLTO.ModuleMap.size();
1079 return RegularLTO.ParallelCodeGenParallelismLevel + ModuleCount;
1080}
1081
1082// If only some of the modules were split, we cannot correctly handle
1083// code that contains type tests or type checked loads.
1084Error LTO::checkPartiallySplit() {
1085 if (!ThinLTO.CombinedIndex.partiallySplitLTOUnits())
1086 return Error::success();
1087
1088 Function *TypeTestFunc = RegularLTO.CombinedModule->getFunction(
1089 Intrinsic::getName(Intrinsic::type_test));
1090 Function *TypeCheckedLoadFunc = RegularLTO.CombinedModule->getFunction(
1091 Intrinsic::getName(Intrinsic::type_checked_load));
1092 Function *TypeCheckedLoadRelativeFunc =
1093 RegularLTO.CombinedModule->getFunction(
1094 Intrinsic::getName(Intrinsic::type_checked_load_relative));
1095
1096 // First check if there are type tests / type checked loads in the
1097 // merged regular LTO module IR.
1098 if ((TypeTestFunc && !TypeTestFunc->use_empty()) ||
1099 (TypeCheckedLoadFunc && !TypeCheckedLoadFunc->use_empty()) ||
1100 (TypeCheckedLoadRelativeFunc &&
1101 !TypeCheckedLoadRelativeFunc->use_empty()))
1102 return make_error<StringError>(
1103 "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",
1105
1106 // Otherwise check if there are any recorded in the combined summary from the
1107 // ThinLTO modules.
1108 for (auto &P : ThinLTO.CombinedIndex) {
1109 for (auto &S : P.second.SummaryList) {
1110 auto *FS = dyn_cast<FunctionSummary>(S.get());
1111 if (!FS)
1112 continue;
1113 if (!FS->type_test_assume_vcalls().empty() ||
1114 !FS->type_checked_load_vcalls().empty() ||
1115 !FS->type_test_assume_const_vcalls().empty() ||
1116 !FS->type_checked_load_const_vcalls().empty() ||
1117 !FS->type_tests().empty())
1118 return make_error<StringError>(
1119 "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)",
1121 }
1122 }
1123 return Error::success();
1124}
1125
1127 // Compute "dead" symbols, we don't want to import/export these!
1128 DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
1129 DenseMap<GlobalValue::GUID, PrevailingType> GUIDPrevailingResolutions;
1130 for (auto &Res : *GlobalResolutions) {
1131 // Normally resolution have IR name of symbol. We can do nothing here
1132 // otherwise. See comments in GlobalResolution struct for more details.
1133 if (Res.second.IRName.empty())
1134 continue;
1135
1137 GlobalValue::dropLLVMManglingEscape(Res.second.IRName));
1138
1139 if (Res.second.VisibleOutsideSummary && Res.second.Prevailing)
1140 GUIDPreservedSymbols.insert(GUID);
1141
1142 if (Res.second.ExportDynamic)
1143 DynamicExportSymbols.insert(GUID);
1144
1145 GUIDPrevailingResolutions[GUID] =
1147 }
1148
1149 auto isPrevailing = [&](GlobalValue::GUID G) {
1150 auto It = GUIDPrevailingResolutions.find(G);
1151 if (It == GUIDPrevailingResolutions.end())
1153 return It->second;
1154 };
1155 computeDeadSymbolsWithConstProp(ThinLTO.CombinedIndex, GUIDPreservedSymbols,
1156 isPrevailing, Conf.OptLevel > 0);
1157
1158 // Setup output file to emit statistics.
1159 auto StatsFileOrErr = setupStatsFile(Conf.StatsFile);
1160 if (!StatsFileOrErr)
1161 return StatsFileOrErr.takeError();
1162 std::unique_ptr<ToolOutputFile> StatsFile = std::move(StatsFileOrErr.get());
1163
1164 // TODO: Ideally this would be controlled automatically by detecting that we
1165 // are linking with an allocator that supports these interfaces, rather than
1166 // an internal option (which would still be needed for tests, however). For
1167 // example, if the library exported a symbol like __malloc_hot_cold the linker
1168 // could recognize that and set a flag in the lto::Config.
1170 ThinLTO.CombinedIndex.setWithSupportsHotColdNew();
1171
1172 Error Result = runRegularLTO(AddStream);
1173 if (!Result)
1174 // This will reset the GlobalResolutions optional once done with it to
1175 // reduce peak memory before importing.
1176 Result = runThinLTO(AddStream, Cache, GUIDPreservedSymbols);
1177
1178 if (StatsFile)
1179 PrintStatisticsJSON(StatsFile->os());
1180
1181 return Result;
1182}
1183
1185 const ModuleSummaryIndex &Index) {
1186 if (Index.withSupportsHotColdNew())
1187 return;
1188
1189 // The profile matcher applies hotness attributes directly for allocations,
1190 // and those will cause us to generate calls to the hot/cold interfaces
1191 // unconditionally. If supports-hot-cold-new was not enabled in the LTO
1192 // link then assume we don't want these calls (e.g. not linking with
1193 // the appropriate library, or otherwise trying to disable this behavior).
1194 for (auto &F : Mod) {
1195 for (auto &BB : F) {
1196 for (auto &I : BB) {
1197 auto *CI = dyn_cast<CallBase>(&I);
1198 if (!CI)
1199 continue;
1200 if (CI->hasFnAttr("memprof"))
1201 CI->removeFnAttr("memprof");
1202 // Strip off all memprof metadata as it is no longer needed.
1203 // Importantly, this avoids the addition of new memprof attributes
1204 // after inlining propagation.
1205 // TODO: If we support additional types of MemProf metadata beyond hot
1206 // and cold, we will need to update the metadata based on the allocator
1207 // APIs supported instead of completely stripping all.
1208 CI->setMetadata(LLVMContext::MD_memprof, nullptr);
1209 CI->setMetadata(LLVMContext::MD_callsite, nullptr);
1210 }
1211 }
1212 }
1213}
1214
1215Error LTO::runRegularLTO(AddStreamFn AddStream) {
1216 // Setup optimization remarks.
1217 auto DiagFileOrErr = lto::setupLLVMOptimizationRemarks(
1218 RegularLTO.CombinedModule->getContext(), Conf.RemarksFilename,
1221 LLVM_DEBUG(dbgs() << "Running regular LTO\n");
1222 if (!DiagFileOrErr)
1223 return DiagFileOrErr.takeError();
1224 DiagnosticOutputFile = std::move(*DiagFileOrErr);
1225
1226 // Finalize linking of regular LTO modules containing summaries now that
1227 // we have computed liveness information.
1228 for (auto &M : RegularLTO.ModsWithSummaries)
1229 if (Error Err = linkRegularLTO(std::move(M),
1230 /*LivenessFromIndex=*/true))
1231 return Err;
1232
1233 // Ensure we don't have inconsistently split LTO units with type tests.
1234 // FIXME: this checks both LTO and ThinLTO. It happens to work as we take
1235 // this path both cases but eventually this should be split into two and
1236 // do the ThinLTO checks in `runThinLTO`.
1237 if (Error Err = checkPartiallySplit())
1238 return Err;
1239
1240 // Make sure commons have the right size/alignment: we kept the largest from
1241 // all the prevailing when adding the inputs, and we apply it here.
1242 const DataLayout &DL = RegularLTO.CombinedModule->getDataLayout();
1243 for (auto &I : RegularLTO.Commons) {
1244 if (!I.second.Prevailing)
1245 // Don't do anything if no instance of this common was prevailing.
1246 continue;
1247 GlobalVariable *OldGV = RegularLTO.CombinedModule->getNamedGlobal(I.first);
1248 if (OldGV && DL.getTypeAllocSize(OldGV->getValueType()) == I.second.Size) {
1249 // Don't create a new global if the type is already correct, just make
1250 // sure the alignment is correct.
1251 OldGV->setAlignment(I.second.Alignment);
1252 continue;
1253 }
1254 ArrayType *Ty =
1255 ArrayType::get(Type::getInt8Ty(RegularLTO.Ctx), I.second.Size);
1256 auto *GV = new GlobalVariable(*RegularLTO.CombinedModule, Ty, false,
1259 GV->setAlignment(I.second.Alignment);
1260 if (OldGV) {
1261 OldGV->replaceAllUsesWith(GV);
1262 GV->takeName(OldGV);
1263 OldGV->eraseFromParent();
1264 } else {
1265 GV->setName(I.first);
1266 }
1267 }
1268
1269 updateMemProfAttributes(*RegularLTO.CombinedModule, ThinLTO.CombinedIndex);
1270
1271 bool WholeProgramVisibilityEnabledInLTO =
1273 // If validation is enabled, upgrade visibility only when all vtables
1274 // have typeinfos.
1276
1277 // This returns true when the name is local or not defined. Locals are
1278 // expected to be handled separately.
1279 auto IsVisibleToRegularObj = [&](StringRef name) {
1280 auto It = GlobalResolutions->find(name);
1281 return (It == GlobalResolutions->end() || It->second.VisibleOutsideSummary);
1282 };
1283
1284 // If allowed, upgrade public vcall visibility metadata to linkage unit
1285 // visibility before whole program devirtualization in the optimizer.
1287 *RegularLTO.CombinedModule, WholeProgramVisibilityEnabledInLTO,
1288 DynamicExportSymbols, Conf.ValidateAllVtablesHaveTypeInfos,
1289 IsVisibleToRegularObj);
1290 updatePublicTypeTestCalls(*RegularLTO.CombinedModule,
1291 WholeProgramVisibilityEnabledInLTO);
1292
1293 if (Conf.PreOptModuleHook &&
1294 !Conf.PreOptModuleHook(0, *RegularLTO.CombinedModule))
1295 return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
1296
1297 if (!Conf.CodeGenOnly) {
1298 for (const auto &R : *GlobalResolutions) {
1299 GlobalValue *GV =
1300 RegularLTO.CombinedModule->getNamedValue(R.second.IRName);
1301 if (!R.second.isPrevailingIRSymbol())
1302 continue;
1303 if (R.second.Partition != 0 &&
1304 R.second.Partition != GlobalResolution::External)
1305 continue;
1306
1307 // Ignore symbols defined in other partitions.
1308 // Also skip declarations, which are not allowed to have internal linkage.
1309 if (!GV || GV->hasLocalLinkage() || GV->isDeclaration())
1310 continue;
1311
1312 // Symbols that are marked DLLImport or DLLExport should not be
1313 // internalized, as they are either externally visible or referencing
1314 // external symbols. Symbols that have AvailableExternally or Appending
1315 // linkage might be used by future passes and should be kept as is.
1316 // These linkages are seen in Unified regular LTO, because the process
1317 // of creating split LTO units introduces symbols with that linkage into
1318 // one of the created modules. Normally, only the ThinLTO backend would
1319 // compile this module, but Unified Regular LTO processes both
1320 // modules created by the splitting process as regular LTO modules.
1321 if ((LTOMode == LTOKind::LTOK_UnifiedRegular) &&
1324 continue;
1325
1326 GV->setUnnamedAddr(R.second.UnnamedAddr ? GlobalValue::UnnamedAddr::Global
1328 if (EnableLTOInternalization && R.second.Partition == 0)
1330 }
1331
1332 if (Conf.PostInternalizeModuleHook &&
1333 !Conf.PostInternalizeModuleHook(0, *RegularLTO.CombinedModule))
1334 return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
1335 }
1336
1337 if (!RegularLTO.EmptyCombinedModule || Conf.AlwaysEmitRegularLTOObj) {
1338 if (Error Err =
1339 backend(Conf, AddStream, RegularLTO.ParallelCodeGenParallelismLevel,
1340 *RegularLTO.CombinedModule, ThinLTO.CombinedIndex))
1341 return Err;
1342 }
1343
1344 return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
1345}
1346
1347static const char *libcallRoutineNames[] = {
1348#define HANDLE_LIBCALL(code, name) name,
1349#include "llvm/IR/RuntimeLibcalls.def"
1350#undef HANDLE_LIBCALL
1351};
1352
1355}
1356
1357/// This class defines the interface to the ThinLTO backend.
1359protected:
1360 const Config &Conf;
1365
1366public:
1368 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1369 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1370 lto::IndexWriteCallback OnWrite, bool ShouldEmitImportsFiles)
1371 : Conf(Conf), CombinedIndex(CombinedIndex),
1372 ModuleToDefinedGVSummaries(ModuleToDefinedGVSummaries),
1373 OnWrite(OnWrite), ShouldEmitImportsFiles(ShouldEmitImportsFiles) {}
1374
1375 virtual ~ThinBackendProc() = default;
1376 virtual Error start(
1377 unsigned Task, BitcodeModule BM,
1378 const FunctionImporter::ImportMapTy &ImportList,
1379 const FunctionImporter::ExportSetTy &ExportList,
1380 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1382 virtual Error wait() = 0;
1383 virtual unsigned getThreadCount() = 0;
1384
1385 // Write sharded indices and (optionally) imports to disk
1387 llvm::StringRef ModulePath,
1388 const std::string &NewModulePath) {
1389 std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
1390 std::error_code EC;
1391 gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries,
1392 ImportList, ModuleToSummariesForIndex);
1393
1394 raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC,
1395 sys::fs::OpenFlags::OF_None);
1396 if (EC)
1397 return errorCodeToError(EC);
1398 writeIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex);
1399
1400 if (ShouldEmitImportsFiles) {
1401 EC = EmitImportsFiles(ModulePath, NewModulePath + ".imports",
1402 ModuleToSummariesForIndex);
1403 if (EC)
1404 return errorCodeToError(EC);
1405 }
1406 return Error::success();
1407 }
1408};
1409
1410namespace {
1411class InProcessThinBackend : public ThinBackendProc {
1412 DefaultThreadPool BackendThreadPool;
1413 AddStreamFn AddStream;
1414 FileCache Cache;
1415 std::set<GlobalValue::GUID> CfiFunctionDefs;
1416 std::set<GlobalValue::GUID> CfiFunctionDecls;
1417
1418 std::optional<Error> Err;
1419 std::mutex ErrMu;
1420
1421 bool ShouldEmitIndexFiles;
1422
1423public:
1424 InProcessThinBackend(
1425 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1426 ThreadPoolStrategy ThinLTOParallelism,
1427 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1428 AddStreamFn AddStream, FileCache Cache, lto::IndexWriteCallback OnWrite,
1429 bool ShouldEmitIndexFiles, bool ShouldEmitImportsFiles)
1430 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries,
1431 OnWrite, ShouldEmitImportsFiles),
1432 BackendThreadPool(ThinLTOParallelism), AddStream(std::move(AddStream)),
1433 Cache(std::move(Cache)), ShouldEmitIndexFiles(ShouldEmitIndexFiles) {
1434 for (auto &Name : CombinedIndex.cfiFunctionDefs())
1435 CfiFunctionDefs.insert(
1437 for (auto &Name : CombinedIndex.cfiFunctionDecls())
1438 CfiFunctionDecls.insert(
1440 }
1441
1442 Error runThinLTOBackendThread(
1443 AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM,
1444 ModuleSummaryIndex &CombinedIndex,
1445 const FunctionImporter::ImportMapTy &ImportList,
1446 const FunctionImporter::ExportSetTy &ExportList,
1447 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1448 const GVSummaryMapTy &DefinedGlobals,
1450 auto RunThinBackend = [&](AddStreamFn AddStream) {
1451 LTOLLVMContext BackendContext(Conf);
1452 Expected<std::unique_ptr<Module>> MOrErr = BM.parseModule(BackendContext);
1453 if (!MOrErr)
1454 return MOrErr.takeError();
1455
1456 return thinBackend(Conf, Task, AddStream, **MOrErr, CombinedIndex,
1457 ImportList, DefinedGlobals, &ModuleMap);
1458 };
1459
1460 auto ModuleID = BM.getModuleIdentifier();
1461
1462 if (ShouldEmitIndexFiles) {
1463 if (auto E = emitFiles(ImportList, ModuleID, ModuleID.str()))
1464 return E;
1465 }
1466
1467 if (!Cache || !CombinedIndex.modulePaths().count(ModuleID) ||
1468 all_of(CombinedIndex.getModuleHash(ModuleID),
1469 [](uint32_t V) { return V == 0; }))
1470 // Cache disabled or no entry for this module in the combined index or
1471 // no module hash.
1472 return RunThinBackend(AddStream);
1473
1475 // The module may be cached, this helps handling it.
1476 computeLTOCacheKey(Key, Conf, CombinedIndex, ModuleID, ImportList,
1477 ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs,
1478 CfiFunctionDecls);
1479 Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, Key, ModuleID);
1480 if (Error Err = CacheAddStreamOrErr.takeError())
1481 return Err;
1482 AddStreamFn &CacheAddStream = *CacheAddStreamOrErr;
1483 if (CacheAddStream)
1484 return RunThinBackend(CacheAddStream);
1485
1486 return Error::success();
1487 }
1488
1489 Error start(
1490 unsigned Task, BitcodeModule BM,
1491 const FunctionImporter::ImportMapTy &ImportList,
1492 const FunctionImporter::ExportSetTy &ExportList,
1493 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1494 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1495 StringRef ModulePath = BM.getModuleIdentifier();
1496 assert(ModuleToDefinedGVSummaries.count(ModulePath));
1497 const GVSummaryMapTy &DefinedGlobals =
1498 ModuleToDefinedGVSummaries.find(ModulePath)->second;
1499 BackendThreadPool.async(
1500 [=](BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
1501 const FunctionImporter::ImportMapTy &ImportList,
1502 const FunctionImporter::ExportSetTy &ExportList,
1503 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>
1504 &ResolvedODR,
1505 const GVSummaryMapTy &DefinedGlobals,
1507 if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled)
1509 "thin backend");
1510 Error E = runThinLTOBackendThread(
1511 AddStream, Cache, Task, BM, CombinedIndex, ImportList, ExportList,
1512 ResolvedODR, DefinedGlobals, ModuleMap);
1513 if (E) {
1514 std::unique_lock<std::mutex> L(ErrMu);
1515 if (Err)
1516 Err = joinErrors(std::move(*Err), std::move(E));
1517 else
1518 Err = std::move(E);
1519 }
1520 if (LLVM_ENABLE_THREADS && Conf.TimeTraceEnabled)
1522 },
1523 BM, std::ref(CombinedIndex), std::ref(ImportList), std::ref(ExportList),
1524 std::ref(ResolvedODR), std::ref(DefinedGlobals), std::ref(ModuleMap));
1525
1526 if (OnWrite)
1527 OnWrite(std::string(ModulePath));
1528 return Error::success();
1529 }
1530
1531 Error wait() override {
1532 BackendThreadPool.wait();
1533 if (Err)
1534 return std::move(*Err);
1535 else
1536 return Error::success();
1537 }
1538
1539 unsigned getThreadCount() override {
1540 return BackendThreadPool.getMaxConcurrency();
1541 }
1542};
1543} // end anonymous namespace
1544
1547 bool ShouldEmitIndexFiles,
1548 bool ShouldEmitImportsFiles) {
1549 return
1550 [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1551 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1552 AddStreamFn AddStream, FileCache Cache) {
1553 return std::make_unique<InProcessThinBackend>(
1554 Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries,
1555 AddStream, Cache, OnWrite, ShouldEmitIndexFiles,
1556 ShouldEmitImportsFiles);
1557 };
1558}
1559
1561 if (!TheTriple.isOSDarwin())
1562 return "";
1563 if (TheTriple.getArch() == Triple::x86_64)
1564 return "core2";
1565 if (TheTriple.getArch() == Triple::x86)
1566 return "yonah";
1567 if (TheTriple.isArm64e())
1568 return "apple-a12";
1569 if (TheTriple.getArch() == Triple::aarch64 ||
1570 TheTriple.getArch() == Triple::aarch64_32)
1571 return "cyclone";
1572 return "";
1573}
1574
1575// Given the original \p Path to an output file, replace any path
1576// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
1577// resulting directory if it does not yet exist.
1579 StringRef NewPrefix) {
1580 if (OldPrefix.empty() && NewPrefix.empty())
1581 return std::string(Path);
1582 SmallString<128> NewPath(Path);
1583 llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix);
1584 StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str());
1585 if (!ParentPath.empty()) {
1586 // Make sure the new directory exists, creating it if necessary.
1587 if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath))
1588 llvm::errs() << "warning: could not create directory '" << ParentPath
1589 << "': " << EC.message() << '\n';
1590 }
1591 return std::string(NewPath);
1592}
1593
1594namespace {
1595class WriteIndexesThinBackend : public ThinBackendProc {
1596 std::string OldPrefix, NewPrefix, NativeObjectPrefix;
1597 raw_fd_ostream *LinkedObjectsFile;
1598
1599public:
1600 WriteIndexesThinBackend(
1601 const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1602 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1603 std::string OldPrefix, std::string NewPrefix,
1604 std::string NativeObjectPrefix, bool ShouldEmitImportsFiles,
1605 raw_fd_ostream *LinkedObjectsFile, lto::IndexWriteCallback OnWrite)
1606 : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries,
1607 OnWrite, ShouldEmitImportsFiles),
1608 OldPrefix(OldPrefix), NewPrefix(NewPrefix),
1609 NativeObjectPrefix(NativeObjectPrefix),
1610 LinkedObjectsFile(LinkedObjectsFile) {}
1611
1612 Error start(
1613 unsigned Task, BitcodeModule BM,
1614 const FunctionImporter::ImportMapTy &ImportList,
1615 const FunctionImporter::ExportSetTy &ExportList,
1616 const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1617 MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1618 StringRef ModulePath = BM.getModuleIdentifier();
1619 std::string NewModulePath =
1620 getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix);
1621
1622 if (LinkedObjectsFile) {
1623 std::string ObjectPrefix =
1624 NativeObjectPrefix.empty() ? NewPrefix : NativeObjectPrefix;
1625 std::string LinkedObjectsFilePath =
1626 getThinLTOOutputFile(ModulePath, OldPrefix, ObjectPrefix);
1627 *LinkedObjectsFile << LinkedObjectsFilePath << '\n';
1628 }
1629
1630 if (auto E = emitFiles(ImportList, ModulePath, NewModulePath))
1631 return E;
1632
1633 if (OnWrite)
1634 OnWrite(std::string(ModulePath));
1635 return Error::success();
1636 }
1637
1638 Error wait() override { return Error::success(); }
1639
1640 // WriteIndexesThinBackend should always return 1 to prevent module
1641 // re-ordering and avoid non-determinism in the final link.
1642 unsigned getThreadCount() override { return 1; }
1643};
1644} // end anonymous namespace
1645
1647 std::string OldPrefix, std::string NewPrefix,
1648 std::string NativeObjectPrefix, bool ShouldEmitImportsFiles,
1649 raw_fd_ostream *LinkedObjectsFile, IndexWriteCallback OnWrite) {
1650 return
1651 [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
1652 const DenseMap<StringRef, GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1653 AddStreamFn AddStream, FileCache Cache) {
1654 return std::make_unique<WriteIndexesThinBackend>(
1655 Conf, CombinedIndex, ModuleToDefinedGVSummaries, OldPrefix,
1656 NewPrefix, NativeObjectPrefix, ShouldEmitImportsFiles,
1657 LinkedObjectsFile, OnWrite);
1658 };
1659}
1660
1661Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache,
1662 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
1663 LLVM_DEBUG(dbgs() << "Running ThinLTO\n");
1664 ThinLTO.CombinedIndex.releaseTemporaryMemory();
1665 timeTraceProfilerBegin("ThinLink", StringRef(""));
1666 auto TimeTraceScopeExit = llvm::make_scope_exit([]() {
1669 });
1670 if (ThinLTO.ModuleMap.empty())
1671 return Error::success();
1672
1673 if (ThinLTO.ModulesToCompile && ThinLTO.ModulesToCompile->empty()) {
1674 llvm::errs() << "warning: [ThinLTO] No module compiled\n";
1675 return Error::success();
1676 }
1677
1678 if (Conf.CombinedIndexHook &&
1679 !Conf.CombinedIndexHook(ThinLTO.CombinedIndex, GUIDPreservedSymbols))
1680 return Error::success();
1681
1682 // Collect for each module the list of function it defines (GUID ->
1683 // Summary).
1684 DenseMap<StringRef, GVSummaryMapTy> ModuleToDefinedGVSummaries(
1685 ThinLTO.ModuleMap.size());
1686 ThinLTO.CombinedIndex.collectDefinedGVSummariesPerModule(
1687 ModuleToDefinedGVSummaries);
1688 // Create entries for any modules that didn't have any GV summaries
1689 // (either they didn't have any GVs to start with, or we suppressed
1690 // generation of the summaries because they e.g. had inline assembly
1691 // uses that couldn't be promoted/renamed on export). This is so
1692 // InProcessThinBackend::start can still launch a backend thread, which
1693 // is passed the map of summaries for the module, without any special
1694 // handling for this case.
1695 for (auto &Mod : ThinLTO.ModuleMap)
1696 if (!ModuleToDefinedGVSummaries.count(Mod.first))
1697 ModuleToDefinedGVSummaries.try_emplace(Mod.first);
1698
1699 // Synthesize entry counts for functions in the CombinedIndex.
1700 computeSyntheticCounts(ThinLTO.CombinedIndex);
1701
1703 ThinLTO.ModuleMap.size());
1705 ThinLTO.ModuleMap.size());
1707
1708 if (DumpThinCGSCCs)
1709 ThinLTO.CombinedIndex.dumpSCCs(outs());
1710
1711 std::set<GlobalValue::GUID> ExportedGUIDs;
1712
1713 bool WholeProgramVisibilityEnabledInLTO =
1715 // If validation is enabled, upgrade visibility only when all vtables
1716 // have typeinfos.
1718 if (hasWholeProgramVisibility(WholeProgramVisibilityEnabledInLTO))
1719 ThinLTO.CombinedIndex.setWithWholeProgramVisibility();
1720
1721 // If we're validating, get the vtable symbols that should not be
1722 // upgraded because they correspond to typeIDs outside of index-based
1723 // WPD info.
1724 DenseSet<GlobalValue::GUID> VisibleToRegularObjSymbols;
1725 if (WholeProgramVisibilityEnabledInLTO &&
1727 // This returns true when the name is local or not defined. Locals are
1728 // expected to be handled separately.
1729 auto IsVisibleToRegularObj = [&](StringRef name) {
1730 auto It = GlobalResolutions->find(name);
1731 return (It == GlobalResolutions->end() ||
1732 It->second.VisibleOutsideSummary);
1733 };
1734
1735 getVisibleToRegularObjVtableGUIDs(ThinLTO.CombinedIndex,
1736 VisibleToRegularObjSymbols,
1737 IsVisibleToRegularObj);
1738 }
1739
1740 // If allowed, upgrade public vcall visibility to linkage unit visibility in
1741 // the summaries before whole program devirtualization below.
1743 ThinLTO.CombinedIndex, WholeProgramVisibilityEnabledInLTO,
1744 DynamicExportSymbols, VisibleToRegularObjSymbols);
1745
1746 // Perform index-based WPD. This will return immediately if there are
1747 // no index entries in the typeIdMetadata map (e.g. if we are instead
1748 // performing IR-based WPD in hybrid regular/thin LTO mode).
1749 std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
1750 runWholeProgramDevirtOnIndex(ThinLTO.CombinedIndex, ExportedGUIDs,
1751 LocalWPDTargetsMap);
1752
1753 auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
1754 return ThinLTO.PrevailingModuleForGUID[GUID] == S->modulePath();
1755 };
1757 MemProfContextDisambiguation ContextDisambiguation;
1758 ContextDisambiguation.run(ThinLTO.CombinedIndex, isPrevailing);
1759 }
1760
1761 // Figure out which symbols need to be internalized. This also needs to happen
1762 // at -O0 because summary-based DCE is implemented using internalization, and
1763 // we must apply DCE consistently with the full LTO module in order to avoid
1764 // undefined references during the final link.
1765 for (auto &Res : *GlobalResolutions) {
1766 // If the symbol does not have external references or it is not prevailing,
1767 // then not need to mark it as exported from a ThinLTO partition.
1768 if (Res.second.Partition != GlobalResolution::External ||
1769 !Res.second.isPrevailingIRSymbol())
1770 continue;
1771 auto GUID = GlobalValue::getGUID(
1772 GlobalValue::dropLLVMManglingEscape(Res.second.IRName));
1773 // Mark exported unless index-based analysis determined it to be dead.
1774 if (ThinLTO.CombinedIndex.isGUIDLive(GUID))
1775 ExportedGUIDs.insert(GUID);
1776 }
1777
1778 // Reset the GlobalResolutions to deallocate the associated memory, as there
1779 // are no further accesses. We specifically want to do this before computing
1780 // cross module importing, which adds to peak memory via the computed import
1781 // and export lists.
1782 GlobalResolutions.reset();
1783
1784 if (Conf.OptLevel > 0)
1785 ComputeCrossModuleImport(ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
1786 isPrevailing, ImportLists, ExportLists);
1787
1788 // Any functions referenced by the jump table in the regular LTO object must
1789 // be exported.
1790 for (auto &Def : ThinLTO.CombinedIndex.cfiFunctionDefs())
1791 ExportedGUIDs.insert(
1793 for (auto &Decl : ThinLTO.CombinedIndex.cfiFunctionDecls())
1794 ExportedGUIDs.insert(
1796
1797 auto isExported = [&](StringRef ModuleIdentifier, ValueInfo VI) {
1798 const auto &ExportList = ExportLists.find(ModuleIdentifier);
1799 return (ExportList != ExportLists.end() && ExportList->second.count(VI)) ||
1800 ExportedGUIDs.count(VI.getGUID());
1801 };
1802
1803 // Update local devirtualized targets that were exported by cross-module
1804 // importing or by other devirtualizations marked in the ExportedGUIDs set.
1805 updateIndexWPDForExports(ThinLTO.CombinedIndex, isExported,
1806 LocalWPDTargetsMap);
1807
1808 thinLTOInternalizeAndPromoteInIndex(ThinLTO.CombinedIndex, isExported,
1809 isPrevailing);
1810
1811 auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1812 GlobalValue::GUID GUID,
1813 GlobalValue::LinkageTypes NewLinkage) {
1814 ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1815 };
1816 thinLTOResolvePrevailingInIndex(Conf, ThinLTO.CombinedIndex, isPrevailing,
1817 recordNewLinkage, GUIDPreservedSymbols);
1818
1819 thinLTOPropagateFunctionAttrs(ThinLTO.CombinedIndex, isPrevailing);
1820
1821 generateParamAccessSummary(ThinLTO.CombinedIndex);
1822
1825
1826 TimeTraceScopeExit.release();
1827
1828 std::unique_ptr<ThinBackendProc> BackendProc =
1829 ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
1830 AddStream, Cache);
1831
1832 auto &ModuleMap =
1833 ThinLTO.ModulesToCompile ? *ThinLTO.ModulesToCompile : ThinLTO.ModuleMap;
1834
1835 auto ProcessOneModule = [&](int I) -> Error {
1836 auto &Mod = *(ModuleMap.begin() + I);
1837 // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for
1838 // combined module and parallel code generation partitions.
1839 return BackendProc->start(RegularLTO.ParallelCodeGenParallelismLevel + I,
1840 Mod.second, ImportLists[Mod.first],
1841 ExportLists[Mod.first], ResolvedODR[Mod.first],
1842 ThinLTO.ModuleMap);
1843 };
1844
1845 if (BackendProc->getThreadCount() == 1) {
1846 // Process the modules in the order they were provided on the command-line.
1847 // It is important for this codepath to be used for WriteIndexesThinBackend,
1848 // to ensure the emitted LinkedObjectsFile lists ThinLTO objects in the same
1849 // order as the inputs, which otherwise would affect the final link order.
1850 for (int I = 0, E = ModuleMap.size(); I != E; ++I)
1851 if (Error E = ProcessOneModule(I))
1852 return E;
1853 } else {
1854 // When executing in parallel, process largest bitsize modules first to
1855 // improve parallelism, and avoid starving the thread pool near the end.
1856 // This saves about 15 sec on a 36-core machine while link `clang.exe` (out
1857 // of 100 sec).
1858 std::vector<BitcodeModule *> ModulesVec;
1859 ModulesVec.reserve(ModuleMap.size());
1860 for (auto &Mod : ModuleMap)
1861 ModulesVec.push_back(&Mod.second);
1862 for (int I : generateModulesOrdering(ModulesVec))
1863 if (Error E = ProcessOneModule(I))
1864 return E;
1865 }
1866 return BackendProc->wait();
1867}
1868
1872 std::optional<uint64_t> RemarksHotnessThreshold, int Count) {
1873 std::string Filename = std::string(RemarksFilename);
1874 // For ThinLTO, file.opt.<format> becomes
1875 // file.opt.<format>.thin.<num>.<format>.
1876 if (!Filename.empty() && Count != -1)
1877 Filename =
1878 (Twine(Filename) + ".thin." + llvm::utostr(Count) + "." + RemarksFormat)
1879 .str();
1880
1881 auto ResultOrErr = llvm::setupLLVMOptimizationRemarks(
1884 if (Error E = ResultOrErr.takeError())
1885 return std::move(E);
1886
1887 if (*ResultOrErr)
1888 (*ResultOrErr)->keep();
1889
1890 return ResultOrErr;
1891}
1892
1895 // Setup output file to emit statistics.
1896 if (StatsFilename.empty())
1897 return nullptr;
1898
1900 std::error_code EC;
1901 auto StatsFile =
1902 std::make_unique<ToolOutputFile>(StatsFilename, EC, sys::fs::OF_None);
1903 if (EC)
1904 return errorCodeToError(EC);
1905
1906 StatsFile->keep();
1907 return std::move(StatsFile);
1908}
1909
1910// Compute the ordering we will process the inputs: the rough heuristic here
1911// is to sort them per size so that the largest module get schedule as soon as
1912// possible. This is purely a compile-time optimization.
1914 auto Seq = llvm::seq<int>(0, R.size());
1915 std::vector<int> ModulesOrdering(Seq.begin(), Seq.end());
1916 llvm::sort(ModulesOrdering, [&](int LeftIndex, int RightIndex) {
1917 auto LSize = R[LeftIndex]->getBuffer().size();
1918 auto RSize = R[RightIndex]->getBuffer().size();
1919 return LSize > RSize;
1920 });
1921 return ModulesOrdering;
1922}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DEBUG(X)
Definition: Debug.h:101
std::string Name
Symbol * Sym
Definition: ELF_riscv.cpp:479
#define DEBUG_TYPE
static void writeToResolutionFile(raw_ostream &OS, InputFile *Input, ArrayRef< SymbolResolution > Res)
Definition: LTO.cpp:669
static void thinLTOResolvePrevailingGUID(const Config &C, ValueInfo VI, DenseSet< GlobalValueSummary * > &GlobalInvolvedWithAlias, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, function_ref< void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)> recordNewLinkage, const DenseSet< GlobalValue::GUID > &GUIDPreservedSymbols)
Definition: LTO.cpp:349
static void handleNonPrevailingComdat(GlobalValue &GV, std::set< const Comdat * > &NonPrevailingComdats)
Definition: LTO.cpp:781
static cl::opt< bool > DumpThinCGSCCs("dump-thin-cg-sccs", cl::init(false), cl::Hidden, cl::desc("Dump the SCCs in the ThinLTO index's callgraph"))
static void thinLTOInternalizeAndPromoteGUID(ValueInfo VI, function_ref< bool(StringRef, ValueInfo)> isExported, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing)
Definition: LTO.cpp:448
static const char * libcallRoutineNames[]
Definition: LTO.cpp:1347
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
This file contains the declarations for metadata subclasses.
LLVMContext & Context
#define P(N)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static const char * name
Definition: SMEABIPass.cpp:49
raw_pwrite_stream & OS
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
This file contains some functions that are useful when dealing with strings.
This pass exposes codegen information to IR-level passes.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
iterator begin() const
Definition: ArrayRef.h:153
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:647
Represents a module in a bitcode file.
StringRef getModuleIdentifier() const
Error readSummary(ModuleSummaryIndex &CombinedIndex, StringRef ModulePath, std::function< bool(GlobalValue::GUID)> IsPrevailing=nullptr)
Parse the specified bitcode buffer and merge its module summary index into CombinedIndex.
Expected< std::unique_ptr< Module > > parseModule(LLVMContext &Context, ParserCallbacks Callbacks={})
Read the entire bitcode module and return it.
Expected< std::unique_ptr< Module > > getLazyModule(LLVMContext &Context, bool ShouldLazyLoadMetadata, bool IsImporting, ParserCallbacks Callbacks={})
Read the bitcode module and prepare for lazy deserialization of function bodies.
static ConstantAggregateZero * get(Type *Ty)
Definition: Constants.cpp:1663
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
Definition: DenseMap.h:235
unsigned size() const
Definition: DenseMap.h:99
iterator begin()
Definition: DenseMap.h:75
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
iterator end()
Definition: DenseMap.h:84
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:334
Tagged union holding either a T or a Error.
Definition: Error.h:474
Error takeError()
Take ownership of the stored error.
Definition: Error.h:601
std::unordered_set< GlobalValue::GUID > FunctionsToImportTy
Set of functions to import from a source module.
void setAlignment(Align Align)
Sets the alignment attribute of the GlobalObject.
Definition: Globals.cpp:128
Function and variable summary information to aid decisions and implementation of importing.
static bool isAppendingLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:399
static bool isExternalWeakLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:411
static bool isLocalLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:408
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:274
void setUnnamedAddr(UnnamedAddr Val)
Definition: GlobalValue.h:231
bool hasLocalLinkage() const
Definition: GlobalValue.h:527
static StringRef dropLLVMManglingEscape(StringRef Name)
If the given string begins with the GlobalValue name mangling escape character '\1',...
Definition: GlobalValue.h:566
const Comdat * getComdat() const
Definition: Globals.cpp:184
static bool isLinkOnceLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:387
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:536
DLLStorageClassTypes
Storage classes of global values for PE targets.
Definition: GlobalValue.h:73
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:594
const GlobalObject * getAliaseeObject() const
Definition: Globals.cpp:368
static bool isExternalLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:375
VisibilityTypes
An enumeration for the kinds of visibility of global values.
Definition: GlobalValue.h:66
@ DefaultVisibility
The GV is visible.
Definition: GlobalValue.h:67
static GUID getGUID(StringRef GlobalName)
Return a 64-bit global unique ID constructed from global value name (i.e.
Definition: GlobalValue.h:590
static LinkageTypes getWeakLinkage(bool ODR)
Definition: GlobalValue.h:371
bool isWeakForLinker() const
Definition: GlobalValue.h:551
bool hasAppendingLinkage() const
Definition: GlobalValue.h:524
bool hasAvailableExternallyLinkage() const
Definition: GlobalValue.h:511
std::string getGlobalIdentifier() const
Return the modified name for this global value suitable to be used as the key for a global lookup (e....
Definition: Globals.cpp:169
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:51
@ CommonLinkage
Tentative definitions.
Definition: GlobalValue.h:62
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:59
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:52
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
Definition: GlobalValue.h:56
@ AvailableExternallyLinkage
Available for inspection, not emission.
Definition: GlobalValue.h:53
DLLStorageClassTypes getDLLStorageClass() const
Definition: GlobalValue.h:275
Type * getValueType() const
Definition: GlobalValue.h:296
static bool isLinkOnceODRLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:384
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition: Globals.cpp:455
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
This class implements a map that also provides access to all stored values in a deterministic order.
Definition: MapVector.h:36
iterator begin()
Definition: MapVector.h:69
size_type size() const
Definition: MapVector.h:60
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFile(const Twine &Filename, bool IsText=false, bool RequiresNullTerminator=true, bool IsVolatile=false, std::optional< Align > Alignment=std::nullopt)
Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful,...
Class to hold module path string table and global value map, and encapsulate methods for operating on...
std::set< std::string > & cfiFunctionDecls()
const ModuleHash & getModuleHash(const StringRef ModPath) const
Get the module SHA1 hash recorded for the given module path.
const StringMap< ModuleHash > & modulePaths() const
Table of modules, containing module hash and id.
std::set< std::string > & cfiFunctionDefs()
static void CollectAsmSymvers(const Module &M, function_ref< void(StringRef, StringRef)> AsmSymver)
Parse inline ASM and collect the symvers directives that are defined in the current module.
uint32_t getSymbolFlags(Symbol S) const
ArrayRef< Symbol > symbols() const
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
A tuple of MDNodes.
Definition: Metadata.h:1729
The optimization diagnostic interface.
Diagnostic information for applied optimization remarks.
A discriminated union of two or more pointer types, with the discriminator in the low bit of the poin...
Definition: PointerUnion.h:118
A class that wrap the SHA1 algorithm.
Definition: SHA1.h:26
void update(ArrayRef< uint8_t > Data)
Digest more data.
Definition: SHA1.cpp:208
std::array< uint8_t, 20 > result()
Return the current raw 160-bits SHA1 for the digested data since the last call to init().
Definition: SHA1.cpp:288
A non-threaded implementation.
Definition: ThreadPool.h:218
void wait() override
Blocking wait for all the tasks to execute first.
Definition: ThreadPool.cpp:201
unsigned getMaxConcurrency() const override
Returns always 1: there is no concurrency.
Definition: ThreadPool.h:233
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:166
bool empty() const
Definition: SmallSet.h:159
bool erase(const T &V)
Definition: SmallSet.h:207
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:179
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition: SmallString.h:26
StringRef str() const
Explicit conversion to StringRef.
Definition: SmallString.h:254
A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...
Definition: StringRef.h:849
StringMapEntry - This is used to represent one value that is inserted into a StringMap.
StringMap - This is an unconventional map that is specialized for handling keys that are "strings",...
Definition: StringMap.h:128
size_type count(StringRef Key) const
count - Return 1 if the element is in the map, 0 otherwise.
Definition: StringMap.h:277
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
bool contains(StringRef Other) const
Return true if the given string is a substring of *this, and false otherwise.
Definition: StringRef.h:420
MCTargetOptions MCOptions
Machine level options.
DebuggerKind DebuggerTuning
Which debugger to tune for.
unsigned FunctionSections
Emit functions into separate sections.
unsigned DataSections
Emit data into separate sections.
auto async(Function &&F, Args &&...ArgList)
Asynchronous submission of a task to the pool.
Definition: ThreadPool.h:78
This tells how a thread pool will be used.
Definition: Threading.h:116
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
bool isArm64e() const
Tests whether the target is the Apple "arm64e" AArch64 subarch.
Definition: Triple.h:1015
@ aarch64_32
Definition: Triple.h:53
ArchType getArch() const
Get the parsed architecture type of this triple.
Definition: Triple.h:361
bool isOSDarwin() const
Is this a "Darwin" OS (macOS, iOS, tvOS, watchOS, XROS, or DriverKit).
Definition: Triple.h:542
bool isOSBinFormatELF() const
Tests whether the OS uses the ELF binary format.
Definition: Triple.h:703
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
static IntegerType * getInt8Ty(LLVMContext &C)
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:377
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:534
bool use_empty() const
Definition: Value.h:344
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:383
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
iterator find(const_arg_type_t< ValueT > V)
Definition: DenseSet.h:179
size_type size() const
Definition: DenseSet.h:81
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:97
An efficient, type-erasing, non-owning reference to a callable.
Ephemeral symbols produced by Reader::symbols() and Reader::module_symbols().
Definition: IRSymtab.h:313
The purpose of this class is to only expose the symbol information that an LTO client should need in ...
Definition: LTO.h:139
An input file.
Definition: LTO.h:111
static Expected< std::unique_ptr< InputFile > > create(MemoryBufferRef Object)
Create an InputFile.
Definition: LTO.cpp:540
ArrayRef< Symbol > symbols() const
A range over the symbols in this InputFile.
Definition: LTO.h:164
StringRef getName() const
Returns the path to the InputFile.
Definition: LTO.cpp:569
BitcodeModule & getSingleBitcodeModule()
Definition: LTO.cpp:573
Error add(std::unique_ptr< InputFile > Obj, ArrayRef< SymbolResolution > Res)
Add an input file to the LTO link, using the provided symbol resolutions.
Definition: LTO.cpp:693
LTO(Config Conf, ThinBackend Backend=nullptr, unsigned ParallelCodeGenParallelismLevel=1, LTOKind LTOMode=LTOK_Default)
Create an LTO object.
Definition: LTO.cpp:591
LTOKind
Unified LTO modes.
Definition: LTO.h:261
@ LTOK_UnifiedRegular
Regular LTO, with Unified LTO enabled.
Definition: LTO.h:266
@ LTOK_Default
Any LTO mode without Unified LTO. The default mode.
Definition: LTO.h:263
@ LTOK_UnifiedThin
ThinLTO, with Unified LTO enabled.
Definition: LTO.h:269
static ArrayRef< const char * > getRuntimeLibcallSymbols()
Static method that returns a list of libcall symbols that can be generated by LTO but might not be vi...
Definition: LTO.cpp:1353
Error run(AddStreamFn AddStream, FileCache Cache=nullptr)
Runs the LTO pipeline.
Definition: LTO.cpp:1126
unsigned getMaxTasks() const
Returns an upper bound on the number of tasks that the client may expect.
Definition: LTO.cpp:1075
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:470
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
This class defines the interface to the ThinLTO backend.
Definition: LTO.cpp:1358
virtual unsigned getThreadCount()=0
const Config & Conf
Definition: LTO.cpp:1360
lto::IndexWriteCallback OnWrite
Definition: LTO.cpp:1363
Error emitFiles(const FunctionImporter::ImportMapTy &ImportList, llvm::StringRef ModulePath, const std::string &NewModulePath)
Definition: LTO.cpp:1386
virtual Error wait()=0
ThinBackendProc(const Config &Conf, ModuleSummaryIndex &CombinedIndex, const DenseMap< StringRef, GVSummaryMapTy > &ModuleToDefinedGVSummaries, lto::IndexWriteCallback OnWrite, bool ShouldEmitImportsFiles)
Definition: LTO.cpp:1367
const DenseMap< StringRef, GVSummaryMapTy > & ModuleToDefinedGVSummaries
Definition: LTO.cpp:1362
ModuleSummaryIndex & CombinedIndex
Definition: LTO.cpp:1361
virtual ~ThinBackendProc()=default
virtual Error start(unsigned Task, BitcodeModule BM, const FunctionImporter::ImportMapTy &ImportList, const FunctionImporter::ExportSetTy &ExportList, const std::map< GlobalValue::GUID, GlobalValue::LinkageTypes > &ResolvedODR, MapVector< StringRef, BitcodeModule > &ModuleMap)=0
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Key
PAL metadata keys.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:1017
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
ThinBackend createInProcessThinBackend(ThreadPoolStrategy Parallelism, IndexWriteCallback OnWrite=nullptr, bool ShouldEmitIndexFiles=false, bool ShouldEmitImportsFiles=false)
Definition: LTO.cpp:1545
Error thinBackend(const Config &C, unsigned Task, AddStreamFn AddStream, Module &M, const ModuleSummaryIndex &CombinedIndex, const FunctionImporter::ImportMapTy &ImportList, const GVSummaryMapTy &DefinedGlobals, MapVector< StringRef, BitcodeModule > *ModuleMap, const std::vector< uint8_t > &CmdArgs=std::vector< uint8_t >())
Runs a ThinLTO backend.
Definition: LTOBackend.cpp:553
std::string getThinLTOOutputFile(StringRef Path, StringRef OldPrefix, StringRef NewPrefix)
Given the original Path to an output file, replace any path prefix matching OldPrefix with NewPrefix.
Definition: LTO.cpp:1578
std::function< std::unique_ptr< ThinBackendProc >(const Config &C, ModuleSummaryIndex &CombinedIndex, DenseMap< StringRef, GVSummaryMapTy > &ModuleToDefinedGVSummaries, AddStreamFn AddStream, FileCache Cache)> ThinBackend
A ThinBackend defines what happens after the thin-link phase during ThinLTO.
Definition: LTO.h:202
std::function< void(const std::string &)> IndexWriteCallback
This ThinBackend runs the individual backend jobs in-process.
Definition: LTO.h:212
StringLiteral getThinLTODefaultCPU(const Triple &TheTriple)
Definition: LTO.cpp:1560
Expected< std::unique_ptr< ToolOutputFile > > setupStatsFile(StringRef StatsFilename)
Setups the output file for saving statistics.
Definition: LTO.cpp:1894
Error backend(const Config &C, AddStreamFn AddStream, unsigned ParallelCodeGenParallelismLevel, Module &M, ModuleSummaryIndex &CombinedIndex)
Runs a regular LTO backend.
Definition: LTOBackend.cpp:506
Error finalizeOptimizationRemarks(std::unique_ptr< ToolOutputFile > DiagOutputFile)
Definition: LTOBackend.cpp:495
ThinBackend createWriteIndexesThinBackend(std::string OldPrefix, std::string NewPrefix, std::string NativeObjectPrefix, bool ShouldEmitImportsFiles, raw_fd_ostream *LinkedObjectsFile, IndexWriteCallback OnWrite)
This ThinBackend writes individual module indexes to files, instead of running the individual backend...
Definition: LTO.cpp:1646
std::vector< int > generateModulesOrdering(ArrayRef< BitcodeModule * > R)
Produces a container ordering for optimal multi-threaded processing.
Definition: LTO.cpp:1913
Expected< std::unique_ptr< ToolOutputFile > > setupLLVMOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses, StringRef RemarksFormat, bool RemarksWithHotness, std::optional< uint64_t > RemarksHotnessThreshold=0, int Count=-1)
Setup optimization remarks.
Definition: LTO.cpp:1869
void updateMemProfAttributes(Module &Mod, const ModuleSummaryIndex &Index)
Updates MemProf attributes (and metadata) based on whether the index has recorded that we are linking...
Definition: LTO.cpp:1184
Expected< IRSymtabFile > readIRSymtab(MemoryBufferRef MBRef)
Reads a bitcode file, creating its irsymtab if necessary.
size_t getThreadCount()
Definition: Parallel.h:55
void write64le(void *P, uint64_t V)
Definition: Endian.h:455
void write32le(void *P, uint32_t V)
Definition: Endian.h:452
std::error_code create_directories(const Twine &path, bool IgnoreExisting=true, perms Perms=owner_all|group_all)
Create all the non-existent directories in path.
Definition: Path.cpp:968
StringRef parent_path(StringRef path, Style style=Style::native)
Get parent path.
Definition: Path.cpp:468
bool replace_path_prefix(SmallVectorImpl< char > &Path, StringRef OldPrefix, StringRef NewPrefix, Style style=Style::native)
Replace matching path prefix with another path.
Definition: Path.cpp:519
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
ThreadPoolStrategy heavyweight_hardware_concurrency(unsigned ThreadCount=0)
Returns a thread strategy for tasks requiring significant memory or other resources.
Definition: Threading.h:162
cl::opt< std::string > RemarksFormat("lto-pass-remarks-format", cl::desc("The format used for serializing remarks (default: YAML)"), cl::value_desc("format"), cl::init("yaml"))
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1731
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1689
void ComputeCrossModuleImport(const ModuleSummaryIndex &Index, const DenseMap< StringRef, GVSummaryMapTy > &ModuleToDefinedGVSummaries, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, DenseMap< StringRef, FunctionImporter::ImportMapTy > &ImportLists, DenseMap< StringRef, FunctionImporter::ExportSetTy > &ExportLists)
Compute all the imports and exports for every module in the Index.
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:59
void timeTraceProfilerInitialize(unsigned TimeTraceGranularity, StringRef ProcName)
Initialize the time trace profiler.
void generateParamAccessSummary(ModuleSummaryIndex &Index)
cl::opt< std::string > RemarksPasses("lto-pass-remarks-filter", cl::desc("Only record optimization remarks from passes whose " "names match the given regular expression"), cl::value_desc("regex"))
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:90
void writeIndexToFile(const ModuleSummaryIndex &Index, raw_ostream &Out, const std::map< std::string, GVSummaryMapTy > *ModuleToSummariesForIndex=nullptr)
Write the specified module summary index to the given raw output stream, where it will be written in ...
raw_fd_ostream & outs()
This returns a reference to a raw_fd_ostream for standard output.
std::function< Expected< std::unique_ptr< CachedFileStream > >(unsigned Task, const Twine &ModuleName)> AddStreamFn
This type defines the callback to add a file that is generated on the fly.
Definition: Caching.h:42
std::error_code EmitImportsFiles(StringRef ModulePath, StringRef OutputFilename, const std::map< std::string, GVSummaryMapTy > &ModuleToSummariesForIndex)
Emit into OutputFilename the files module ModulePath will import from.
bool thinLTOPropagateFunctionAttrs(ModuleSummaryIndex &Index, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing)
Propagate function attributes for function summaries along the index's callgraph during thinlink.
bool hasWholeProgramVisibility(bool WholeProgramVisibilityEnabledInLTO)
void EnableStatistics(bool DoPrintOnExit=true)
Enable the collection and printing of statistics.
Definition: Statistic.cpp:134
void thinLTOInternalizeAndPromoteInIndex(ModuleSummaryIndex &Index, function_ref< bool(StringRef, ValueInfo)> isExported, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing)
Update the linkages in the given Index to mark exported values as external and non-exported values as...
Definition: LTO.cpp:527
void computeLTOCacheKey(SmallString< 40 > &Key, const lto::Config &Conf, const ModuleSummaryIndex &Index, StringRef ModuleID, const FunctionImporter::ImportMapTy &ImportList, const FunctionImporter::ExportSetTy &ExportList, const std::map< GlobalValue::GUID, GlobalValue::LinkageTypes > &ResolvedODR, const GVSummaryMapTy &DefinedGlobals, const std::set< GlobalValue::GUID > &CfiFunctionDefs={}, const std::set< GlobalValue::GUID > &CfiFunctionDecls={})
Computes a unique hash for the Module considering the current list of export/import and other global ...
Definition: LTO.cpp:89
void timeTraceProfilerFinishThread()
Finish a time trace profiler running on a worker thread.
void timeTraceProfilerBegin(StringRef Name, StringRef Detail)
Manually begin a time section, with the given Name and Detail.
Error joinErrors(Error E1, Error E2)
Concatenate errors.
Definition: Error.h:431
void updatePublicTypeTestCalls(Module &M, bool WholeProgramVisibilityEnabledInLTO)
void getVisibleToRegularObjVtableGUIDs(ModuleSummaryIndex &Index, DenseSet< GlobalValue::GUID > &VisibleToRegularObjSymbols, function_ref< bool(StringRef)> IsVisibleToRegularObj)
Based on typeID string, get all associated vtable GUIDS that are visible to regular objects.
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1656
bool timeTraceProfilerEnabled()
Is the time trace profiler enabled, i.e. initialized?
Definition: TimeProfiler.h:102
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
Expected< std::unique_ptr< ToolOutputFile > > setupLLVMOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses, StringRef RemarksFormat, bool RemarksWithHotness, std::optional< uint64_t > RemarksHotnessThreshold=0)
Setup optimization remarks that output to a file.
cl::opt< bool > EnableLTOInternalization
Enable global value internalization in LTO.
cl::opt< bool > RemarksWithHotness("lto-pass-remarks-with-hotness", cl::desc("With PGO, include profile count in optimization remarks"), cl::Hidden)
void computeSyntheticCounts(ModuleSummaryIndex &Index)
Compute synthetic function entry counts.
void timeTraceProfilerEnd()
Manually end the last time section.
cl::opt< std::string > RemarksFilename("lto-pass-remarks-output", cl::desc("Output filename for pass remarks"), cl::value_desc("filename"))
cl::opt< bool > SupportsHotColdNew
Indicate we are linking with an allocator that supports hot/cold operator new interfaces.
void updateIndexWPDForExports(ModuleSummaryIndex &Summary, function_ref< bool(StringRef, ValueInfo)> isExported, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap)
Call after cross-module importing to update the recorded single impl devirt target names for any loca...
void thinLTOResolvePrevailingInIndex(const lto::Config &C, ModuleSummaryIndex &Index, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, function_ref< void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)> recordNewLinkage, const DenseSet< GlobalValue::GUID > &GUIDPreservedSymbols)
Resolve linkage for prevailing symbols in the Index.
Definition: LTO.cpp:426
void gatherImportedSummariesForModule(StringRef ModulePath, const DenseMap< StringRef, GVSummaryMapTy > &ModuleToDefinedGVSummaries, const FunctionImporter::ImportMapTy &ImportList, std::map< std::string, GVSummaryMapTy > &ModuleToSummariesForIndex)
Compute the set of summaries needed for a ThinLTO backend compilation of ModulePath.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Mod
The access may modify the value stored in memory.
cl::opt< bool > EnableMemProfContextDisambiguation
Enable MemProf context disambiguation for thin link.
void runWholeProgramDevirtOnIndex(ModuleSummaryIndex &Summary, std::set< GlobalValue::GUID > &ExportedGUIDs, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap)
Perform index-based whole program devirtualization on the Summary index.
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1858
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
Definition: STLExtras.h:1930
cl::opt< std::optional< uint64_t >, false, remarks::HotnessThresholdParser > RemarksHotnessThreshold("lto-pass-remarks-hotness-threshold", cl::desc("Minimum profile count required for an " "optimization remark to be output." " Use 'auto' to apply the threshold from profile summary."), cl::value_desc("uint or 'auto'"), cl::init(0), cl::Hidden)
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Definition: Error.cpp:103
bool UpgradeDebugInfo(Module &M)
Check the debug info version number, if it is out-dated, drop the debug info.
std::array< uint32_t, 5 > ModuleHash
160 bits SHA1
std::function< Expected< AddStreamFn >(unsigned Task, StringRef Key, const Twine &ModuleName)> FileCache
This is the type of a file cache.
Definition: Caching.h:58
void PrintStatisticsJSON(raw_ostream &OS)
Print statistics in JSON format.
Definition: Statistic.cpp:203
void computeDeadSymbolsWithConstProp(ModuleSummaryIndex &Index, const DenseSet< GlobalValue::GUID > &GUIDPreservedSymbols, function_ref< PrevailingType(GlobalValue::GUID)> isPrevailing, bool ImportEnabled)
Compute dead symbols and run constant propagation in combined index after that.
@ Keep
No function return thunk.
void updateVCallVisibilityInModule(Module &M, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols, bool ValidateAllVtablesHaveTypeInfos, function_ref< bool(StringRef)> IsVisibleToRegularObj)
If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...
void updateVCallVisibilityInIndex(ModuleSummaryIndex &Index, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols, const DenseSet< GlobalValue::GUID > &VisibleToRegularObjSymbols)
If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...
Implement std::hash so that hash_code can be used in STL containers.
Definition: BitVector.h:858
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Used in the streaming interface as the general argument type.
Struct that holds a reference to a particular GUID in a global value summary.
LTO configuration.
Definition: Config.h:41
bool HasWholeProgramVisibility
Asserts whether we can assume whole program visibility during the LTO link.
Definition: Config.h:78
bool ValidateAllVtablesHaveTypeInfos
We're validating that all native vtables have corresponding type infos.
Definition: Config.h:81
std::optional< uint64_t > RemarksHotnessThreshold
The minimum hotness value a diagnostic needs in order to be included in optimization diagnostics.
Definition: Config.h:160
std::string StatsFile
Statistics output file path.
Definition: Config.h:169
ModuleHookFn PreOptModuleHook
This module hook is called after linking (regular LTO) or loading (ThinLTO) the module,...
Definition: Config.h:223
CombinedIndexHookFn CombinedIndexHook
Definition: Config.h:255
std::optional< CodeModel::Model > CodeModel
Definition: Config.h:56
std::string AAPipeline
Definition: Config.h:105
bool CodeGenOnly
Disable entirely the optimizer, including importing for ThinLTO.
Definition: Config.h:68
std::vector< std::string > MAttrs
Definition: Config.h:50
std::vector< std::string > MllvmArgs
Definition: Config.h:51
std::vector< std::string > ThinLTOModulesToCompile
Specific thinLTO modules to compile.
Definition: Config.h:172
CodeGenOptLevel CGOptLevel
Definition: Config.h:57
std::unique_ptr< raw_ostream > ResolutionFile
If this field is set, LTO will write input file paths and symbol resolutions here in llvm-lto2 comman...
Definition: Config.h:190
std::string DefaultTriple
Setting this field will replace unspecified target triples in input files with this triple.
Definition: Config.h:113
bool AlwaysEmitRegularLTOObj
Always emit a Regular LTO object even when it is empty because no Regular LTO modules were linked.
Definition: Config.h:89
std::string CPU
Definition: Config.h:48
std::string DwoDir
The directory to store .dwo files.
Definition: Config.h:125
std::string RemarksFilename
Optimization remarks file path.
Definition: Config.h:139
VisScheme VisibilityScheme
Allows non-imported definitions to get the potentially more constraining visibility from the prevaili...
Definition: Config.h:95
std::string OverrideTriple
Setting this field will replace target triples in input files with this triple.
Definition: Config.h:109
std::string ProfileRemapping
Name remapping file for profile data.
Definition: Config.h:122
bool AllVtablesHaveTypeInfos
If all native vtables have corresponding type infos, allow usage of RTTI to block devirtualization on...
Definition: Config.h:84
TargetOptions Options
Definition: Config.h:49
bool TimeTraceEnabled
Time trace enabled.
Definition: Config.h:175
std::string RemarksPasses
Optimization remarks pass filter.
Definition: Config.h:142
std::string OptPipeline
If this field is set, the set of passes run in the middle-end optimizer will be the one specified by ...
Definition: Config.h:100
ModuleHookFn PostInternalizeModuleHook
This hook is called after internalizing the module.
Definition: Config.h:230
unsigned TimeTraceGranularity
Time trace granularity.
Definition: Config.h:178
unsigned OptLevel
Definition: Config.h:59
bool RemarksWithHotness
Whether to emit optimization remarks with hotness informations.
Definition: Config.h:145
std::optional< Reloc::Model > RelocModel
Definition: Config.h:55
CodeGenFileType CGFileType
Definition: Config.h:58
bool Freestanding
Flag to indicate that the optimizer should not assume builtins are present on the target.
Definition: Config.h:65
std::string SampleProfile
Sample PGO profile path.
Definition: Config.h:119
std::string RemarksFormat
The format used for serializing remarks (default: YAML).
Definition: Config.h:163
A derived class of LLVMContext that initializes itself according to a given Config object.
Definition: Config.h:294
The resolution for a symbol.
Definition: LTO.h:457
unsigned FinalDefinitionInLinkageUnit
The definition of this symbol is unpreemptable at runtime and is known to be in this linkage unit.
Definition: LTO.h:467
unsigned ExportDynamic
The symbol was exported dynamically, and therefore could be referenced by a shared library not visibl...
Definition: LTO.h:474
unsigned Prevailing
The linker has chosen this definition of the symbol.
Definition: LTO.h:463
unsigned LinkerRedefined
Linker redefined version of the symbol which appeared in -wrap or -defsym linker option.
Definition: LTO.h:478
unsigned VisibleToRegularObj
The definition of this symbol is visible outside of the LTO unit.
Definition: LTO.h:470