Line data Source code
1 : //===-LTO.cpp - LLVM Link Time Optimizer ----------------------------------===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // This file implements functions and classes used to support LTO.
11 : //
12 : //===----------------------------------------------------------------------===//
13 :
14 : #include "llvm/LTO/LTO.h"
15 : #include "llvm/ADT/Statistic.h"
16 : #include "llvm/Analysis/TargetLibraryInfo.h"
17 : #include "llvm/Analysis/TargetTransformInfo.h"
18 : #include "llvm/Bitcode/BitcodeReader.h"
19 : #include "llvm/Bitcode/BitcodeWriter.h"
20 : #include "llvm/CodeGen/Analysis.h"
21 : #include "llvm/Config/llvm-config.h"
22 : #include "llvm/IR/AutoUpgrade.h"
23 : #include "llvm/IR/DiagnosticPrinter.h"
24 : #include "llvm/IR/LegacyPassManager.h"
25 : #include "llvm/IR/Mangler.h"
26 : #include "llvm/IR/Metadata.h"
27 : #include "llvm/LTO/LTOBackend.h"
28 : #include "llvm/Linker/IRMover.h"
29 : #include "llvm/Object/IRObjectFile.h"
30 : #include "llvm/Support/Error.h"
31 : #include "llvm/Support/ManagedStatic.h"
32 : #include "llvm/Support/MemoryBuffer.h"
33 : #include "llvm/Support/Path.h"
34 : #include "llvm/Support/SHA1.h"
35 : #include "llvm/Support/SourceMgr.h"
36 : #include "llvm/Support/TargetRegistry.h"
37 : #include "llvm/Support/ThreadPool.h"
38 : #include "llvm/Support/Threading.h"
39 : #include "llvm/Support/VCSRevision.h"
40 : #include "llvm/Support/raw_ostream.h"
41 : #include "llvm/Target/TargetMachine.h"
42 : #include "llvm/Target/TargetOptions.h"
43 : #include "llvm/Transforms/IPO.h"
44 : #include "llvm/Transforms/IPO/PassManagerBuilder.h"
45 : #include "llvm/Transforms/Utils/SplitModule.h"
46 :
47 : #include <set>
48 :
49 : using namespace llvm;
50 : using namespace lto;
51 : using namespace object;
52 :
53 : #define DEBUG_TYPE "lto"
54 :
55 : static cl::opt<bool>
56 : DumpThinCGSCCs("dump-thin-cg-sccs", cl::init(false), cl::Hidden,
57 : cl::desc("Dump the SCCs in the ThinLTO index's callgraph"));
58 :
59 : // Returns a unique hash for the Module considering the current list of
60 : // export/import and other global analysis results.
61 : // The hash is produced in \p Key.
62 80 : static void computeCacheKey(
63 : SmallString<40> &Key, const Config &Conf, const ModuleSummaryIndex &Index,
64 : StringRef ModuleID, const FunctionImporter::ImportMapTy &ImportList,
65 : const FunctionImporter::ExportSetTy &ExportList,
66 : const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
67 : const GVSummaryMapTy &DefinedGlobals,
68 : const std::set<GlobalValue::GUID> &CfiFunctionDefs,
69 : const std::set<GlobalValue::GUID> &CfiFunctionDecls) {
70 : // Compute the unique hash for this entry.
71 : // This is based on the current compiler version, the module itself, the
72 : // export list, the hash for every single module in the import list, the
73 : // list of ResolvedODR for the module, and the list of preserved symbols.
74 : SHA1 Hasher;
75 :
76 : // Start with the compiler revision
77 : Hasher.update(LLVM_VERSION_STRING);
78 : #ifdef LLVM_REVISION
79 : Hasher.update(LLVM_REVISION);
80 : #endif
81 :
82 : // Include the parts of the LTO configuration that affect code generation.
83 : auto AddString = [&](StringRef Str) {
84 : Hasher.update(Str);
85 : Hasher.update(ArrayRef<uint8_t>{0});
86 80 : };
87 : auto AddUnsigned = [&](unsigned I) {
88 : uint8_t Data[4];
89 79 : Data[0] = I;
90 79 : Data[1] = I >> 8;
91 79 : Data[2] = I >> 16;
92 79 : Data[3] = I >> 24;
93 79 : Hasher.update(ArrayRef<uint8_t>{Data, 4});
94 80 : };
95 : auto AddUint64 = [&](uint64_t I) {
96 : uint8_t Data[8];
97 : Data[0] = I;
98 : Data[1] = I >> 8;
99 : Data[2] = I >> 16;
100 : Data[3] = I >> 24;
101 : Data[4] = I >> 32;
102 : Data[5] = I >> 40;
103 : Data[6] = I >> 48;
104 : Data[7] = I >> 56;
105 : Hasher.update(ArrayRef<uint8_t>{Data, 8});
106 80 : };
107 80 : AddString(Conf.CPU);
108 : // FIXME: Hash more of Options. For now all clients initialize Options from
109 : // command-line flags (which is unsupported in production), but may set
110 : // RelaxELFRelocations. The clang driver can also pass FunctionSections,
111 : // DataSections and DebuggerTuning via command line flags.
112 80 : AddUnsigned(Conf.Options.RelaxELFRelocations);
113 80 : AddUnsigned(Conf.Options.FunctionSections);
114 80 : AddUnsigned(Conf.Options.DataSections);
115 80 : AddUnsigned((unsigned)Conf.Options.DebuggerTuning);
116 81 : for (auto &A : Conf.MAttrs)
117 1 : AddString(A);
118 80 : if (Conf.RelocModel)
119 80 : AddUnsigned(*Conf.RelocModel);
120 : else
121 0 : AddUnsigned(-1);
122 80 : if (Conf.CodeModel)
123 1 : AddUnsigned(*Conf.CodeModel);
124 : else
125 79 : AddUnsigned(-1);
126 80 : AddUnsigned(Conf.CGOptLevel);
127 80 : AddUnsigned(Conf.CGFileType);
128 80 : AddUnsigned(Conf.OptLevel);
129 80 : AddUnsigned(Conf.UseNewPM);
130 80 : AddString(Conf.OptPipeline);
131 80 : AddString(Conf.AAPipeline);
132 80 : AddString(Conf.OverrideTriple);
133 80 : AddString(Conf.DefaultTriple);
134 80 : AddString(Conf.DwoDir);
135 :
136 : // Include the hash for the current module
137 80 : auto ModHash = Index.getModuleHash(ModuleID);
138 80 : Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
139 117 : for (auto F : ExportList)
140 : // The export list can impact the internalization, be conservative here
141 37 : Hasher.update(ArrayRef<uint8_t>((uint8_t *)&F, sizeof(F)));
142 :
143 : // Include the hash for every module we import functions from. The set of
144 : // imported symbols for each module may affect code generation and is
145 : // sensitive to link order, so include that as well.
146 188 : for (auto &Entry : ImportList) {
147 28 : auto ModHash = Index.getModuleHash(Entry.first());
148 28 : Hasher.update(ArrayRef<uint8_t>((uint8_t *)&ModHash[0], sizeof(ModHash)));
149 :
150 28 : AddUint64(Entry.second.size());
151 63 : for (auto &Fn : Entry.second)
152 35 : AddUint64(Fn);
153 : }
154 :
155 : // Include the hash for the resolved ODR.
156 84 : for (auto &Entry : ResolvedODR) {
157 8 : Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.first,
158 : sizeof(GlobalValue::GUID)));
159 8 : Hasher.update(ArrayRef<uint8_t>((const uint8_t *)&Entry.second,
160 : sizeof(GlobalValue::LinkageTypes)));
161 : }
162 :
163 : // Members of CfiFunctionDefs and CfiFunctionDecls that are referenced or
164 : // defined in this module.
165 : std::set<GlobalValue::GUID> UsedCfiDefs;
166 : std::set<GlobalValue::GUID> UsedCfiDecls;
167 :
168 : // Typeids used in this module.
169 : std::set<GlobalValue::GUID> UsedTypeIds;
170 :
171 : auto AddUsedCfiGlobal = [&](GlobalValue::GUID ValueGUID) {
172 : if (CfiFunctionDefs.count(ValueGUID))
173 : UsedCfiDefs.insert(ValueGUID);
174 : if (CfiFunctionDecls.count(ValueGUID))
175 : UsedCfiDecls.insert(ValueGUID);
176 80 : };
177 :
178 : auto AddUsedThings = [&](GlobalValueSummary *GS) {
179 : if (!GS) return;
180 : AddUnsigned(GS->isLive());
181 : for (const ValueInfo &VI : GS->refs()) {
182 : AddUnsigned(VI.isDSOLocal());
183 : AddUsedCfiGlobal(VI.getGUID());
184 : }
185 : if (auto *FS = dyn_cast<FunctionSummary>(GS)) {
186 : for (auto &TT : FS->type_tests())
187 : UsedTypeIds.insert(TT);
188 : for (auto &TT : FS->type_test_assume_vcalls())
189 : UsedTypeIds.insert(TT.GUID);
190 : for (auto &TT : FS->type_checked_load_vcalls())
191 : UsedTypeIds.insert(TT.GUID);
192 : for (auto &TT : FS->type_test_assume_const_vcalls())
193 : UsedTypeIds.insert(TT.VFunc.GUID);
194 : for (auto &TT : FS->type_checked_load_const_vcalls())
195 : UsedTypeIds.insert(TT.VFunc.GUID);
196 : for (auto &ET : FS->calls()) {
197 : AddUnsigned(ET.first.isDSOLocal());
198 : AddUsedCfiGlobal(ET.first.getGUID());
199 : }
200 : }
201 80 : };
202 :
203 : // Include the hash for the linkage type to reflect internalization and weak
204 : // resolution, and collect any used type identifier resolutions.
205 179 : for (auto &GS : DefinedGlobals) {
206 198 : GlobalValue::LinkageTypes Linkage = GS.second->linkage();
207 99 : Hasher.update(
208 : ArrayRef<uint8_t>((const uint8_t *)&Linkage, sizeof(Linkage)));
209 99 : AddUsedCfiGlobal(GS.first);
210 99 : AddUsedThings(GS.second);
211 : }
212 :
213 : // Imported functions may introduce new uses of type identifier resolutions,
214 : // so we need to collect their used resolutions as well.
215 188 : for (auto &ImpM : ImportList)
216 63 : for (auto &ImpF : ImpM.second)
217 35 : AddUsedThings(Index.findSummaryInModule(ImpF, ImpM.first()));
218 :
219 : auto AddTypeIdSummary = [&](StringRef TId, const TypeIdSummary &S) {
220 : AddString(TId);
221 :
222 : AddUnsigned(S.TTRes.TheKind);
223 : AddUnsigned(S.TTRes.SizeM1BitWidth);
224 :
225 : AddUint64(S.TTRes.AlignLog2);
226 : AddUint64(S.TTRes.SizeM1);
227 : AddUint64(S.TTRes.BitMask);
228 : AddUint64(S.TTRes.InlineBits);
229 :
230 : AddUint64(S.WPDRes.size());
231 : for (auto &WPD : S.WPDRes) {
232 : AddUnsigned(WPD.first);
233 : AddUnsigned(WPD.second.TheKind);
234 : AddString(WPD.second.SingleImplName);
235 :
236 : AddUint64(WPD.second.ResByArg.size());
237 : for (auto &ByArg : WPD.second.ResByArg) {
238 : AddUint64(ByArg.first.size());
239 : for (uint64_t Arg : ByArg.first)
240 : AddUint64(Arg);
241 : AddUnsigned(ByArg.second.TheKind);
242 : AddUint64(ByArg.second.Info);
243 : AddUnsigned(ByArg.second.Byte);
244 : AddUnsigned(ByArg.second.Bit);
245 : }
246 : }
247 80 : };
248 :
249 : // Include the hash for all type identifiers used by this module.
250 102 : for (GlobalValue::GUID TId : UsedTypeIds) {
251 : auto TidIter = Index.typeIds().equal_range(TId);
252 30 : for (auto It = TidIter.first; It != TidIter.second; ++It)
253 16 : AddTypeIdSummary(It->second.first, It->second.second);
254 : }
255 :
256 80 : AddUnsigned(UsedCfiDefs.size());
257 82 : for (auto &V : UsedCfiDefs)
258 2 : AddUint64(V);
259 :
260 80 : AddUnsigned(UsedCfiDecls.size());
261 81 : for (auto &V : UsedCfiDecls)
262 1 : AddUint64(V);
263 :
264 80 : if (!Conf.SampleProfile.empty()) {
265 0 : auto FileOrErr = MemoryBuffer::getFile(Conf.SampleProfile);
266 0 : if (FileOrErr) {
267 : Hasher.update(FileOrErr.get()->getBuffer());
268 :
269 0 : if (!Conf.ProfileRemapping.empty()) {
270 0 : FileOrErr = MemoryBuffer::getFile(Conf.ProfileRemapping);
271 0 : if (FileOrErr)
272 : Hasher.update(FileOrErr.get()->getBuffer());
273 : }
274 : }
275 : }
276 :
277 160 : Key = toHex(Hasher.result());
278 80 : }
279 :
280 849 : static void thinLTOResolveWeakForLinkerGUID(
281 : GlobalValueSummaryList &GVSummaryList, GlobalValue::GUID GUID,
282 : DenseSet<GlobalValueSummary *> &GlobalInvolvedWithAlias,
283 : function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
284 : isPrevailing,
285 : function_ref<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)>
286 : recordNewLinkage) {
287 1698 : for (auto &S : GVSummaryList) {
288 : GlobalValue::LinkageTypes OriginalLinkage = S->linkage();
289 : if (!GlobalValue::isWeakForLinker(OriginalLinkage))
290 : continue;
291 : // We need to emit only one of these. The prevailing module will keep it,
292 : // but turned into a weak, while the others will drop it when possible.
293 : // This is both a compile-time optimization and a correctness
294 : // transformation. This is necessary for correctness when we have exported
295 : // a reference - we need to convert the linkonce to weak to
296 : // ensure a copy is kept to satisfy the exported reference.
297 : // FIXME: We may want to split the compile time and correctness
298 : // aspects into separate routines.
299 262 : if (isPrevailing(GUID, S.get())) {
300 181 : if (GlobalValue::isLinkOnceLinkage(OriginalLinkage))
301 : S->setLinkage(GlobalValue::getWeakLinkage(
302 : GlobalValue::isLinkOnceODRLinkage(OriginalLinkage)));
303 : }
304 : // Alias and aliasee can't be turned into available_externally.
305 81 : else if (!isa<AliasSummary>(S.get()) &&
306 : !GlobalInvolvedWithAlias.count(S.get()))
307 : S->setLinkage(GlobalValue::AvailableExternallyLinkage);
308 262 : if (S->linkage() != OriginalLinkage)
309 126 : recordNewLinkage(S->modulePath(), GUID, S->linkage());
310 : }
311 849 : }
312 :
313 : // Resolve Weak and LinkOnce values in the \p Index.
314 : //
315 : // We'd like to drop these functions if they are no longer referenced in the
316 : // current module. However there is a chance that another module is still
317 : // referencing them because of the import. We make sure we always emit at least
318 : // one copy.
319 233 : void llvm::thinLTOResolveWeakForLinkerInIndex(
320 : ModuleSummaryIndex &Index,
321 : function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
322 : isPrevailing,
323 : function_ref<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)>
324 : recordNewLinkage) {
325 : // We won't optimize the globals that are referenced by an alias for now
326 : // Ideally we should turn the alias into a global and duplicate the definition
327 : // when needed.
328 : DenseSet<GlobalValueSummary *> GlobalInvolvedWithAlias;
329 1082 : for (auto &I : Index)
330 1698 : for (auto &S : I.second.SummaryList)
331 : if (auto AS = dyn_cast<AliasSummary>(S.get()))
332 169 : GlobalInvolvedWithAlias.insert(&AS->getAliasee());
333 :
334 1082 : for (auto &I : Index)
335 849 : thinLTOResolveWeakForLinkerGUID(I.second.SummaryList, I.first,
336 : GlobalInvolvedWithAlias, isPrevailing,
337 : recordNewLinkage);
338 233 : }
339 :
340 862 : static void thinLTOInternalizeAndPromoteGUID(
341 : GlobalValueSummaryList &GVSummaryList, GlobalValue::GUID GUID,
342 : function_ref<bool(StringRef, GlobalValue::GUID)> isExported) {
343 1732 : for (auto &S : GVSummaryList) {
344 870 : if (isExported(S->modulePath(), GUID)) {
345 : if (GlobalValue::isLocalLinkage(S->linkage()))
346 : S->setLinkage(GlobalValue::ExternalLinkage);
347 : } else if (!GlobalValue::isLocalLinkage(S->linkage()))
348 : S->setLinkage(GlobalValue::InternalLinkage);
349 : }
350 862 : }
351 :
352 : // Update the linkages in the given \p Index to mark exported values
353 : // as external and non-exported values as internal.
354 236 : void llvm::thinLTOInternalizeAndPromoteInIndex(
355 : ModuleSummaryIndex &Index,
356 : function_ref<bool(StringRef, GlobalValue::GUID)> isExported) {
357 1098 : for (auto &I : Index)
358 862 : thinLTOInternalizeAndPromoteGUID(I.second.SummaryList, I.first, isExported);
359 236 : }
360 :
361 : // Requires a destructor for std::vector<InputModule>.
362 : InputFile::~InputFile() = default;
363 :
364 918 : Expected<std::unique_ptr<InputFile>> InputFile::create(MemoryBufferRef Object) {
365 1836 : std::unique_ptr<InputFile> File(new InputFile);
366 :
367 1836 : Expected<IRSymtabFile> FOrErr = readIRSymtab(Object);
368 918 : if (!FOrErr)
369 : return FOrErr.takeError();
370 :
371 907 : File->TargetTriple = FOrErr->TheReader.getTargetTriple();
372 907 : File->SourceFileName = FOrErr->TheReader.getSourceFileName();
373 907 : File->COFFLinkerOpts = FOrErr->TheReader.getCOFFLinkerOpts();
374 1814 : File->ComdatTable = FOrErr->TheReader.getComdatTable();
375 :
376 3666 : for (unsigned I = 0; I != FOrErr->Mods.size(); ++I) {
377 926 : size_t Begin = File->Symbols.size();
378 : for (const irsymtab::Reader::SymbolRef &Sym :
379 3852 : FOrErr->TheReader.module_symbols(I))
380 : // Skip symbols that are irrelevant to LTO. Note that this condition needs
381 : // to match the one in Skip() in LTO::addRegularLTO().
382 4000 : if (Sym.isGlobal() && !Sym.isFormatSpecific())
383 1819 : File->Symbols.push_back(Sym);
384 926 : File->ModuleSymIndices.push_back({Begin, File->Symbols.size()});
385 : }
386 :
387 907 : File->Mods = FOrErr->Mods;
388 : File->Strtab = std::move(FOrErr->Strtab);
389 : return std::move(File);
390 : }
391 :
392 291 : StringRef InputFile::getName() const {
393 291 : return Mods[0].getModuleIdentifier();
394 : }
395 :
396 486 : LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel,
397 486 : Config &Conf)
398 : : ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel),
399 : Ctx(Conf), CombinedModule(llvm::make_unique<Module>("ld-temp.o", Ctx)),
400 486 : Mover(llvm::make_unique<IRMover>(*CombinedModule)) {}
401 :
402 486 : LTO::ThinLTOState::ThinLTOState(ThinBackend Backend)
403 1155 : : Backend(Backend), CombinedIndex(/*HaveGVs*/ false) {
404 486 : if (!Backend)
405 : this->Backend =
406 606 : createInProcessThinBackend(llvm::heavyweight_hardware_concurrency());
407 486 : }
408 :
409 486 : LTO::LTO(Config Conf, ThinBackend Backend,
410 486 : unsigned ParallelCodeGenParallelismLevel)
411 : : Conf(std::move(Conf)),
412 : RegularLTO(ParallelCodeGenParallelismLevel, this->Conf),
413 486 : ThinLTO(std::move(Backend)) {}
414 :
415 : // Requires a destructor for MapVector<BitcodeModule>.
416 : LTO::~LTO() = default;
417 :
418 : // Add the symbols in the given module to the GlobalResolutions map, and resolve
419 : // their partitions.
420 726 : void LTO::addModuleToGlobalRes(ArrayRef<InputFile::Symbol> Syms,
421 : ArrayRef<SymbolResolution> Res,
422 : unsigned Partition, bool InSummary) {
423 : auto *ResI = Res.begin();
424 : auto *ResE = Res.end();
425 : (void)ResE;
426 2131 : for (const InputFile::Symbol &Sym : Syms) {
427 : assert(ResI != ResE);
428 1405 : SymbolResolution Res = *ResI++;
429 :
430 1405 : StringRef Name = Sym.getName();
431 1405 : Triple TT(RegularLTO.CombinedModule->getTargetTriple());
432 : // Strip the __imp_ prefix from COFF dllimport symbols (similar to the
433 : // way they are handled by lld), otherwise we can end up with two
434 : // global resolutions (one with and one for a copy of the symbol without).
435 1405 : if (TT.isOSBinFormatCOFF() && Name.startswith("__imp_"))
436 3 : Name = Name.substr(strlen("__imp_"));
437 1405 : auto &GlobalRes = GlobalResolutions[Name];
438 1405 : GlobalRes.UnnamedAddr &= Sym.isUnnamedAddr();
439 1405 : if (Res.Prevailing) {
440 : assert(!GlobalRes.Prevailing &&
441 : "Multiple prevailing defs are not allowed");
442 973 : GlobalRes.Prevailing = true;
443 1946 : GlobalRes.IRName = Sym.getIRName();
444 432 : } else if (!GlobalRes.Prevailing && GlobalRes.IRName.empty()) {
445 : // Sometimes it can be two copies of symbol in a module and prevailing
446 : // symbol can have no IR name. That might happen if symbol is defined in
447 : // module level inline asm block. In case we have multiple modules with
448 : // the same symbol we want to use IR name of the prevailing symbol.
449 : // Otherwise, if we haven't seen a prevailing symbol, set the name so that
450 : // we can later use it to check if there is any prevailing copy in IR.
451 616 : GlobalRes.IRName = Sym.getIRName();
452 : }
453 :
454 : // Set the partition to external if we know it is re-defined by the linker
455 : // with -defsym or -wrap options, used elsewhere, e.g. it is visible to a
456 : // regular object, is referenced from llvm.compiler_used, or was already
457 : // recorded as being referenced from a different partition.
458 1405 : if (Res.LinkerRedefined || Res.VisibleToRegularObj || Sym.isUsed() ||
459 549 : (GlobalRes.Partition != GlobalResolution::Unknown &&
460 : GlobalRes.Partition != Partition)) {
461 988 : GlobalRes.Partition = GlobalResolution::External;
462 : } else
463 : // First recorded reference, save the current partition.
464 417 : GlobalRes.Partition = Partition;
465 :
466 : // Flag as visible outside of summary if visible from a regular object or
467 : // from a module that does not have a summary.
468 1405 : GlobalRes.VisibleOutsideSummary |=
469 1405 : (Res.VisibleToRegularObj || Sym.isUsed() || !InSummary);
470 : }
471 726 : }
472 :
473 0 : static void writeToResolutionFile(raw_ostream &OS, InputFile *Input,
474 : ArrayRef<SymbolResolution> Res) {
475 0 : StringRef Path = Input->getName();
476 0 : OS << Path << '\n';
477 0 : auto ResI = Res.begin();
478 0 : for (const InputFile::Symbol &Sym : Input->symbols()) {
479 : assert(ResI != Res.end());
480 0 : SymbolResolution Res = *ResI++;
481 :
482 0 : OS << "-r=" << Path << ',' << Sym.getName() << ',';
483 0 : if (Res.Prevailing)
484 : OS << 'p';
485 0 : if (Res.FinalDefinitionInLinkageUnit)
486 : OS << 'l';
487 0 : if (Res.VisibleToRegularObj)
488 : OS << 'x';
489 0 : if (Res.LinkerRedefined)
490 : OS << 'r';
491 : OS << '\n';
492 : }
493 : OS.flush();
494 : assert(ResI == Res.end());
495 0 : }
496 :
497 707 : Error LTO::add(std::unique_ptr<InputFile> Input,
498 : ArrayRef<SymbolResolution> Res) {
499 : assert(!CalledGetMaxTasks);
500 :
501 707 : if (Conf.ResolutionFile)
502 265 : writeToResolutionFile(*Conf.ResolutionFile, Input.get(), Res);
503 :
504 707 : if (RegularLTO.CombinedModule->getTargetTriple().empty())
505 496 : RegularLTO.CombinedModule->setTargetTriple(Input->getTargetTriple());
506 :
507 707 : const SymbolResolution *ResI = Res.begin();
508 2843 : for (unsigned I = 0; I != Input->Mods.size(); ++I)
509 2178 : if (Error Err = addModule(*Input, I, ResI, Res.end()))
510 : return Err;
511 :
512 : assert(ResI == Res.end());
513 : return Error::success();
514 : }
515 :
516 726 : Error LTO::addModule(InputFile &Input, unsigned ModI,
517 : const SymbolResolution *&ResI,
518 : const SymbolResolution *ResE) {
519 1452 : Expected<BitcodeLTOInfo> LTOInfo = Input.Mods[ModI].getLTOInfo();
520 726 : if (!LTOInfo)
521 : return LTOInfo.takeError();
522 :
523 1452 : BitcodeModule BM = Input.Mods[ModI];
524 726 : auto ModSyms = Input.module_symbols(ModI);
525 1452 : addModuleToGlobalRes(ModSyms, {ResI, ResE},
526 315 : LTOInfo->IsThinLTO ? ThinLTO.ModuleMap.size() + 1 : 0,
527 726 : LTOInfo->HasSummary);
528 :
529 726 : if (LTOInfo->IsThinLTO)
530 315 : return addThinLTO(BM, ModSyms, ResI, ResE);
531 :
532 : Expected<RegularLTOState::AddedModule> ModOrErr =
533 822 : addRegularLTO(BM, ModSyms, ResI, ResE);
534 411 : if (!ModOrErr)
535 : return ModOrErr.takeError();
536 :
537 411 : if (!LTOInfo->HasSummary)
538 387 : return linkRegularLTO(std::move(*ModOrErr), /*LivenessFromIndex=*/false);
539 :
540 : // Regular LTO module summaries are added to a dummy module that represents
541 : // the combined regular LTO module.
542 24 : if (Error Err = BM.readSummary(ThinLTO.CombinedIndex, "", -1ull))
543 : return Err;
544 24 : RegularLTO.ModsWithSummaries.push_back(std::move(*ModOrErr));
545 : return Error::success();
546 : }
547 :
548 : // Checks whether the given global value is in a non-prevailing comdat
549 : // (comdat containing values the linker indicated were not prevailing,
550 : // which we then dropped to available_externally), and if so, removes
551 : // it from the comdat. This is called for all global values to ensure the
552 : // comdat is empty rather than leaving an incomplete comdat. It is needed for
553 : // regular LTO modules, in case we are in a mixed-LTO mode (both regular
554 : // and thin LTO modules) compilation. Since the regular LTO module will be
555 : // linked first in the final native link, we want to make sure the linker
556 : // doesn't select any of these incomplete comdats that would be left
557 : // in the regular LTO module without this cleanup.
558 : static void
559 111 : handleNonPrevailingComdat(GlobalValue &GV,
560 : std::set<const Comdat *> &NonPrevailingComdats) {
561 : Comdat *C = GV.getComdat();
562 111 : if (!C)
563 : return;
564 :
565 57 : if (!NonPrevailingComdats.count(C))
566 53 : return;
567 :
568 : // Additionally need to drop externally visible global values from the comdat
569 : // to available_externally, so that there aren't multiply defined linker
570 : // errors.
571 : if (!GV.hasLocalLinkage())
572 : GV.setLinkage(GlobalValue::AvailableExternallyLinkage);
573 :
574 : if (auto GO = dyn_cast<GlobalObject>(&GV))
575 : GO->setComdat(nullptr);
576 : }
577 :
578 : // Add a regular LTO object to the link.
579 : // The resulting module needs to be linked into the combined LTO module with
580 : // linkRegularLTO.
581 : Expected<LTO::RegularLTOState::AddedModule>
582 411 : LTO::addRegularLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
583 : const SymbolResolution *&ResI,
584 : const SymbolResolution *ResE) {
585 411 : RegularLTOState::AddedModule Mod;
586 : Expected<std::unique_ptr<Module>> MOrErr =
587 : BM.getLazyModule(RegularLTO.Ctx, /*ShouldLazyLoadMetadata*/ true,
588 822 : /*IsImporting*/ false);
589 411 : if (!MOrErr)
590 : return MOrErr.takeError();
591 : Module &M = **MOrErr;
592 411 : Mod.M = std::move(*MOrErr);
593 :
594 822 : if (Error Err = M.materializeMetadata())
595 : return std::move(Err);
596 411 : UpgradeDebugInfo(M);
597 :
598 822 : ModuleSymbolTable SymTab;
599 411 : SymTab.addModule(&M);
600 :
601 622 : for (GlobalVariable &GV : M.globals())
602 211 : if (GV.hasAppendingLinkage())
603 13 : Mod.Keep.push_back(&GV);
604 :
605 : DenseSet<GlobalObject *> AliasedGlobals;
606 441 : for (auto &GA : M.aliases())
607 30 : if (GlobalObject *GO = GA.getBaseObject())
608 : AliasedGlobals.insert(GO);
609 :
610 : // In this function we need IR GlobalValues matching the symbols in Syms
611 : // (which is not backed by a module), so we need to enumerate them in the same
612 : // order. The symbol enumeration order of a ModuleSymbolTable intentionally
613 : // matches the order of an irsymtab, but when we read the irsymtab in
614 : // InputFile::create we omit some symbols that are irrelevant to LTO. The
615 : // Skip() function skips the same symbols from the module as InputFile does
616 : // from the symbol table.
617 411 : auto MsymI = SymTab.symbols().begin(), MsymE = SymTab.symbols().end();
618 : auto Skip = [&]() {
619 : while (MsymI != MsymE) {
620 : auto Flags = SymTab.getSymbolFlags(*MsymI);
621 : if ((Flags & object::BasicSymbolRef::SF_Global) &&
622 : !(Flags & object::BasicSymbolRef::SF_FormatSpecific))
623 : return;
624 : ++MsymI;
625 : }
626 411 : };
627 411 : Skip();
628 :
629 : std::set<const Comdat *> NonPrevailingComdats;
630 1192 : for (const InputFile::Symbol &Sym : Syms) {
631 : assert(ResI != ResE);
632 781 : SymbolResolution Res = *ResI++;
633 :
634 : assert(MsymI != MsymE);
635 781 : ModuleSymbolTable::Symbol Msym = *MsymI++;
636 781 : Skip();
637 :
638 781 : if (GlobalValue *GV = Msym.dyn_cast<GlobalValue *>()) {
639 761 : if (Res.Prevailing) {
640 1126 : if (Sym.isUndefined())
641 5 : continue;
642 558 : Mod.Keep.push_back(GV);
643 : // For symbols re-defined with linker -wrap and -defsym options,
644 : // set the linkage to weak to inhibit IPO. The linkage will be
645 : // restored by the linker.
646 558 : if (Res.LinkerRedefined)
647 5 : GV->setLinkage(GlobalValue::WeakAnyLinkage);
648 :
649 558 : GlobalValue::LinkageTypes OriginalLinkage = GV->getLinkage();
650 558 : if (GlobalValue::isLinkOnceLinkage(OriginalLinkage))
651 : GV->setLinkage(GlobalValue::getWeakLinkage(
652 : GlobalValue::isLinkOnceODRLinkage(OriginalLinkage)));
653 192 : } else if (isa<GlobalObject>(GV) &&
654 176 : (GV->hasLinkOnceODRLinkage() || GV->hasWeakODRLinkage() ||
655 : GV->hasAvailableExternallyLinkage()) &&
656 : !AliasedGlobals.count(cast<GlobalObject>(GV))) {
657 : // Any of the above three types of linkage indicates that the
658 : // chosen prevailing symbol will have the same semantics as this copy of
659 : // the symbol, so we may be able to link it with available_externally
660 : // linkage. We will decide later whether to do that when we link this
661 : // module (in linkRegularLTO), based on whether it is undefined.
662 30 : Mod.Keep.push_back(GV);
663 30 : GV->setLinkage(GlobalValue::AvailableExternallyLinkage);
664 30 : if (GV->hasComdat())
665 10 : NonPrevailingComdats.insert(GV->getComdat());
666 30 : cast<GlobalObject>(GV)->setComdat(nullptr);
667 : }
668 :
669 : // Set the 'local' flag based on the linker resolution for this symbol.
670 756 : if (Res.FinalDefinitionInLinkageUnit)
671 183 : GV->setDSOLocal(true);
672 : }
673 : // Common resolution: collect the maximum size/alignment over all commons.
674 : // We also record if we see an instance of a common as prevailing, so that
675 : // if none is prevailing we can ignore it later.
676 1552 : if (Sym.isCommon()) {
677 : // FIXME: We should figure out what to do about commons defined by asm.
678 : // For now they aren't reported correctly by ModuleSymbolTable.
679 29 : auto &CommonRes = RegularLTO.Commons[Sym.getIRName()];
680 58 : CommonRes.Size = std::max(CommonRes.Size, Sym.getCommonSize());
681 29 : CommonRes.Align = std::max(CommonRes.Align, Sym.getCommonAlignment());
682 29 : CommonRes.Prevailing |= Res.Prevailing;
683 : }
684 :
685 : }
686 411 : if (!M.getComdatSymbolTable().empty())
687 175 : for (GlobalValue &GV : M.global_values())
688 111 : handleNonPrevailingComdat(GV, NonPrevailingComdats);
689 : assert(MsymI == MsymE);
690 : return std::move(Mod);
691 : }
692 :
693 411 : Error LTO::linkRegularLTO(RegularLTOState::AddedModule Mod,
694 : bool LivenessFromIndex) {
695 : std::vector<GlobalValue *> Keep;
696 1012 : for (GlobalValue *GV : Mod.Keep) {
697 601 : if (LivenessFromIndex && !ThinLTO.CombinedIndex.isGUIDLive(GV->getGUID()))
698 : continue;
699 :
700 1192 : if (!GV->hasAvailableExternallyLinkage()) {
701 566 : Keep.push_back(GV);
702 566 : continue;
703 : }
704 :
705 : // Only link available_externally definitions if we don't already have a
706 : // definition.
707 : GlobalValue *CombinedGV =
708 30 : RegularLTO.CombinedModule->getNamedValue(GV->getName());
709 30 : if (CombinedGV && !CombinedGV->isDeclaration())
710 : continue;
711 :
712 14 : Keep.push_back(GV);
713 : }
714 :
715 : return RegularLTO.Mover->move(std::move(Mod.M), Keep,
716 : [](GlobalValue &, IRMover::ValueAdder) {},
717 411 : /* IsPerformingImport */ false);
718 : }
719 :
720 : // Add a ThinLTO module to the link.
721 315 : Error LTO::addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
722 : const SymbolResolution *&ResI,
723 : const SymbolResolution *ResE) {
724 315 : if (Error Err =
725 315 : BM.readSummary(ThinLTO.CombinedIndex, BM.getModuleIdentifier(),
726 315 : ThinLTO.ModuleMap.size()))
727 : return Err;
728 :
729 939 : for (const InputFile::Symbol &Sym : Syms) {
730 : assert(ResI != ResE);
731 624 : SymbolResolution Res = *ResI++;
732 :
733 624 : if (!Sym.getIRName().empty()) {
734 616 : auto GUID = GlobalValue::getGUID(GlobalValue::getGlobalIdentifier(
735 616 : Sym.getIRName(), GlobalValue::ExternalLinkage, ""));
736 616 : if (Res.Prevailing) {
737 395 : ThinLTO.PrevailingModuleForGUID[GUID] = BM.getModuleIdentifier();
738 :
739 : // For linker redefined symbols (via --wrap or --defsym) we want to
740 : // switch the linkage to `weak` to prevent IPOs from happening.
741 : // Find the summary in the module for this very GV and record the new
742 : // linkage so that we can switch it when we import the GV.
743 395 : if (Res.LinkerRedefined)
744 7 : if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(
745 : GUID, BM.getModuleIdentifier()))
746 : S->setLinkage(GlobalValue::WeakAnyLinkage);
747 : }
748 :
749 : // If the linker resolved the symbol to a local definition then mark it
750 : // as local in the summary for the module we are adding.
751 616 : if (Res.FinalDefinitionInLinkageUnit) {
752 294 : if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(
753 : GUID, BM.getModuleIdentifier())) {
754 : S->setDSOLocal(true);
755 : }
756 : }
757 : }
758 : }
759 :
760 630 : if (!ThinLTO.ModuleMap.insert({BM.getModuleIdentifier(), BM}).second)
761 : return make_error<StringError>(
762 : "Expected at most one ThinLTO module per bitcode file",
763 2 : inconvertibleErrorCode());
764 :
765 : return Error::success();
766 : }
767 :
768 337 : unsigned LTO::getMaxTasks() const {
769 337 : CalledGetMaxTasks = true;
770 674 : return RegularLTO.ParallelCodeGenParallelismLevel + ThinLTO.ModuleMap.size();
771 : }
772 :
773 479 : Error LTO::run(AddStreamFn AddStream, NativeObjectCache Cache) {
774 : // Compute "dead" symbols, we don't want to import/export these!
775 : DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
776 : DenseMap<GlobalValue::GUID, PrevailingType> GUIDPrevailingResolutions;
777 2067 : for (auto &Res : GlobalResolutions) {
778 : // Normally resolution have IR name of symbol. We can do nothing here
779 : // otherwise. See comments in GlobalResolution struct for more details.
780 1109 : if (Res.second.IRName.empty())
781 19 : continue;
782 :
783 1090 : GlobalValue::GUID GUID = GlobalValue::getGUID(
784 1090 : GlobalValue::dropLLVMManglingEscape(Res.second.IRName));
785 :
786 1090 : if (Res.second.VisibleOutsideSummary && Res.second.Prevailing)
787 1648 : GUIDPreservedSymbols.insert(GlobalValue::getGUID(
788 : GlobalValue::dropLLVMManglingEscape(Res.second.IRName)));
789 :
790 1090 : GUIDPrevailingResolutions[GUID] =
791 1090 : Res.second.Prevailing ? PrevailingType::Yes : PrevailingType::No;
792 : }
793 :
794 : auto isPrevailing = [&](GlobalValue::GUID G) {
795 : auto It = GUIDPrevailingResolutions.find(G);
796 : if (It == GUIDPrevailingResolutions.end())
797 : return PrevailingType::Unknown;
798 : return It->second;
799 479 : };
800 479 : computeDeadSymbols(ThinLTO.CombinedIndex, GUIDPreservedSymbols, isPrevailing);
801 :
802 : // Setup output file to emit statistics.
803 : std::unique_ptr<ToolOutputFile> StatsFile = nullptr;
804 478 : if (!Conf.StatsFile.empty()) {
805 0 : EnableStatistics(false);
806 : std::error_code EC;
807 : StatsFile =
808 0 : llvm::make_unique<ToolOutputFile>(Conf.StatsFile, EC, sys::fs::F_None);
809 0 : if (EC)
810 0 : return errorCodeToError(EC);
811 : StatsFile->keep();
812 : }
813 :
814 951 : Error Result = runRegularLTO(AddStream);
815 473 : if (!Result)
816 1000 : Result = runThinLTO(AddStream, Cache);
817 :
818 473 : if (StatsFile)
819 0 : PrintStatisticsJSON(StatsFile->os());
820 :
821 : return Result;
822 : }
823 :
824 478 : Error LTO::runRegularLTO(AddStreamFn AddStream) {
825 502 : for (auto &M : RegularLTO.ModsWithSummaries)
826 24 : if (Error Err = linkRegularLTO(std::move(M),
827 48 : /*LivenessFromIndex=*/true))
828 : return Err;
829 :
830 : // Make sure commons have the right size/alignment: we kept the largest from
831 : // all the prevailing when adding the inputs, and we apply it here.
832 478 : const DataLayout &DL = RegularLTO.CombinedModule->getDataLayout();
833 496 : for (auto &I : RegularLTO.Commons) {
834 18 : if (!I.second.Prevailing)
835 : // Don't do anything if no instance of this common was prevailing.
836 : continue;
837 : GlobalVariable *OldGV = RegularLTO.CombinedModule->getNamedGlobal(I.first);
838 14 : if (OldGV && DL.getTypeAllocSize(OldGV->getValueType()) == I.second.Size) {
839 : // Don't create a new global if the type is already correct, just make
840 : // sure the alignment is correct.
841 10 : OldGV->setAlignment(I.second.Align);
842 10 : continue;
843 : }
844 : ArrayType *Ty =
845 4 : ArrayType::get(Type::getInt8Ty(RegularLTO.Ctx), I.second.Size);
846 : auto *GV = new GlobalVariable(*RegularLTO.CombinedModule, Ty, false,
847 : GlobalValue::CommonLinkage,
848 8 : ConstantAggregateZero::get(Ty), "");
849 4 : GV->setAlignment(I.second.Align);
850 4 : if (OldGV) {
851 8 : OldGV->replaceAllUsesWith(ConstantExpr::getBitCast(GV, OldGV->getType()));
852 4 : GV->takeName(OldGV);
853 4 : OldGV->eraseFromParent();
854 : } else {
855 0 : GV->setName(I.first);
856 : }
857 : }
858 :
859 478 : if (Conf.PreOptModuleHook &&
860 170 : !Conf.PreOptModuleHook(0, *RegularLTO.CombinedModule))
861 : return Error::success();
862 :
863 477 : if (!Conf.CodeGenOnly) {
864 2043 : for (const auto &R : GlobalResolutions) {
865 : if (!R.second.isPrevailingIRSymbol())
866 : continue;
867 938 : if (R.second.Partition != 0 &&
868 : R.second.Partition != GlobalResolution::External)
869 : continue;
870 :
871 : GlobalValue *GV =
872 891 : RegularLTO.CombinedModule->getNamedValue(R.second.IRName);
873 : // Ignore symbols defined in other partitions.
874 : // Also skip declarations, which are not allowed to have internal linkage.
875 1442 : if (!GV || GV->hasLocalLinkage() || GV->isDeclaration())
876 355 : continue;
877 536 : GV->setUnnamedAddr(R.second.UnnamedAddr ? GlobalValue::UnnamedAddr::Global
878 : : GlobalValue::UnnamedAddr::None);
879 536 : if (R.second.Partition == 0)
880 : GV->setLinkage(GlobalValue::InternalLinkage);
881 : }
882 :
883 477 : if (Conf.PostInternalizeModuleHook &&
884 198 : !Conf.PostInternalizeModuleHook(0, *RegularLTO.CombinedModule))
885 : return Error::success();
886 : }
887 448 : return backend(Conf, AddStream, RegularLTO.ParallelCodeGenParallelismLevel,
888 1787 : std::move(RegularLTO.CombinedModule), ThinLTO.CombinedIndex);
889 : }
890 :
891 : /// This class defines the interface to the ThinLTO backend.
892 : class lto::ThinBackendProc {
893 : protected:
894 : Config &Conf;
895 : ModuleSummaryIndex &CombinedIndex;
896 : const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries;
897 :
898 : public:
899 : ThinBackendProc(Config &Conf, ModuleSummaryIndex &CombinedIndex,
900 : const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries)
901 187 : : Conf(Conf), CombinedIndex(CombinedIndex),
902 187 : ModuleToDefinedGVSummaries(ModuleToDefinedGVSummaries) {}
903 :
904 0 : virtual ~ThinBackendProc() {}
905 : virtual Error start(
906 : unsigned Task, BitcodeModule BM,
907 : const FunctionImporter::ImportMapTy &ImportList,
908 : const FunctionImporter::ExportSetTy &ExportList,
909 : const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
910 : MapVector<StringRef, BitcodeModule> &ModuleMap) = 0;
911 : virtual Error wait() = 0;
912 : };
913 :
914 : namespace {
915 0 : class InProcessThinBackend : public ThinBackendProc {
916 : ThreadPool BackendThreadPool;
917 : AddStreamFn AddStream;
918 : NativeObjectCache Cache;
919 : std::set<GlobalValue::GUID> CfiFunctionDefs;
920 : std::set<GlobalValue::GUID> CfiFunctionDecls;
921 :
922 : Optional<Error> Err;
923 : std::mutex ErrMu;
924 :
925 : public:
926 153 : InProcessThinBackend(
927 : Config &Conf, ModuleSummaryIndex &CombinedIndex,
928 : unsigned ThinLTOParallelismLevel,
929 : const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
930 : AddStreamFn AddStream, NativeObjectCache Cache)
931 153 : : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries),
932 : BackendThreadPool(ThinLTOParallelismLevel),
933 153 : AddStream(std::move(AddStream)), Cache(std::move(Cache)) {
934 155 : for (auto &Name : CombinedIndex.cfiFunctionDefs())
935 : CfiFunctionDefs.insert(
936 4 : GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
937 154 : for (auto &Name : CombinedIndex.cfiFunctionDecls())
938 : CfiFunctionDecls.insert(
939 2 : GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
940 153 : }
941 :
942 255 : Error runThinLTOBackendThread(
943 : AddStreamFn AddStream, NativeObjectCache Cache, unsigned Task,
944 : BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
945 : const FunctionImporter::ImportMapTy &ImportList,
946 : const FunctionImporter::ExportSetTy &ExportList,
947 : const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
948 : const GVSummaryMapTy &DefinedGlobals,
949 : MapVector<StringRef, BitcodeModule> &ModuleMap) {
950 : auto RunThinBackend = [&](AddStreamFn AddStream) {
951 : LTOLLVMContext BackendContext(Conf);
952 : Expected<std::unique_ptr<Module>> MOrErr = BM.parseModule(BackendContext);
953 : if (!MOrErr)
954 : return MOrErr.takeError();
955 :
956 : return thinBackend(Conf, Task, AddStream, **MOrErr, CombinedIndex,
957 : ImportList, DefinedGlobals, ModuleMap);
958 255 : };
959 :
960 : auto ModuleID = BM.getModuleIdentifier();
961 :
962 342 : if (!Cache || !CombinedIndex.modulePaths().count(ModuleID) ||
963 87 : all_of(CombinedIndex.getModuleHash(ModuleID),
964 : [](uint32_t V) { return V == 0; }))
965 : // Cache disabled or no entry for this module in the combined index or
966 : // no module hash.
967 350 : return RunThinBackend(AddStream);
968 :
969 : SmallString<40> Key;
970 : // The module may be cached, this helps handling it.
971 80 : computeCacheKey(Key, Conf, CombinedIndex, ModuleID, ImportList, ExportList,
972 80 : ResolvedODR, DefinedGlobals, CfiFunctionDefs,
973 80 : CfiFunctionDecls);
974 240 : if (AddStreamFn CacheAddStream = Cache(Task, Key))
975 112 : return RunThinBackend(CacheAddStream);
976 :
977 : return Error::success();
978 : }
979 :
980 255 : Error start(
981 : unsigned Task, BitcodeModule BM,
982 : const FunctionImporter::ImportMapTy &ImportList,
983 : const FunctionImporter::ExportSetTy &ExportList,
984 : const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
985 : MapVector<StringRef, BitcodeModule> &ModuleMap) override {
986 255 : StringRef ModulePath = BM.getModuleIdentifier();
987 : assert(ModuleToDefinedGVSummaries.count(ModulePath));
988 : const GVSummaryMapTy &DefinedGlobals =
989 255 : ModuleToDefinedGVSummaries.find(ModulePath)->second;
990 765 : BackendThreadPool.async(
991 : [=](BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
992 : const FunctionImporter::ImportMapTy &ImportList,
993 : const FunctionImporter::ExportSetTy &ExportList,
994 : const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>
995 : &ResolvedODR,
996 : const GVSummaryMapTy &DefinedGlobals,
997 : MapVector<StringRef, BitcodeModule> &ModuleMap) {
998 : Error E = runThinLTOBackendThread(
999 : AddStream, Cache, Task, BM, CombinedIndex, ImportList, ExportList,
1000 : ResolvedODR, DefinedGlobals, ModuleMap);
1001 : if (E) {
1002 : std::unique_lock<std::mutex> L(ErrMu);
1003 : if (Err)
1004 : Err = joinErrors(std::move(*Err), std::move(E));
1005 : else
1006 : Err = std::move(E);
1007 : }
1008 : },
1009 510 : BM, std::ref(CombinedIndex), std::ref(ImportList), std::ref(ExportList),
1010 255 : std::ref(ResolvedODR), std::ref(DefinedGlobals), std::ref(ModuleMap));
1011 255 : return Error::success();
1012 : }
1013 :
1014 153 : Error wait() override {
1015 153 : BackendThreadPool.wait();
1016 153 : if (Err)
1017 : return std::move(*Err);
1018 : else
1019 : return Error::success();
1020 : }
1021 : };
1022 : } // end anonymous namespace
1023 :
1024 450 : ThinBackend lto::createInProcessThinBackend(unsigned ParallelismLevel) {
1025 : return [=](Config &Conf, ModuleSummaryIndex &CombinedIndex,
1026 : const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1027 : AddStreamFn AddStream, NativeObjectCache Cache) {
1028 : return llvm::make_unique<InProcessThinBackend>(
1029 : Conf, CombinedIndex, ParallelismLevel, ModuleToDefinedGVSummaries,
1030 153 : AddStream, Cache);
1031 450 : };
1032 : }
1033 :
1034 : // Given the original \p Path to an output file, replace any path
1035 : // prefix matching \p OldPrefix with \p NewPrefix. Also, create the
1036 : // resulting directory if it does not yet exist.
1037 67 : std::string lto::getThinLTOOutputFile(const std::string &Path,
1038 : const std::string &OldPrefix,
1039 : const std::string &NewPrefix) {
1040 67 : if (OldPrefix.empty() && NewPrefix.empty())
1041 : return Path;
1042 : SmallString<128> NewPath(Path);
1043 2 : llvm::sys::path::replace_path_prefix(NewPath, OldPrefix, NewPrefix);
1044 2 : StringRef ParentPath = llvm::sys::path::parent_path(NewPath.str());
1045 2 : if (!ParentPath.empty()) {
1046 : // Make sure the new directory exists, creating it if necessary.
1047 2 : if (std::error_code EC = llvm::sys::fs::create_directories(ParentPath))
1048 0 : llvm::errs() << "warning: could not create directory '" << ParentPath
1049 0 : << "': " << EC.message() << '\n';
1050 : }
1051 : return NewPath.str();
1052 : }
1053 :
1054 : namespace {
1055 0 : class WriteIndexesThinBackend : public ThinBackendProc {
1056 : std::string OldPrefix, NewPrefix;
1057 : bool ShouldEmitImportsFiles;
1058 : raw_fd_ostream *LinkedObjectsFile;
1059 : lto::IndexWriteCallback OnWrite;
1060 :
1061 : public:
1062 34 : WriteIndexesThinBackend(
1063 : Config &Conf, ModuleSummaryIndex &CombinedIndex,
1064 : const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1065 : std::string OldPrefix, std::string NewPrefix, bool ShouldEmitImportsFiles,
1066 : raw_fd_ostream *LinkedObjectsFile, lto::IndexWriteCallback OnWrite)
1067 34 : : ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries),
1068 : OldPrefix(OldPrefix), NewPrefix(NewPrefix),
1069 : ShouldEmitImportsFiles(ShouldEmitImportsFiles),
1070 34 : LinkedObjectsFile(LinkedObjectsFile), OnWrite(OnWrite) {}
1071 :
1072 56 : Error start(
1073 : unsigned Task, BitcodeModule BM,
1074 : const FunctionImporter::ImportMapTy &ImportList,
1075 : const FunctionImporter::ExportSetTy &ExportList,
1076 : const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
1077 : MapVector<StringRef, BitcodeModule> &ModuleMap) override {
1078 : StringRef ModulePath = BM.getModuleIdentifier();
1079 : std::string NewModulePath =
1080 112 : getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix);
1081 :
1082 56 : if (LinkedObjectsFile)
1083 6 : *LinkedObjectsFile << NewModulePath << '\n';
1084 :
1085 : std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
1086 56 : gatherImportedSummariesForModule(ModulePath, ModuleToDefinedGVSummaries,
1087 : ImportList, ModuleToSummariesForIndex);
1088 :
1089 : std::error_code EC;
1090 56 : raw_fd_ostream OS(NewModulePath + ".thinlto.bc", EC,
1091 112 : sys::fs::OpenFlags::F_None);
1092 56 : if (EC)
1093 1 : return errorCodeToError(EC);
1094 55 : WriteIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex);
1095 :
1096 55 : if (ShouldEmitImportsFiles) {
1097 29 : EC = EmitImportsFiles(ModulePath, NewModulePath + ".imports",
1098 29 : ModuleToSummariesForIndex);
1099 29 : if (EC)
1100 1 : return errorCodeToError(EC);
1101 : }
1102 :
1103 54 : if (OnWrite)
1104 36 : OnWrite(ModulePath);
1105 : return Error::success();
1106 : }
1107 :
1108 64 : Error wait() override { return Error::success(); }
1109 : };
1110 : } // end anonymous namespace
1111 :
1112 37 : ThinBackend lto::createWriteIndexesThinBackend(
1113 : std::string OldPrefix, std::string NewPrefix, bool ShouldEmitImportsFiles,
1114 : raw_fd_ostream *LinkedObjectsFile, IndexWriteCallback OnWrite) {
1115 : return [=](Config &Conf, ModuleSummaryIndex &CombinedIndex,
1116 : const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
1117 : AddStreamFn AddStream, NativeObjectCache Cache) {
1118 : return llvm::make_unique<WriteIndexesThinBackend>(
1119 34 : Conf, CombinedIndex, ModuleToDefinedGVSummaries, OldPrefix, NewPrefix,
1120 34 : ShouldEmitImportsFiles, LinkedObjectsFile, OnWrite);
1121 62 : };
1122 : }
1123 :
1124 473 : Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache) {
1125 473 : if (ThinLTO.ModuleMap.empty())
1126 : return Error::success();
1127 :
1128 187 : if (Conf.CombinedIndexHook && !Conf.CombinedIndexHook(ThinLTO.CombinedIndex))
1129 : return Error::success();
1130 :
1131 : // Collect for each module the list of function it defines (GUID ->
1132 : // Summary).
1133 : StringMap<GVSummaryMapTy>
1134 374 : ModuleToDefinedGVSummaries(ThinLTO.ModuleMap.size());
1135 187 : ThinLTO.CombinedIndex.collectDefinedGVSummariesPerModule(
1136 : ModuleToDefinedGVSummaries);
1137 : // Create entries for any modules that didn't have any GV summaries
1138 : // (either they didn't have any GVs to start with, or we suppressed
1139 : // generation of the summaries because they e.g. had inline assembly
1140 : // uses that couldn't be promoted/renamed on export). This is so
1141 : // InProcessThinBackend::start can still launch a backend thread, which
1142 : // is passed the map of summaries for the module, without any special
1143 : // handling for this case.
1144 498 : for (auto &Mod : ThinLTO.ModuleMap)
1145 296 : if (!ModuleToDefinedGVSummaries.count(Mod.first))
1146 15 : ModuleToDefinedGVSummaries.try_emplace(Mod.first);
1147 :
1148 : StringMap<FunctionImporter::ImportMapTy> ImportLists(
1149 374 : ThinLTO.ModuleMap.size());
1150 : StringMap<FunctionImporter::ExportSetTy> ExportLists(
1151 374 : ThinLTO.ModuleMap.size());
1152 187 : StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
1153 :
1154 187 : if (DumpThinCGSCCs)
1155 1 : ThinLTO.CombinedIndex.dumpSCCs(outs());
1156 :
1157 187 : if (Conf.OptLevel > 0)
1158 183 : ComputeCrossModuleImport(ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
1159 : ImportLists, ExportLists);
1160 :
1161 : // Figure out which symbols need to be internalized. This also needs to happen
1162 : // at -O0 because summary-based DCE is implemented using internalization, and
1163 : // we must apply DCE consistently with the full LTO module in order to avoid
1164 : // undefined references during the final link.
1165 : std::set<GlobalValue::GUID> ExportedGUIDs;
1166 851 : for (auto &Res : GlobalResolutions) {
1167 : // If the symbol does not have external references or it is not prevailing,
1168 : // then not need to mark it as exported from a ThinLTO partition.
1169 477 : if (Res.second.Partition != GlobalResolution::External ||
1170 : !Res.second.isPrevailingIRSymbol())
1171 95 : continue;
1172 382 : auto GUID = GlobalValue::getGUID(
1173 382 : GlobalValue::dropLLVMManglingEscape(Res.second.IRName));
1174 : // Mark exported unless index-based analysis determined it to be dead.
1175 382 : if (ThinLTO.CombinedIndex.isGUIDLive(GUID))
1176 : ExportedGUIDs.insert(GUID);
1177 : }
1178 :
1179 : // Any functions referenced by the jump table in the regular LTO object must
1180 : // be exported.
1181 190 : for (auto &Def : ThinLTO.CombinedIndex.cfiFunctionDefs())
1182 : ExportedGUIDs.insert(
1183 6 : GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Def)));
1184 :
1185 : auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
1186 : const auto &ExportList = ExportLists.find(ModuleIdentifier);
1187 : return (ExportList != ExportLists.end() &&
1188 : ExportList->second.count(GUID)) ||
1189 : ExportedGUIDs.count(GUID);
1190 187 : };
1191 187 : thinLTOInternalizeAndPromoteInIndex(ThinLTO.CombinedIndex, isExported);
1192 :
1193 : auto isPrevailing = [&](GlobalValue::GUID GUID,
1194 : const GlobalValueSummary *S) {
1195 : return ThinLTO.PrevailingModuleForGUID[GUID] == S->modulePath();
1196 187 : };
1197 : auto recordNewLinkage = [&](StringRef ModuleIdentifier,
1198 : GlobalValue::GUID GUID,
1199 : GlobalValue::LinkageTypes NewLinkage) {
1200 : ResolvedODR[ModuleIdentifier][GUID] = NewLinkage;
1201 187 : };
1202 187 : thinLTOResolveWeakForLinkerInIndex(ThinLTO.CombinedIndex, isPrevailing,
1203 : recordNewLinkage);
1204 :
1205 : std::unique_ptr<ThinBackendProc> BackendProc =
1206 187 : ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
1207 187 : AddStream, Cache);
1208 :
1209 : // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for combined
1210 : // module and parallel code generation partitions.
1211 187 : unsigned Task = RegularLTO.ParallelCodeGenParallelismLevel;
1212 496 : for (auto &Mod : ThinLTO.ModuleMap) {
1213 311 : if (Error E = BackendProc->start(Task, Mod.second, ImportLists[Mod.first],
1214 311 : ExportLists[Mod.first],
1215 622 : ResolvedODR[Mod.first], ThinLTO.ModuleMap))
1216 : return E;
1217 309 : ++Task;
1218 : }
1219 :
1220 185 : return BackendProc->wait();
1221 : }
1222 :
1223 : Expected<std::unique_ptr<ToolOutputFile>>
1224 793 : lto::setupOptimizationRemarks(LLVMContext &Context,
1225 : StringRef LTORemarksFilename,
1226 : bool LTOPassRemarksWithHotness, int Count) {
1227 793 : if (LTOPassRemarksWithHotness)
1228 9 : Context.setDiagnosticsHotnessRequested(true);
1229 793 : if (LTORemarksFilename.empty())
1230 : return nullptr;
1231 :
1232 : std::string Filename = LTORemarksFilename;
1233 15 : if (Count != -1)
1234 10 : Filename += ".thin." + llvm::utostr(Count) + ".yaml";
1235 :
1236 : std::error_code EC;
1237 : auto DiagnosticFile =
1238 15 : llvm::make_unique<ToolOutputFile>(Filename, EC, sys::fs::F_None);
1239 15 : if (EC)
1240 0 : return errorCodeToError(EC);
1241 15 : Context.setDiagnosticsOutputFile(
1242 30 : llvm::make_unique<yaml::Output>(DiagnosticFile->os()));
1243 : DiagnosticFile->keep();
1244 : return std::move(DiagnosticFile);
1245 : }
|
