LLVM  6.0.0svn
LTO.h
Go to the documentation of this file.
1 //===-LTO.h - LLVM Link Time Optimizer ------------------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file declares functions and classes used to support LTO. It is intended
11 // to be used both by LTO classes as well as by clients (gold-plugin) that
12 // don't utilize the LTO code generator interfaces.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #ifndef LLVM_LTO_LTO_H
17 #define LLVM_LTO_LTO_H
18 
19 #include "llvm/ADT/MapVector.h"
20 #include "llvm/ADT/StringMap.h"
21 #include "llvm/ADT/StringSet.h"
23 #include "llvm/IR/DiagnosticInfo.h"
25 #include "llvm/LTO/Config.h"
26 #include "llvm/Linker/IRMover.h"
27 #include "llvm/Object/IRSymtab.h"
28 #include "llvm/Support/Error.h"
30 #include "llvm/Support/thread.h"
33 
34 namespace llvm {
35 
36 class BitcodeModule;
37 class Error;
38 class LLVMContext;
39 class MemoryBufferRef;
40 class Module;
41 class Target;
42 class raw_pwrite_stream;
43 
44 /// Resolve Weak and LinkOnce values in the \p Index. Linkage changes recorded
45 /// in the index and the ThinLTO backends must apply the changes to the Module
46 /// via thinLTOResolveWeakForLinkerModule.
47 ///
48 /// This is done for correctness (if value exported, ensure we always
49 /// emit a copy), and compile-time optimization (allow drop of duplicates).
51  ModuleSummaryIndex &Index,
52  function_ref<bool(GlobalValue::GUID, const GlobalValueSummary *)>
53  isPrevailing,
54  function_ref<void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)>
55  recordNewLinkage);
56 
57 /// Update the linkages in the given \p Index to mark exported values
58 /// as external and non-exported values as internal. The ThinLTO backends
59 /// must apply the changes to the Module via thinLTOInternalizeModule.
61  ModuleSummaryIndex &Index,
62  function_ref<bool(StringRef, GlobalValue::GUID)> isExported);
63 
64 namespace lto {
65 
66 /// Given the original \p Path to an output file, replace any path
67 /// prefix matching \p OldPrefix with \p NewPrefix. Also, create the
68 /// resulting directory if it does not yet exist.
69 std::string getThinLTOOutputFile(const std::string &Path,
70  const std::string &OldPrefix,
71  const std::string &NewPrefix);
72 
73 /// Setup optimization remarks.
74 Expected<std::unique_ptr<ToolOutputFile>>
76  bool LTOPassRemarksWithHotness, int Count = -1);
77 
78 class LTO;
79 struct SymbolResolution;
80 class ThinBackendProc;
81 
82 /// An input file. This is a symbol table wrapper that only exposes the
83 /// information that an LTO client should need in order to do symbol resolution.
84 class InputFile {
85 public:
86  class Symbol;
87 
88 private:
89  // FIXME: Remove LTO class friendship once we have bitcode symbol tables.
90  friend LTO;
91  InputFile() = default;
92 
93  std::vector<BitcodeModule> Mods;
94  SmallVector<char, 0> Strtab;
95  std::vector<Symbol> Symbols;
96 
97  // [begin, end) for each module
98  std::vector<std::pair<size_t, size_t>> ModuleSymIndices;
99 
100  StringRef TargetTriple, SourceFileName, COFFLinkerOpts;
101  std::vector<StringRef> ComdatTable;
102 
103 public:
104  ~InputFile();
105 
106  /// Create an InputFile.
108 
109  /// The purpose of this class is to only expose the symbol information that an
110  /// LTO client should need in order to do symbol resolution.
112  friend LTO;
113 
114  public:
115  Symbol(const irsymtab::Symbol &S) : irsymtab::Symbol(S) {}
116 
131  };
132 
133  /// A range over the symbols in this InputFile.
134  ArrayRef<Symbol> symbols() const { return Symbols; }
135 
136  /// Returns linker options specified in the input file.
137  StringRef getCOFFLinkerOpts() const { return COFFLinkerOpts; }
138 
139  /// Returns the path to the InputFile.
140  StringRef getName() const;
141 
142  /// Returns the input file's target triple.
143  StringRef getTargetTriple() const { return TargetTriple; }
144 
145  /// Returns the source file path specified at compile time.
146  StringRef getSourceFileName() const { return SourceFileName; }
147 
148  // Returns a table with all the comdats used by this file.
149  ArrayRef<StringRef> getComdatTable() const { return ComdatTable; }
150 
151 private:
152  ArrayRef<Symbol> module_symbols(unsigned I) const {
153  const auto &Indices = ModuleSymIndices[I];
154  return {Symbols.data() + Indices.first, Symbols.data() + Indices.second};
155  }
156 };
157 
158 /// This class wraps an output stream for a native object. Most clients should
159 /// just be able to return an instance of this base class from the stream
160 /// callback, but if a client needs to perform some action after the stream is
161 /// written to, that can be done by deriving from this class and overriding the
162 /// destructor.
164 public:
165  NativeObjectStream(std::unique_ptr<raw_pwrite_stream> OS) : OS(std::move(OS)) {}
166  std::unique_ptr<raw_pwrite_stream> OS;
167  virtual ~NativeObjectStream() = default;
168 };
169 
170 /// This type defines the callback to add a native object that is generated on
171 /// the fly.
172 ///
173 /// Stream callbacks must be thread safe.
174 typedef std::function<std::unique_ptr<NativeObjectStream>(unsigned Task)>
176 
177 /// This is the type of a native object cache. To request an item from the
178 /// cache, pass a unique string as the Key. For hits, the cached file will be
179 /// added to the link and this function will return AddStreamFn(). For misses,
180 /// the cache will return a stream callback which must be called at most once to
181 /// produce content for the stream. The native object stream produced by the
182 /// stream callback will add the file to the link after the stream is written
183 /// to.
184 ///
185 /// Clients generally look like this:
186 ///
187 /// if (AddStreamFn AddStream = Cache(Task, Key))
188 /// ProduceContent(AddStream);
189 typedef std::function<AddStreamFn(unsigned Task, StringRef Key)>
191 
192 /// A ThinBackend defines what happens after the thin-link phase during ThinLTO.
193 /// The details of this type definition aren't important; clients can only
194 /// create a ThinBackend using one of the create*ThinBackend() functions below.
195 typedef std::function<std::unique_ptr<ThinBackendProc>(
196  Config &C, ModuleSummaryIndex &CombinedIndex,
197  StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
198  AddStreamFn AddStream, NativeObjectCache Cache)>
200 
201 /// This ThinBackend runs the individual backend jobs in-process.
202 ThinBackend createInProcessThinBackend(unsigned ParallelismLevel);
203 
204 /// This ThinBackend writes individual module indexes to files, instead of
205 /// running the individual backend jobs. This backend is for distributed builds
206 /// where separate processes will invoke the real backends.
207 ///
208 /// To find the path to write the index to, the backend checks if the path has a
209 /// prefix of OldPrefix; if so, it replaces that prefix with NewPrefix. It then
210 /// appends ".thinlto.bc" and writes the index to that path. If
211 /// ShouldEmitImportsFiles is true it also writes a list of imported files to a
212 /// similar path with ".imports" appended instead.
213 ThinBackend createWriteIndexesThinBackend(std::string OldPrefix,
214  std::string NewPrefix,
215  bool ShouldEmitImportsFiles,
216  std::string LinkedObjectsFile);
217 
218 /// This class implements a resolution-based interface to LLVM's LTO
219 /// functionality. It supports regular LTO, parallel LTO code generation and
220 /// ThinLTO. You can use it from a linker in the following way:
221 /// - Set hooks and code generation options (see lto::Config struct defined in
222 /// Config.h), and use the lto::Config object to create an lto::LTO object.
223 /// - Create lto::InputFile objects using lto::InputFile::create(), then use
224 /// the symbols() function to enumerate its symbols and compute a resolution
225 /// for each symbol (see SymbolResolution below).
226 /// - After the linker has visited each input file (and each regular object
227 /// file) and computed a resolution for each symbol, take each lto::InputFile
228 /// and pass it and an array of symbol resolutions to the add() function.
229 /// - Call the getMaxTasks() function to get an upper bound on the number of
230 /// native object files that LTO may add to the link.
231 /// - Call the run() function. This function will use the supplied AddStream
232 /// and Cache functions to add up to getMaxTasks() native object files to
233 /// the link.
234 class LTO {
235  friend InputFile;
236 
237 public:
238  /// Create an LTO object. A default constructed LTO object has a reasonable
239  /// production configuration, but you can customize it by passing arguments to
240  /// this constructor.
241  /// FIXME: We do currently require the DiagHandler field to be set in Conf.
242  /// Until that is fixed, a Config argument is required.
243  LTO(Config Conf, ThinBackend Backend = nullptr,
244  unsigned ParallelCodeGenParallelismLevel = 1);
245  ~LTO();
246 
247  /// Add an input file to the LTO link, using the provided symbol resolutions.
248  /// The symbol resolutions must appear in the enumeration order given by
249  /// InputFile::symbols().
250  Error add(std::unique_ptr<InputFile> Obj, ArrayRef<SymbolResolution> Res);
251 
252  /// Returns an upper bound on the number of tasks that the client may expect.
253  /// This may only be called after all IR object files have been added. For a
254  /// full description of tasks see LTOBackend.h.
255  unsigned getMaxTasks() const;
256 
257  /// Runs the LTO pipeline. This function calls the supplied AddStream
258  /// function to add native object files to the link.
259  ///
260  /// The Cache parameter is optional. If supplied, it will be used to cache
261  /// native object files and add them to the link.
262  ///
263  /// The client will receive at most one callback (via either AddStream or
264  /// Cache) for each task identifier.
265  Error run(AddStreamFn AddStream, NativeObjectCache Cache = nullptr);
266 
267 private:
268  Config Conf;
269 
270  struct RegularLTOState {
271  RegularLTOState(unsigned ParallelCodeGenParallelismLevel, Config &Conf);
273  uint64_t Size = 0;
274  unsigned Align = 0;
275  /// Record if at least one instance of the common was marked as prevailing
276  bool Prevailing = false;
277  };
278  std::map<std::string, CommonResolution> Commons;
279 
280  unsigned ParallelCodeGenParallelismLevel;
281  LTOLLVMContext Ctx;
282  bool HasModule = false;
283  std::unique_ptr<Module> CombinedModule;
284  std::unique_ptr<IRMover> Mover;
285 
286  // This stores the information about a regular LTO module that we have added
287  // to the link. It will either be linked immediately (for modules without
288  // summaries) or after summary-based dead stripping (for modules with
289  // summaries).
290  struct AddedModule {
291  std::unique_ptr<Module> M;
292  std::vector<GlobalValue *> Keep;
293  };
294  std::vector<AddedModule> ModsWithSummaries;
295  } RegularLTO;
296 
297  struct ThinLTOState {
298  ThinLTOState(ThinBackend Backend);
299 
300  ThinBackend Backend;
301  ModuleSummaryIndex CombinedIndex;
303  DenseMap<GlobalValue::GUID, StringRef> PrevailingModuleForGUID;
304  } ThinLTO;
305 
306  // The global resolution for a particular (mangled) symbol name. This is in
307  // particular necessary to track whether each symbol can be internalized.
308  // Because any input file may introduce a new cross-partition reference, we
309  // cannot make any final internalization decisions until all input files have
310  // been added and the client has called run(). During run() we apply
311  // internalization decisions either directly to the module (for regular LTO)
312  // or to the combined index (for ThinLTO).
313  struct GlobalResolution {
314  /// The unmangled name of the global.
315  std::string IRName;
316 
317  /// Keep track if the symbol is visible outside of a module with a summary
318  /// (i.e. in either a regular object or a regular LTO module without a
319  /// summary).
320  bool VisibleOutsideSummary = false;
321 
322  bool UnnamedAddr = true;
323 
324  /// This field keeps track of the partition number of this global. The
325  /// regular LTO object is partition 0, while each ThinLTO object has its own
326  /// partition number from 1 onwards.
327  ///
328  /// Any global that is defined or used by more than one partition, or that
329  /// is referenced externally, may not be internalized.
330  ///
331  /// Partitions generally have a one-to-one correspondence with tasks, except
332  /// that we use partition 0 for all parallel LTO code generation partitions.
333  /// Any partitioning of the combined LTO object is done internally by the
334  /// LTO backend.
335  unsigned Partition = Unknown;
336 
337  /// Special partition numbers.
338  enum : unsigned {
339  /// A partition number has not yet been assigned to this global.
340  Unknown = -1u,
341 
342  /// This global is either used by more than one partition or has an
343  /// external reference, and therefore cannot be internalized.
344  External = -2u,
345 
346  /// The RegularLTO partition
347  RegularLTO = 0,
348  };
349  };
350 
351  // Global mapping from mangled symbol names to resolutions.
352  StringMap<GlobalResolution> GlobalResolutions;
353 
354  void addModuleToGlobalRes(ArrayRef<InputFile::Symbol> Syms,
355  ArrayRef<SymbolResolution> Res, unsigned Partition,
356  bool InSummary);
357 
358  // These functions take a range of symbol resolutions [ResI, ResE) and consume
359  // the resolutions used by a single input module by incrementing ResI. After
360  // these functions return, [ResI, ResE) will refer to the resolution range for
361  // the remaining modules in the InputFile.
362  Error addModule(InputFile &Input, unsigned ModI,
363  const SymbolResolution *&ResI, const SymbolResolution *ResE);
364 
366  addRegularLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
367  const SymbolResolution *&ResI, const SymbolResolution *ResE);
368  Error linkRegularLTO(RegularLTOState::AddedModule Mod,
369  bool LivenessFromIndex);
370 
371  Error addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
372  const SymbolResolution *&ResI, const SymbolResolution *ResE);
373 
374  Error runRegularLTO(AddStreamFn AddStream);
375  Error runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
376  bool HasRegularLTO);
377 
378  mutable bool CalledGetMaxTasks = false;
379 };
380 
381 /// The resolution for a symbol. The linker must provide a SymbolResolution for
382 /// each global symbol based on its internal resolution of that symbol.
385  : Prevailing(0), FinalDefinitionInLinkageUnit(0), VisibleToRegularObj(0),
386  LinkerRedefined(0) {}
387 
388  /// The linker has chosen this definition of the symbol.
389  unsigned Prevailing : 1;
390 
391  /// The definition of this symbol is unpreemptable at runtime and is known to
392  /// be in this linkage unit.
394 
395  /// The definition of this symbol is visible outside of the LTO unit.
396  unsigned VisibleToRegularObj : 1;
397 
398  /// Linker redefined version of the symbol which appeared in -wrap or -defsym
399  /// linker option.
400  unsigned LinkerRedefined : 1;
401 };
402 
403 } // namespace lto
404 } // namespace llvm
405 
406 #endif
NativeObjectStream(std::unique_ptr< raw_pwrite_stream > OS)
Definition: LTO.h:165
uint64_t CallInst * C
GlobalValue::VisibilityTypes getVisibility() const
Definition: IRSymtab.h:186
ArrayRef< Symbol > symbols() const
A range over the symbols in this InputFile.
Definition: LTO.h:134
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
StringRef getCOFFWeakExternalFallback() const
COFF-specific: for weak externals, returns the name of the symbol that is used as a fallback if the w...
Definition: IRSymtab.h:218
ArrayRef< StringRef > getComdatTable() const
Definition: LTO.h:149
Expected< std::unique_ptr< ToolOutputFile > > setupOptimizationRemarks(LLVMContext &Context, StringRef LTORemarksFilename, bool LTOPassRemarksWithHotness, int Count=-1)
Setup optimization remarks.
Definition: LTO.cpp:1184
LLVMContext & Context
unsigned Prevailing
The linker has chosen this definition of the symbol.
Definition: LTO.h:389
bool isTLS() const
Definition: IRSymtab.h:195
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
Definition: GlobalValue.h:483
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
Symbol(const irsymtab::Symbol &S)
Definition: LTO.h:115
cl::opt< std::string > LTORemarksFilename("lto-pass-remarks-output", cl::desc("Output filename for pass remarks"), cl::value_desc("filename"))
Contains the information needed by linkers for symbol resolution, as well as by the LTO implementatio...
Definition: IRSymtab.h:87
The purpose of this class is to only expose the symbol information that an LTO client should need in ...
Definition: LTO.h:111
unsigned LinkerRedefined
Linker redefined version of the symbol which appeared in -wrap or -defsym linker option.
Definition: LTO.h:400
bool isExecutable() const
Definition: IRSymtab.h:204
The resolution for a symbol.
Definition: LTO.h:383
This class implements a map that also provides access to all stored values in a deterministic order...
Definition: MapVector.h:38
static Expected< std::unique_ptr< InputFile > > create(MemoryBufferRef Object)
Create an InputFile.
Definition: LTO.cpp:353
bool canBeOmittedFromSymbolTable() const
Definition: IRSymtab.h:197
std::unique_ptr< raw_pwrite_stream > OS
Definition: LTO.h:166
std::function< std::unique_ptr< NativeObjectStream >unsigned Task)> AddStreamFn
This type defines the callback to add a native object that is generated on the fly.
Definition: LTO.h:175
Definition: BitVector.h:920
Represents a module in a bitcode file.
Definition: BitcodeReader.h:57
StringRef getSourceFileName() const
Returns the source file path specified at compile time.
Definition: LTO.h:146
Tagged union holding either a T or a Error.
Definition: CachePruning.h:23
bool isCommon() const
Definition: IRSymtab.h:192
StringRef getTargetTriple() const
Returns the input file&#39;s target triple.
Definition: LTO.h:143
cl::opt< bool > LTOPassRemarksWithHotness("lto-pass-remarks-with-hotness", cl::desc("With PGO, include profile count in optimization remarks"), cl::Hidden)
bool isUndefined() const
Definition: IRSymtab.h:190
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
An input file.
Definition: LTO.h:84
This class implements a resolution-based interface to LLVM&#39;s LTO functionality.
Definition: LTO.h:234
Class to hold module path string table and global value map, and encapsulate methods for operating on...
StringRef getName() const
Returns the mangled symbol name.
Definition: IRSymtab.h:174
std::unique_ptr< Module > M
Definition: LTO.h:291
std::function< AddStreamFn(unsigned Task, StringRef Key)> NativeObjectCache
This is the type of a native object cache.
Definition: LTO.h:190
uint32_t getCommonAlignment() const
Definition: IRSymtab.h:211
This class wraps an output stream for a native object.
Definition: LTO.h:163
bool isIndirect() const
Definition: IRSymtab.h:193
ThinBackend createWriteIndexesThinBackend(std::string OldPrefix, std::string NewPrefix, bool ShouldEmitImportsFiles, std::string LinkedObjectsFile)
This ThinBackend writes individual module indexes to files, instead of running the individual backend...
Definition: LTO.cpp:1068
LTO configuration.
Definition: Config.h:36
This represents a symbol that has been read from a storage::Symbol and possibly a storage::Uncommon...
Definition: IRSymtab.h:162
const T * data() const
Definition: ArrayRef.h:146
StringRef getSectionName() const
Definition: IRSymtab.h:223
std::function< std::unique_ptr< ThinBackendProc > Config &C, ModuleSummaryIndex &CombinedIndex, StringMap< GVSummaryMapTy > &ModuleToDefinedGVSummaries, AddStreamFn AddStream, NativeObjectCache Cache)> ThinBackend
A ThinBackend defines what happens after the thin-link phase during ThinLTO.
Definition: LTO.h:199
int getComdatIndex() const
Returns the index into the comdat table (see Reader::getComdatTable()), or -1 if not a comdat member...
Definition: IRSymtab.h:182
StringMap - This is an unconventional map that is specialized for handling keys that are "strings"...
Definition: StringMap.h:224
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:48
ThinBackend createInProcessThinBackend(unsigned ParallelismLevel)
This ThinBackend runs the individual backend jobs in-process.
Definition: LTO.cpp:978
bool isWeak() const
Definition: IRSymtab.h:191
std::vector< GlobalValue * > Keep
Definition: LTO.h:292
void thinLTOResolveWeakForLinkerInIndex(ModuleSummaryIndex &Index, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, function_ref< void(StringRef, GlobalValue::GUID, GlobalValue::LinkageTypes)> recordNewLinkage)
Resolve Weak and LinkOnce values in the Index.
Definition: LTO.cpp:308
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned VisibleToRegularObj
The definition of this symbol is visible outside of the LTO unit.
Definition: LTO.h:396
ModuleSummaryIndex.h This file contains the declarations the classes that hold the module index and s...
StringRef getCOFFLinkerOpts() const
Returns linker options specified in the input file.
Definition: LTO.h:137
constexpr char Size[]
Key for Kernel::Arg::Metadata::mSize.
Lightweight error class with error context and mandatory checking.
Definition: Error.h:156
uint64_t getCommonSize() const
Definition: IRSymtab.h:206
A derived class of LLVMContext that initializes itself according to a given Config object...
Definition: Config.h:188
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
unsigned FinalDefinitionInLinkageUnit
The definition of this symbol is unpreemptable at runtime and is known to be in this linkage unit...
Definition: LTO.h:393
std::string getThinLTOOutputFile(const std::string &Path, const std::string &OldPrefix, const std::string &NewPrefix)
Given the original Path to an output file, replace any path prefix matching OldPrefix with NewPrefix...
Definition: LTO.cpp:991
void thinLTOInternalizeAndPromoteInIndex(ModuleSummaryIndex &Index, function_ref< bool(StringRef, GlobalValue::GUID)> isExported)
Update the linkages in the given Index to mark exported values as external and non-exported values as...
Definition: LTO.cpp:343