LLVM  9.0.0svn
RTDyldObjectLinkingLayer.h
Go to the documentation of this file.
1 //===- RTDyldObjectLinkingLayer.h - RTDyld-based jit linking ---*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Contains the definition for an RTDyld-based, in-process object linking layer.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #ifndef LLVM_EXECUTIONENGINE_ORC_RTDYLDOBJECTLINKINGLAYER_H
14 #define LLVM_EXECUTIONENGINE_ORC_RTDYLDOBJECTLINKINGLAYER_H
15 
16 #include "llvm/ADT/STLExtras.h"
17 #include "llvm/ADT/StringMap.h"
18 #include "llvm/ADT/StringRef.h"
24 #include "llvm/Object/ObjectFile.h"
25 #include "llvm/Support/Error.h"
26 #include <algorithm>
27 #include <cassert>
28 #include <functional>
29 #include <list>
30 #include <memory>
31 #include <string>
32 #include <utility>
33 #include <vector>
34 
35 namespace llvm {
36 namespace orc {
37 
39 public:
40  /// Functor for receiving object-loaded notifications.
41  using NotifyLoadedFunction =
44 
45  /// Functor for receiving finalization notifications.
46  using NotifyEmittedFunction =
47  std::function<void(VModuleKey, std::unique_ptr<MemoryBuffer>)>;
48 
50  std::function<std::unique_ptr<RuntimeDyld::MemoryManager>()>;
51 
52  /// Construct an ObjectLinkingLayer with the given NotifyLoaded,
53  /// and NotifyEmitted functors.
55  GetMemoryManagerFunction GetMemoryManager);
56 
57  /// Emit the object.
59  std::unique_ptr<MemoryBuffer> O) override;
60 
61  /// Set the NotifyLoaded callback.
63  this->NotifyLoaded = std::move(NotifyLoaded);
64  return *this;
65  }
66 
67  /// Set the NotifyEmitted callback.
70  this->NotifyEmitted = std::move(NotifyEmitted);
71  return *this;
72  }
73 
74  /// Set the 'ProcessAllSections' flag.
75  ///
76  /// If set to true, all sections in each object file will be allocated using
77  /// the memory manager, rather than just the sections required for execution.
78  ///
79  /// This is kludgy, and may be removed in the future.
80  RTDyldObjectLinkingLayer &setProcessAllSections(bool ProcessAllSections) {
81  this->ProcessAllSections = ProcessAllSections;
82  return *this;
83  }
84 
85  /// Instructs this RTDyldLinkingLayer2 instance to override the symbol flags
86  /// returned by RuntimeDyld for any given object file with the flags supplied
87  /// by the MaterializationResponsibility instance. This is a workaround to
88  /// support symbol visibility in COFF, which does not use the libObject's
89  /// SF_Exported flag. Use only when generating / adding COFF object files.
90  ///
91  /// FIXME: We should be able to remove this if/when COFF properly tracks
92  /// exported symbols.
95  this->OverrideObjectFlags = OverrideObjectFlags;
96  return *this;
97  }
98 
99  /// If set, this RTDyldObjectLinkingLayer instance will claim responsibility
100  /// for any symbols provided by a given object file that were not already in
101  /// the MaterializationResponsibility instance. Setting this flag allows
102  /// higher-level program representations (e.g. LLVM IR) to be added based on
103  /// only a subset of the symbols they provide, without having to write
104  /// intervening layers to scan and add the additional symbols. This trades
105  /// diagnostic quality for convenience however: If all symbols are enumerated
106  /// up-front then clashes can be detected and reported early (and usually
107  /// deterministically). If this option is set, clashes for the additional
108  /// symbols may not be detected until late, and detection may depend on
109  /// the flow of control through JIT'd code. Use with care.
111  setAutoClaimResponsibilityForObjectSymbols(bool AutoClaimObjectSymbols) {
112  this->AutoClaimObjectSymbols = AutoClaimObjectSymbols;
113  return *this;
114  }
115 
116 private:
118  object::ObjectFile &Obj,
119  std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo,
120  std::map<StringRef, JITEvaluatedSymbol> Resolved,
121  std::set<StringRef> &InternalSymbols);
122 
123  void onObjEmit(VModuleKey K, std::unique_ptr<MemoryBuffer> ObjBuffer,
125 
126  mutable std::mutex RTDyldLayerMutex;
127  GetMemoryManagerFunction GetMemoryManager;
128  NotifyLoadedFunction NotifyLoaded;
129  NotifyEmittedFunction NotifyEmitted;
130  bool ProcessAllSections = false;
131  bool OverrideObjectFlags = false;
132  bool AutoClaimObjectSymbols = false;
133  std::vector<std::unique_ptr<RuntimeDyld::MemoryManager>> MemMgrs;
134 };
135 
137 public:
138  using ObjectPtr = std::unique_ptr<MemoryBuffer>;
139 
140 protected:
141 
142  /// Holds an object to be allocated/linked as a unit in the JIT.
143  ///
144  /// An instance of this class will be created for each object added
145  /// via JITObjectLayer::addObject. Deleting the instance (via
146  /// removeObject) frees its memory, removing all symbol definitions that
147  /// had been provided by this instance. Higher level layers are responsible
148  /// for taking any action required to handle the missing symbols.
149  class LinkedObject {
150  public:
151  LinkedObject() = default;
152  LinkedObject(const LinkedObject&) = delete;
153  void operator=(const LinkedObject&) = delete;
154  virtual ~LinkedObject() = default;
155 
156  virtual Error finalize() = 0;
157 
159  getSymbolMaterializer(std::string Name) = 0;
160 
161  virtual void mapSectionAddress(const void *LocalAddress,
162  JITTargetAddress TargetAddr) const = 0;
163 
164  JITSymbol getSymbol(StringRef Name, bool ExportedSymbolsOnly) {
165  auto SymEntry = SymbolTable.find(Name);
166  if (SymEntry == SymbolTable.end())
167  return nullptr;
168  if (!SymEntry->second.getFlags().isExported() && ExportedSymbolsOnly)
169  return nullptr;
170  if (!Finalized)
171  return JITSymbol(getSymbolMaterializer(Name),
172  SymEntry->second.getFlags());
173  return JITSymbol(SymEntry->second);
174  }
175 
176  protected:
178  bool Finalized = false;
179  };
180 };
181 
182 /// Bare bones object linking layer.
183 ///
184 /// This class is intended to be used as the base layer for a JIT. It allows
185 /// object files to be loaded into memory, linked, and the addresses of their
186 /// symbols queried. All objects added to this layer can see each other's
187 /// symbols.
189 public:
190 
192 
193  /// Functor for receiving object-loaded notifications.
194  using NotifyLoadedFtor =
197 
198  /// Functor for receiving finalization notifications.
199  using NotifyFinalizedFtor =
202 
203  /// Functor for receiving deallocation notifications.
204  using NotifyFreedFtor = std::function<void(VModuleKey, const object::ObjectFile &Obj)>;
205 
206 private:
208 
209  template <typename MemoryManagerPtrT>
210  class ConcreteLinkedObject : public LinkedObject {
211  public:
212  ConcreteLinkedObject(LegacyRTDyldObjectLinkingLayer &Parent, VModuleKey K,
213  OwnedObject Obj, MemoryManagerPtrT MemMgr,
214  std::shared_ptr<SymbolResolver> Resolver,
215  bool ProcessAllSections)
216  : K(std::move(K)),
217  Parent(Parent),
218  MemMgr(std::move(MemMgr)),
219  PFC(llvm::make_unique<PreFinalizeContents>(
220  std::move(Obj), std::move(Resolver),
221  ProcessAllSections)) {
222  buildInitialSymbolTable(PFC->Obj);
223  }
224 
225  ~ConcreteLinkedObject() override {
226  if (this->Parent.NotifyFreed && ObjForNotify.getBinary())
227  this->Parent.NotifyFreed(K, *ObjForNotify.getBinary());
228 
229  MemMgr->deregisterEHFrames();
230  }
231 
232  Error finalize() override {
233  assert(PFC && "mapSectionAddress called on finalized LinkedObject");
234 
235  JITSymbolResolverAdapter ResolverAdapter(Parent.ES, *PFC->Resolver,
236  nullptr);
237  PFC->RTDyld = llvm::make_unique<RuntimeDyld>(*MemMgr, ResolverAdapter);
238  PFC->RTDyld->setProcessAllSections(PFC->ProcessAllSections);
239 
240  Finalized = true;
241 
242  std::unique_ptr<RuntimeDyld::LoadedObjectInfo> Info =
243  PFC->RTDyld->loadObject(*PFC->Obj.getBinary());
244 
245  // Copy the symbol table out of the RuntimeDyld instance.
246  {
247  auto SymTab = PFC->RTDyld->getSymbolTable();
248  for (auto &KV : SymTab)
249  SymbolTable[KV.first] = KV.second;
250  }
251 
252  if (Parent.NotifyLoaded)
253  Parent.NotifyLoaded(K, *PFC->Obj.getBinary(), *Info);
254 
255  PFC->RTDyld->finalizeWithMemoryManagerLocking();
256 
257  if (PFC->RTDyld->hasError())
258  return make_error<StringError>(PFC->RTDyld->getErrorString(),
260 
261  if (Parent.NotifyFinalized)
262  Parent.NotifyFinalized(K, *PFC->Obj.getBinary(), *Info);
263 
264  // Release resources.
265  if (this->Parent.NotifyFreed)
266  ObjForNotify = std::move(PFC->Obj); // needed for callback
267  PFC = nullptr;
268  return Error::success();
269  }
270 
271  JITSymbol::GetAddressFtor getSymbolMaterializer(std::string Name) override {
272  return [this, Name]() -> Expected<JITTargetAddress> {
273  // The symbol may be materialized between the creation of this lambda
274  // and its execution, so we need to double check.
275  if (!this->Finalized)
276  if (auto Err = this->finalize())
277  return std::move(Err);
278  return this->getSymbol(Name, false).getAddress();
279  };
280  }
281 
282  void mapSectionAddress(const void *LocalAddress,
283  JITTargetAddress TargetAddr) const override {
284  assert(PFC && "mapSectionAddress called on finalized LinkedObject");
285  assert(PFC->RTDyld && "mapSectionAddress called on raw LinkedObject");
286  PFC->RTDyld->mapSectionAddress(LocalAddress, TargetAddr);
287  }
288 
289  private:
290  void buildInitialSymbolTable(const OwnedObject &Obj) {
291  for (auto &Symbol : Obj.getBinary()->symbols()) {
292  if (Symbol.getFlags() & object::SymbolRef::SF_Undefined)
293  continue;
295  // FIXME: Raise an error for bad symbols.
296  if (!SymbolName) {
297  consumeError(SymbolName.takeError());
298  continue;
299  }
300  // FIXME: Raise an error for bad symbols.
302  if (!Flags) {
303  consumeError(Flags.takeError());
304  continue;
305  }
306  SymbolTable.insert(
307  std::make_pair(*SymbolName, JITEvaluatedSymbol(0, *Flags)));
308  }
309  }
310 
311  // Contains the information needed prior to finalization: the object files,
312  // memory manager, resolver, and flags needed for RuntimeDyld.
313  struct PreFinalizeContents {
314  PreFinalizeContents(OwnedObject Obj,
315  std::shared_ptr<SymbolResolver> Resolver,
316  bool ProcessAllSections)
317  : Obj(std::move(Obj)),
318  Resolver(std::move(Resolver)),
319  ProcessAllSections(ProcessAllSections) {}
320 
321  OwnedObject Obj;
322  std::shared_ptr<SymbolResolver> Resolver;
323  bool ProcessAllSections;
324  std::unique_ptr<RuntimeDyld> RTDyld;
325  };
326 
327  VModuleKey K;
329  MemoryManagerPtrT MemMgr;
330  OwnedObject ObjForNotify;
331  std::unique_ptr<PreFinalizeContents> PFC;
332  };
333 
334  template <typename MemoryManagerPtrT>
335  std::unique_ptr<ConcreteLinkedObject<MemoryManagerPtrT>>
336  createLinkedObject(LegacyRTDyldObjectLinkingLayer &Parent, VModuleKey K,
337  OwnedObject Obj, MemoryManagerPtrT MemMgr,
338  std::shared_ptr<SymbolResolver> Resolver,
339  bool ProcessAllSections) {
340  using LOS = ConcreteLinkedObject<MemoryManagerPtrT>;
341  return llvm::make_unique<LOS>(Parent, std::move(K), std::move(Obj),
342  std::move(MemMgr), std::move(Resolver),
343  ProcessAllSections);
344  }
345 
346 public:
347  struct Resources {
348  std::shared_ptr<RuntimeDyld::MemoryManager> MemMgr;
349  std::shared_ptr<SymbolResolver> Resolver;
350  };
351 
352  using ResourcesGetter = std::function<Resources(VModuleKey)>;
353 
354  /// Construct an ObjectLinkingLayer with the given NotifyLoaded,
355  /// and NotifyFinalized functors.
357  ExecutionSession &ES, ResourcesGetter GetResources,
358  NotifyLoadedFtor NotifyLoaded = NotifyLoadedFtor(),
359  NotifyFinalizedFtor NotifyFinalized = NotifyFinalizedFtor(),
360  NotifyFreedFtor NotifyFreed = NotifyFreedFtor())
361  : ES(ES), GetResources(std::move(GetResources)),
362  NotifyLoaded(std::move(NotifyLoaded)),
363  NotifyFinalized(std::move(NotifyFinalized)),
364  NotifyFreed(std::move(NotifyFreed)),
365  ProcessAllSections(false) {
366  }
367 
368  /// Set the 'ProcessAllSections' flag.
369  ///
370  /// If set to true, all sections in each object file will be allocated using
371  /// the memory manager, rather than just the sections required for execution.
372  ///
373  /// This is kludgy, and may be removed in the future.
374  void setProcessAllSections(bool ProcessAllSections) {
375  this->ProcessAllSections = ProcessAllSections;
376  }
377 
378  /// Add an object to the JIT.
380 
381  auto Obj =
382  object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef());
383  if (!Obj)
384  return Obj.takeError();
385 
386  assert(!LinkedObjects.count(K) && "VModuleKey already in use");
387 
388  auto R = GetResources(K);
389 
390  LinkedObjects[K] = createLinkedObject(
391  *this, K, OwnedObject(std::move(*Obj), std::move(ObjBuffer)),
392  std::move(R.MemMgr), std::move(R.Resolver), ProcessAllSections);
393 
394  return Error::success();
395  }
396 
397  /// Remove the object associated with VModuleKey K.
398  ///
399  /// All memory allocated for the object will be freed, and the sections and
400  /// symbols it provided will no longer be available. No attempt is made to
401  /// re-emit the missing symbols, and any use of these symbols (directly or
402  /// indirectly) will result in undefined behavior. If dependence tracking is
403  /// required to detect or resolve such issues it should be added at a higher
404  /// layer.
406  assert(LinkedObjects.count(K) && "VModuleKey not associated with object");
407  // How do we invalidate the symbols in H?
408  LinkedObjects.erase(K);
409  return Error::success();
410  }
411 
412  /// Search for the given named symbol.
413  /// @param Name The name of the symbol to search for.
414  /// @param ExportedSymbolsOnly If true, search only for exported symbols.
415  /// @return A handle for the given named symbol, if it exists.
416  JITSymbol findSymbol(StringRef Name, bool ExportedSymbolsOnly) {
417  for (auto &KV : LinkedObjects)
418  if (auto Sym = KV.second->getSymbol(Name, ExportedSymbolsOnly))
419  return Sym;
420  else if (auto Err = Sym.takeError())
421  return std::move(Err);
422 
423  return nullptr;
424  }
425 
426  /// Search for the given named symbol in the context of the loaded
427  /// object represented by the VModuleKey K.
428  /// @param K The VModuleKey for the object to search in.
429  /// @param Name The name of the symbol to search for.
430  /// @param ExportedSymbolsOnly If true, search only for exported symbols.
431  /// @return A handle for the given named symbol, if it is found in the
432  /// given object.
434  bool ExportedSymbolsOnly) {
435  assert(LinkedObjects.count(K) && "VModuleKey not associated with object");
436  return LinkedObjects[K]->getSymbol(Name, ExportedSymbolsOnly);
437  }
438 
439  /// Map section addresses for the object associated with the
440  /// VModuleKey K.
441  void mapSectionAddress(VModuleKey K, const void *LocalAddress,
442  JITTargetAddress TargetAddr) {
443  assert(LinkedObjects.count(K) && "VModuleKey not associated with object");
444  LinkedObjects[K]->mapSectionAddress(LocalAddress, TargetAddr);
445  }
446 
447  /// Immediately emit and finalize the object represented by the given
448  /// VModuleKey.
449  /// @param K VModuleKey for object to emit/finalize.
451  assert(LinkedObjects.count(K) && "VModuleKey not associated with object");
452  return LinkedObjects[K]->finalize();
453  }
454 
455 private:
456  ExecutionSession &ES;
457 
458  ResourcesGetter GetResources;
459  NotifyLoadedFtor NotifyLoaded;
460  NotifyFinalizedFtor NotifyFinalized;
461  NotifyFreedFtor NotifyFreed;
462 
463  // NB! `LinkedObjects` needs to be destroyed before `NotifyFreed` because
464  // `~ConcreteLinkedObject` calls `NotifyFreed`
465  std::map<VModuleKey, std::unique_ptr<LinkedObject>> LinkedObjects;
466  bool ProcessAllSections = false;
467 };
468 
469 } // end namespace orc
470 } // end namespace llvm
471 
472 #endif // LLVM_EXECUTIONENGINE_ORC_RTDYLDOBJECTLINKINGLAYER_H
static Expected< OwningBinary< ObjectFile > > createObjectFile(StringRef ObjectPath)
Definition: ObjectFile.cpp:161
Information about the loaded object.
Definition: RuntimeDyld.h:69
Represents a symbol in the JIT.
Definition: JITSymbol.h:237
std::function< void(VModuleKey, std::unique_ptr< MemoryBuffer >)> NotifyEmittedFunction
Functor for receiving finalization notifications.
RTDyldObjectLinkingLayer & setNotifyEmitted(NotifyEmittedFunction NotifyEmitted)
Set the NotifyEmitted callback.
This class represents lattice values for constants.
Definition: AllocatorList.h:23
arc branch finalize
Error removeObject(VModuleKey K)
Remove the object associated with VModuleKey K.
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
RTDyldObjectLinkingLayer & setProcessAllSections(bool ProcessAllSections)
Set the &#39;ProcessAllSections&#39; flag.
std::function< Resources(VModuleKey)> ResourcesGetter
This class is the base class for all object file types.
Definition: ObjectFile.h:226
std::function< void(VModuleKey, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &)> NotifyLoadedFunction
Functor for receiving object-loaded notifications.
static Expected< JITSymbolFlags > fromObjectSymbol(const object::SymbolRef &Symbol)
Construct a JITSymbolFlags value based on the flags of the given libobject symbol.
Definition: JITSymbol.cpp:40
Error takeError()
Take ownership of the stored error.
Definition: Error.h:552
uint64_t VModuleKey
VModuleKey provides a unique identifier (allocated and managed by ExecutionSessions) for a module add...
Definition: Core.h:39
JITSymbol getSymbol(StringRef Name, bool ExportedSymbolsOnly)
Definition: BitVector.h:937
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
RTDyldObjectLinkingLayer(ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager)
Construct an ObjectLinkingLayer with the given NotifyLoaded, and NotifyEmitted functors.
Tagged union holding either a T or a Error.
Definition: CachePruning.h:22
JITSymbol findSymbolIn(VModuleKey K, StringRef Name, bool ExportedSymbolsOnly)
Search for the given named symbol in the context of the loaded object represented by the VModuleKey K...
Tracks responsibility for materialization, and mediates interactions between MaterializationUnits and...
Definition: Core.h:154
LegacyRTDyldObjectLinkingLayer(ExecutionSession &ES, ResourcesGetter GetResources, NotifyLoadedFtor NotifyLoaded=NotifyLoadedFtor(), NotifyFinalizedFtor NotifyFinalized=NotifyFinalizedFtor(), NotifyFreedFtor NotifyFreed=NotifyFreedFtor())
Construct an ObjectLinkingLayer with the given NotifyLoaded, and NotifyFinalized functors.
std::function< void(VModuleKey, const object::ObjectFile &Obj)> NotifyFreedFtor
Functor for receiving deallocation notifications.
Legacy adapter. Remove once we kill off the old ORC layers.
Definition: Legacy.h:93
uint64_t JITTargetAddress
Represents an address in the target process&#39;s address space.
Definition: JITSymbol.h:40
Expected< const typename ELFT::Sym * > getSymbol(typename ELFT::SymRange Symbols, uint32_t Index)
Definition: ELF.h:334
Analysis containing CSE Info
Definition: CSEInfo.cpp:20
void mapSectionAddress(VModuleKey K, const void *LocalAddress, JITTargetAddress TargetAddr)
Map section addresses for the object associated with the VModuleKey K.
Interface for looking up the initializer for a variable name, used by Init::resolveReferences.
Definition: Record.h:1856
void consumeError(Error Err)
Consume a Error without doing anything.
Definition: Error.h:981
RTDyldObjectLinkingLayer & setAutoClaimResponsibilityForObjectSymbols(bool AutoClaimObjectSymbols)
If set, this RTDyldObjectLinkingLayer instance will claim responsibility for any symbols provided by ...
std::shared_ptr< RuntimeDyld::MemoryManager > MemMgr
static ErrorSuccess success()
Create a success value.
Definition: Error.h:326
std::function< void(VModuleKey, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &)> NotifyLoadedFtor
Functor for receiving object-loaded notifications.
Error addObject(VModuleKey K, ObjectPtr ObjBuffer)
Add an object to the JIT.
std::function< std::unique_ptr< RuntimeDyld::MemoryManager >()> GetMemoryManagerFunction
JITSymbol findSymbol(StringRef Name, bool ExportedSymbolsOnly)
Search for the given named symbol.
An ExecutionSession represents a running JIT program.
Definition: Core.h:696
StringMap - This is an unconventional map that is specialized for handling keys that are "strings"...
Definition: StringMap.h:219
Represents a symbol that has been evaluated to an address already.
Definition: JITSymbol.h:208
std::function< Expected< JITTargetAddress >()> GetAddressFtor
Definition: JITSymbol.h:239
symbol_iterator_range symbols() const
Definition: ObjectFile.h:307
RTDyldObjectLinkingLayer & setNotifyLoaded(NotifyLoadedFunction NotifyLoaded)
Set the NotifyLoaded callback.
void setProcessAllSections(bool ProcessAllSections)
Set the &#39;ProcessAllSections&#39; flag.
Holds an object to be allocated/linked as a unit in the JIT.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
std::function< void(VModuleKey, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &)> NotifyFinalizedFtor
Functor for receiving finalization notifications.
Lightweight error class with error context and mandatory checking.
Definition: Error.h:157
Interface for Layers that accept object files.
Definition: Layer.h:113
print Print MemDeps of function
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
RTDyldObjectLinkingLayer & setOverrideObjectFlagsWithResponsibilityFlags(bool OverrideObjectFlags)
Instructs this RTDyldLinkingLayer2 instance to override the symbol flags returned by RuntimeDyld for ...
void emit(MaterializationResponsibility R, std::unique_ptr< MemoryBuffer > O) override
Emit the object.
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:77
Error emitAndFinalize(VModuleKey K)
Immediately emit and finalize the object represented by the given VModuleKey.