LLVM  8.0.0svn
RTDyldObjectLinkingLayer.h
Go to the documentation of this file.
1 //===- RTDyldObjectLinkingLayer.h - RTDyld-based jit linking ---*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // Contains the definition for an RTDyld-based, in-process object linking layer.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_EXECUTIONENGINE_ORC_RTDYLDOBJECTLINKINGLAYER_H
15 #define LLVM_EXECUTIONENGINE_ORC_RTDYLDOBJECTLINKINGLAYER_H
16 
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/StringMap.h"
19 #include "llvm/ADT/StringRef.h"
25 #include "llvm/Object/ObjectFile.h"
26 #include "llvm/Support/Error.h"
27 #include <algorithm>
28 #include <cassert>
29 #include <functional>
30 #include <list>
31 #include <memory>
32 #include <string>
33 #include <utility>
34 #include <vector>
35 
36 namespace llvm {
37 namespace orc {
38 
40 public:
41  /// Functor for receiving object-loaded notifications.
42  using NotifyLoadedFunction =
45 
46  /// Functor for receiving finalization notifications.
47  using NotifyEmittedFunction = std::function<void(VModuleKey)>;
48 
50  std::function<std::unique_ptr<RuntimeDyld::MemoryManager>(VModuleKey)>;
51 
52  /// Construct an ObjectLinkingLayer with the given NotifyLoaded,
53  /// and NotifyEmitted functors.
55  ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager,
58 
59  /// Emit the object.
61  std::unique_ptr<MemoryBuffer> O) override;
62 
63  /// Set the 'ProcessAllSections' flag.
64  ///
65  /// If set to true, all sections in each object file will be allocated using
66  /// the memory manager, rather than just the sections required for execution.
67  ///
68  /// This is kludgy, and may be removed in the future.
70  this->ProcessAllSections = ProcessAllSections;
71  return *this;
72  }
73 
74  /// Instructs this RTDyldLinkingLayer2 instance to override the symbol flags
75  /// returned by RuntimeDyld for any given object file with the flags supplied
76  /// by the MaterializationResponsibility instance. This is a workaround to
77  /// support symbol visibility in COFF, which does not use the libObject's
78  /// SF_Exported flag. Use only when generating / adding COFF object files.
79  ///
80  /// FIXME: We should be able to remove this if/when COFF properly tracks
81  /// exported symbols.
84  this->OverrideObjectFlags = OverrideObjectFlags;
85  return *this;
86  }
87 
88  /// If set, this RTDyldObjectLinkingLayer2 instance will claim responsibility
89  /// for any symbols provided by a given object file that were not already in
90  /// the MaterializationResponsibility instance. Setting this flag allows
91  /// higher-level program representations (e.g. LLVM IR) to be added based on
92  /// only a subset of the symbols they provide, without having to write
93  /// intervening layers to scan and add the additional symbols. This trades
94  /// diagnostic quality for convenience however: If all symbols are enumerated
95  /// up-front then clashes can be detected and reported early (and usually
96  /// deterministically). If this option is set, clashes for the additional
97  /// symbols may not be detected until late, and detection may depend on
98  /// the flow of control through JIT'd code. Use with care.
100  setAutoClaimResponsibilityForObjectSymbols(bool AutoClaimObjectSymbols) {
101  this->AutoClaimObjectSymbols = AutoClaimObjectSymbols;
102  return *this;
103  }
104 
105 private:
106  mutable std::mutex RTDyldLayerMutex;
107  GetMemoryManagerFunction GetMemoryManager;
108  NotifyLoadedFunction NotifyLoaded;
109  NotifyEmittedFunction NotifyEmitted;
110  bool ProcessAllSections = false;
111  bool OverrideObjectFlags = false;
112  bool AutoClaimObjectSymbols = false;
113  std::map<VModuleKey, std::shared_ptr<RuntimeDyld::MemoryManager>> MemMgrs;
114 };
115 
117 public:
118  using ObjectPtr = std::unique_ptr<MemoryBuffer>;
119 
120 protected:
121 
122  /// Holds an object to be allocated/linked as a unit in the JIT.
123  ///
124  /// An instance of this class will be created for each object added
125  /// via JITObjectLayer::addObject. Deleting the instance (via
126  /// removeObject) frees its memory, removing all symbol definitions that
127  /// had been provided by this instance. Higher level layers are responsible
128  /// for taking any action required to handle the missing symbols.
129  class LinkedObject {
130  public:
131  LinkedObject() = default;
132  LinkedObject(const LinkedObject&) = delete;
133  void operator=(const LinkedObject&) = delete;
134  virtual ~LinkedObject() = default;
135 
136  virtual Error finalize() = 0;
137 
139  getSymbolMaterializer(std::string Name) = 0;
140 
141  virtual void mapSectionAddress(const void *LocalAddress,
142  JITTargetAddress TargetAddr) const = 0;
143 
144  JITSymbol getSymbol(StringRef Name, bool ExportedSymbolsOnly) {
145  auto SymEntry = SymbolTable.find(Name);
146  if (SymEntry == SymbolTable.end())
147  return nullptr;
148  if (!SymEntry->second.getFlags().isExported() && ExportedSymbolsOnly)
149  return nullptr;
150  if (!Finalized)
151  return JITSymbol(getSymbolMaterializer(Name),
152  SymEntry->second.getFlags());
153  return JITSymbol(SymEntry->second);
154  }
155 
156  protected:
158  bool Finalized = false;
159  };
160 };
161 
162 /// Bare bones object linking layer.
163 ///
164 /// This class is intended to be used as the base layer for a JIT. It allows
165 /// object files to be loaded into memory, linked, and the addresses of their
166 /// symbols queried. All objects added to this layer can see each other's
167 /// symbols.
169 public:
170 
172 
173  /// Functor for receiving object-loaded notifications.
174  using NotifyLoadedFtor =
177 
178  /// Functor for receiving finalization notifications.
179  using NotifyFinalizedFtor =
182 
183  /// Functor for receiving deallocation notifications.
184  using NotifyFreedFtor = std::function<void(VModuleKey, const object::ObjectFile &Obj)>;
185 
186 private:
188 
189  template <typename MemoryManagerPtrT>
190  class ConcreteLinkedObject : public LinkedObject {
191  public:
192  ConcreteLinkedObject(RTDyldObjectLinkingLayer &Parent, VModuleKey K,
193  OwnedObject Obj, MemoryManagerPtrT MemMgr,
194  std::shared_ptr<SymbolResolver> Resolver,
195  bool ProcessAllSections)
196  : K(std::move(K)),
197  Parent(Parent),
198  MemMgr(std::move(MemMgr)),
199  PFC(llvm::make_unique<PreFinalizeContents>(
200  std::move(Obj), std::move(Resolver),
201  ProcessAllSections)) {
202  buildInitialSymbolTable(PFC->Obj);
203  }
204 
205  ~ConcreteLinkedObject() override {
206  if (this->Parent.NotifyFreed && ObjForNotify.getBinary())
207  this->Parent.NotifyFreed(K, *ObjForNotify.getBinary());
208 
209  MemMgr->deregisterEHFrames();
210  }
211 
212  Error finalize() override {
213  assert(PFC && "mapSectionAddress called on finalized LinkedObject");
214 
215  JITSymbolResolverAdapter ResolverAdapter(Parent.ES, *PFC->Resolver,
216  nullptr);
217  PFC->RTDyld = llvm::make_unique<RuntimeDyld>(*MemMgr, ResolverAdapter);
218  PFC->RTDyld->setProcessAllSections(PFC->ProcessAllSections);
219 
220  Finalized = true;
221 
222  std::unique_ptr<RuntimeDyld::LoadedObjectInfo> Info =
223  PFC->RTDyld->loadObject(*PFC->Obj.getBinary());
224 
225  // Copy the symbol table out of the RuntimeDyld instance.
226  {
227  auto SymTab = PFC->RTDyld->getSymbolTable();
228  for (auto &KV : SymTab)
229  SymbolTable[KV.first] = KV.second;
230  }
231 
232  if (Parent.NotifyLoaded)
233  Parent.NotifyLoaded(K, *PFC->Obj.getBinary(), *Info);
234 
235  PFC->RTDyld->finalizeWithMemoryManagerLocking();
236 
237  if (PFC->RTDyld->hasError())
238  return make_error<StringError>(PFC->RTDyld->getErrorString(),
240 
241  if (Parent.NotifyFinalized)
242  Parent.NotifyFinalized(K, *PFC->Obj.getBinary(), *Info);
243 
244  // Release resources.
245  if (this->Parent.NotifyFreed)
246  ObjForNotify = std::move(PFC->Obj); // needed for callback
247  PFC = nullptr;
248  return Error::success();
249  }
250 
251  JITSymbol::GetAddressFtor getSymbolMaterializer(std::string Name) override {
252  return [this, Name]() -> Expected<JITTargetAddress> {
253  // The symbol may be materialized between the creation of this lambda
254  // and its execution, so we need to double check.
255  if (!this->Finalized)
256  if (auto Err = this->finalize())
257  return std::move(Err);
258  return this->getSymbol(Name, false).getAddress();
259  };
260  }
261 
262  void mapSectionAddress(const void *LocalAddress,
263  JITTargetAddress TargetAddr) const override {
264  assert(PFC && "mapSectionAddress called on finalized LinkedObject");
265  assert(PFC->RTDyld && "mapSectionAddress called on raw LinkedObject");
266  PFC->RTDyld->mapSectionAddress(LocalAddress, TargetAddr);
267  }
268 
269  private:
270  void buildInitialSymbolTable(const OwnedObject &Obj) {
271  for (auto &Symbol : Obj.getBinary()->symbols()) {
272  if (Symbol.getFlags() & object::SymbolRef::SF_Undefined)
273  continue;
275  // FIXME: Raise an error for bad symbols.
276  if (!SymbolName) {
277  consumeError(SymbolName.takeError());
278  continue;
279  }
280  // FIXME: Raise an error for bad symbols.
282  if (!Flags) {
283  consumeError(Flags.takeError());
284  continue;
285  }
286  SymbolTable.insert(
287  std::make_pair(*SymbolName, JITEvaluatedSymbol(0, *Flags)));
288  }
289  }
290 
291  // Contains the information needed prior to finalization: the object files,
292  // memory manager, resolver, and flags needed for RuntimeDyld.
293  struct PreFinalizeContents {
294  PreFinalizeContents(OwnedObject Obj,
295  std::shared_ptr<SymbolResolver> Resolver,
296  bool ProcessAllSections)
297  : Obj(std::move(Obj)),
298  Resolver(std::move(Resolver)),
299  ProcessAllSections(ProcessAllSections) {}
300 
301  OwnedObject Obj;
302  std::shared_ptr<SymbolResolver> Resolver;
303  bool ProcessAllSections;
304  std::unique_ptr<RuntimeDyld> RTDyld;
305  };
306 
307  VModuleKey K;
308  RTDyldObjectLinkingLayer &Parent;
309  MemoryManagerPtrT MemMgr;
310  OwnedObject ObjForNotify;
311  std::unique_ptr<PreFinalizeContents> PFC;
312  };
313 
314  template <typename MemoryManagerPtrT>
315  std::unique_ptr<ConcreteLinkedObject<MemoryManagerPtrT>>
316  createLinkedObject(RTDyldObjectLinkingLayer &Parent, VModuleKey K,
317  OwnedObject Obj, MemoryManagerPtrT MemMgr,
318  std::shared_ptr<SymbolResolver> Resolver,
319  bool ProcessAllSections) {
320  using LOS = ConcreteLinkedObject<MemoryManagerPtrT>;
321  return llvm::make_unique<LOS>(Parent, std::move(K), std::move(Obj),
322  std::move(MemMgr), std::move(Resolver),
323  ProcessAllSections);
324  }
325 
326 public:
327  struct Resources {
328  std::shared_ptr<RuntimeDyld::MemoryManager> MemMgr;
329  std::shared_ptr<SymbolResolver> Resolver;
330  };
331 
332  using ResourcesGetter = std::function<Resources(VModuleKey)>;
333 
334  /// Construct an ObjectLinkingLayer with the given NotifyLoaded,
335  /// and NotifyFinalized functors.
337  ExecutionSession &ES, ResourcesGetter GetResources,
338  NotifyLoadedFtor NotifyLoaded = NotifyLoadedFtor(),
339  NotifyFinalizedFtor NotifyFinalized = NotifyFinalizedFtor(),
340  NotifyFreedFtor NotifyFreed = NotifyFreedFtor())
341  : ES(ES), GetResources(std::move(GetResources)),
342  NotifyLoaded(std::move(NotifyLoaded)),
343  NotifyFinalized(std::move(NotifyFinalized)),
344  NotifyFreed(std::move(NotifyFreed)),
345  ProcessAllSections(false) {
346  }
347 
348  /// Set the 'ProcessAllSections' flag.
349  ///
350  /// If set to true, all sections in each object file will be allocated using
351  /// the memory manager, rather than just the sections required for execution.
352  ///
353  /// This is kludgy, and may be removed in the future.
354  void setProcessAllSections(bool ProcessAllSections) {
355  this->ProcessAllSections = ProcessAllSections;
356  }
357 
358  /// Add an object to the JIT.
360 
361  auto Obj =
362  object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef());
363  if (!Obj)
364  return Obj.takeError();
365 
366  assert(!LinkedObjects.count(K) && "VModuleKey already in use");
367 
368  auto R = GetResources(K);
369 
370  LinkedObjects[K] = createLinkedObject(
371  *this, K, OwnedObject(std::move(*Obj), std::move(ObjBuffer)),
372  std::move(R.MemMgr), std::move(R.Resolver), ProcessAllSections);
373 
374  return Error::success();
375  }
376 
377  /// Remove the object associated with VModuleKey K.
378  ///
379  /// All memory allocated for the object will be freed, and the sections and
380  /// symbols it provided will no longer be available. No attempt is made to
381  /// re-emit the missing symbols, and any use of these symbols (directly or
382  /// indirectly) will result in undefined behavior. If dependence tracking is
383  /// required to detect or resolve such issues it should be added at a higher
384  /// layer.
386  assert(LinkedObjects.count(K) && "VModuleKey not associated with object");
387  // How do we invalidate the symbols in H?
388  LinkedObjects.erase(K);
389  return Error::success();
390  }
391 
392  /// Search for the given named symbol.
393  /// @param Name The name of the symbol to search for.
394  /// @param ExportedSymbolsOnly If true, search only for exported symbols.
395  /// @return A handle for the given named symbol, if it exists.
396  JITSymbol findSymbol(StringRef Name, bool ExportedSymbolsOnly) {
397  for (auto &KV : LinkedObjects)
398  if (auto Sym = KV.second->getSymbol(Name, ExportedSymbolsOnly))
399  return Sym;
400  else if (auto Err = Sym.takeError())
401  return std::move(Err);
402 
403  return nullptr;
404  }
405 
406  /// Search for the given named symbol in the context of the loaded
407  /// object represented by the VModuleKey K.
408  /// @param K The VModuleKey for the object to search in.
409  /// @param Name The name of the symbol to search for.
410  /// @param ExportedSymbolsOnly If true, search only for exported symbols.
411  /// @return A handle for the given named symbol, if it is found in the
412  /// given object.
414  bool ExportedSymbolsOnly) {
415  assert(LinkedObjects.count(K) && "VModuleKey not associated with object");
416  return LinkedObjects[K]->getSymbol(Name, ExportedSymbolsOnly);
417  }
418 
419  /// Map section addresses for the object associated with the
420  /// VModuleKey K.
421  void mapSectionAddress(VModuleKey K, const void *LocalAddress,
422  JITTargetAddress TargetAddr) {
423  assert(LinkedObjects.count(K) && "VModuleKey not associated with object");
424  LinkedObjects[K]->mapSectionAddress(LocalAddress, TargetAddr);
425  }
426 
427  /// Immediately emit and finalize the object represented by the given
428  /// VModuleKey.
429  /// @param K VModuleKey for object to emit/finalize.
431  assert(LinkedObjects.count(K) && "VModuleKey not associated with object");
432  return LinkedObjects[K]->finalize();
433  }
434 
435 private:
436  ExecutionSession &ES;
437 
438  std::map<VModuleKey, std::unique_ptr<LinkedObject>> LinkedObjects;
439  ResourcesGetter GetResources;
440  NotifyLoadedFtor NotifyLoaded;
441  NotifyFinalizedFtor NotifyFinalized;
442  NotifyFreedFtor NotifyFreed;
443  bool ProcessAllSections = false;
444 };
445 
446 } // end namespace orc
447 } // end namespace llvm
448 
449 #endif // LLVM_EXECUTIONENGINE_ORC_RTDYLDOBJECTLINKINGLAYER_H
static Expected< OwningBinary< ObjectFile > > createObjectFile(StringRef ObjectPath)
Definition: ObjectFile.cpp:153
Information about the loaded object.
Definition: RuntimeDyld.h:69
Represents a symbol in the JIT.
Definition: JITSymbol.h:226
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
arc branch finalize
JITSymbol getSymbol(StringRef Name, bool ExportedSymbolsOnly)
Holds an object to be allocated/linked as a unit in the JIT.
Error emitAndFinalize(VModuleKey K)
Immediately emit and finalize the object represented by the given VModuleKey.
Bare bones object linking layer.
This class is the base class for all object file types.
Definition: ObjectFile.h:190
static Expected< JITSymbolFlags > fromObjectSymbol(const object::SymbolRef &Symbol)
Construct a JITSymbolFlags value based on the flags of the given libobject symbol.
Definition: JITSymbol.cpp:35
Error takeError()
Take ownership of the stored error.
Definition: Error.h:553
uint64_t VModuleKey
VModuleKey provides a unique identifier (allocated and managed by ExecutionSessions) for a module add...
Definition: Core.h:40
Definition: BitVector.h:938
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
std::function< void(VModuleKey, const object::ObjectFile &Obj)> NotifyFreedFtor
Functor for receiving deallocation notifications.
RTDyldObjectLinkingLayer2 & setProcessAllSections(bool ProcessAllSections)
Set the &#39;ProcessAllSections&#39; flag.
void mapSectionAddress(VModuleKey K, const void *LocalAddress, JITTargetAddress TargetAddr)
Map section addresses for the object associated with the VModuleKey K.
Tagged union holding either a T or a Error.
Definition: CachePruning.h:23
Tracks responsibility for materialization, and mediates interactions between MaterializationUnits and...
Definition: Core.h:125
std::function< std::unique_ptr< RuntimeDyld::MemoryManager >(VModuleKey)> GetMemoryManagerFunction
std::function< void(VModuleKey, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &)> NotifyLoadedFtor
Functor for receiving object-loaded notifications.
uint64_t JITTargetAddress
Represents an address in the target process&#39;s address space.
Definition: JITSymbol.h:41
Expected< const typename ELFT::Sym * > getSymbol(typename ELFT::SymRange Symbols, uint32_t Index)
Definition: ELF.h:337
void emit(MaterializationResponsibility R, VModuleKey K, std::unique_ptr< MemoryBuffer > O) override
Emit the object.
Interface for looking up the initializer for a variable name, used by Init::resolveReferences.
Definition: Record.h:1774
RTDyldObjectLinkingLayer(ExecutionSession &ES, ResourcesGetter GetResources, NotifyLoadedFtor NotifyLoaded=NotifyLoadedFtor(), NotifyFinalizedFtor NotifyFinalized=NotifyFinalizedFtor(), NotifyFreedFtor NotifyFreed=NotifyFreedFtor())
Construct an ObjectLinkingLayer with the given NotifyLoaded, and NotifyFinalized functors.
std::function< Resources(VModuleKey)> ResourcesGetter
void consumeError(Error Err)
Consume a Error without doing anything.
Definition: Error.h:978
static ErrorSuccess success()
Create a success value.
Definition: Error.h:327
std::function< void(VModuleKey, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &)> NotifyFinalizedFtor
Functor for receiving finalization notifications.
An ExecutionSession represents a running JIT program.
Definition: Core.h:619
StringMap - This is an unconventional map that is specialized for handling keys that are "strings"...
Definition: StringMap.h:220
RTDyldObjectLinkingLayer2 & setAutoClaimResponsibilityForObjectSymbols(bool AutoClaimObjectSymbols)
If set, this RTDyldObjectLinkingLayer2 instance will claim responsibility for any symbols provided by...
void setProcessAllSections(bool ProcessAllSections)
Set the &#39;ProcessAllSections&#39; flag.
Represents a symbol that has been evaluated to an address already.
Definition: JITSymbol.h:197
JITSymbol findSymbolIn(VModuleKey K, StringRef Name, bool ExportedSymbolsOnly)
Search for the given named symbol in the context of the loaded object represented by the VModuleKey K...
RTDyldObjectLinkingLayer2 & setOverrideObjectFlagsWithResponsibilityFlags(bool OverrideObjectFlags)
Instructs this RTDyldLinkingLayer2 instance to override the symbol flags returned by RuntimeDyld for ...
std::function< Expected< JITTargetAddress >()> GetAddressFtor
Definition: JITSymbol.h:228
symbol_iterator_range symbols() const
Definition: ObjectFile.h:270
std::shared_ptr< RuntimeDyld::MemoryManager > MemMgr
JITSymbol findSymbol(StringRef Name, bool ExportedSymbolsOnly)
Search for the given named symbol.
Error addObject(VModuleKey K, ObjectPtr ObjBuffer)
Add an object to the JIT.
std::function< void(VModuleKey)> NotifyEmittedFunction
Functor for receiving finalization notifications.
Error removeObject(VModuleKey K)
Remove the object associated with VModuleKey K.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Lightweight error class with error context and mandatory checking.
Definition: Error.h:158
Interface for Layers that accept object files.
Definition: Layer.h:94
std::function< void(VModuleKey, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &)> NotifyLoadedFunction
Functor for receiving object-loaded notifications.
print Print MemDeps of function
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
RTDyldObjectLinkingLayer2(ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager, NotifyLoadedFunction NotifyLoaded=NotifyLoadedFunction(), NotifyEmittedFunction NotifyEmitted=NotifyEmittedFunction())
Construct an ObjectLinkingLayer with the given NotifyLoaded, and NotifyEmitted functors.
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:77