LLVM  8.0.0svn
RTDyldObjectLinkingLayer.h
Go to the documentation of this file.
1 //===- RTDyldObjectLinkingLayer.h - RTDyld-based jit linking ---*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // Contains the definition for an RTDyld-based, in-process object linking layer.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_EXECUTIONENGINE_ORC_RTDYLDOBJECTLINKINGLAYER_H
15 #define LLVM_EXECUTIONENGINE_ORC_RTDYLDOBJECTLINKINGLAYER_H
16 
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/ADT/StringMap.h"
19 #include "llvm/ADT/StringRef.h"
25 #include "llvm/Object/ObjectFile.h"
26 #include "llvm/Support/Error.h"
27 #include <algorithm>
28 #include <cassert>
29 #include <functional>
30 #include <list>
31 #include <memory>
32 #include <string>
33 #include <utility>
34 #include <vector>
35 
36 namespace llvm {
37 namespace orc {
38 
40 public:
41  /// Functor for receiving object-loaded notifications.
42  using NotifyLoadedFunction =
45 
46  /// Functor for receiving finalization notifications.
47  using NotifyEmittedFunction = std::function<void(VModuleKey)>;
48 
50  std::function<std::unique_ptr<RuntimeDyld::MemoryManager>()>;
51 
52  /// Construct an ObjectLinkingLayer with the given NotifyLoaded,
53  /// and NotifyEmitted functors.
55  ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager,
58 
59  /// Emit the object.
61  std::unique_ptr<MemoryBuffer> O) override;
62 
63  /// Set the 'ProcessAllSections' flag.
64  ///
65  /// If set to true, all sections in each object file will be allocated using
66  /// the memory manager, rather than just the sections required for execution.
67  ///
68  /// This is kludgy, and may be removed in the future.
69  RTDyldObjectLinkingLayer &setProcessAllSections(bool ProcessAllSections) {
70  this->ProcessAllSections = ProcessAllSections;
71  return *this;
72  }
73 
74  /// Instructs this RTDyldLinkingLayer2 instance to override the symbol flags
75  /// returned by RuntimeDyld for any given object file with the flags supplied
76  /// by the MaterializationResponsibility instance. This is a workaround to
77  /// support symbol visibility in COFF, which does not use the libObject's
78  /// SF_Exported flag. Use only when generating / adding COFF object files.
79  ///
80  /// FIXME: We should be able to remove this if/when COFF properly tracks
81  /// exported symbols.
84  this->OverrideObjectFlags = OverrideObjectFlags;
85  return *this;
86  }
87 
88  /// If set, this RTDyldObjectLinkingLayer instance will claim responsibility
89  /// for any symbols provided by a given object file that were not already in
90  /// the MaterializationResponsibility instance. Setting this flag allows
91  /// higher-level program representations (e.g. LLVM IR) to be added based on
92  /// only a subset of the symbols they provide, without having to write
93  /// intervening layers to scan and add the additional symbols. This trades
94  /// diagnostic quality for convenience however: If all symbols are enumerated
95  /// up-front then clashes can be detected and reported early (and usually
96  /// deterministically). If this option is set, clashes for the additional
97  /// symbols may not be detected until late, and detection may depend on
98  /// the flow of control through JIT'd code. Use with care.
100  setAutoClaimResponsibilityForObjectSymbols(bool AutoClaimObjectSymbols) {
101  this->AutoClaimObjectSymbols = AutoClaimObjectSymbols;
102  return *this;
103  }
104 
105 private:
107  object::ObjectFile &Obj,
108  std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo,
109  std::map<StringRef, JITEvaluatedSymbol> Resolved,
110  std::set<StringRef> &InternalSymbols);
111 
112  void onObjEmit(VModuleKey K, MaterializationResponsibility &R, Error Err);
113 
114  mutable std::mutex RTDyldLayerMutex;
115  GetMemoryManagerFunction GetMemoryManager;
116  NotifyLoadedFunction NotifyLoaded;
117  NotifyEmittedFunction NotifyEmitted;
118  bool ProcessAllSections = false;
119  bool OverrideObjectFlags = false;
120  bool AutoClaimObjectSymbols = false;
121  std::vector<std::unique_ptr<RuntimeDyld::MemoryManager>> MemMgrs;
122 };
123 
125 public:
126  using ObjectPtr = std::unique_ptr<MemoryBuffer>;
127 
128 protected:
129 
130  /// Holds an object to be allocated/linked as a unit in the JIT.
131  ///
132  /// An instance of this class will be created for each object added
133  /// via JITObjectLayer::addObject. Deleting the instance (via
134  /// removeObject) frees its memory, removing all symbol definitions that
135  /// had been provided by this instance. Higher level layers are responsible
136  /// for taking any action required to handle the missing symbols.
137  class LinkedObject {
138  public:
139  LinkedObject() = default;
140  LinkedObject(const LinkedObject&) = delete;
141  void operator=(const LinkedObject&) = delete;
142  virtual ~LinkedObject() = default;
143 
144  virtual Error finalize() = 0;
145 
147  getSymbolMaterializer(std::string Name) = 0;
148 
149  virtual void mapSectionAddress(const void *LocalAddress,
150  JITTargetAddress TargetAddr) const = 0;
151 
152  JITSymbol getSymbol(StringRef Name, bool ExportedSymbolsOnly) {
153  auto SymEntry = SymbolTable.find(Name);
154  if (SymEntry == SymbolTable.end())
155  return nullptr;
156  if (!SymEntry->second.getFlags().isExported() && ExportedSymbolsOnly)
157  return nullptr;
158  if (!Finalized)
159  return JITSymbol(getSymbolMaterializer(Name),
160  SymEntry->second.getFlags());
161  return JITSymbol(SymEntry->second);
162  }
163 
164  protected:
166  bool Finalized = false;
167  };
168 };
169 
170 /// Bare bones object linking layer.
171 ///
172 /// This class is intended to be used as the base layer for a JIT. It allows
173 /// object files to be loaded into memory, linked, and the addresses of their
174 /// symbols queried. All objects added to this layer can see each other's
175 /// symbols.
177 public:
178 
180 
181  /// Functor for receiving object-loaded notifications.
182  using NotifyLoadedFtor =
185 
186  /// Functor for receiving finalization notifications.
187  using NotifyFinalizedFtor =
190 
191  /// Functor for receiving deallocation notifications.
192  using NotifyFreedFtor = std::function<void(VModuleKey, const object::ObjectFile &Obj)>;
193 
194 private:
196 
197  template <typename MemoryManagerPtrT>
198  class ConcreteLinkedObject : public LinkedObject {
199  public:
200  ConcreteLinkedObject(LegacyRTDyldObjectLinkingLayer &Parent, VModuleKey K,
201  OwnedObject Obj, MemoryManagerPtrT MemMgr,
202  std::shared_ptr<SymbolResolver> Resolver,
203  bool ProcessAllSections)
204  : K(std::move(K)),
205  Parent(Parent),
206  MemMgr(std::move(MemMgr)),
207  PFC(llvm::make_unique<PreFinalizeContents>(
208  std::move(Obj), std::move(Resolver),
209  ProcessAllSections)) {
210  buildInitialSymbolTable(PFC->Obj);
211  }
212 
213  ~ConcreteLinkedObject() override {
214  if (this->Parent.NotifyFreed && ObjForNotify.getBinary())
215  this->Parent.NotifyFreed(K, *ObjForNotify.getBinary());
216 
217  MemMgr->deregisterEHFrames();
218  }
219 
220  Error finalize() override {
221  assert(PFC && "mapSectionAddress called on finalized LinkedObject");
222 
223  JITSymbolResolverAdapter ResolverAdapter(Parent.ES, *PFC->Resolver,
224  nullptr);
225  PFC->RTDyld = llvm::make_unique<RuntimeDyld>(*MemMgr, ResolverAdapter);
226  PFC->RTDyld->setProcessAllSections(PFC->ProcessAllSections);
227 
228  Finalized = true;
229 
230  std::unique_ptr<RuntimeDyld::LoadedObjectInfo> Info =
231  PFC->RTDyld->loadObject(*PFC->Obj.getBinary());
232 
233  // Copy the symbol table out of the RuntimeDyld instance.
234  {
235  auto SymTab = PFC->RTDyld->getSymbolTable();
236  for (auto &KV : SymTab)
237  SymbolTable[KV.first] = KV.second;
238  }
239 
240  if (Parent.NotifyLoaded)
241  Parent.NotifyLoaded(K, *PFC->Obj.getBinary(), *Info);
242 
243  PFC->RTDyld->finalizeWithMemoryManagerLocking();
244 
245  if (PFC->RTDyld->hasError())
246  return make_error<StringError>(PFC->RTDyld->getErrorString(),
248 
249  if (Parent.NotifyFinalized)
250  Parent.NotifyFinalized(K, *PFC->Obj.getBinary(), *Info);
251 
252  // Release resources.
253  if (this->Parent.NotifyFreed)
254  ObjForNotify = std::move(PFC->Obj); // needed for callback
255  PFC = nullptr;
256  return Error::success();
257  }
258 
259  JITSymbol::GetAddressFtor getSymbolMaterializer(std::string Name) override {
260  return [this, Name]() -> Expected<JITTargetAddress> {
261  // The symbol may be materialized between the creation of this lambda
262  // and its execution, so we need to double check.
263  if (!this->Finalized)
264  if (auto Err = this->finalize())
265  return std::move(Err);
266  return this->getSymbol(Name, false).getAddress();
267  };
268  }
269 
270  void mapSectionAddress(const void *LocalAddress,
271  JITTargetAddress TargetAddr) const override {
272  assert(PFC && "mapSectionAddress called on finalized LinkedObject");
273  assert(PFC->RTDyld && "mapSectionAddress called on raw LinkedObject");
274  PFC->RTDyld->mapSectionAddress(LocalAddress, TargetAddr);
275  }
276 
277  private:
278  void buildInitialSymbolTable(const OwnedObject &Obj) {
279  for (auto &Symbol : Obj.getBinary()->symbols()) {
280  if (Symbol.getFlags() & object::SymbolRef::SF_Undefined)
281  continue;
283  // FIXME: Raise an error for bad symbols.
284  if (!SymbolName) {
285  consumeError(SymbolName.takeError());
286  continue;
287  }
288  // FIXME: Raise an error for bad symbols.
290  if (!Flags) {
291  consumeError(Flags.takeError());
292  continue;
293  }
294  SymbolTable.insert(
295  std::make_pair(*SymbolName, JITEvaluatedSymbol(0, *Flags)));
296  }
297  }
298 
299  // Contains the information needed prior to finalization: the object files,
300  // memory manager, resolver, and flags needed for RuntimeDyld.
301  struct PreFinalizeContents {
302  PreFinalizeContents(OwnedObject Obj,
303  std::shared_ptr<SymbolResolver> Resolver,
304  bool ProcessAllSections)
305  : Obj(std::move(Obj)),
306  Resolver(std::move(Resolver)),
307  ProcessAllSections(ProcessAllSections) {}
308 
309  OwnedObject Obj;
310  std::shared_ptr<SymbolResolver> Resolver;
311  bool ProcessAllSections;
312  std::unique_ptr<RuntimeDyld> RTDyld;
313  };
314 
315  VModuleKey K;
317  MemoryManagerPtrT MemMgr;
318  OwnedObject ObjForNotify;
319  std::unique_ptr<PreFinalizeContents> PFC;
320  };
321 
322  template <typename MemoryManagerPtrT>
323  std::unique_ptr<ConcreteLinkedObject<MemoryManagerPtrT>>
324  createLinkedObject(LegacyRTDyldObjectLinkingLayer &Parent, VModuleKey K,
325  OwnedObject Obj, MemoryManagerPtrT MemMgr,
326  std::shared_ptr<SymbolResolver> Resolver,
327  bool ProcessAllSections) {
328  using LOS = ConcreteLinkedObject<MemoryManagerPtrT>;
329  return llvm::make_unique<LOS>(Parent, std::move(K), std::move(Obj),
330  std::move(MemMgr), std::move(Resolver),
331  ProcessAllSections);
332  }
333 
334 public:
335  struct Resources {
336  std::shared_ptr<RuntimeDyld::MemoryManager> MemMgr;
337  std::shared_ptr<SymbolResolver> Resolver;
338  };
339 
340  using ResourcesGetter = std::function<Resources(VModuleKey)>;
341 
342  /// Construct an ObjectLinkingLayer with the given NotifyLoaded,
343  /// and NotifyFinalized functors.
345  ExecutionSession &ES, ResourcesGetter GetResources,
346  NotifyLoadedFtor NotifyLoaded = NotifyLoadedFtor(),
347  NotifyFinalizedFtor NotifyFinalized = NotifyFinalizedFtor(),
348  NotifyFreedFtor NotifyFreed = NotifyFreedFtor())
349  : ES(ES), GetResources(std::move(GetResources)),
350  NotifyLoaded(std::move(NotifyLoaded)),
351  NotifyFinalized(std::move(NotifyFinalized)),
352  NotifyFreed(std::move(NotifyFreed)),
353  ProcessAllSections(false) {
354  }
355 
356  /// Set the 'ProcessAllSections' flag.
357  ///
358  /// If set to true, all sections in each object file will be allocated using
359  /// the memory manager, rather than just the sections required for execution.
360  ///
361  /// This is kludgy, and may be removed in the future.
362  void setProcessAllSections(bool ProcessAllSections) {
363  this->ProcessAllSections = ProcessAllSections;
364  }
365 
366  /// Add an object to the JIT.
368 
369  auto Obj =
370  object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef());
371  if (!Obj)
372  return Obj.takeError();
373 
374  assert(!LinkedObjects.count(K) && "VModuleKey already in use");
375 
376  auto R = GetResources(K);
377 
378  LinkedObjects[K] = createLinkedObject(
379  *this, K, OwnedObject(std::move(*Obj), std::move(ObjBuffer)),
380  std::move(R.MemMgr), std::move(R.Resolver), ProcessAllSections);
381 
382  return Error::success();
383  }
384 
385  /// Remove the object associated with VModuleKey K.
386  ///
387  /// All memory allocated for the object will be freed, and the sections and
388  /// symbols it provided will no longer be available. No attempt is made to
389  /// re-emit the missing symbols, and any use of these symbols (directly or
390  /// indirectly) will result in undefined behavior. If dependence tracking is
391  /// required to detect or resolve such issues it should be added at a higher
392  /// layer.
394  assert(LinkedObjects.count(K) && "VModuleKey not associated with object");
395  // How do we invalidate the symbols in H?
396  LinkedObjects.erase(K);
397  return Error::success();
398  }
399 
400  /// Search for the given named symbol.
401  /// @param Name The name of the symbol to search for.
402  /// @param ExportedSymbolsOnly If true, search only for exported symbols.
403  /// @return A handle for the given named symbol, if it exists.
404  JITSymbol findSymbol(StringRef Name, bool ExportedSymbolsOnly) {
405  for (auto &KV : LinkedObjects)
406  if (auto Sym = KV.second->getSymbol(Name, ExportedSymbolsOnly))
407  return Sym;
408  else if (auto Err = Sym.takeError())
409  return std::move(Err);
410 
411  return nullptr;
412  }
413 
414  /// Search for the given named symbol in the context of the loaded
415  /// object represented by the VModuleKey K.
416  /// @param K The VModuleKey for the object to search in.
417  /// @param Name The name of the symbol to search for.
418  /// @param ExportedSymbolsOnly If true, search only for exported symbols.
419  /// @return A handle for the given named symbol, if it is found in the
420  /// given object.
422  bool ExportedSymbolsOnly) {
423  assert(LinkedObjects.count(K) && "VModuleKey not associated with object");
424  return LinkedObjects[K]->getSymbol(Name, ExportedSymbolsOnly);
425  }
426 
427  /// Map section addresses for the object associated with the
428  /// VModuleKey K.
429  void mapSectionAddress(VModuleKey K, const void *LocalAddress,
430  JITTargetAddress TargetAddr) {
431  assert(LinkedObjects.count(K) && "VModuleKey not associated with object");
432  LinkedObjects[K]->mapSectionAddress(LocalAddress, TargetAddr);
433  }
434 
435  /// Immediately emit and finalize the object represented by the given
436  /// VModuleKey.
437  /// @param K VModuleKey for object to emit/finalize.
439  assert(LinkedObjects.count(K) && "VModuleKey not associated with object");
440  return LinkedObjects[K]->finalize();
441  }
442 
443 private:
444  ExecutionSession &ES;
445 
446  std::map<VModuleKey, std::unique_ptr<LinkedObject>> LinkedObjects;
447  ResourcesGetter GetResources;
448  NotifyLoadedFtor NotifyLoaded;
449  NotifyFinalizedFtor NotifyFinalized;
450  NotifyFreedFtor NotifyFreed;
451  bool ProcessAllSections = false;
452 };
453 
454 } // end namespace orc
455 } // end namespace llvm
456 
457 #endif // LLVM_EXECUTIONENGINE_ORC_RTDYLDOBJECTLINKINGLAYER_H
static Expected< OwningBinary< ObjectFile > > createObjectFile(StringRef ObjectPath)
Definition: ObjectFile.cpp:161
Information about the loaded object.
Definition: RuntimeDyld.h:69
Represents a symbol in the JIT.
Definition: JITSymbol.h:238
This class represents lattice values for constants.
Definition: AllocatorList.h:24
arc branch finalize
Error removeObject(VModuleKey K)
Remove the object associated with VModuleKey K.
RTDyldObjectLinkingLayer & setProcessAllSections(bool ProcessAllSections)
Set the &#39;ProcessAllSections&#39; flag.
std::function< Resources(VModuleKey)> ResourcesGetter
This class is the base class for all object file types.
Definition: ObjectFile.h:202
std::function< void(VModuleKey, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &)> NotifyLoadedFunction
Functor for receiving object-loaded notifications.
static Expected< JITSymbolFlags > fromObjectSymbol(const object::SymbolRef &Symbol)
Construct a JITSymbolFlags value based on the flags of the given libobject symbol.
Definition: JITSymbol.cpp:41
Error takeError()
Take ownership of the stored error.
Definition: Error.h:553
uint64_t VModuleKey
VModuleKey provides a unique identifier (allocated and managed by ExecutionSessions) for a module add...
Definition: Core.h:40
JITSymbol getSymbol(StringRef Name, bool ExportedSymbolsOnly)
amdgpu Simplify well known AMD library false Value Value const Twine & Name
Definition: BitVector.h:938
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
Tagged union holding either a T or a Error.
Definition: CachePruning.h:23
RTDyldObjectLinkingLayer(ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager, NotifyLoadedFunction NotifyLoaded=NotifyLoadedFunction(), NotifyEmittedFunction NotifyEmitted=NotifyEmittedFunction())
Construct an ObjectLinkingLayer with the given NotifyLoaded, and NotifyEmitted functors.
JITSymbol findSymbolIn(VModuleKey K, StringRef Name, bool ExportedSymbolsOnly)
Search for the given named symbol in the context of the loaded object represented by the VModuleKey K...
Tracks responsibility for materialization, and mediates interactions between MaterializationUnits and...
Definition: Core.h:155
LegacyRTDyldObjectLinkingLayer(ExecutionSession &ES, ResourcesGetter GetResources, NotifyLoadedFtor NotifyLoaded=NotifyLoadedFtor(), NotifyFinalizedFtor NotifyFinalized=NotifyFinalizedFtor(), NotifyFreedFtor NotifyFreed=NotifyFreedFtor())
Construct an ObjectLinkingLayer with the given NotifyLoaded, and NotifyFinalized functors.
std::function< void(VModuleKey, const object::ObjectFile &Obj)> NotifyFreedFtor
Functor for receiving deallocation notifications.
Legacy adapter. Remove once we kill off the old ORC layers.
Definition: Legacy.h:94
uint64_t JITTargetAddress
Represents an address in the target process&#39;s address space.
Definition: JITSymbol.h:41
Expected< const typename ELFT::Sym * > getSymbol(typename ELFT::SymRange Symbols, uint32_t Index)
Definition: ELF.h:337
void mapSectionAddress(VModuleKey K, const void *LocalAddress, JITTargetAddress TargetAddr)
Map section addresses for the object associated with the VModuleKey K.
Interface for looking up the initializer for a variable name, used by Init::resolveReferences.
Definition: Record.h:1774
std::function< void(VModuleKey)> NotifyEmittedFunction
Functor for receiving finalization notifications.
void consumeError(Error Err)
Consume a Error without doing anything.
Definition: Error.h:982
RTDyldObjectLinkingLayer & setAutoClaimResponsibilityForObjectSymbols(bool AutoClaimObjectSymbols)
If set, this RTDyldObjectLinkingLayer instance will claim responsibility for any symbols provided by ...
std::shared_ptr< RuntimeDyld::MemoryManager > MemMgr
static ErrorSuccess success()
Create a success value.
Definition: Error.h:327
std::function< void(VModuleKey, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &)> NotifyLoadedFtor
Functor for receiving object-loaded notifications.
Error addObject(VModuleKey K, ObjectPtr ObjBuffer)
Add an object to the JIT.
std::function< std::unique_ptr< RuntimeDyld::MemoryManager >()> GetMemoryManagerFunction
JITSymbol findSymbol(StringRef Name, bool ExportedSymbolsOnly)
Search for the given named symbol.
An ExecutionSession represents a running JIT program.
Definition: Core.h:697
StringMap - This is an unconventional map that is specialized for handling keys that are "strings"...
Definition: StringMap.h:220
Represents a symbol that has been evaluated to an address already.
Definition: JITSymbol.h:209
std::function< Expected< JITTargetAddress >()> GetAddressFtor
Definition: JITSymbol.h:240
symbol_iterator_range symbols() const
Definition: ObjectFile.h:284
void setProcessAllSections(bool ProcessAllSections)
Set the &#39;ProcessAllSections&#39; flag.
Holds an object to be allocated/linked as a unit in the JIT.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
std::function< void(VModuleKey, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &)> NotifyFinalizedFtor
Functor for receiving finalization notifications.
Lightweight error class with error context and mandatory checking.
Definition: Error.h:158
Interface for Layers that accept object files.
Definition: Layer.h:114
print Print MemDeps of function
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
RTDyldObjectLinkingLayer & setOverrideObjectFlagsWithResponsibilityFlags(bool OverrideObjectFlags)
Instructs this RTDyldLinkingLayer2 instance to override the symbol flags returned by RuntimeDyld for ...
void emit(MaterializationResponsibility R, std::unique_ptr< MemoryBuffer > O) override
Emit the object.
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:77
Error emitAndFinalize(VModuleKey K)
Immediately emit and finalize the object represented by the given VModuleKey.