LLVM  9.0.0svn
RTDyldObjectLinkingLayer.h
Go to the documentation of this file.
1 //===- RTDyldObjectLinkingLayer.h - RTDyld-based jit linking ---*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Contains the definition for an RTDyld-based, in-process object linking layer.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #ifndef LLVM_EXECUTIONENGINE_ORC_RTDYLDOBJECTLINKINGLAYER_H
14 #define LLVM_EXECUTIONENGINE_ORC_RTDYLDOBJECTLINKINGLAYER_H
15 
16 #include "llvm/ADT/STLExtras.h"
17 #include "llvm/ADT/StringMap.h"
18 #include "llvm/ADT/StringRef.h"
24 #include "llvm/Object/ObjectFile.h"
25 #include "llvm/Support/Error.h"
26 #include <algorithm>
27 #include <cassert>
28 #include <functional>
29 #include <list>
30 #include <memory>
31 #include <string>
32 #include <utility>
33 #include <vector>
34 
35 namespace llvm {
36 namespace orc {
37 
39 public:
40  /// Functor for receiving object-loaded notifications.
41  using NotifyLoadedFunction =
44 
45  /// Functor for receiving finalization notifications.
46  using NotifyEmittedFunction = std::function<void(VModuleKey)>;
47 
49  std::function<std::unique_ptr<RuntimeDyld::MemoryManager>()>;
50 
51  /// Construct an ObjectLinkingLayer with the given NotifyLoaded,
52  /// and NotifyEmitted functors.
54  ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager,
57 
58  /// Emit the object.
60  std::unique_ptr<MemoryBuffer> O) override;
61 
62  /// Set the 'ProcessAllSections' flag.
63  ///
64  /// If set to true, all sections in each object file will be allocated using
65  /// the memory manager, rather than just the sections required for execution.
66  ///
67  /// This is kludgy, and may be removed in the future.
68  RTDyldObjectLinkingLayer &setProcessAllSections(bool ProcessAllSections) {
69  this->ProcessAllSections = ProcessAllSections;
70  return *this;
71  }
72 
73  /// Instructs this RTDyldLinkingLayer2 instance to override the symbol flags
74  /// returned by RuntimeDyld for any given object file with the flags supplied
75  /// by the MaterializationResponsibility instance. This is a workaround to
76  /// support symbol visibility in COFF, which does not use the libObject's
77  /// SF_Exported flag. Use only when generating / adding COFF object files.
78  ///
79  /// FIXME: We should be able to remove this if/when COFF properly tracks
80  /// exported symbols.
83  this->OverrideObjectFlags = OverrideObjectFlags;
84  return *this;
85  }
86 
87  /// If set, this RTDyldObjectLinkingLayer instance will claim responsibility
88  /// for any symbols provided by a given object file that were not already in
89  /// the MaterializationResponsibility instance. Setting this flag allows
90  /// higher-level program representations (e.g. LLVM IR) to be added based on
91  /// only a subset of the symbols they provide, without having to write
92  /// intervening layers to scan and add the additional symbols. This trades
93  /// diagnostic quality for convenience however: If all symbols are enumerated
94  /// up-front then clashes can be detected and reported early (and usually
95  /// deterministically). If this option is set, clashes for the additional
96  /// symbols may not be detected until late, and detection may depend on
97  /// the flow of control through JIT'd code. Use with care.
99  setAutoClaimResponsibilityForObjectSymbols(bool AutoClaimObjectSymbols) {
100  this->AutoClaimObjectSymbols = AutoClaimObjectSymbols;
101  return *this;
102  }
103 
104 private:
106  object::ObjectFile &Obj,
107  std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo,
108  std::map<StringRef, JITEvaluatedSymbol> Resolved,
109  std::set<StringRef> &InternalSymbols);
110 
111  void onObjEmit(VModuleKey K, MaterializationResponsibility &R, Error Err);
112 
113  mutable std::mutex RTDyldLayerMutex;
114  GetMemoryManagerFunction GetMemoryManager;
115  NotifyLoadedFunction NotifyLoaded;
116  NotifyEmittedFunction NotifyEmitted;
117  bool ProcessAllSections = false;
118  bool OverrideObjectFlags = false;
119  bool AutoClaimObjectSymbols = false;
120  std::vector<std::unique_ptr<RuntimeDyld::MemoryManager>> MemMgrs;
121 };
122 
124 public:
125  using ObjectPtr = std::unique_ptr<MemoryBuffer>;
126 
127 protected:
128 
129  /// Holds an object to be allocated/linked as a unit in the JIT.
130  ///
131  /// An instance of this class will be created for each object added
132  /// via JITObjectLayer::addObject. Deleting the instance (via
133  /// removeObject) frees its memory, removing all symbol definitions that
134  /// had been provided by this instance. Higher level layers are responsible
135  /// for taking any action required to handle the missing symbols.
136  class LinkedObject {
137  public:
138  LinkedObject() = default;
139  LinkedObject(const LinkedObject&) = delete;
140  void operator=(const LinkedObject&) = delete;
141  virtual ~LinkedObject() = default;
142 
143  virtual Error finalize() = 0;
144 
146  getSymbolMaterializer(std::string Name) = 0;
147 
148  virtual void mapSectionAddress(const void *LocalAddress,
149  JITTargetAddress TargetAddr) const = 0;
150 
151  JITSymbol getSymbol(StringRef Name, bool ExportedSymbolsOnly) {
152  auto SymEntry = SymbolTable.find(Name);
153  if (SymEntry == SymbolTable.end())
154  return nullptr;
155  if (!SymEntry->second.getFlags().isExported() && ExportedSymbolsOnly)
156  return nullptr;
157  if (!Finalized)
158  return JITSymbol(getSymbolMaterializer(Name),
159  SymEntry->second.getFlags());
160  return JITSymbol(SymEntry->second);
161  }
162 
163  protected:
165  bool Finalized = false;
166  };
167 };
168 
169 /// Bare bones object linking layer.
170 ///
171 /// This class is intended to be used as the base layer for a JIT. It allows
172 /// object files to be loaded into memory, linked, and the addresses of their
173 /// symbols queried. All objects added to this layer can see each other's
174 /// symbols.
176 public:
177 
179 
180  /// Functor for receiving object-loaded notifications.
181  using NotifyLoadedFtor =
184 
185  /// Functor for receiving finalization notifications.
186  using NotifyFinalizedFtor =
189 
190  /// Functor for receiving deallocation notifications.
191  using NotifyFreedFtor = std::function<void(VModuleKey, const object::ObjectFile &Obj)>;
192 
193 private:
195 
196  template <typename MemoryManagerPtrT>
197  class ConcreteLinkedObject : public LinkedObject {
198  public:
199  ConcreteLinkedObject(LegacyRTDyldObjectLinkingLayer &Parent, VModuleKey K,
200  OwnedObject Obj, MemoryManagerPtrT MemMgr,
201  std::shared_ptr<SymbolResolver> Resolver,
202  bool ProcessAllSections)
203  : K(std::move(K)),
204  Parent(Parent),
205  MemMgr(std::move(MemMgr)),
206  PFC(llvm::make_unique<PreFinalizeContents>(
207  std::move(Obj), std::move(Resolver),
208  ProcessAllSections)) {
209  buildInitialSymbolTable(PFC->Obj);
210  }
211 
212  ~ConcreteLinkedObject() override {
213  if (this->Parent.NotifyFreed && ObjForNotify.getBinary())
214  this->Parent.NotifyFreed(K, *ObjForNotify.getBinary());
215 
216  MemMgr->deregisterEHFrames();
217  }
218 
219  Error finalize() override {
220  assert(PFC && "mapSectionAddress called on finalized LinkedObject");
221 
222  JITSymbolResolverAdapter ResolverAdapter(Parent.ES, *PFC->Resolver,
223  nullptr);
224  PFC->RTDyld = llvm::make_unique<RuntimeDyld>(*MemMgr, ResolverAdapter);
225  PFC->RTDyld->setProcessAllSections(PFC->ProcessAllSections);
226 
227  Finalized = true;
228 
229  std::unique_ptr<RuntimeDyld::LoadedObjectInfo> Info =
230  PFC->RTDyld->loadObject(*PFC->Obj.getBinary());
231 
232  // Copy the symbol table out of the RuntimeDyld instance.
233  {
234  auto SymTab = PFC->RTDyld->getSymbolTable();
235  for (auto &KV : SymTab)
236  SymbolTable[KV.first] = KV.second;
237  }
238 
239  if (Parent.NotifyLoaded)
240  Parent.NotifyLoaded(K, *PFC->Obj.getBinary(), *Info);
241 
242  PFC->RTDyld->finalizeWithMemoryManagerLocking();
243 
244  if (PFC->RTDyld->hasError())
245  return make_error<StringError>(PFC->RTDyld->getErrorString(),
247 
248  if (Parent.NotifyFinalized)
249  Parent.NotifyFinalized(K, *PFC->Obj.getBinary(), *Info);
250 
251  // Release resources.
252  if (this->Parent.NotifyFreed)
253  ObjForNotify = std::move(PFC->Obj); // needed for callback
254  PFC = nullptr;
255  return Error::success();
256  }
257 
258  JITSymbol::GetAddressFtor getSymbolMaterializer(std::string Name) override {
259  return [this, Name]() -> Expected<JITTargetAddress> {
260  // The symbol may be materialized between the creation of this lambda
261  // and its execution, so we need to double check.
262  if (!this->Finalized)
263  if (auto Err = this->finalize())
264  return std::move(Err);
265  return this->getSymbol(Name, false).getAddress();
266  };
267  }
268 
269  void mapSectionAddress(const void *LocalAddress,
270  JITTargetAddress TargetAddr) const override {
271  assert(PFC && "mapSectionAddress called on finalized LinkedObject");
272  assert(PFC->RTDyld && "mapSectionAddress called on raw LinkedObject");
273  PFC->RTDyld->mapSectionAddress(LocalAddress, TargetAddr);
274  }
275 
276  private:
277  void buildInitialSymbolTable(const OwnedObject &Obj) {
278  for (auto &Symbol : Obj.getBinary()->symbols()) {
279  if (Symbol.getFlags() & object::SymbolRef::SF_Undefined)
280  continue;
282  // FIXME: Raise an error for bad symbols.
283  if (!SymbolName) {
284  consumeError(SymbolName.takeError());
285  continue;
286  }
287  // FIXME: Raise an error for bad symbols.
289  if (!Flags) {
290  consumeError(Flags.takeError());
291  continue;
292  }
293  SymbolTable.insert(
294  std::make_pair(*SymbolName, JITEvaluatedSymbol(0, *Flags)));
295  }
296  }
297 
298  // Contains the information needed prior to finalization: the object files,
299  // memory manager, resolver, and flags needed for RuntimeDyld.
300  struct PreFinalizeContents {
301  PreFinalizeContents(OwnedObject Obj,
302  std::shared_ptr<SymbolResolver> Resolver,
303  bool ProcessAllSections)
304  : Obj(std::move(Obj)),
305  Resolver(std::move(Resolver)),
306  ProcessAllSections(ProcessAllSections) {}
307 
308  OwnedObject Obj;
309  std::shared_ptr<SymbolResolver> Resolver;
310  bool ProcessAllSections;
311  std::unique_ptr<RuntimeDyld> RTDyld;
312  };
313 
314  VModuleKey K;
316  MemoryManagerPtrT MemMgr;
317  OwnedObject ObjForNotify;
318  std::unique_ptr<PreFinalizeContents> PFC;
319  };
320 
321  template <typename MemoryManagerPtrT>
322  std::unique_ptr<ConcreteLinkedObject<MemoryManagerPtrT>>
323  createLinkedObject(LegacyRTDyldObjectLinkingLayer &Parent, VModuleKey K,
324  OwnedObject Obj, MemoryManagerPtrT MemMgr,
325  std::shared_ptr<SymbolResolver> Resolver,
326  bool ProcessAllSections) {
327  using LOS = ConcreteLinkedObject<MemoryManagerPtrT>;
328  return llvm::make_unique<LOS>(Parent, std::move(K), std::move(Obj),
329  std::move(MemMgr), std::move(Resolver),
330  ProcessAllSections);
331  }
332 
333 public:
334  struct Resources {
335  std::shared_ptr<RuntimeDyld::MemoryManager> MemMgr;
336  std::shared_ptr<SymbolResolver> Resolver;
337  };
338 
339  using ResourcesGetter = std::function<Resources(VModuleKey)>;
340 
341  /// Construct an ObjectLinkingLayer with the given NotifyLoaded,
342  /// and NotifyFinalized functors.
344  ExecutionSession &ES, ResourcesGetter GetResources,
345  NotifyLoadedFtor NotifyLoaded = NotifyLoadedFtor(),
346  NotifyFinalizedFtor NotifyFinalized = NotifyFinalizedFtor(),
347  NotifyFreedFtor NotifyFreed = NotifyFreedFtor())
348  : ES(ES), GetResources(std::move(GetResources)),
349  NotifyLoaded(std::move(NotifyLoaded)),
350  NotifyFinalized(std::move(NotifyFinalized)),
351  NotifyFreed(std::move(NotifyFreed)),
352  ProcessAllSections(false) {
353  }
354 
355  /// Set the 'ProcessAllSections' flag.
356  ///
357  /// If set to true, all sections in each object file will be allocated using
358  /// the memory manager, rather than just the sections required for execution.
359  ///
360  /// This is kludgy, and may be removed in the future.
361  void setProcessAllSections(bool ProcessAllSections) {
362  this->ProcessAllSections = ProcessAllSections;
363  }
364 
365  /// Add an object to the JIT.
367 
368  auto Obj =
369  object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef());
370  if (!Obj)
371  return Obj.takeError();
372 
373  assert(!LinkedObjects.count(K) && "VModuleKey already in use");
374 
375  auto R = GetResources(K);
376 
377  LinkedObjects[K] = createLinkedObject(
378  *this, K, OwnedObject(std::move(*Obj), std::move(ObjBuffer)),
379  std::move(R.MemMgr), std::move(R.Resolver), ProcessAllSections);
380 
381  return Error::success();
382  }
383 
384  /// Remove the object associated with VModuleKey K.
385  ///
386  /// All memory allocated for the object will be freed, and the sections and
387  /// symbols it provided will no longer be available. No attempt is made to
388  /// re-emit the missing symbols, and any use of these symbols (directly or
389  /// indirectly) will result in undefined behavior. If dependence tracking is
390  /// required to detect or resolve such issues it should be added at a higher
391  /// layer.
393  assert(LinkedObjects.count(K) && "VModuleKey not associated with object");
394  // How do we invalidate the symbols in H?
395  LinkedObjects.erase(K);
396  return Error::success();
397  }
398 
399  /// Search for the given named symbol.
400  /// @param Name The name of the symbol to search for.
401  /// @param ExportedSymbolsOnly If true, search only for exported symbols.
402  /// @return A handle for the given named symbol, if it exists.
403  JITSymbol findSymbol(StringRef Name, bool ExportedSymbolsOnly) {
404  for (auto &KV : LinkedObjects)
405  if (auto Sym = KV.second->getSymbol(Name, ExportedSymbolsOnly))
406  return Sym;
407  else if (auto Err = Sym.takeError())
408  return std::move(Err);
409 
410  return nullptr;
411  }
412 
413  /// Search for the given named symbol in the context of the loaded
414  /// object represented by the VModuleKey K.
415  /// @param K The VModuleKey for the object to search in.
416  /// @param Name The name of the symbol to search for.
417  /// @param ExportedSymbolsOnly If true, search only for exported symbols.
418  /// @return A handle for the given named symbol, if it is found in the
419  /// given object.
421  bool ExportedSymbolsOnly) {
422  assert(LinkedObjects.count(K) && "VModuleKey not associated with object");
423  return LinkedObjects[K]->getSymbol(Name, ExportedSymbolsOnly);
424  }
425 
426  /// Map section addresses for the object associated with the
427  /// VModuleKey K.
428  void mapSectionAddress(VModuleKey K, const void *LocalAddress,
429  JITTargetAddress TargetAddr) {
430  assert(LinkedObjects.count(K) && "VModuleKey not associated with object");
431  LinkedObjects[K]->mapSectionAddress(LocalAddress, TargetAddr);
432  }
433 
434  /// Immediately emit and finalize the object represented by the given
435  /// VModuleKey.
436  /// @param K VModuleKey for object to emit/finalize.
438  assert(LinkedObjects.count(K) && "VModuleKey not associated with object");
439  return LinkedObjects[K]->finalize();
440  }
441 
442 private:
443  ExecutionSession &ES;
444 
445  ResourcesGetter GetResources;
446  NotifyLoadedFtor NotifyLoaded;
447  NotifyFinalizedFtor NotifyFinalized;
448  NotifyFreedFtor NotifyFreed;
449 
450  // NB! `LinkedObjects` needs to be destroyed before `NotifyFreed` because
451  // `~ConcreteLinkedObject` calls `NotifyFreed`
452  std::map<VModuleKey, std::unique_ptr<LinkedObject>> LinkedObjects;
453  bool ProcessAllSections = false;
454 };
455 
456 } // end namespace orc
457 } // end namespace llvm
458 
459 #endif // LLVM_EXECUTIONENGINE_ORC_RTDYLDOBJECTLINKINGLAYER_H
static Expected< OwningBinary< ObjectFile > > createObjectFile(StringRef ObjectPath)
Definition: ObjectFile.cpp:160
Information about the loaded object.
Definition: RuntimeDyld.h:68
Represents a symbol in the JIT.
Definition: JITSymbol.h:237
This class represents lattice values for constants.
Definition: AllocatorList.h:23
arc branch finalize
Error removeObject(VModuleKey K)
Remove the object associated with VModuleKey K.
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
RTDyldObjectLinkingLayer & setProcessAllSections(bool ProcessAllSections)
Set the &#39;ProcessAllSections&#39; flag.
std::function< Resources(VModuleKey)> ResourcesGetter
This class is the base class for all object file types.
Definition: ObjectFile.h:225
std::function< void(VModuleKey, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &)> NotifyLoadedFunction
Functor for receiving object-loaded notifications.
static Expected< JITSymbolFlags > fromObjectSymbol(const object::SymbolRef &Symbol)
Construct a JITSymbolFlags value based on the flags of the given libobject symbol.
Definition: JITSymbol.cpp:40
Error takeError()
Take ownership of the stored error.
Definition: Error.h:552
uint64_t VModuleKey
VModuleKey provides a unique identifier (allocated and managed by ExecutionSessions) for a module add...
Definition: Core.h:39
JITSymbol getSymbol(StringRef Name, bool ExportedSymbolsOnly)
Definition: BitVector.h:937
constexpr char SymbolName[]
Key for Kernel::Metadata::mSymbolName.
Tagged union holding either a T or a Error.
Definition: CachePruning.h:22
RTDyldObjectLinkingLayer(ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager, NotifyLoadedFunction NotifyLoaded=NotifyLoadedFunction(), NotifyEmittedFunction NotifyEmitted=NotifyEmittedFunction())
Construct an ObjectLinkingLayer with the given NotifyLoaded, and NotifyEmitted functors.
JITSymbol findSymbolIn(VModuleKey K, StringRef Name, bool ExportedSymbolsOnly)
Search for the given named symbol in the context of the loaded object represented by the VModuleKey K...
Tracks responsibility for materialization, and mediates interactions between MaterializationUnits and...
Definition: Core.h:154
LegacyRTDyldObjectLinkingLayer(ExecutionSession &ES, ResourcesGetter GetResources, NotifyLoadedFtor NotifyLoaded=NotifyLoadedFtor(), NotifyFinalizedFtor NotifyFinalized=NotifyFinalizedFtor(), NotifyFreedFtor NotifyFreed=NotifyFreedFtor())
Construct an ObjectLinkingLayer with the given NotifyLoaded, and NotifyFinalized functors.
std::function< void(VModuleKey, const object::ObjectFile &Obj)> NotifyFreedFtor
Functor for receiving deallocation notifications.
Legacy adapter. Remove once we kill off the old ORC layers.
Definition: Legacy.h:93
uint64_t JITTargetAddress
Represents an address in the target process&#39;s address space.
Definition: JITSymbol.h:40
Expected< const typename ELFT::Sym * > getSymbol(typename ELFT::SymRange Symbols, uint32_t Index)
Definition: ELF.h:336
Analysis containing CSE Info
Definition: CSEInfo.cpp:20
void mapSectionAddress(VModuleKey K, const void *LocalAddress, JITTargetAddress TargetAddr)
Map section addresses for the object associated with the VModuleKey K.
Interface for looking up the initializer for a variable name, used by Init::resolveReferences.
Definition: Record.h:1854
std::function< void(VModuleKey)> NotifyEmittedFunction
Functor for receiving finalization notifications.
void consumeError(Error Err)
Consume a Error without doing anything.
Definition: Error.h:981
RTDyldObjectLinkingLayer & setAutoClaimResponsibilityForObjectSymbols(bool AutoClaimObjectSymbols)
If set, this RTDyldObjectLinkingLayer instance will claim responsibility for any symbols provided by ...
std::shared_ptr< RuntimeDyld::MemoryManager > MemMgr
static ErrorSuccess success()
Create a success value.
Definition: Error.h:326
std::function< void(VModuleKey, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &)> NotifyLoadedFtor
Functor for receiving object-loaded notifications.
Error addObject(VModuleKey K, ObjectPtr ObjBuffer)
Add an object to the JIT.
std::function< std::unique_ptr< RuntimeDyld::MemoryManager >()> GetMemoryManagerFunction
JITSymbol findSymbol(StringRef Name, bool ExportedSymbolsOnly)
Search for the given named symbol.
An ExecutionSession represents a running JIT program.
Definition: Core.h:696
StringMap - This is an unconventional map that is specialized for handling keys that are "strings"...
Definition: StringMap.h:219
Represents a symbol that has been evaluated to an address already.
Definition: JITSymbol.h:208
std::function< Expected< JITTargetAddress >()> GetAddressFtor
Definition: JITSymbol.h:239
symbol_iterator_range symbols() const
Definition: ObjectFile.h:307
void setProcessAllSections(bool ProcessAllSections)
Set the &#39;ProcessAllSections&#39; flag.
Holds an object to be allocated/linked as a unit in the JIT.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
std::function< void(VModuleKey, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &)> NotifyFinalizedFtor
Functor for receiving finalization notifications.
Lightweight error class with error context and mandatory checking.
Definition: Error.h:157
Interface for Layers that accept object files.
Definition: Layer.h:113
print Print MemDeps of function
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
RTDyldObjectLinkingLayer & setOverrideObjectFlagsWithResponsibilityFlags(bool OverrideObjectFlags)
Instructs this RTDyldLinkingLayer2 instance to override the symbol flags returned by RuntimeDyld for ...
void emit(MaterializationResponsibility R, std::unique_ptr< MemoryBuffer > O) override
Emit the object.
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:77
Error emitAndFinalize(VModuleKey K)
Immediately emit and finalize the object represented by the given VModuleKey.