LLVM  8.0.0svn
IndirectionUtils.h
Go to the documentation of this file.
1 //===- IndirectionUtils.h - Utilities for adding indirections ---*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // Contains utilities for adding indirections and breaking up modules.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
15 #define LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
16 
17 #include "llvm/ADT/StringMap.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/ADT/Twine.h"
22 #include "llvm/Support/Error.h"
23 #include "llvm/Support/Memory.h"
24 #include "llvm/Support/Process.h"
26 #include <algorithm>
27 #include <cassert>
28 #include <cstdint>
29 #include <functional>
30 #include <map>
31 #include <memory>
32 #include <system_error>
33 #include <utility>
34 #include <vector>
35 
36 namespace llvm {
37 
38 class Constant;
39 class Function;
40 class FunctionType;
41 class GlobalAlias;
42 class GlobalVariable;
43 class Module;
44 class PointerType;
45 class Triple;
46 class Value;
47 
48 namespace orc {
49 
50 /// Base class for pools of compiler re-entry trampolines.
51 /// These trampolines are callable addresses that save all register state
52 /// before calling a supplied function to return the trampoline landing
53 /// address, then restore all state before jumping to that address. They
54 /// are used by various ORC APIs to support lazy compilation
56 public:
57  virtual ~TrampolinePool() {}
58 
59  /// Get an available trampoline address.
60  /// Returns an error if no trampoline can be created.
62 
63 private:
64  virtual void anchor();
65 };
66 
67 /// A trampoline pool for trampolines within the current process.
68 template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool {
69 public:
71  std::function<JITTargetAddress(JITTargetAddress TrampolineAddr)>;
72 
73  /// Creates a LocalTrampolinePool with the given RunCallback function.
74  /// Returns an error if this function is unable to correctly allocate, write
75  /// and protect the resolver code block.
77  Create(GetTrampolineLandingFunction GetTrampolineLanding) {
78  Error Err = Error::success();
79 
80  auto LTP = std::unique_ptr<LocalTrampolinePool>(
81  new LocalTrampolinePool(std::move(GetTrampolineLanding), Err));
82 
83  if (Err)
84  return std::move(Err);
85  return std::move(LTP);
86  }
87 
88  /// Get a free trampoline. Returns an error if one can not be provide (e.g.
89  /// because the pool is empty and can not be grown).
91  std::lock_guard<std::mutex> Lock(LTPMutex);
92  if (AvailableTrampolines.empty()) {
93  if (auto Err = grow())
94  return std::move(Err);
95  }
96  assert(!AvailableTrampolines.empty() && "Failed to grow trampoline pool");
97  auto TrampolineAddr = AvailableTrampolines.back();
98  AvailableTrampolines.pop_back();
99  return TrampolineAddr;
100  }
101 
102  /// Returns the given trampoline to the pool for re-use.
103  void releaseTrampoline(JITTargetAddress TrampolineAddr) {
104  std::lock_guard<std::mutex> Lock(LTPMutex);
105  AvailableTrampolines.push_back(TrampolineAddr);
106  }
107 
108 private:
109  static JITTargetAddress reenter(void *TrampolinePoolPtr, void *TrampolineId) {
111  static_cast<LocalTrampolinePool *>(TrampolinePoolPtr);
112  return TrampolinePool->GetTrampolineLanding(static_cast<JITTargetAddress>(
113  reinterpret_cast<uintptr_t>(TrampolineId)));
114  }
115 
117  Error &Err)
118  : GetTrampolineLanding(std::move(GetTrampolineLanding)) {
119 
120  ErrorAsOutParameter _(&Err);
121 
122  /// Try to set up the resolver block.
123  std::error_code EC;
125  ORCABI::ResolverCodeSize, nullptr,
127  if (EC) {
128  Err = errorCodeToError(EC);
129  return;
130  }
131 
132  ORCABI::writeResolverCode(static_cast<uint8_t *>(ResolverBlock.base()),
133  &reenter, this);
134 
135  EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(),
138  if (EC) {
139  Err = errorCodeToError(EC);
140  return;
141  }
142  }
143 
144  Error grow() {
145  assert(this->AvailableTrampolines.empty() && "Growing prematurely?");
146 
147  std::error_code EC;
148  auto TrampolineBlock =
150  sys::Process::getPageSize(), nullptr,
152  if (EC)
153  return errorCodeToError(EC);
154 
155  unsigned NumTrampolines =
156  (sys::Process::getPageSize() - ORCABI::PointerSize) /
157  ORCABI::TrampolineSize;
158 
159  uint8_t *TrampolineMem = static_cast<uint8_t *>(TrampolineBlock.base());
160  ORCABI::writeTrampolines(TrampolineMem, ResolverBlock.base(),
161  NumTrampolines);
162 
163  for (unsigned I = 0; I < NumTrampolines; ++I)
164  this->AvailableTrampolines.push_back(
165  static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(
166  TrampolineMem + (I * ORCABI::TrampolineSize))));
167 
168  if (auto EC = sys::Memory::protectMappedMemory(
169  TrampolineBlock.getMemoryBlock(),
171  return errorCodeToError(EC);
172 
173  TrampolineBlocks.push_back(std::move(TrampolineBlock));
174  return Error::success();
175  }
176 
177  GetTrampolineLandingFunction GetTrampolineLanding;
178 
179  std::mutex LTPMutex;
180  sys::OwningMemoryBlock ResolverBlock;
181  std::vector<sys::OwningMemoryBlock> TrampolineBlocks;
182  std::vector<JITTargetAddress> AvailableTrampolines;
183 };
184 
185 /// Target-independent base class for compile callback management.
187 public:
188  using CompileFunction = std::function<JITTargetAddress()>;
189 
190  virtual ~JITCompileCallbackManager() = default;
191 
192  /// Reserve a compile callback.
193  Expected<JITTargetAddress> getCompileCallback(CompileFunction Compile);
194 
195  /// Execute the callback for the given trampoline id. Called by the JIT
196  /// to compile functions on demand.
197  JITTargetAddress executeCompileCallback(JITTargetAddress TrampolineAddr);
198 
199 protected:
200  /// Construct a JITCompileCallbackManager.
201  JITCompileCallbackManager(std::unique_ptr<TrampolinePool> TP,
202  ExecutionSession &ES,
203  JITTargetAddress ErrorHandlerAddress)
204  : TP(std::move(TP)), ES(ES),
205  CallbacksJD(ES.createJITDylib("<Callbacks>")),
206  ErrorHandlerAddress(ErrorHandlerAddress) {}
207 
208  void setTrampolinePool(std::unique_ptr<TrampolinePool> TP) {
209  this->TP = std::move(TP);
210  }
211 
212 private:
213  std::mutex CCMgrMutex;
214  std::unique_ptr<TrampolinePool> TP;
215  ExecutionSession &ES;
216  JITDylib &CallbacksJD;
217  JITTargetAddress ErrorHandlerAddress;
218  std::map<JITTargetAddress, SymbolStringPtr> AddrToSymbol;
219  size_t NextCallbackId = 0;
220 };
221 
222 /// Manage compile callbacks for in-process JITs.
223 template <typename ORCABI>
225 public:
226  /// Create a new LocalJITCompileCallbackManager.
228  Create(ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) {
229  Error Err = Error::success();
230  auto CCMgr = std::unique_ptr<LocalJITCompileCallbackManager>(
231  new LocalJITCompileCallbackManager(ES, ErrorHandlerAddress, Err));
232  if (Err)
233  return std::move(Err);
234  return std::move(CCMgr);
235  }
236 
237 private:
238  /// Construct a InProcessJITCompileCallbackManager.
239  /// @param ErrorHandlerAddress The address of an error handler in the target
240  /// process to be used if a compile callback fails.
242  JITTargetAddress ErrorHandlerAddress,
243  Error &Err)
244  : JITCompileCallbackManager(nullptr, ES, ErrorHandlerAddress) {
245  ErrorAsOutParameter _(&Err);
247  [this](JITTargetAddress TrampolineAddr) {
248  return executeCompileCallback(TrampolineAddr);
249  });
250 
251  if (!TP) {
252  Err = TP.takeError();
253  return;
254  }
255 
256  setTrampolinePool(std::move(*TP));
257  }
258 };
259 
260 /// Base class for managing collections of named indirect stubs.
262 public:
263  /// Map type for initializing the manager. See init.
265 
266  virtual ~IndirectStubsManager() = default;
267 
268  /// Create a single stub with the given name, target address and flags.
269  virtual Error createStub(StringRef StubName, JITTargetAddress StubAddr,
270  JITSymbolFlags StubFlags) = 0;
271 
272  /// Create StubInits.size() stubs with the given names, target
273  /// addresses, and flags.
274  virtual Error createStubs(const StubInitsMap &StubInits) = 0;
275 
276  /// Find the stub with the given name. If ExportedStubsOnly is true,
277  /// this will only return a result if the stub's flags indicate that it
278  /// is exported.
279  virtual JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) = 0;
280 
281  /// Find the implementation-pointer for the stub.
282  virtual JITEvaluatedSymbol findPointer(StringRef Name) = 0;
283 
284  /// Change the value of the implementation pointer for the stub.
285  virtual Error updatePointer(StringRef Name, JITTargetAddress NewAddr) = 0;
286 
287 private:
288  virtual void anchor();
289 };
290 
291 /// IndirectStubsManager implementation for the host architecture, e.g.
292 /// OrcX86_64. (See OrcArchitectureSupport.h).
293 template <typename TargetT>
295 public:
297  JITSymbolFlags StubFlags) override {
298  std::lock_guard<std::mutex> Lock(StubsMutex);
299  if (auto Err = reserveStubs(1))
300  return Err;
301 
302  createStubInternal(StubName, StubAddr, StubFlags);
303 
304  return Error::success();
305  }
306 
307  Error createStubs(const StubInitsMap &StubInits) override {
308  std::lock_guard<std::mutex> Lock(StubsMutex);
309  if (auto Err = reserveStubs(StubInits.size()))
310  return Err;
311 
312  for (auto &Entry : StubInits)
313  createStubInternal(Entry.first(), Entry.second.first,
314  Entry.second.second);
315 
316  return Error::success();
317  }
318 
319  JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override {
320  std::lock_guard<std::mutex> Lock(StubsMutex);
321  auto I = StubIndexes.find(Name);
322  if (I == StubIndexes.end())
323  return nullptr;
324  auto Key = I->second.first;
325  void *StubAddr = IndirectStubsInfos[Key.first].getStub(Key.second);
326  assert(StubAddr && "Missing stub address");
327  auto StubTargetAddr =
328  static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(StubAddr));
329  auto StubSymbol = JITEvaluatedSymbol(StubTargetAddr, I->second.second);
330  if (ExportedStubsOnly && !StubSymbol.getFlags().isExported())
331  return nullptr;
332  return StubSymbol;
333  }
334 
336  std::lock_guard<std::mutex> Lock(StubsMutex);
337  auto I = StubIndexes.find(Name);
338  if (I == StubIndexes.end())
339  return nullptr;
340  auto Key = I->second.first;
341  void *PtrAddr = IndirectStubsInfos[Key.first].getPtr(Key.second);
342  assert(PtrAddr && "Missing pointer address");
343  auto PtrTargetAddr =
344  static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(PtrAddr));
345  return JITEvaluatedSymbol(PtrTargetAddr, I->second.second);
346  }
347 
349  using AtomicIntPtr = std::atomic<uintptr_t>;
350 
351  std::lock_guard<std::mutex> Lock(StubsMutex);
352  auto I = StubIndexes.find(Name);
353  assert(I != StubIndexes.end() && "No stub pointer for symbol");
354  auto Key = I->second.first;
355  AtomicIntPtr *AtomicStubPtr = reinterpret_cast<AtomicIntPtr *>(
356  IndirectStubsInfos[Key.first].getPtr(Key.second));
357  *AtomicStubPtr = static_cast<uintptr_t>(NewAddr);
358  return Error::success();
359  }
360 
361 private:
362  Error reserveStubs(unsigned NumStubs) {
363  if (NumStubs <= FreeStubs.size())
364  return Error::success();
365 
366  unsigned NewStubsRequired = NumStubs - FreeStubs.size();
367  unsigned NewBlockId = IndirectStubsInfos.size();
368  typename TargetT::IndirectStubsInfo ISI;
369  if (auto Err =
370  TargetT::emitIndirectStubsBlock(ISI, NewStubsRequired, nullptr))
371  return Err;
372  for (unsigned I = 0; I < ISI.getNumStubs(); ++I)
373  FreeStubs.push_back(std::make_pair(NewBlockId, I));
374  IndirectStubsInfos.push_back(std::move(ISI));
375  return Error::success();
376  }
377 
378  void createStubInternal(StringRef StubName, JITTargetAddress InitAddr,
379  JITSymbolFlags StubFlags) {
380  auto Key = FreeStubs.back();
381  FreeStubs.pop_back();
382  *IndirectStubsInfos[Key.first].getPtr(Key.second) =
383  reinterpret_cast<void *>(static_cast<uintptr_t>(InitAddr));
384  StubIndexes[StubName] = std::make_pair(Key, StubFlags);
385  }
386 
387  std::mutex StubsMutex;
388  std::vector<typename TargetT::IndirectStubsInfo> IndirectStubsInfos;
389  using StubKey = std::pair<uint16_t, uint16_t>;
390  std::vector<StubKey> FreeStubs;
392 };
393 
394 /// Create a local compile callback manager.
395 ///
396 /// The given target triple will determine the ABI, and the given
397 /// ErrorHandlerAddress will be used by the resulting compile callback
398 /// manager if a compile callback fails.
401  JITTargetAddress ErrorHandlerAddress);
402 
403 /// Create a local indriect stubs manager builder.
404 ///
405 /// The given target triple will determine the ABI.
406 std::function<std::unique_ptr<IndirectStubsManager>()>
408 
409 /// Build a function pointer of FunctionType with the given constant
410 /// address.
411 ///
412 /// Usage example: Turn a trampoline address into a function pointer constant
413 /// for use in a stub.
415 
416 /// Create a function pointer with the given type, name, and initializer
417 /// in the given Module.
419  Constant *Initializer);
420 
421 /// Turn a function declaration into a stub function that makes an
422 /// indirect call using the given function pointer.
423 void makeStub(Function &F, Value &ImplPointer);
424 
425 /// Promotes private symbols to global hidden, and renames to prevent clashes
426 /// with other promoted symbols. The same SymbolPromoter instance should be
427 /// used for all symbols to be added to a single JITDylib.
429 public:
430  /// Promote symbols in the given module. Returns the set of global values
431  /// that have been renamed/promoted.
432  std::vector<GlobalValue *> operator()(Module &M);
433 
434 private:
435  unsigned NextId = 0;
436 };
437 
438 /// Clone a function declaration into a new module.
439 ///
440 /// This function can be used as the first step towards creating a callback
441 /// stub (see makeStub), or moving a function body (see moveFunctionBody).
442 ///
443 /// If the VMap argument is non-null, a mapping will be added between F and
444 /// the new declaration, and between each of F's arguments and the new
445 /// declaration's arguments. This map can then be passed in to moveFunction to
446 /// move the function body if required. Note: When moving functions between
447 /// modules with these utilities, all decls should be cloned (and added to a
448 /// single VMap) before any bodies are moved. This will ensure that references
449 /// between functions all refer to the versions in the new module.
450 Function *cloneFunctionDecl(Module &Dst, const Function &F,
451  ValueToValueMapTy *VMap = nullptr);
452 
453 /// Move the body of function 'F' to a cloned function declaration in a
454 /// different module (See related cloneFunctionDecl).
455 ///
456 /// If the target function declaration is not supplied via the NewF parameter
457 /// then it will be looked up via the VMap.
458 ///
459 /// This will delete the body of function 'F' from its original parent module,
460 /// but leave its declaration.
461 void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
462  ValueMaterializer *Materializer = nullptr,
463  Function *NewF = nullptr);
464 
465 /// Clone a global variable declaration into a new module.
467  ValueToValueMapTy *VMap = nullptr);
468 
469 /// Move global variable GV from its parent module to cloned global
470 /// declaration in a different module.
471 ///
472 /// If the target global declaration is not supplied via the NewGV parameter
473 /// then it will be looked up via the VMap.
474 ///
475 /// This will delete the initializer of GV from its original parent module,
476 /// but leave its declaration.
478  ValueToValueMapTy &VMap,
479  ValueMaterializer *Materializer = nullptr,
480  GlobalVariable *NewGV = nullptr);
481 
482 /// Clone a global alias declaration into a new module.
484  ValueToValueMapTy &VMap);
485 
486 /// Clone module flags metadata into the destination module.
487 void cloneModuleFlagsMetadata(Module &Dst, const Module &Src,
488  ValueToValueMapTy &VMap);
489 
490 } // end namespace orc
491 
492 } // end namespace llvm
493 
494 #endif // LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
IndirectStubsManager implementation for the host architecture, e.g.
Base class for managing collections of named indirect stubs.
std::function< JITTargetAddress(JITTargetAddress TrampolineAddr)> GetTrampolineLandingFunction
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
std::function< std::unique_ptr< IndirectStubsManager >)> createLocalIndirectStubsManagerBuilder(const Triple &T)
Create a local indriect stubs manager builder.
GlobalAlias * cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, ValueToValueMapTy &VMap)
Clone a global alias declaration into a new module.
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:64
A trampoline pool for trampolines within the current process.
void cloneModuleFlagsMetadata(Module &Dst, const Module &Src, ValueToValueMapTy &VMap)
Clone module flags metadata into the destination module.
static sys::Mutex Lock
F(f)
Manage compile callbacks for in-process JITs.
static MemoryBlock allocateMappedMemory(size_t NumBytes, const MemoryBlock *const NearBlock, unsigned Flags, std::error_code &EC)
This method allocates a block of memory that is suitable for loading dynamically generated code (e...
void setTrampolinePool(std::unique_ptr< TrampolinePool > TP)
Promotes private symbols to global hidden, and renames to prevent clashes with other promoted symbols...
Target-independent base class for compile callback management.
static Expected< std::unique_ptr< LocalTrampolinePool > > Create(GetTrampolineLandingFunction GetTrampolineLanding)
Creates a LocalTrampolinePool with the given RunCallback function.
virtual Expected< JITTargetAddress > getTrampoline()=0
Get an available trampoline address.
amdgpu Simplify well known AMD library false Value Value const Twine & Name
Definition: BitVector.h:938
unsigned size() const
Definition: StringMap.h:112
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override
Change the value of the implementation pointer for the stub.
GlobalVariable * createImplPointer(PointerType &PT, Module &M, const Twine &Name, Constant *Initializer)
Create a function pointer with the given type, name, and initializer in the given Module...
void moveGlobalVariableInitializer(GlobalVariable &OrigGV, ValueToValueMapTy &VMap, ValueMaterializer *Materializer=nullptr, GlobalVariable *NewGV=nullptr)
Move global variable GV from its parent module to cloned global declaration in a different module...
Base class for pools of compiler re-entry trampolines.
Tagged union holding either a T or a Error.
Definition: CachePruning.h:23
JITEvaluatedSymbol findPointer(StringRef Name) override
Find the implementation-pointer for the stub.
void makeStub(Function &F, Value &ImplPointer)
Turn a function declaration into a stub function that makes an indirect call using the given function...
Key
PAL metadata keys.
Class to represent function types.
Definition: DerivedTypes.h:103
uint64_t JITTargetAddress
Represents an address in the target process&#39;s address space.
Definition: JITSymbol.h:41
GlobalVariable * cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, ValueToValueMapTy *VMap=nullptr)
Clone a global variable declaration into a new module.
Class to represent pointers.
Definition: DerivedTypes.h:467
void releaseTrampoline(JITTargetAddress TrampolineAddr)
Returns the given trampoline to the pool for re-use.
This is a class that can be implemented by clients to materialize Values on demand.
Definition: ValueMapper.h:51
Expected< JITTargetAddress > getTrampoline() override
Get a free trampoline.
This is an important base class in LLVM.
Definition: Constant.h:42
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Definition: Error.cpp:87
JITCompileCallbackManager(std::unique_ptr< TrampolinePool > TP, ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress)
Construct a JITCompileCallbackManager.
Flags for symbols in the JIT.
Definition: JITSymbol.h:44
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
static unsigned getPageSize()
Constant * createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr)
Build a function pointer of FunctionType with the given constant address.
std::function< JITTargetAddress()> CompileFunction
static ErrorSuccess success()
Create a success value.
Definition: Error.h:327
static Expected< std::unique_ptr< LocalJITCompileCallbackManager > > Create(ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress)
Create a new LocalJITCompileCallbackManager.
Function * cloneFunctionDecl(Module &Dst, const Function &F, ValueToValueMapTy *VMap=nullptr)
Clone a function declaration into a new module.
Expected< std::unique_ptr< JITCompileCallbackManager > > createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress)
Create a local compile callback manager.
Error createStub(StringRef StubName, JITTargetAddress StubAddr, JITSymbolFlags StubFlags) override
Create a single stub with the given name, target address and flags.
Owning version of MemoryBlock.
Definition: Memory.h:119
An ExecutionSession represents a running JIT program.
Definition: Core.h:689
StringMap - This is an unconventional map that is specialized for handling keys that are "strings"...
Definition: StringMap.h:220
Represents a symbol that has been evaluated to an address already.
Definition: JITSymbol.h:197
Helper for Errors used as out-parameters.
Definition: Error.h:1018
Provides a library for accessing information about this process and other processes on the operating ...
#define I(x, y, z)
Definition: MD5.cpp:58
Error createStubs(const StubInitsMap &StubInits) override
Create StubInits.size() stubs with the given names, target addresses, and flags.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override
Find the stub with the given name.
LLVM Value Representation.
Definition: Value.h:73
Lightweight error class with error context and mandatory checking.
Definition: Error.h:158
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, ValueMaterializer *Materializer=nullptr, Function *NewF=nullptr)
Move the body of function &#39;F&#39; to a cloned function declaration in a different module (See related clo...
#define _
A symbol table that supports asynchoronous symbol queries.
Definition: Core.h:490
static std::error_code protectMappedMemory(const MemoryBlock &Block, unsigned Flags)
This method sets the protection flags for a block of memory to the state specified by /p Flags...