LLVM  9.0.0svn
IndirectionUtils.h
Go to the documentation of this file.
1 //===- IndirectionUtils.h - Utilities for adding indirections ---*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Contains utilities for adding indirections and breaking up modules.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #ifndef LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
14 #define LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
15 
16 #include "llvm/ADT/StringMap.h"
17 #include "llvm/ADT/StringRef.h"
18 #include "llvm/ADT/Twine.h"
21 #include "llvm/Support/Error.h"
22 #include "llvm/Support/Memory.h"
23 #include "llvm/Support/Process.h"
25 #include <algorithm>
26 #include <cassert>
27 #include <cstdint>
28 #include <functional>
29 #include <map>
30 #include <memory>
31 #include <system_error>
32 #include <utility>
33 #include <vector>
34 
35 namespace llvm {
36 
37 class Constant;
38 class Function;
39 class FunctionType;
40 class GlobalAlias;
41 class GlobalVariable;
42 class Module;
43 class PointerType;
44 class Triple;
45 class Value;
46 
47 namespace orc {
48 
49 /// Base class for pools of compiler re-entry trampolines.
50 /// These trampolines are callable addresses that save all register state
51 /// before calling a supplied function to return the trampoline landing
52 /// address, then restore all state before jumping to that address. They
53 /// are used by various ORC APIs to support lazy compilation
55 public:
56  virtual ~TrampolinePool() {}
57 
58  /// Get an available trampoline address.
59  /// Returns an error if no trampoline can be created.
61 
62 private:
63  virtual void anchor();
64 };
65 
66 /// A trampoline pool for trampolines within the current process.
67 template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool {
68 public:
70  std::function<JITTargetAddress(JITTargetAddress TrampolineAddr)>;
71 
72  /// Creates a LocalTrampolinePool with the given RunCallback function.
73  /// Returns an error if this function is unable to correctly allocate, write
74  /// and protect the resolver code block.
76  Create(GetTrampolineLandingFunction GetTrampolineLanding) {
77  Error Err = Error::success();
78 
79  auto LTP = std::unique_ptr<LocalTrampolinePool>(
80  new LocalTrampolinePool(std::move(GetTrampolineLanding), Err));
81 
82  if (Err)
83  return std::move(Err);
84  return std::move(LTP);
85  }
86 
87  /// Get a free trampoline. Returns an error if one can not be provide (e.g.
88  /// because the pool is empty and can not be grown).
90  std::lock_guard<std::mutex> Lock(LTPMutex);
91  if (AvailableTrampolines.empty()) {
92  if (auto Err = grow())
93  return std::move(Err);
94  }
95  assert(!AvailableTrampolines.empty() && "Failed to grow trampoline pool");
96  auto TrampolineAddr = AvailableTrampolines.back();
97  AvailableTrampolines.pop_back();
98  return TrampolineAddr;
99  }
100 
101  /// Returns the given trampoline to the pool for re-use.
102  void releaseTrampoline(JITTargetAddress TrampolineAddr) {
103  std::lock_guard<std::mutex> Lock(LTPMutex);
104  AvailableTrampolines.push_back(TrampolineAddr);
105  }
106 
107 private:
108  static JITTargetAddress reenter(void *TrampolinePoolPtr, void *TrampolineId) {
110  static_cast<LocalTrampolinePool *>(TrampolinePoolPtr);
111  return TrampolinePool->GetTrampolineLanding(static_cast<JITTargetAddress>(
112  reinterpret_cast<uintptr_t>(TrampolineId)));
113  }
114 
116  Error &Err)
117  : GetTrampolineLanding(std::move(GetTrampolineLanding)) {
118 
119  ErrorAsOutParameter _(&Err);
120 
121  /// Try to set up the resolver block.
122  std::error_code EC;
124  ORCABI::ResolverCodeSize, nullptr,
126  if (EC) {
127  Err = errorCodeToError(EC);
128  return;
129  }
130 
131  ORCABI::writeResolverCode(static_cast<uint8_t *>(ResolverBlock.base()),
132  &reenter, this);
133 
134  EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(),
137  if (EC) {
138  Err = errorCodeToError(EC);
139  return;
140  }
141  }
142 
143  Error grow() {
144  assert(this->AvailableTrampolines.empty() && "Growing prematurely?");
145 
146  std::error_code EC;
147  auto TrampolineBlock =
151  if (EC)
152  return errorCodeToError(EC);
153 
154  unsigned NumTrampolines =
155  (sys::Process::getPageSizeEstimate() - ORCABI::PointerSize) /
156  ORCABI::TrampolineSize;
157 
158  uint8_t *TrampolineMem = static_cast<uint8_t *>(TrampolineBlock.base());
159  ORCABI::writeTrampolines(TrampolineMem, ResolverBlock.base(),
160  NumTrampolines);
161 
162  for (unsigned I = 0; I < NumTrampolines; ++I)
163  this->AvailableTrampolines.push_back(
164  static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(
165  TrampolineMem + (I * ORCABI::TrampolineSize))));
166 
167  if (auto EC = sys::Memory::protectMappedMemory(
168  TrampolineBlock.getMemoryBlock(),
170  return errorCodeToError(EC);
171 
172  TrampolineBlocks.push_back(std::move(TrampolineBlock));
173  return Error::success();
174  }
175 
176  GetTrampolineLandingFunction GetTrampolineLanding;
177 
178  std::mutex LTPMutex;
179  sys::OwningMemoryBlock ResolverBlock;
180  std::vector<sys::OwningMemoryBlock> TrampolineBlocks;
181  std::vector<JITTargetAddress> AvailableTrampolines;
182 };
183 
184 /// Target-independent base class for compile callback management.
186 public:
187  using CompileFunction = std::function<JITTargetAddress()>;
188 
189  virtual ~JITCompileCallbackManager() = default;
190 
191  /// Reserve a compile callback.
192  Expected<JITTargetAddress> getCompileCallback(CompileFunction Compile);
193 
194  /// Execute the callback for the given trampoline id. Called by the JIT
195  /// to compile functions on demand.
196  JITTargetAddress executeCompileCallback(JITTargetAddress TrampolineAddr);
197 
198 protected:
199  /// Construct a JITCompileCallbackManager.
200  JITCompileCallbackManager(std::unique_ptr<TrampolinePool> TP,
201  ExecutionSession &ES,
202  JITTargetAddress ErrorHandlerAddress)
203  : TP(std::move(TP)), ES(ES),
204  CallbacksJD(ES.createJITDylib("<Callbacks>")),
205  ErrorHandlerAddress(ErrorHandlerAddress) {}
206 
207  void setTrampolinePool(std::unique_ptr<TrampolinePool> TP) {
208  this->TP = std::move(TP);
209  }
210 
211 private:
212  std::mutex CCMgrMutex;
213  std::unique_ptr<TrampolinePool> TP;
214  ExecutionSession &ES;
215  JITDylib &CallbacksJD;
216  JITTargetAddress ErrorHandlerAddress;
217  std::map<JITTargetAddress, SymbolStringPtr> AddrToSymbol;
218  size_t NextCallbackId = 0;
219 };
220 
221 /// Manage compile callbacks for in-process JITs.
222 template <typename ORCABI>
224 public:
225  /// Create a new LocalJITCompileCallbackManager.
227  Create(ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) {
228  Error Err = Error::success();
229  auto CCMgr = std::unique_ptr<LocalJITCompileCallbackManager>(
230  new LocalJITCompileCallbackManager(ES, ErrorHandlerAddress, Err));
231  if (Err)
232  return std::move(Err);
233  return std::move(CCMgr);
234  }
235 
236 private:
237  /// Construct a InProcessJITCompileCallbackManager.
238  /// @param ErrorHandlerAddress The address of an error handler in the target
239  /// process to be used if a compile callback fails.
241  JITTargetAddress ErrorHandlerAddress,
242  Error &Err)
243  : JITCompileCallbackManager(nullptr, ES, ErrorHandlerAddress) {
244  ErrorAsOutParameter _(&Err);
246  [this](JITTargetAddress TrampolineAddr) {
247  return executeCompileCallback(TrampolineAddr);
248  });
249 
250  if (!TP) {
251  Err = TP.takeError();
252  return;
253  }
254 
255  setTrampolinePool(std::move(*TP));
256  }
257 };
258 
259 /// Base class for managing collections of named indirect stubs.
261 public:
262  /// Map type for initializing the manager. See init.
264 
265  virtual ~IndirectStubsManager() = default;
266 
267  /// Create a single stub with the given name, target address and flags.
268  virtual Error createStub(StringRef StubName, JITTargetAddress StubAddr,
269  JITSymbolFlags StubFlags) = 0;
270 
271  /// Create StubInits.size() stubs with the given names, target
272  /// addresses, and flags.
273  virtual Error createStubs(const StubInitsMap &StubInits) = 0;
274 
275  /// Find the stub with the given name. If ExportedStubsOnly is true,
276  /// this will only return a result if the stub's flags indicate that it
277  /// is exported.
278  virtual JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) = 0;
279 
280  /// Find the implementation-pointer for the stub.
281  virtual JITEvaluatedSymbol findPointer(StringRef Name) = 0;
282 
283  /// Change the value of the implementation pointer for the stub.
284  virtual Error updatePointer(StringRef Name, JITTargetAddress NewAddr) = 0;
285 
286 private:
287  virtual void anchor();
288 };
289 
290 /// IndirectStubsManager implementation for the host architecture, e.g.
291 /// OrcX86_64. (See OrcArchitectureSupport.h).
292 template <typename TargetT>
294 public:
296  JITSymbolFlags StubFlags) override {
297  std::lock_guard<std::mutex> Lock(StubsMutex);
298  if (auto Err = reserveStubs(1))
299  return Err;
300 
301  createStubInternal(StubName, StubAddr, StubFlags);
302 
303  return Error::success();
304  }
305 
306  Error createStubs(const StubInitsMap &StubInits) override {
307  std::lock_guard<std::mutex> Lock(StubsMutex);
308  if (auto Err = reserveStubs(StubInits.size()))
309  return Err;
310 
311  for (auto &Entry : StubInits)
312  createStubInternal(Entry.first(), Entry.second.first,
313  Entry.second.second);
314 
315  return Error::success();
316  }
317 
318  JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override {
319  std::lock_guard<std::mutex> Lock(StubsMutex);
320  auto I = StubIndexes.find(Name);
321  if (I == StubIndexes.end())
322  return nullptr;
323  auto Key = I->second.first;
324  void *StubAddr = IndirectStubsInfos[Key.first].getStub(Key.second);
325  assert(StubAddr && "Missing stub address");
326  auto StubTargetAddr =
327  static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(StubAddr));
328  auto StubSymbol = JITEvaluatedSymbol(StubTargetAddr, I->second.second);
329  if (ExportedStubsOnly && !StubSymbol.getFlags().isExported())
330  return nullptr;
331  return StubSymbol;
332  }
333 
335  std::lock_guard<std::mutex> Lock(StubsMutex);
336  auto I = StubIndexes.find(Name);
337  if (I == StubIndexes.end())
338  return nullptr;
339  auto Key = I->second.first;
340  void *PtrAddr = IndirectStubsInfos[Key.first].getPtr(Key.second);
341  assert(PtrAddr && "Missing pointer address");
342  auto PtrTargetAddr =
343  static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(PtrAddr));
344  return JITEvaluatedSymbol(PtrTargetAddr, I->second.second);
345  }
346 
348  using AtomicIntPtr = std::atomic<uintptr_t>;
349 
350  std::lock_guard<std::mutex> Lock(StubsMutex);
351  auto I = StubIndexes.find(Name);
352  assert(I != StubIndexes.end() && "No stub pointer for symbol");
353  auto Key = I->second.first;
354  AtomicIntPtr *AtomicStubPtr = reinterpret_cast<AtomicIntPtr *>(
355  IndirectStubsInfos[Key.first].getPtr(Key.second));
356  *AtomicStubPtr = static_cast<uintptr_t>(NewAddr);
357  return Error::success();
358  }
359 
360 private:
361  Error reserveStubs(unsigned NumStubs) {
362  if (NumStubs <= FreeStubs.size())
363  return Error::success();
364 
365  unsigned NewStubsRequired = NumStubs - FreeStubs.size();
366  unsigned NewBlockId = IndirectStubsInfos.size();
367  typename TargetT::IndirectStubsInfo ISI;
368  if (auto Err =
369  TargetT::emitIndirectStubsBlock(ISI, NewStubsRequired, nullptr))
370  return Err;
371  for (unsigned I = 0; I < ISI.getNumStubs(); ++I)
372  FreeStubs.push_back(std::make_pair(NewBlockId, I));
373  IndirectStubsInfos.push_back(std::move(ISI));
374  return Error::success();
375  }
376 
377  void createStubInternal(StringRef StubName, JITTargetAddress InitAddr,
378  JITSymbolFlags StubFlags) {
379  auto Key = FreeStubs.back();
380  FreeStubs.pop_back();
381  *IndirectStubsInfos[Key.first].getPtr(Key.second) =
382  reinterpret_cast<void *>(static_cast<uintptr_t>(InitAddr));
383  StubIndexes[StubName] = std::make_pair(Key, StubFlags);
384  }
385 
386  std::mutex StubsMutex;
387  std::vector<typename TargetT::IndirectStubsInfo> IndirectStubsInfos;
388  using StubKey = std::pair<uint16_t, uint16_t>;
389  std::vector<StubKey> FreeStubs;
391 };
392 
393 /// Create a local compile callback manager.
394 ///
395 /// The given target triple will determine the ABI, and the given
396 /// ErrorHandlerAddress will be used by the resulting compile callback
397 /// manager if a compile callback fails.
400  JITTargetAddress ErrorHandlerAddress);
401 
402 /// Create a local indriect stubs manager builder.
403 ///
404 /// The given target triple will determine the ABI.
405 std::function<std::unique_ptr<IndirectStubsManager>()>
407 
408 /// Build a function pointer of FunctionType with the given constant
409 /// address.
410 ///
411 /// Usage example: Turn a trampoline address into a function pointer constant
412 /// for use in a stub.
414 
415 /// Create a function pointer with the given type, name, and initializer
416 /// in the given Module.
418  Constant *Initializer);
419 
420 /// Turn a function declaration into a stub function that makes an
421 /// indirect call using the given function pointer.
422 void makeStub(Function &F, Value &ImplPointer);
423 
424 /// Promotes private symbols to global hidden, and renames to prevent clashes
425 /// with other promoted symbols. The same SymbolPromoter instance should be
426 /// used for all symbols to be added to a single JITDylib.
428 public:
429  /// Promote symbols in the given module. Returns the set of global values
430  /// that have been renamed/promoted.
431  std::vector<GlobalValue *> operator()(Module &M);
432 
433 private:
434  unsigned NextId = 0;
435 };
436 
437 /// Clone a function declaration into a new module.
438 ///
439 /// This function can be used as the first step towards creating a callback
440 /// stub (see makeStub), or moving a function body (see moveFunctionBody).
441 ///
442 /// If the VMap argument is non-null, a mapping will be added between F and
443 /// the new declaration, and between each of F's arguments and the new
444 /// declaration's arguments. This map can then be passed in to moveFunction to
445 /// move the function body if required. Note: When moving functions between
446 /// modules with these utilities, all decls should be cloned (and added to a
447 /// single VMap) before any bodies are moved. This will ensure that references
448 /// between functions all refer to the versions in the new module.
449 Function *cloneFunctionDecl(Module &Dst, const Function &F,
450  ValueToValueMapTy *VMap = nullptr);
451 
452 /// Move the body of function 'F' to a cloned function declaration in a
453 /// different module (See related cloneFunctionDecl).
454 ///
455 /// If the target function declaration is not supplied via the NewF parameter
456 /// then it will be looked up via the VMap.
457 ///
458 /// This will delete the body of function 'F' from its original parent module,
459 /// but leave its declaration.
460 void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
461  ValueMaterializer *Materializer = nullptr,
462  Function *NewF = nullptr);
463 
464 /// Clone a global variable declaration into a new module.
466  ValueToValueMapTy *VMap = nullptr);
467 
468 /// Move global variable GV from its parent module to cloned global
469 /// declaration in a different module.
470 ///
471 /// If the target global declaration is not supplied via the NewGV parameter
472 /// then it will be looked up via the VMap.
473 ///
474 /// This will delete the initializer of GV from its original parent module,
475 /// but leave its declaration.
477  ValueToValueMapTy &VMap,
478  ValueMaterializer *Materializer = nullptr,
479  GlobalVariable *NewGV = nullptr);
480 
481 /// Clone a global alias declaration into a new module.
483  ValueToValueMapTy &VMap);
484 
485 /// Clone module flags metadata into the destination module.
486 void cloneModuleFlagsMetadata(Module &Dst, const Module &Src,
487  ValueToValueMapTy &VMap);
488 
489 } // end namespace orc
490 
491 } // end namespace llvm
492 
493 #endif // LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
IndirectStubsManager implementation for the host architecture, e.g.
Base class for managing collections of named indirect stubs.
std::function< JITTargetAddress(JITTargetAddress TrampolineAddr)> GetTrampolineLandingFunction
This class represents lattice values for constants.
Definition: AllocatorList.h:23
std::function< std::unique_ptr< IndirectStubsManager >)> createLocalIndirectStubsManagerBuilder(const Triple &T)
Create a local indriect stubs manager builder.
GlobalAlias * cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, ValueToValueMapTy &VMap)
Clone a global alias declaration into a new module.
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:65
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
A trampoline pool for trampolines within the current process.
void cloneModuleFlagsMetadata(Module &Dst, const Module &Src, ValueToValueMapTy &VMap)
Clone module flags metadata into the destination module.
static sys::Mutex Lock
F(f)
Manage compile callbacks for in-process JITs.
static MemoryBlock allocateMappedMemory(size_t NumBytes, const MemoryBlock *const NearBlock, unsigned Flags, std::error_code &EC)
This method allocates a block of memory that is suitable for loading dynamically generated code (e...
void setTrampolinePool(std::unique_ptr< TrampolinePool > TP)
Promotes private symbols to global hidden, and renames to prevent clashes with other promoted symbols...
Target-independent base class for compile callback management.
static Expected< std::unique_ptr< LocalTrampolinePool > > Create(GetTrampolineLandingFunction GetTrampolineLanding)
Creates a LocalTrampolinePool with the given RunCallback function.
virtual Expected< JITTargetAddress > getTrampoline()=0
Get an available trampoline address.
Definition: BitVector.h:937
unsigned size() const
Definition: StringMap.h:111
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override
Change the value of the implementation pointer for the stub.
GlobalVariable * createImplPointer(PointerType &PT, Module &M, const Twine &Name, Constant *Initializer)
Create a function pointer with the given type, name, and initializer in the given Module...
void moveGlobalVariableInitializer(GlobalVariable &OrigGV, ValueToValueMapTy &VMap, ValueMaterializer *Materializer=nullptr, GlobalVariable *NewGV=nullptr)
Move global variable GV from its parent module to cloned global declaration in a different module...
Base class for pools of compiler re-entry trampolines.
Tagged union holding either a T or a Error.
Definition: CachePruning.h:22
JITEvaluatedSymbol findPointer(StringRef Name) override
Find the implementation-pointer for the stub.
void makeStub(Function &F, Value &ImplPointer)
Turn a function declaration into a stub function that makes an indirect call using the given function...
Key
PAL metadata keys.
Class to represent function types.
Definition: DerivedTypes.h:102
static unsigned getPageSizeEstimate()
Get the process&#39;s estimated page size.
Definition: Process.h:56
uint64_t JITTargetAddress
Represents an address in the target process&#39;s address space.
Definition: JITSymbol.h:40
GlobalVariable * cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, ValueToValueMapTy *VMap=nullptr)
Clone a global variable declaration into a new module.
Class to represent pointers.
Definition: DerivedTypes.h:498
void releaseTrampoline(JITTargetAddress TrampolineAddr)
Returns the given trampoline to the pool for re-use.
This is a class that can be implemented by clients to materialize Values on demand.
Definition: ValueMapper.h:50
Expected< JITTargetAddress > getTrampoline() override
Get a free trampoline.
This is an important base class in LLVM.
Definition: Constant.h:41
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Definition: Error.cpp:87
JITCompileCallbackManager(std::unique_ptr< TrampolinePool > TP, ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress)
Construct a JITCompileCallbackManager.
Flags for symbols in the JIT.
Definition: JITSymbol.h:55
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:43
Constant * createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr)
Build a function pointer of FunctionType with the given constant address.
std::function< JITTargetAddress()> CompileFunction
static ErrorSuccess success()
Create a success value.
Definition: Error.h:326
static Expected< std::unique_ptr< LocalJITCompileCallbackManager > > Create(ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress)
Create a new LocalJITCompileCallbackManager.
Function * cloneFunctionDecl(Module &Dst, const Function &F, ValueToValueMapTy *VMap=nullptr)
Clone a function declaration into a new module.
Expected< std::unique_ptr< JITCompileCallbackManager > > createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress)
Create a local compile callback manager.
Error createStub(StringRef StubName, JITTargetAddress StubAddr, JITSymbolFlags StubFlags) override
Create a single stub with the given name, target address and flags.
Owning version of MemoryBlock.
Definition: Memory.h:128
An ExecutionSession represents a running JIT program.
Definition: Core.h:696
StringMap - This is an unconventional map that is specialized for handling keys that are "strings"...
Definition: StringMap.h:219
Represents a symbol that has been evaluated to an address already.
Definition: JITSymbol.h:208
Helper for Errors used as out-parameters.
Definition: Error.h:1021
Provides a library for accessing information about this process and other processes on the operating ...
#define I(x, y, z)
Definition: MD5.cpp:58
Error createStubs(const StubInitsMap &StubInits) override
Create StubInits.size() stubs with the given names, target addresses, and flags.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override
Find the stub with the given name.
LLVM Value Representation.
Definition: Value.h:72
Lightweight error class with error context and mandatory checking.
Definition: Error.h:157
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, ValueMaterializer *Materializer=nullptr, Function *NewF=nullptr)
Move the body of function &#39;F&#39; to a cloned function declaration in a different module (See related clo...
#define _
A symbol table that supports asynchoronous symbol queries.
Definition: Core.h:495
static std::error_code protectMappedMemory(const MemoryBlock &Block, unsigned Flags)
This method sets the protection flags for a block of memory to the state specified by /p Flags...