Line data Source code
1 : //===- IndirectionUtils.h - Utilities for adding indirections ---*- C++ -*-===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // Contains utilities for adding indirections and breaking up modules.
11 : //
12 : //===----------------------------------------------------------------------===//
13 :
14 : #ifndef LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
15 : #define LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
16 :
17 : #include "llvm/ADT/StringMap.h"
18 : #include "llvm/ADT/StringRef.h"
19 : #include "llvm/ADT/Twine.h"
20 : #include "llvm/ExecutionEngine/JITSymbol.h"
21 : #include "llvm/ExecutionEngine/Orc/Core.h"
22 : #include "llvm/Support/Error.h"
23 : #include "llvm/Support/Memory.h"
24 : #include "llvm/Support/Process.h"
25 : #include "llvm/Transforms/Utils/ValueMapper.h"
26 : #include <algorithm>
27 : #include <cassert>
28 : #include <cstdint>
29 : #include <functional>
30 : #include <map>
31 : #include <memory>
32 : #include <system_error>
33 : #include <utility>
34 : #include <vector>
35 :
36 : namespace llvm {
37 :
38 : class Constant;
39 : class Function;
40 : class FunctionType;
41 : class GlobalAlias;
42 : class GlobalVariable;
43 : class Module;
44 : class PointerType;
45 : class Triple;
46 : class Value;
47 :
48 : namespace orc {
49 :
50 : /// Base class for pools of compiler re-entry trampolines.
51 : /// These trampolines are callable addresses that save all register state
52 : /// before calling a supplied function to return the trampoline landing
53 : /// address, then restore all state before jumping to that address. They
54 : /// are used by various ORC APIs to support lazy compilation
55 : class TrampolinePool {
56 : public:
57 0 : virtual ~TrampolinePool() {}
58 :
59 : /// Get an available trampoline address.
60 : /// Returns an error if no trampoline can be created.
61 : virtual Expected<JITTargetAddress> getTrampoline() = 0;
62 :
63 : private:
64 : virtual void anchor();
65 : };
66 :
67 : /// A trampoline pool for trampolines within the current process.
68 : template <typename ORCABI> class LocalTrampolinePool : public TrampolinePool {
69 : public:
70 : using GetTrampolineLandingFunction =
71 : std::function<JITTargetAddress(JITTargetAddress TrampolineAddr)>;
72 :
73 : /// Creates a LocalTrampolinePool with the given RunCallback function.
74 : /// Returns an error if this function is unable to correctly allocate, write
75 : /// and protect the resolver code block.
76 : static Expected<std::unique_ptr<LocalTrampolinePool>>
77 25 : Create(GetTrampolineLandingFunction GetTrampolineLanding) {
78 : Error Err = Error::success();
79 :
80 25 : auto LTP = std::unique_ptr<LocalTrampolinePool>(
81 25 : new LocalTrampolinePool(std::move(GetTrampolineLanding), Err));
82 :
83 25 : if (Err)
84 : return std::move(Err);
85 : return std::move(LTP);
86 : }
87 25 :
88 : /// Get a free trampoline. Returns an error if one can not be provide (e.g.
89 : /// because the pool is empty and can not be grown).
90 25 : Expected<JITTargetAddress> getTrampoline() override {
91 25 : std::lock_guard<std::mutex> Lock(LTPMutex);
92 : if (AvailableTrampolines.empty()) {
93 25 : if (auto Err = grow())
94 : return std::move(Err);
95 : }
96 : assert(!AvailableTrampolines.empty() && "Failed to grow trampoline pool");
97 0 : auto TrampolineAddr = AvailableTrampolines.back();
98 : AvailableTrampolines.pop_back();
99 : return TrampolineAddr;
100 0 : }
101 0 :
102 : /// Returns the given trampoline to the pool for re-use.
103 0 : void releaseTrampoline(JITTargetAddress TrampolineAddr) {
104 : std::lock_guard<std::mutex> Lock(LTPMutex);
105 : AvailableTrampolines.push_back(TrampolineAddr);
106 : }
107 0 :
108 : private:
109 : static JITTargetAddress reenter(void *TrampolinePoolPtr, void *TrampolineId) {
110 0 : LocalTrampolinePool<ORCABI> *TrampolinePool =
111 0 : static_cast<LocalTrampolinePool *>(TrampolinePoolPtr);
112 : return TrampolinePool->GetTrampolineLanding(static_cast<JITTargetAddress>(
113 0 : reinterpret_cast<uintptr_t>(TrampolineId)));
114 : }
115 :
116 : LocalTrampolinePool(GetTrampolineLandingFunction GetTrampolineLanding,
117 0 : Error &Err)
118 : : GetTrampolineLanding(std::move(GetTrampolineLanding)) {
119 :
120 0 : ErrorAsOutParameter _(&Err);
121 0 :
122 : /// Try to set up the resolver block.
123 0 : std::error_code EC;
124 : ResolverBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(
125 : ORCABI::ResolverCodeSize, nullptr,
126 : sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));
127 0 : if (EC) {
128 : Err = errorCodeToError(EC);
129 : return;
130 0 : }
131 0 :
132 : ORCABI::writeResolverCode(static_cast<uint8_t *>(ResolverBlock.base()),
133 0 : &reenter, this);
134 :
135 : EC = sys::Memory::protectMappedMemory(ResolverBlock.getMemoryBlock(),
136 : sys::Memory::MF_READ |
137 0 : sys::Memory::MF_EXEC);
138 : if (EC) {
139 : Err = errorCodeToError(EC);
140 0 : return;
141 0 : }
142 : }
143 0 :
144 : Error grow() {
145 : assert(this->AvailableTrampolines.empty() && "Growing prematurely?");
146 :
147 0 : std::error_code EC;
148 : auto TrampolineBlock =
149 : sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory(
150 0 : sys::Process::getPageSize(), nullptr,
151 0 : sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC));
152 : if (EC)
153 0 : return errorCodeToError(EC);
154 :
155 : unsigned NumTrampolines =
156 : (sys::Process::getPageSize() - ORCABI::PointerSize) /
157 : ORCABI::TrampolineSize;
158 :
159 : uint8_t *TrampolineMem = static_cast<uint8_t *>(TrampolineBlock.base());
160 22 : ORCABI::writeTrampolines(TrampolineMem, ResolverBlock.base(),
161 22 : NumTrampolines);
162 22 :
163 30 : for (unsigned I = 0; I < NumTrampolines; ++I)
164 : this->AvailableTrampolines.push_back(
165 : static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(
166 : TrampolineMem + (I * ORCABI::TrampolineSize))));
167 22 :
168 : if (auto EC = sys::Memory::protectMappedMemory(
169 : TrampolineBlock.getMemoryBlock(),
170 : sys::Memory::MF_READ | sys::Memory::MF_EXEC))
171 0 : return errorCodeToError(EC);
172 0 :
173 0 : TrampolineBlocks.push_back(std::move(TrampolineBlock));
174 0 : return Error::success();
175 : }
176 :
177 : GetTrampolineLandingFunction GetTrampolineLanding;
178 0 :
179 : std::mutex LTPMutex;
180 : sys::OwningMemoryBlock ResolverBlock;
181 : std::vector<sys::OwningMemoryBlock> TrampolineBlocks;
182 0 : std::vector<JITTargetAddress> AvailableTrampolines;
183 0 : };
184 0 :
185 0 : /// Target-independent base class for compile callback management.
186 : class JITCompileCallbackManager {
187 : public:
188 : using CompileFunction = std::function<JITTargetAddress()>;
189 0 :
190 1 : virtual ~JITCompileCallbackManager() = default;
191 :
192 : /// Reserve a compile callback.
193 0 : Expected<JITTargetAddress> getCompileCallback(CompileFunction Compile);
194 0 :
195 0 : /// Execute the callback for the given trampoline id. Called by the JIT
196 0 : /// to compile functions on demand.
197 : JITTargetAddress executeCompileCallback(JITTargetAddress TrampolineAddr);
198 :
199 : protected:
200 0 : /// Construct a JITCompileCallbackManager.
201 1 : JITCompileCallbackManager(std::unique_ptr<TrampolinePool> TP,
202 : ExecutionSession &ES,
203 : JITTargetAddress ErrorHandlerAddress)
204 1 : : TP(std::move(TP)), ES(ES),
205 1 : CallbacksJD(ES.createJITDylib("<Callbacks>")),
206 3 : ErrorHandlerAddress(ErrorHandlerAddress) {}
207 0 :
208 : void setTrampolinePool(std::unique_ptr<TrampolinePool> TP) {
209 : this->TP = std::move(TP);
210 : }
211 0 :
212 : private:
213 : std::mutex CCMgrMutex;
214 : std::unique_ptr<TrampolinePool> TP;
215 0 : ExecutionSession &ES;
216 0 : JITDylib &CallbacksJD;
217 0 : JITTargetAddress ErrorHandlerAddress;
218 0 : std::map<JITTargetAddress, SymbolStringPtr> AddrToSymbol;
219 : size_t NextCallbackId = 0;
220 : };
221 :
222 0 : /// Manage compile callbacks for in-process JITs.
223 : template <typename ORCABI>
224 : class LocalJITCompileCallbackManager : public JITCompileCallbackManager {
225 : public:
226 0 : /// Create a new LocalJITCompileCallbackManager.
227 0 : static Expected<std::unique_ptr<LocalJITCompileCallbackManager>>
228 0 : Create(ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) {
229 0 : Error Err = Error::success();
230 : auto CCMgr = std::unique_ptr<LocalJITCompileCallbackManager>(
231 : new LocalJITCompileCallbackManager(ES, ErrorHandlerAddress, Err));
232 : if (Err)
233 0 : return std::move(Err);
234 : return std::move(CCMgr);
235 : }
236 :
237 22 : private:
238 22 : /// Construct a InProcessJITCompileCallbackManager.
239 22 : /// @param ErrorHandlerAddress The address of an error handler in the target
240 30 : /// process to be used if a compile callback fails.
241 : LocalJITCompileCallbackManager(ExecutionSession &ES,
242 : JITTargetAddress ErrorHandlerAddress,
243 : Error &Err)
244 22 : : JITCompileCallbackManager(nullptr, ES, ErrorHandlerAddress) {
245 : ErrorAsOutParameter _(&Err);
246 : auto TP = LocalTrampolinePool<ORCABI>::Create(
247 : [this](JITTargetAddress TrampolineAddr) {
248 : return executeCompileCallback(TrampolineAddr);
249 : });
250 :
251 : if (!TP) {
252 : Err = TP.takeError();
253 : return;
254 : }
255 :
256 22 : setTrampolinePool(std::move(*TP));
257 : }
258 : };
259 :
260 22 : /// Base class for managing collections of named indirect stubs.
261 : class IndirectStubsManager {
262 22 : public:
263 : /// Map type for initializing the manager. See init.
264 : using StubInitsMap = StringMap<std::pair<JITTargetAddress, JITSymbolFlags>>;
265 :
266 22 : virtual ~IndirectStubsManager() = default;
267 :
268 0 : /// Create a single stub with the given name, target address and flags.
269 : virtual Error createStub(StringRef StubName, JITTargetAddress StubAddr,
270 : JITSymbolFlags StubFlags) = 0;
271 :
272 0 : /// Create StubInits.size() stubs with the given names, target
273 : /// addresses, and flags.
274 0 : virtual Error createStubs(const StubInitsMap &StubInits) = 0;
275 :
276 : /// Find the stub with the given name. If ExportedStubsOnly is true,
277 : /// this will only return a result if the stub's flags indicate that it
278 0 : /// is exported.
279 : virtual JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) = 0;
280 0 :
281 : /// Find the implementation-pointer for the stub.
282 : virtual JITEvaluatedSymbol findPointer(StringRef Name) = 0;
283 :
284 0 : /// Change the value of the implementation pointer for the stub.
285 : virtual Error updatePointer(StringRef Name, JITTargetAddress NewAddr) = 0;
286 0 :
287 : private:
288 : virtual void anchor();
289 : };
290 0 :
291 : /// IndirectStubsManager implementation for the host architecture, e.g.
292 0 : /// OrcX86_64. (See OrcArchitectureSupport.h).
293 : template <typename TargetT>
294 : class LocalIndirectStubsManager : public IndirectStubsManager {
295 : public:
296 0 : Error createStub(StringRef StubName, JITTargetAddress StubAddr,
297 : JITSymbolFlags StubFlags) override {
298 0 : std::lock_guard<std::mutex> Lock(StubsMutex);
299 : if (auto Err = reserveStubs(1))
300 : return Err;
301 :
302 0 : createStubInternal(StubName, StubAddr, StubFlags);
303 :
304 : return Error::success();
305 25 : }
306 :
307 25 : Error createStubs(const StubInitsMap &StubInits) override {
308 : std::lock_guard<std::mutex> Lock(StubsMutex);
309 : if (auto Err = reserveStubs(StubInits.size()))
310 : return Err;
311 :
312 : for (auto &Entry : StubInits)
313 25 : createStubInternal(Entry.first(), Entry.second.first,
314 : Entry.second.second);
315 :
316 25 : return Error::success();
317 0 : }
318 0 :
319 : JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override {
320 : std::lock_guard<std::mutex> Lock(StubsMutex);
321 25 : auto I = StubIndexes.find(Name);
322 : if (I == StubIndexes.end())
323 : return nullptr;
324 25 : auto Key = I->second.first;
325 : void *StubAddr = IndirectStubsInfos[Key.first].getStub(Key.second);
326 : assert(StubAddr && "Missing stub address");
327 25 : auto StubTargetAddr =
328 0 : static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(StubAddr));
329 0 : auto StubSymbol = JITEvaluatedSymbol(StubTargetAddr, I->second.second);
330 : if (ExportedStubsOnly && !StubSymbol.getFlags().isExported())
331 : return nullptr;
332 25 : return StubSymbol;
333 : }
334 25 :
335 : JITEvaluatedSymbol findPointer(StringRef Name) override {
336 : std::lock_guard<std::mutex> Lock(StubsMutex);
337 : auto I = StubIndexes.find(Name);
338 : if (I == StubIndexes.end())
339 : return nullptr;
340 25 : auto Key = I->second.first;
341 : void *PtrAddr = IndirectStubsInfos[Key.first].getPtr(Key.second);
342 : assert(PtrAddr && "Missing pointer address");
343 25 : auto PtrTargetAddr =
344 0 : static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(PtrAddr));
345 0 : return JITEvaluatedSymbol(PtrTargetAddr, I->second.second);
346 : }
347 :
348 25 : Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override {
349 : using AtomicIntPtr = std::atomic<uintptr_t>;
350 :
351 25 : std::lock_guard<std::mutex> Lock(StubsMutex);
352 : auto I = StubIndexes.find(Name);
353 : assert(I != StubIndexes.end() && "No stub pointer for symbol");
354 25 : auto Key = I->second.first;
355 0 : AtomicIntPtr *AtomicStubPtr = reinterpret_cast<AtomicIntPtr *>(
356 0 : IndirectStubsInfos[Key.first].getPtr(Key.second));
357 : *AtomicStubPtr = static_cast<uintptr_t>(NewAddr);
358 : return Error::success();
359 0 : }
360 :
361 0 : private:
362 : Error reserveStubs(unsigned NumStubs) {
363 : if (NumStubs <= FreeStubs.size())
364 : return Error::success();
365 :
366 : unsigned NewStubsRequired = NumStubs - FreeStubs.size();
367 0 : unsigned NewBlockId = IndirectStubsInfos.size();
368 : typename TargetT::IndirectStubsInfo ISI;
369 : if (auto Err =
370 0 : TargetT::emitIndirectStubsBlock(ISI, NewStubsRequired, nullptr))
371 0 : return Err;
372 0 : for (unsigned I = 0; I < ISI.getNumStubs(); ++I)
373 : FreeStubs.push_back(std::make_pair(NewBlockId, I));
374 : IndirectStubsInfos.push_back(std::move(ISI));
375 0 : return Error::success();
376 : }
377 :
378 0 : void createStubInternal(StringRef StubName, JITTargetAddress InitAddr,
379 : JITSymbolFlags StubFlags) {
380 : auto Key = FreeStubs.back();
381 0 : FreeStubs.pop_back();
382 0 : *IndirectStubsInfos[Key.first].getPtr(Key.second) =
383 0 : reinterpret_cast<void *>(static_cast<uintptr_t>(InitAddr));
384 : StubIndexes[StubName] = std::make_pair(Key, StubFlags);
385 : }
386 0 :
387 : std::mutex StubsMutex;
388 0 : std::vector<typename TargetT::IndirectStubsInfo> IndirectStubsInfos;
389 : using StubKey = std::pair<uint16_t, uint16_t>;
390 : std::vector<StubKey> FreeStubs;
391 : StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes;
392 : };
393 :
394 0 : /// Create a local compile callback manager.
395 : ///
396 : /// The given target triple will determine the ABI, and the given
397 0 : /// ErrorHandlerAddress will be used by the resulting compile callback
398 0 : /// manager if a compile callback fails.
399 0 : Expected<std::unique_ptr<JITCompileCallbackManager>>
400 : createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
401 : JITTargetAddress ErrorHandlerAddress);
402 0 :
403 : /// Create a local indriect stubs manager builder.
404 : ///
405 0 : /// The given target triple will determine the ABI.
406 : std::function<std::unique_ptr<IndirectStubsManager>()>
407 : createLocalIndirectStubsManagerBuilder(const Triple &T);
408 0 :
409 0 : /// Build a function pointer of FunctionType with the given constant
410 0 : /// address.
411 : ///
412 : /// Usage example: Turn a trampoline address into a function pointer constant
413 0 : /// for use in a stub.
414 : Constant *createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr);
415 0 :
416 : /// Create a function pointer with the given type, name, and initializer
417 : /// in the given Module.
418 : GlobalVariable *createImplPointer(PointerType &PT, Module &M, const Twine &Name,
419 : Constant *Initializer);
420 :
421 0 : /// Turn a function declaration into a stub function that makes an
422 : /// indirect call using the given function pointer.
423 : void makeStub(Function &F, Value &ImplPointer);
424 0 :
425 0 : /// Promotes private symbols to global hidden, and renames to prevent clashes
426 0 : /// with other promoted symbols. The same SymbolPromoter instance should be
427 : /// used for all symbols to be added to a single JITDylib.
428 14 : class SymbolLinkagePromoter {
429 : public:
430 : /// Promote symbols in the given module. Returns the set of global values
431 : /// that have been renamed/promoted.
432 0 : std::vector<GlobalValue *> operator()(Module &M);
433 :
434 : private:
435 0 : unsigned NextId = 0;
436 0 : };
437 0 :
438 : /// Clone a function declaration into a new module.
439 : ///
440 0 : /// This function can be used as the first step towards creating a callback
441 : /// stub (see makeStub), or moving a function body (see moveFunctionBody).
442 0 : ///
443 : /// If the VMap argument is non-null, a mapping will be added between F and
444 : /// the new declaration, and between each of F's arguments and the new
445 : /// declaration's arguments. This map can then be passed in to moveFunction to
446 : /// move the function body if required. Note: When moving functions between
447 : /// modules with these utilities, all decls should be cloned (and added to a
448 0 : /// single VMap) before any bodies are moved. This will ensure that references
449 : /// between functions all refer to the versions in the new module.
450 : Function *cloneFunctionDecl(Module &Dst, const Function &F,
451 0 : ValueToValueMapTy *VMap = nullptr);
452 0 :
453 0 : /// Move the body of function 'F' to a cloned function declaration in a
454 : /// different module (See related cloneFunctionDecl).
455 : ///
456 : /// If the target function declaration is not supplied via the NewF parameter
457 : /// then it will be looked up via the VMap.
458 : ///
459 0 : /// This will delete the body of function 'F' from its original parent module,
460 : /// but leave its declaration.
461 : void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap,
462 0 : ValueMaterializer *Materializer = nullptr,
463 0 : Function *NewF = nullptr);
464 0 :
465 : /// Clone a global variable declaration into a new module.
466 : GlobalVariable *cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV,
467 0 : ValueToValueMapTy *VMap = nullptr);
468 :
469 0 : /// Move global variable GV from its parent module to cloned global
470 : /// declaration in a different module.
471 : ///
472 : /// If the target global declaration is not supplied via the NewGV parameter
473 : /// then it will be looked up via the VMap.
474 : ///
475 0 : /// This will delete the initializer of GV from its original parent module,
476 : /// but leave its declaration.
477 : void moveGlobalVariableInitializer(GlobalVariable &OrigGV,
478 0 : ValueToValueMapTy &VMap,
479 0 : ValueMaterializer *Materializer = nullptr,
480 0 : GlobalVariable *NewGV = nullptr);
481 :
482 : /// Clone a global alias declaration into a new module.
483 0 : GlobalAlias *cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA,
484 : ValueToValueMapTy &VMap);
485 :
486 0 : /// Clone module flags metadata into the destination module.
487 : void cloneModuleFlagsMetadata(Module &Dst, const Module &Src,
488 : ValueToValueMapTy &VMap);
489 0 :
490 0 : } // end namespace orc
491 0 :
492 : } // end namespace llvm
493 :
494 0 : #endif // LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
|
