13#ifndef LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
14#define LLVM_EXECUTIONENGINE_ORC_INDIRECTIONUTILS_H
32#include <system_error>
77 std::lock_guard<std::mutex> Lock(
TPMutex);
79 if (
auto Err =
grow())
80 return std::move(Err);
85 return TrampolineAddr;
90 std::lock_guard<std::mutex> Lock(
TPMutex);
111 auto LTP = std::unique_ptr<LocalTrampolinePool>(
115 return std::move(Err);
116 return std::move(LTP);
120 static JITTargetAddress reenter(
void *TrampolinePoolPtr,
void *TrampolineId) {
124 std::promise<JITTargetAddress> LandingAddressP;
125 auto LandingAddressF = LandingAddressP.get_future();
129 LandingAddressP.set_value(LandingAddress);
131 return LandingAddressF.get();
135 : ResolveLanding(
std::
move(ResolveLanding)) {
137 ErrorAsOutParameter
_(&Err);
142 ORCABI::ResolverCodeSize,
nullptr,
149 ORCABI::writeResolverCode(
static_cast<char *
>(ResolverBlock.
base()),
163 Error grow()
override {
167 auto TrampolineBlock =
174 unsigned NumTrampolines =
176 ORCABI::TrampolineSize;
178 char *TrampolineMem =
static_cast<char *
>(TrampolineBlock.base());
179 ORCABI::writeTrampolines(
183 for (
unsigned I = 0;
I < NumTrampolines; ++
I)
185 TrampolineMem + (
I * ORCABI::TrampolineSize)));
188 TrampolineBlock.getMemoryBlock(),
192 TrampolineBlocks.push_back(std::move(TrampolineBlock));
198 sys::OwningMemoryBlock ResolverBlock;
199 std::vector<sys::OwningMemoryBlock> TrampolineBlocks;
222 CallbacksJD(ES.createBareJITDylib(
"<Callbacks>")),
223 ErrorHandlerAddress(ErrorHandlerAddress) {}
226 this->TP = std::move(TP);
230 std::mutex CCMgrMutex;
231 std::unique_ptr<TrampolinePool> TP;
235 std::map<JITTargetAddress, SymbolStringPtr> AddrToSymbol;
236 size_t NextCallbackId = 0;
240template <
typename ORCABI>
247 auto CCMgr = std::unique_ptr<LocalJITCompileCallbackManager>(
250 return std::move(Err);
251 return std::move(CCMgr);
262 using NotifyLandingResolvedFunction =
268 NotifyLandingResolvedFunction NotifyLandingResolved) {
273 Err = TP.takeError();
309 virtual void anchor();
315 : NumStubs(NumStubs), StubsMem(
std::
move(StubsMem)) {}
319 auto ISAS = getIndirectStubsBlockSizes<ORCABI>(MinStubs,
PageSize);
322 "StubBytes is not a page size multiple");
327 auto StubsAndPtrsMem =
329 ISAS.StubBytes + PointerAlloc,
nullptr,
335 auto StubsBlockMem =
static_cast<char *
>(StubsAndPtrsMem.base());
336 auto PtrBlockAddress =
339 ORCABI::writeIndirectStubsBlock(StubsBlockMem,
341 PtrBlockAddress, ISAS.NumStubs);
353 return static_cast<char *
>(StubsMem.
base()) +
Idx * ORCABI::StubSize;
358 static_cast<char *
>(StubsMem.
base()) + NumStubs * ORCABI::StubSize;
359 return reinterpret_cast<void **
>(PtrsBase) +
Idx;
363 unsigned NumStubs = 0;
369template <
typename TargetT>
374 std::lock_guard<std::mutex> Lock(StubsMutex);
375 if (
auto Err = reserveStubs(1))
378 createStubInternal(StubName, StubAddr, StubFlags);
384 std::lock_guard<std::mutex> Lock(StubsMutex);
385 if (
auto Err = reserveStubs(StubInits.
size()))
388 for (
const auto &Entry : StubInits)
389 createStubInternal(Entry.first(), Entry.second.first,
390 Entry.second.second);
396 std::lock_guard<std::mutex> Lock(StubsMutex);
397 auto I = StubIndexes.find(
Name);
398 if (
I == StubIndexes.end())
400 auto Key =
I->second.first;
401 void *StubAddr = IndirectStubsInfos[Key.first].getStub(Key.second);
402 assert(StubAddr &&
"Missing stub address");
403 auto StubTargetAddr =
406 if (ExportedStubsOnly && !StubSymbol.getFlags().isExported())
412 std::lock_guard<std::mutex> Lock(StubsMutex);
413 auto I = StubIndexes.find(
Name);
414 if (
I == StubIndexes.end())
416 auto Key =
I->second.first;
417 void *PtrAddr = IndirectStubsInfos[Key.first].getPtr(Key.second);
418 assert(PtrAddr &&
"Missing pointer address");
425 using AtomicIntPtr = std::atomic<uintptr_t>;
427 std::lock_guard<std::mutex> Lock(StubsMutex);
428 auto I = StubIndexes.find(
Name);
429 assert(
I != StubIndexes.end() &&
"No stub pointer for symbol");
430 auto Key =
I->second.first;
431 AtomicIntPtr *AtomicStubPtr =
reinterpret_cast<AtomicIntPtr *
>(
432 IndirectStubsInfos[Key.first].getPtr(Key.second));
433 *AtomicStubPtr =
static_cast<uintptr_t
>(NewAddr);
438 Error reserveStubs(
unsigned NumStubs) {
439 if (NumStubs <= FreeStubs.size())
442 unsigned NewStubsRequired = NumStubs - FreeStubs.size();
443 unsigned NewBlockId = IndirectStubsInfos.size();
447 return ISI.takeError();
448 for (
unsigned I = 0;
I < ISI->getNumStubs(); ++
I)
449 FreeStubs.push_back(std::make_pair(NewBlockId,
I));
450 IndirectStubsInfos.push_back(std::move(*ISI));
456 auto Key = FreeStubs.back();
457 FreeStubs.pop_back();
458 *IndirectStubsInfos[Key.first].getPtr(Key.second) =
459 jitTargetAddressToPointer<void *>(InitAddr);
460 StubIndexes[StubName] = std::make_pair(Key, StubFlags);
464 std::mutex StubsMutex;
465 std::vector<LocalIndirectStubsInfo<TargetT>> IndirectStubsInfos;
466 using StubKey = std::pair<uint16_t, uint16_t>;
467 std::vector<StubKey> FreeStubs;
468 StringMap<std::pair<StubKey, JITSymbolFlags>> StubIndexes;
476Expected<std::unique_ptr<JITCompileCallbackManager>>
483std::function<std::unique_ptr<IndirectStubsManager>()>
496 Constant *Initializer);
500void makeStub(Function &
F, Value &ImplPointer);
This file defines the StringMap class.
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static cl::opt< int > PageSize("imp-null-check-page-size", cl::desc("The page size of the target in bytes"), cl::init(4096), cl::Hidden)
Machine Check Debug Module
Provides a library for accessing information about this process and other processes on the operating ...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Helper for Errors used as out-parameters.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
Represents a symbol that has been evaluated to an address already.
Flags for symbols in the JIT.
Superclass for all disassemblers.
A Module instance is used to store all the information related to an LLVM module.
StringMap - This is an unconventional map that is specialized for handling keys that are "strings",...
StringRef - Represent a constant reference to a string, i.e.
This is a class that can be implemented by clients to materialize Values on demand.
An ExecutionSession represents a running JIT program.
Base class for managing collections of named indirect stubs.
virtual Error updatePointer(StringRef Name, JITTargetAddress NewAddr)=0
Change the value of the implementation pointer for the stub.
virtual JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly)=0
Find the stub with the given name.
virtual JITEvaluatedSymbol findPointer(StringRef Name)=0
Find the implementation-pointer for the stub.
virtual Error createStub(StringRef StubName, JITTargetAddress StubAddr, JITSymbolFlags StubFlags)=0
Create a single stub with the given name, target address and flags.
virtual ~IndirectStubsManager()=default
virtual Error createStubs(const StubInitsMap &StubInits)=0
Create StubInits.size() stubs with the given names, target addresses, and flags.
Target-independent base class for compile callback management.
std::function< JITTargetAddress()> CompileFunction
JITTargetAddress executeCompileCallback(JITTargetAddress TrampolineAddr)
Execute the callback for the given trampoline id.
JITCompileCallbackManager(std::unique_ptr< TrampolinePool > TP, ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress)
Construct a JITCompileCallbackManager.
Expected< JITTargetAddress > getCompileCallback(CompileFunction Compile)
Reserve a compile callback.
virtual ~JITCompileCallbackManager()=default
void setTrampolinePool(std::unique_ptr< TrampolinePool > TP)
Represents a JIT'd dynamic library.
unsigned getNumStubs() const
void * getStub(unsigned Idx) const
LocalIndirectStubsInfo(unsigned NumStubs, sys::OwningMemoryBlock StubsMem)
static Expected< LocalIndirectStubsInfo > create(unsigned MinStubs, unsigned PageSize)
void ** getPtr(unsigned Idx) const
IndirectStubsManager implementation for the host architecture, e.g.
Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override
Change the value of the implementation pointer for the stub.
Error createStub(StringRef StubName, JITTargetAddress StubAddr, JITSymbolFlags StubFlags) override
Create a single stub with the given name, target address and flags.
JITEvaluatedSymbol findPointer(StringRef Name) override
Find the implementation-pointer for the stub.
Error createStubs(const StubInitsMap &StubInits) override
Create StubInits.size() stubs with the given names, target addresses, and flags.
JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override
Find the stub with the given name.
Manage compile callbacks for in-process JITs.
static Expected< std::unique_ptr< LocalJITCompileCallbackManager > > Create(ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress)
Create a new LocalJITCompileCallbackManager.
A trampoline pool for trampolines within the current process.
static Expected< std::unique_ptr< LocalTrampolinePool > > Create(ResolveLandingFunction ResolveLanding)
Creates a LocalTrampolinePool with the given RunCallback function.
Base class for pools of compiler re-entry trampolines.
void releaseTrampoline(JITTargetAddress TrampolineAddr)
Returns the given trampoline to the pool for re-use.
unique_function< void(JITTargetAddress TrampolineAddr, NotifyLandingResolvedFunction OnLandingResolved) const > ResolveLandingFunction
virtual ~TrampolinePool()
unique_function< void(JITTargetAddress) const > NotifyLandingResolvedFunction
std::vector< JITTargetAddress > AvailableTrampolines
Expected< JITTargetAddress > getTrampoline()
Get an available trampoline address.
This class encapsulates the notion of a memory block which has an address and a size.
static MemoryBlock allocateMappedMemory(size_t NumBytes, const MemoryBlock *const NearBlock, unsigned Flags, std::error_code &EC)
This method allocates a block of memory that is suitable for loading dynamically generated code (e....
static std::error_code protectMappedMemory(const MemoryBlock &Block, unsigned Flags)
This method sets the protection flags for a block of memory to the state specified by /p Flags.
Owning version of MemoryBlock.
MemoryBlock getMemoryBlock() const
static unsigned getPageSizeEstimate()
Get the process's estimated page size.
unique_function is a type-erasing functor similar to std::function.
void moveGlobalVariableInitializer(GlobalVariable &OrigGV, ValueToValueMapTy &VMap, ValueMaterializer *Materializer=nullptr, GlobalVariable *NewGV=nullptr)
Move global variable GV from its parent module to cloned global declaration in a different module.
void cloneModuleFlagsMetadata(Module &Dst, const Module &Src, ValueToValueMapTy &VMap)
Clone module flags metadata into the destination module.
Expected< std::unique_ptr< JITCompileCallbackManager > > createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress)
Create a local compile callback manager.
void makeStub(Function &F, Value &ImplPointer)
Turn a function declaration into a stub function that makes an indirect call using the given function...
Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym, jitlink::LinkGraph &G, MCDisassembler &Disassembler, MCInstrAnalysis &MIA)
Introduce relocations to Sym in its own definition if there are any pointers formed via PC-relative a...
GlobalVariable * cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, ValueToValueMapTy *VMap=nullptr)
Clone a global variable declaration into a new module.
Function * cloneFunctionDecl(Module &Dst, const Function &F, ValueToValueMapTy *VMap=nullptr)
Clone a function declaration into a new module.
void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, ValueMaterializer *Materializer=nullptr, Function *NewF=nullptr)
Move the body of function 'F' to a cloned function declaration in a different module (See related clo...
std::function< std::unique_ptr< IndirectStubsManager >()> createLocalIndirectStubsManagerBuilder(const Triple &T)
Create a local indriect stubs manager builder.
GlobalAlias * cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, ValueToValueMapTy &VMap)
Clone a global alias declaration into a new module.
Constant * createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr)
Build a function pointer of FunctionType with the given constant address.
GlobalVariable * createImplPointer(PointerType &PT, Module &M, const Twine &Name, Constant *Initializer)
Create a function pointer with the given type, name, and initializer in the given Module.
This is an optimization pass for GlobalISel generic memory operations.
uint64_t JITTargetAddress
Represents an address in the target process's address space.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
JITTargetAddress pointerToJITTargetAddress(T *Ptr)
Convert a pointer to a JITTargetAddress.