LLVM 17.0.0git
EPCIndirectionUtils.cpp
Go to the documentation of this file.
1//===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
10
13
14#include <future>
15
16using namespace llvm;
17using namespace llvm::orc;
18
19namespace llvm {
20namespace orc {
21
23public:
24 using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo;
25 using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector;
26
28 getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) {
29 return EPCIU.getIndirectStubs(NumStubs);
30 };
31};
32
33} // end namespace orc
34} // end namespace llvm
35
36namespace {
37
38class EPCTrampolinePool : public TrampolinePool {
39public:
40 EPCTrampolinePool(EPCIndirectionUtils &EPCIU);
41 Error deallocatePool();
42
43protected:
44 Error grow() override;
45
47
49 unsigned TrampolineSize = 0;
50 unsigned TrampolinesPerPage = 0;
51 std::vector<FinalizedAlloc> TrampolineBlocks;
52};
53
54class EPCIndirectStubsManager : public IndirectStubsManager,
56public:
57 EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {}
58
59 Error deallocateStubs();
60
61 Error createStub(StringRef StubName, JITTargetAddress StubAddr,
62 JITSymbolFlags StubFlags) override;
63
64 Error createStubs(const StubInitsMap &StubInits) override;
65
66 JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override;
67
69
71
72private:
73 using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>;
74
75 std::mutex ISMMutex;
77 StringMap<StubInfo> StubInfos;
78};
79
80EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU)
81 : EPCIU(EPCIU) {
82 auto &EPC = EPCIU.getExecutorProcessControl();
83 auto &ABI = EPCIU.getABISupport();
84
85 TrampolineSize = ABI.getTrampolineSize();
86 TrampolinesPerPage =
87 (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize;
88}
89
90Error EPCTrampolinePool::deallocatePool() {
91 std::promise<MSVCPError> DeallocResultP;
92 auto DeallocResultF = DeallocResultP.get_future();
93
95 std::move(TrampolineBlocks),
96 [&](Error Err) { DeallocResultP.set_value(std::move(Err)); });
97
98 return DeallocResultF.get();
99}
100
101Error EPCTrampolinePool::grow() {
102 using namespace jitlink;
103
104 assert(AvailableTrampolines.empty() &&
105 "Grow called with trampolines still available");
106
107 auto ResolverAddress = EPCIU.getResolverBlockAddress();
108 assert(ResolverAddress && "Resolver address can not be null");
109
110 auto &EPC = EPCIU.getExecutorProcessControl();
111 auto PageSize = EPC.getPageSize();
112 auto Alloc = SimpleSegmentAlloc::Create(
113 EPC.getMemMgr(), nullptr,
114 {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}});
115 if (!Alloc)
116 return Alloc.takeError();
117
118 unsigned NumTrampolines = TrampolinesPerPage;
119
120 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
121 EPCIU.getABISupport().writeTrampolines(SegInfo.WorkingMem.data(),
122 SegInfo.Addr.getValue(),
123 ResolverAddress, NumTrampolines);
124 for (unsigned I = 0; I < NumTrampolines; ++I)
125 AvailableTrampolines.push_back(SegInfo.Addr.getValue() +
126 (I * TrampolineSize));
127
128 auto FA = Alloc->finalize();
129 if (!FA)
130 return FA.takeError();
131
132 TrampolineBlocks.push_back(std::move(*FA));
133
134 return Error::success();
135}
136
137Error EPCIndirectStubsManager::createStub(StringRef StubName,
138 JITTargetAddress StubAddr,
139 JITSymbolFlags StubFlags) {
140 StubInitsMap SIM;
141 SIM[StubName] = std::make_pair(StubAddr, StubFlags);
142 return createStubs(SIM);
143}
144
145Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) {
146 auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size());
147 if (!AvailableStubInfos)
148 return AvailableStubInfos.takeError();
149
150 {
151 std::lock_guard<std::mutex> Lock(ISMMutex);
152 unsigned ASIdx = 0;
153 for (auto &SI : StubInits) {
154 auto &A = (*AvailableStubInfos)[ASIdx++];
155 StubInfos[SI.first()] = std::make_pair(A, SI.second.second);
156 }
157 }
158
159 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
160 switch (EPCIU.getABISupport().getPointerSize()) {
161 case 4: {
162 unsigned ASIdx = 0;
163 std::vector<tpctypes::UInt32Write> PtrUpdates;
164 for (auto &SI : StubInits)
165 PtrUpdates.push_back(
166 {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress),
167 static_cast<uint32_t>(SI.second.first)});
168 return MemAccess.writeUInt32s(PtrUpdates);
169 }
170 case 8: {
171 unsigned ASIdx = 0;
172 std::vector<tpctypes::UInt64Write> PtrUpdates;
173 for (auto &SI : StubInits)
174 PtrUpdates.push_back(
175 {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress),
176 static_cast<uint64_t>(SI.second.first)});
177 return MemAccess.writeUInt64s(PtrUpdates);
178 }
179 default:
180 return make_error<StringError>("Unsupported pointer size",
182 }
183}
184
185JITEvaluatedSymbol EPCIndirectStubsManager::findStub(StringRef Name,
186 bool ExportedStubsOnly) {
187 std::lock_guard<std::mutex> Lock(ISMMutex);
188 auto I = StubInfos.find(Name);
189 if (I == StubInfos.end())
190 return nullptr;
191 return {I->second.first.StubAddress, I->second.second};
192}
193
194JITEvaluatedSymbol EPCIndirectStubsManager::findPointer(StringRef Name) {
195 std::lock_guard<std::mutex> Lock(ISMMutex);
196 auto I = StubInfos.find(Name);
197 if (I == StubInfos.end())
198 return nullptr;
199 return {I->second.first.PointerAddress, I->second.second};
200}
201
202Error EPCIndirectStubsManager::updatePointer(StringRef Name,
203 JITTargetAddress NewAddr) {
204
205 JITTargetAddress PtrAddr = 0;
206 {
207 std::lock_guard<std::mutex> Lock(ISMMutex);
208 auto I = StubInfos.find(Name);
209 if (I == StubInfos.end())
210 return make_error<StringError>("Unknown stub name",
212 PtrAddr = I->second.first.PointerAddress;
213 }
214
215 auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess();
216 switch (EPCIU.getABISupport().getPointerSize()) {
217 case 4: {
218 tpctypes::UInt32Write PUpdate(ExecutorAddr(PtrAddr), NewAddr);
219 return MemAccess.writeUInt32s(PUpdate);
220 }
221 case 8: {
222 tpctypes::UInt64Write PUpdate(ExecutorAddr(PtrAddr), NewAddr);
223 return MemAccess.writeUInt64s(PUpdate);
224 }
225 default:
226 return make_error<StringError>("Unsupported pointer size",
228 }
229}
230
231} // end anonymous namespace.
232
233namespace llvm {
234namespace orc {
235
237
240 const auto &TT = EPC.getTargetTriple();
241 switch (TT.getArch()) {
242 default:
243 return make_error<StringError>(
244 std::string("No EPCIndirectionUtils available for ") + TT.str(),
246 case Triple::aarch64:
248 return CreateWithABI<OrcAArch64>(EPC);
249
250 case Triple::x86:
251 return CreateWithABI<OrcI386>(EPC);
252
254 return CreateWithABI<OrcLoongArch64>(EPC);
255
256 case Triple::mips:
257 return CreateWithABI<OrcMips32Be>(EPC);
258
259 case Triple::mipsel:
260 return CreateWithABI<OrcMips32Le>(EPC);
261
262 case Triple::mips64:
263 case Triple::mips64el:
264 return CreateWithABI<OrcMips64>(EPC);
265
266 case Triple::riscv64:
267 return CreateWithABI<OrcRiscv64>(EPC);
268
269 case Triple::x86_64:
270 if (TT.getOS() == Triple::OSType::Win32)
271 return CreateWithABI<OrcX86_64_Win32>(EPC);
272 else
273 return CreateWithABI<OrcX86_64_SysV>(EPC);
274 }
275}
276
278
279 auto &MemMgr = EPC.getMemMgr();
280 auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs));
281
282 if (TP)
283 Err = joinErrors(std::move(Err),
284 static_cast<EPCTrampolinePool &>(*TP).deallocatePool());
285
286 if (ResolverBlock)
287 Err =
288 joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock)));
289
290 return Err;
291}
292
295 JITTargetAddress ReentryCtxAddr) {
296 using namespace jitlink;
297
298 assert(ABI && "ABI can not be null");
299 auto ResolverSize = ABI->getResolverCodeSize();
300
301 auto Alloc =
302 SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr,
303 {{MemProt::Read | MemProt::Exec,
304 {ResolverSize, Align(EPC.getPageSize())}}});
305
306 if (!Alloc)
307 return Alloc.takeError();
308
309 auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec);
310 ResolverBlockAddr = SegInfo.Addr.getValue();
311 ABI->writeResolverCode(SegInfo.WorkingMem.data(), ResolverBlockAddr,
312 ReentryFnAddr, ReentryCtxAddr);
313
314 auto FA = Alloc->finalize();
315 if (!FA)
316 return FA.takeError();
317
318 ResolverBlock = std::move(*FA);
319 return ResolverBlockAddr;
320}
321
322std::unique_ptr<IndirectStubsManager>
324 return std::make_unique<EPCIndirectStubsManager>(*this);
325}
326
328 if (!TP)
329 TP = std::make_unique<EPCTrampolinePool>(*this);
330 return *TP;
331}
332
334 ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) {
335 assert(!LCTM &&
336 "createLazyCallThroughManager can not have been called before");
337 LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr,
338 &getTrampolinePool());
339 return *LCTM;
340}
341
342EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC,
343 std::unique_ptr<ABISupport> ABI)
344 : EPC(EPC), ABI(std::move(ABI)) {
345 assert(this->ABI && "ABI can not be null");
346
347 assert(EPC.getPageSize() > getABISupport().getStubSize() &&
348 "Stubs larger than one page are not supported");
349}
350
352EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) {
353 using namespace jitlink;
354
355 std::lock_guard<std::mutex> Lock(EPCUIMutex);
356
357 // If there aren't enough stubs available then allocate some more.
358 if (NumStubs > AvailableIndirectStubs.size()) {
359 auto NumStubsToAllocate = NumStubs;
360 auto PageSize = EPC.getPageSize();
361 auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize);
362 NumStubsToAllocate = StubBytes / ABI->getStubSize();
363 auto PtrBytes =
364 alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize);
365
366 auto StubProt = MemProt::Read | MemProt::Exec;
367 auto PtrProt = MemProt::Read | MemProt::Write;
368
369 auto Alloc = SimpleSegmentAlloc::Create(
370 EPC.getMemMgr(), nullptr,
371 {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}},
372 {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}});
373
374 if (!Alloc)
375 return Alloc.takeError();
376
377 auto StubSeg = Alloc->getSegInfo(StubProt);
378 auto PtrSeg = Alloc->getSegInfo(PtrProt);
379
380 ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(),
381 StubSeg.Addr.getValue(),
382 PtrSeg.Addr.getValue(), NumStubsToAllocate);
383
384 auto FA = Alloc->finalize();
385 if (!FA)
386 return FA.takeError();
387
388 IndirectStubAllocs.push_back(std::move(*FA));
389
390 auto StubExecutorAddr = StubSeg.Addr;
391 auto PtrExecutorAddr = PtrSeg.Addr;
392 for (unsigned I = 0; I != NumStubsToAllocate; ++I) {
393 AvailableIndirectStubs.push_back(IndirectStubInfo(
394 StubExecutorAddr.getValue(), PtrExecutorAddr.getValue()));
395 StubExecutorAddr += ABI->getStubSize();
396 PtrExecutorAddr += ABI->getPointerSize();
397 }
398 }
399
400 assert(NumStubs <= AvailableIndirectStubs.size() &&
401 "Sufficient stubs should have been allocated above");
402
403 IndirectStubInfoVector Result;
404 while (NumStubs--) {
405 Result.push_back(AvailableIndirectStubs.back());
406 AvailableIndirectStubs.pop_back();
407 }
408
409 return std::move(Result);
410}
411
413 JITTargetAddress TrampolineAddr) {
414 auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr);
415 std::promise<JITTargetAddress> LandingAddrP;
416 auto LandingAddrF = LandingAddrP.get_future();
417 LCTM.resolveTrampolineLandingAddress(
418 TrampolineAddr,
419 [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); });
420 return LandingAddrF.get();
421}
422
424 auto &LCTM = EPCIU.getLazyCallThroughManager();
425 return EPCIU
428 .takeError();
429}
430
431} // end namespace orc
432} // end namespace llvm
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
uint64_t Addr
std::string Name
static cl::opt< int > PageSize("imp-null-check-page-size", cl::desc("The page size of the target in bytes"), cl::init(4096), cl::Hidden)
#define I(x, y, z)
Definition: MD5.cpp:58
@ SI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
@ ABI
Definition: TextStubV5.cpp:100
Lightweight error class with error context and mandatory checking.
Definition: Error.h:156
static ErrorSuccess success()
Create a success value.
Definition: Error.h:330
Tagged union holding either a T or a Error.
Definition: Error.h:470
Represents a symbol that has been evaluated to an address already.
Definition: JITSymbol.h:229
Flags for symbols in the JIT.
Definition: JITSymbol.h:74
StringMap - This is an unconventional map that is specialized for handling keys that are "strings",...
Definition: StringMap.h:111
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
@ loongarch64
Definition: Triple.h:62
@ mips64el
Definition: Triple.h:67
@ aarch64_32
Definition: Triple.h:53
EPCIndirectionUtils::IndirectStubInfo IndirectStubInfo
static Expected< IndirectStubInfoVector > getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs)
EPCIndirectionUtils::IndirectStubInfoVector IndirectStubInfoVector
virtual void writeTrampolines(char *TrampolineBlockWorkingMem, JITTargetAddress TrampolineBlockTragetAddr, JITTargetAddress ResolverAddr, unsigned NumTrampolines) const =0
Provides ExecutorProcessControl based indirect stubs, trampoline pool and lazy call through manager.
std::unique_ptr< IndirectStubsManager > createIndirectStubsManager()
Create an IndirectStubsManager for the executor process.
LazyCallThroughManager & getLazyCallThroughManager()
Create a LazyCallThroughManager for the executor process.
ExecutorProcessControl & getExecutorProcessControl() const
Return a reference to the ExecutorProcessControl object.
LazyCallThroughManager & createLazyCallThroughManager(ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr)
Create a LazyCallThroughManager.
Error cleanup()
Release memory for resources held by this instance.
Expected< JITTargetAddress > writeResolverBlock(JITTargetAddress ReentryFnAddr, JITTargetAddress ReentryCtxAddr)
Write resolver code to the executor process and return its address.
TrampolinePool & getTrampolinePool()
Create a TrampolinePool for the executor process.
static Expected< std::unique_ptr< EPCIndirectionUtils > > Create(ExecutorProcessControl &EPC)
Create based on the ExecutorProcessControl triple.
ABISupport & getABISupport() const
Return a reference to the ABISupport object for this instance.
JITTargetAddress getResolverBlockAddress() const
Returns the address of the Resolver block.
An ExecutionSession represents a running JIT program.
Definition: Core.h:1373
Represents an address in the executor process.
ExecutorProcessControl supports interaction with a JIT target process.
jitlink::JITLinkMemoryManager & getMemMgr() const
Return a JITLinkMemoryManager for the target process.
const Triple & getTargetTriple() const
Return the Triple for the target process.
unsigned getPageSize() const
Get the page size for the target process.
MemoryAccess & getMemoryAccess() const
Return a MemoryAccess object for the target process.
Base class for managing collections of named indirect stubs.
virtual Error updatePointer(StringRef Name, JITTargetAddress NewAddr)=0
Change the value of the implementation pointer for the stub.
virtual JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly)=0
Find the stub with the given name.
virtual JITEvaluatedSymbol findPointer(StringRef Name)=0
Find the implementation-pointer for the stub.
virtual Error createStub(StringRef StubName, JITTargetAddress StubAddr, JITSymbolFlags StubFlags)=0
Create a single stub with the given name, target address and flags.
virtual Error createStubs(const StubInitsMap &StubInits)=0
Create StubInits.size() stubs with the given names, target addresses, and flags.
Manages a set of 'lazy call-through' trampolines.
Definition: LazyReexports.h:38
Base class for pools of compiler re-entry trampolines.
virtual Error grow()=0
Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU)
This will call writeResolver on the given EPCIndirectionUtils instance to set up re-entry via a funct...
static JITTargetAddress reentry(JITTargetAddress LCTMAddr, JITTargetAddress TrampolineAddr)
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:79
uint64_t JITTargetAddress
Represents an address in the target process's address space.
Definition: JITSymbol.h:42
Error joinErrors(Error E1, Error E2)
Concatenate errors.
Definition: Error.h:427
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:155
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1909
JITTargetAddress pointerToJITTargetAddress(T *Ptr)
Convert a pointer to a JITTargetAddress.
Definition: JITSymbol.h:69
Definition: BitVector.h:851