LLVM 22.0.0git
MemoryMapper.cpp
Go to the documentation of this file.
1//===- MemoryMapper.cpp - Cross-process memory mapper ------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
10
11#include "llvm/Config/llvm-config.h" // for LLVM_ON_UNIX
14
15#if defined(LLVM_ON_UNIX) && !defined(__ANDROID__)
16#include <fcntl.h>
17#include <sys/mman.h>
18#if defined(__MVS__)
19#include "llvm/Support/BLAKE3.h"
20#include <sys/shm.h>
21#endif
22#include <unistd.h>
23#elif defined(_WIN32)
24#include <windows.h>
25#endif
26
27namespace llvm {
28namespace orc {
29
31
33 : PageSize(PageSize) {}
34
37 auto PageSize = sys::Process::getPageSize();
38 if (!PageSize)
39 return PageSize.takeError();
40 return std::make_unique<InProcessMemoryMapper>(*PageSize);
41}
42
43void InProcessMemoryMapper::reserve(size_t NumBytes,
44 OnReservedFunction OnReserved) {
45 std::error_code EC;
47 NumBytes, nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC);
48
49 if (EC)
50 return OnReserved(errorCodeToError(EC));
51
52 {
53 std::lock_guard<std::mutex> Lock(Mutex);
54 Reservations[MB.base()].Size = MB.allocatedSize();
55 }
56
57 OnReserved(
58 ExecutorAddrRange(ExecutorAddr::fromPtr(MB.base()), MB.allocatedSize()));
59}
60
62 size_t ContentSize) {
63 return Addr.toPtr<char *>();
64}
65
67 OnInitializedFunction OnInitialized) {
68 ExecutorAddr MinAddr(~0ULL);
69 ExecutorAddr MaxAddr(0);
70
71 // FIXME: Release finalize lifetime segments.
72 for (auto &Segment : AI.Segments) {
73 auto Base = AI.MappingBase + Segment.Offset;
74 auto Size = Segment.ContentSize + Segment.ZeroFillSize;
75
76 if (Base < MinAddr)
77 MinAddr = Base;
78
79 if (Base + Size > MaxAddr)
80 MaxAddr = Base + Size;
81
82 std::memset((Base + Segment.ContentSize).toPtr<void *>(), 0,
83 Segment.ZeroFillSize);
84
86 {Base.toPtr<void *>(), Size},
87 toSysMemoryProtectionFlags(Segment.AG.getMemProt()))) {
88 return OnInitialized(errorCodeToError(EC));
89 }
90 if ((Segment.AG.getMemProt() & MemProt::Exec) == MemProt::Exec)
92 }
93
94 auto DeinitializeActions = shared::runFinalizeActions(AI.Actions);
95 if (!DeinitializeActions)
96 return OnInitialized(DeinitializeActions.takeError());
97
98 {
99 std::lock_guard<std::mutex> Lock(Mutex);
100
101 // This is the maximum range whose permission have been possibly modified
102 auto &Alloc = Allocations[MinAddr];
103 Alloc.Size = MaxAddr - MinAddr;
104 Alloc.DeinitializationActions = std::move(*DeinitializeActions);
105 Reservations[AI.MappingBase.toPtr<void *>()].Allocations.push_back(MinAddr);
106 }
107
108 OnInitialized(MinAddr);
109}
110
114 Error AllErr = Error::success();
115
116 {
117 std::lock_guard<std::mutex> Lock(Mutex);
118
119 for (auto Base : llvm::reverse(Bases)) {
120
122 Allocations[Base].DeinitializationActions)) {
123 AllErr = joinErrors(std::move(AllErr), std::move(Err));
124 }
125
126 // Reset protections to read/write so the area can be reused
128 {Base.toPtr<void *>(), Allocations[Base].Size},
131 AllErr = joinErrors(std::move(AllErr), errorCodeToError(EC));
132 }
133
134 Allocations.erase(Base);
135 }
136 }
137
138 OnDeinitialized(std::move(AllErr));
139}
140
142 OnReleasedFunction OnReleased) {
143 Error Err = Error::success();
144
145 for (auto Base : Bases) {
146 std::vector<ExecutorAddr> AllocAddrs;
147 size_t Size;
148 {
149 std::lock_guard<std::mutex> Lock(Mutex);
150 auto &R = Reservations[Base.toPtr<void *>()];
151 Size = R.Size;
152 AllocAddrs.swap(R.Allocations);
153 }
154
155 // deinitialize sub allocations
156 std::promise<MSVCPError> P;
157 auto F = P.get_future();
158 deinitialize(AllocAddrs, [&](Error Err) { P.set_value(std::move(Err)); });
159 if (Error E = F.get()) {
160 Err = joinErrors(std::move(Err), std::move(E));
161 }
162
163 // free the memory
164 auto MB = sys::MemoryBlock(Base.toPtr<void *>(), Size);
165
167 if (EC) {
168 Err = joinErrors(std::move(Err), errorCodeToError(EC));
169 }
170
171 std::lock_guard<std::mutex> Lock(Mutex);
172 Reservations.erase(Base.toPtr<void *>());
173 }
174
175 OnReleased(std::move(Err));
176}
177
179 std::vector<ExecutorAddr> ReservationAddrs;
180 {
181 std::lock_guard<std::mutex> Lock(Mutex);
182
183 ReservationAddrs.reserve(Reservations.size());
184 for (const auto &R : Reservations) {
185 ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst()));
186 }
187 }
188
189 std::promise<MSVCPError> P;
190 auto F = P.get_future();
191 release(ReservationAddrs, [&](Error Err) { P.set_value(std::move(Err)); });
192 cantFail(F.get());
193}
194
195// SharedMemoryMapper
196
198 SymbolAddrs SAs, size_t PageSize)
199 : EPC(EPC), SAs(SAs), PageSize(PageSize) {
200#if (!defined(LLVM_ON_UNIX) || defined(__ANDROID__)) && !defined(_WIN32)
201 llvm_unreachable("SharedMemoryMapper is not supported on this platform yet");
202#endif
203}
204
207#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
208 auto PageSize = sys::Process::getPageSize();
209 if (!PageSize)
210 return PageSize.takeError();
211
212 return std::make_unique<SharedMemoryMapper>(EPC, SAs, *PageSize);
213#else
215 "SharedMemoryMapper is not supported on this platform yet",
217#endif
218}
219
220void SharedMemoryMapper::reserve(size_t NumBytes,
221 OnReservedFunction OnReserved) {
222#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
223
224 int SharedMemoryId = -1;
225 EPC.callSPSWrapperAsync<
227 SAs.Reserve,
228 [this, NumBytes, OnReserved = std::move(OnReserved), SharedMemoryId](
229 Error SerializationErr,
231 if (SerializationErr) {
232 cantFail(Result.takeError());
233 return OnReserved(std::move(SerializationErr));
234 }
235
236 if (!Result)
237 return OnReserved(Result.takeError());
238
239 ExecutorAddr RemoteAddr;
240 std::string SharedMemoryName;
241 std::tie(RemoteAddr, SharedMemoryName) = std::move(*Result);
242
243 void *LocalAddr = nullptr;
244
245#if defined(LLVM_ON_UNIX)
246
247#if defined(__MVS__)
249 reinterpret_cast<const uint8_t *>(SharedMemoryName.c_str()),
250 SharedMemoryName.size());
251 auto HashedName = BLAKE3::hash<sizeof(key_t)>(Data);
252 key_t Key = *reinterpret_cast<key_t *>(HashedName.data());
253 SharedMemoryId =
254 shmget(Key, NumBytes, IPC_CREAT | __IPC_SHAREAS | 0700);
255 if (SharedMemoryId < 0) {
256 return OnReserved(errorCodeToError(
257 std::error_code(errno, std::generic_category())));
258 }
259 LocalAddr = shmat(SharedMemoryId, nullptr, 0);
260 if (LocalAddr == reinterpret_cast<void *>(-1)) {
261 return OnReserved(errorCodeToError(
262 std::error_code(errno, std::generic_category())));
263 }
264#else
265 int SharedMemoryFile = shm_open(SharedMemoryName.c_str(), O_RDWR, 0700);
266 if (SharedMemoryFile < 0) {
267 return OnReserved(errorCodeToError(errnoAsErrorCode()));
268 }
269
270 // this prevents other processes from accessing it by name
271 shm_unlink(SharedMemoryName.c_str());
272
273 LocalAddr = mmap(nullptr, NumBytes, PROT_READ | PROT_WRITE, MAP_SHARED,
274 SharedMemoryFile, 0);
275 if (LocalAddr == MAP_FAILED) {
276 return OnReserved(errorCodeToError(errnoAsErrorCode()));
277 }
278
279 close(SharedMemoryFile);
280#endif
281
282#elif defined(_WIN32)
283
284 std::wstring WideSharedMemoryName(SharedMemoryName.begin(),
285 SharedMemoryName.end());
286 HANDLE SharedMemoryFile = OpenFileMappingW(
287 FILE_MAP_ALL_ACCESS, FALSE, WideSharedMemoryName.c_str());
288 if (!SharedMemoryFile)
289 return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
290
291 LocalAddr =
292 MapViewOfFile(SharedMemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0);
293 if (!LocalAddr) {
294 CloseHandle(SharedMemoryFile);
295 return OnReserved(errorCodeToError(mapWindowsError(GetLastError())));
296 }
297
298 CloseHandle(SharedMemoryFile);
299
300#endif
301 {
302 std::lock_guard<std::mutex> Lock(Mutex);
303 Reservations.insert(
304 {RemoteAddr, {LocalAddr, NumBytes, SharedMemoryId}});
305 }
306
307 OnReserved(ExecutorAddrRange(RemoteAddr, NumBytes));
308 },
309 SAs.Instance, static_cast<uint64_t>(NumBytes));
310
311#else
312 OnReserved(make_error<StringError>(
313 "SharedMemoryMapper is not supported on this platform yet",
315#endif
316}
317
319 size_t ContentSize) {
320 auto R = Reservations.upper_bound(Addr);
321 assert(R != Reservations.begin() && "Attempt to prepare unreserved range");
322 R--;
323
324 ExecutorAddrDiff Offset = Addr - R->first;
325
326 return static_cast<char *>(R->second.LocalAddr) + Offset;
327}
328
330 OnInitializedFunction OnInitialized) {
331 auto Reservation = Reservations.upper_bound(AI.MappingBase);
332 assert(Reservation != Reservations.begin() && "Attempt to initialize unreserved range");
333 Reservation--;
334
335 auto AllocationOffset = AI.MappingBase - Reservation->first;
336
338
339 AI.Actions.swap(FR.Actions);
340
341 FR.Segments.reserve(AI.Segments.size());
342
343 for (auto Segment : AI.Segments) {
344 char *Base = static_cast<char *>(Reservation->second.LocalAddr) +
345 AllocationOffset + Segment.Offset;
346 std::memset(Base + Segment.ContentSize, 0, Segment.ZeroFillSize);
347
349 SegReq.RAG = {Segment.AG.getMemProt(),
350 Segment.AG.getMemLifetime() == MemLifetime::Finalize};
351 SegReq.Addr = AI.MappingBase + Segment.Offset;
352 SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize;
353
354 FR.Segments.push_back(SegReq);
355 }
356
357 EPC.callSPSWrapperAsync<
359 SAs.Initialize,
360 [OnInitialized = std::move(OnInitialized)](
361 Error SerializationErr, Expected<ExecutorAddr> Result) mutable {
362 if (SerializationErr) {
363 cantFail(Result.takeError());
364 return OnInitialized(std::move(SerializationErr));
365 }
366
367 OnInitialized(std::move(Result));
368 },
369 SAs.Instance, Reservation->first, std::move(FR));
370}
371
373 ArrayRef<ExecutorAddr> Allocations,
375 EPC.callSPSWrapperAsync<
377 SAs.Deinitialize,
378 [OnDeinitialized = std::move(OnDeinitialized)](Error SerializationErr,
379 Error Result) mutable {
380 if (SerializationErr) {
381 cantFail(std::move(Result));
382 return OnDeinitialized(std::move(SerializationErr));
383 }
384
385 OnDeinitialized(std::move(Result));
386 },
387 SAs.Instance, Allocations);
388}
389
391 OnReleasedFunction OnReleased) {
392#if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32)
393 Error Err = Error::success();
394
395 {
396 std::lock_guard<std::mutex> Lock(Mutex);
397
398 for (auto Base : Bases) {
399
400#if defined(LLVM_ON_UNIX)
401
402#if defined(__MVS__)
403 if (shmdt(Reservations[Base].LocalAddr) < 0 ||
404 shmctl(Reservations[Base].SharedMemoryId, IPC_RMID, NULL) < 0)
405 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
406#else
407 if (munmap(Reservations[Base].LocalAddr, Reservations[Base].Size) != 0)
408 Err = joinErrors(std::move(Err), errorCodeToError(errnoAsErrorCode()));
409#endif
410
411#elif defined(_WIN32)
412
413 if (!UnmapViewOfFile(Reservations[Base].LocalAddr))
414 Err = joinErrors(std::move(Err),
415 errorCodeToError(mapWindowsError(GetLastError())));
416
417#endif
418
419 Reservations.erase(Base);
420 }
421 }
422
423 EPC.callSPSWrapperAsync<
425 SAs.Release,
426 [OnReleased = std::move(OnReleased),
427 Err = std::move(Err)](Error SerializationErr, Error Result) mutable {
428 if (SerializationErr) {
429 cantFail(std::move(Result));
430 return OnReleased(
431 joinErrors(std::move(Err), std::move(SerializationErr)));
432 }
433
434 return OnReleased(joinErrors(std::move(Err), std::move(Result)));
435 },
436 SAs.Instance, Bases);
437#else
438 OnReleased(make_error<StringError>(
439 "SharedMemoryMapper is not supported on this platform yet",
441#endif
442}
443
445 std::lock_guard<std::mutex> Lock(Mutex);
446 for (const auto &R : Reservations) {
447
448#if defined(LLVM_ON_UNIX) && !defined(__ANDROID__)
449
450#if defined(__MVS__)
451 shmdt(R.second.LocalAddr);
452#else
453 munmap(R.second.LocalAddr, R.second.Size);
454#endif
455
456#elif defined(_WIN32)
457
458 UnmapViewOfFile(R.second.LocalAddr);
459
460#else
461
462 (void)R;
463
464#endif
465 }
466}
467
468} // namespace orc
469
470} // namespace llvm
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define F(x, y, z)
Definition MD5.cpp:55
#define G(x, y, z)
Definition MD5.cpp:56
#define P(N)
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:41
static BLAKE3Result< NumBytes > hash(ArrayRef< uint8_t > Data)
Returns a BLAKE3 hash for the given data.
Definition BLAKE3.h:86
Lightweight error class with error context and mandatory checking.
Definition Error.h:159
static ErrorSuccess success()
Create a success value.
Definition Error.h:336
Tagged union holding either a T or a Error.
Definition Error.h:485
Represents an address in the executor process.
static ExecutorAddr fromPtr(T *Ptr, UnwrapFn &&Unwrap=UnwrapFn())
Create an ExecutorAddr from the given pointer.
std::enable_if_t< std::is_pointer< T >::value, T > toPtr(WrapFn &&Wrap=WrapFn()) const
Cast this ExecutorAddr to a pointer of the given type.
ExecutorProcessControl supports interaction with a JIT target process.
void initialize(AllocInfo &AI, OnInitializedFunction OnInitialized) override
Ensures executor memory is synchronized with working copy memory, sends functions to be called after ...
void reserve(size_t NumBytes, OnReservedFunction OnReserved) override
Reserves address space in executor process.
char * prepare(jitlink::LinkGraph &G, ExecutorAddr Addr, size_t ContentSize) override
Provides working memory The LinkGraph parameter is included to allow implementations to allocate work...
void deinitialize(ArrayRef< ExecutorAddr > Allocations, OnDeinitializedFunction OnDeInitialized) override
Runs previously specified deinitialization actions Executor addresses returned by initialize should b...
static Expected< std::unique_ptr< InProcessMemoryMapper > > Create()
void release(ArrayRef< ExecutorAddr > Reservations, OnReleasedFunction OnRelease) override
Release address space acquired through reserve()
unique_function< void(Error)> OnReleasedFunction
unique_function< void(Expected< ExecutorAddr >)> OnInitializedFunction
unique_function< void(Expected< ExecutorAddrRange >)> OnReservedFunction
unique_function< void(Error)> OnDeinitializedFunction
static Expected< std::unique_ptr< SharedMemoryMapper > > Create(ExecutorProcessControl &EPC, SymbolAddrs SAs)
char * prepare(jitlink::LinkGraph &G, ExecutorAddr Addr, size_t ContentSize) override
Provides working memory The LinkGraph parameter is included to allow implementations to allocate work...
void reserve(size_t NumBytes, OnReservedFunction OnReserved) override
Reserves address space in executor process.
void deinitialize(ArrayRef< ExecutorAddr > Allocations, OnDeinitializedFunction OnDeInitialized) override
Runs previously specified deinitialization actions Executor addresses returned by initialize should b...
void initialize(AllocInfo &AI, OnInitializedFunction OnInitialized) override
Ensures executor memory is synchronized with working copy memory, sends functions to be called after ...
void release(ArrayRef< ExecutorAddr > Reservations, OnReleasedFunction OnRelease) override
Release address space acquired through reserve()
SharedMemoryMapper(ExecutorProcessControl &EPC, SymbolAddrs SAs, size_t PageSize)
This class encapsulates the notion of a memory block which has an address and a size.
Definition Memory.h:33
static LLVM_ABI std::error_code protectMappedMemory(const MemoryBlock &Block, unsigned Flags)
This method sets the protection flags for a block of memory to the state specified by /p Flags.
static LLVM_ABI std::error_code releaseMappedMemory(MemoryBlock &Block)
This method releases a block of memory that was allocated with the allocateMappedMemory method.
static LLVM_ABI void InvalidateInstructionCache(const void *Addr, size_t Len)
InvalidateInstructionCache - Before the JIT can run a block of code that has been emitted it must inv...
static LLVM_ABI MemoryBlock allocateMappedMemory(size_t NumBytes, const MemoryBlock *const NearBlock, unsigned Flags, std::error_code &EC)
This method allocates a block of memory that is suitable for loading dynamically generated code (e....
static LLVM_ABI Expected< unsigned > getPageSize()
Get the process's page size.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
shared::SPSExpected< shared::SPSTuple< shared::SPSExecutorAddr, shared::SPSString > >( shared::SPSExecutorAddr, uint64_t) SPSExecutorSharedMemoryMapperServiceReserveSignature
Definition OrcRTBridge.h:86
shared::SPSExpected< shared::SPSExecutorAddr >( shared::SPSExecutorAddr, shared::SPSExecutorAddr, shared::SPSSharedMemoryFinalizeRequest) SPSExecutorSharedMemoryMapperServiceInitializeSignature
Definition OrcRTBridge.h:90
shared::SPSError(shared::SPSExecutorAddr, shared::SPSSequence< shared::SPSExecutorAddr >) SPSExecutorSharedMemoryMapperServiceDeinitializeSignature
Definition OrcRTBridge.h:94
shared::SPSError( shared::SPSExecutorAddr, shared::SPSSequence< shared::SPSExecutorAddr >) SPSExecutorSharedMemoryMapperServiceReleaseSignature
Definition OrcRTBridge.h:97
LLVM_ABI Error runDeallocActions(ArrayRef< WrapperFunctionCall > DAs)
Run deallocation actions.
LLVM_ABI Expected< std::vector< WrapperFunctionCall > > runFinalizeActions(AllocActions &AAs)
Run finalize actions.
uint64_t ExecutorAddrDiff
@ Finalize
Finalize memory should be allocated by the allocator, and then be overwritten and deallocated after a...
Definition MemoryFlags.h:83
sys::Memory::ProtectionFlags toSysMemoryProtectionFlags(MemProt MP)
Convert a MemProt value to a corresponding sys::Memory::ProtectionFlags value.
Definition MemoryFlags.h:44
This is an optimization pass for GlobalISel generic memory operations.
@ Offset
Definition DWP.cpp:477
LLVM_ABI std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition Error.cpp:98
auto reverse(ContainerTy &&C)
Definition STLExtras.h:406
Error joinErrors(Error E1, Error E2)
Concatenate errors.
Definition Error.h:442
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
Error make_error(ArgTs &&... Args)
Make a Error instance representing failure using the given error info type.
Definition Error.h:340
void cantFail(Error Err, const char *Msg=nullptr)
Report a fatal error if Err is a failure value.
Definition Error.h:769
FunctionAddr VTableAddr uintptr_t uintptr_t Data
Definition InstrProf.h:189
LLVM_ABI Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Definition Error.cpp:111
LLVM_ABI std::error_code mapWindowsError(unsigned EV)
std::error_code errnoAsErrorCode()
Helper to get errno as an std::error_code.
Definition Error.h:1240
Represents an address range in the exceutor process.
Represents a single allocation containing multiple segments and initialization and deinitialization a...
std::vector< SegInfo > Segments
std::vector< SharedMemorySegFinalizeRequest > Segments