LLVM 19.0.0git
JITLinkMemoryManager.cpp
Go to the documentation of this file.
1//===--- JITLinkMemoryManager.cpp - JITLinkMemoryManager implementation ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
13
14#define DEBUG_TYPE "jitlink"
15
16using namespace llvm;
17
18namespace llvm {
19namespace jitlink {
20
23
25
26 for (auto &Sec : G.sections()) {
27 // Skip empty sections, and sections with NoAlloc lifetime policies.
28 if (Sec.blocks().empty() ||
29 Sec.getMemLifetime() == orc::MemLifetime::NoAlloc)
30 continue;
31
32 auto &Seg = Segments[{Sec.getMemProt(), Sec.getMemLifetime()}];
33 for (auto *B : Sec.blocks())
34 if (LLVM_LIKELY(!B->isZeroFill()))
35 Seg.ContentBlocks.push_back(B);
36 else
37 Seg.ZeroFillBlocks.push_back(B);
38 }
39
40 // Build Segments map.
41 auto CompareBlocks = [](const Block *LHS, const Block *RHS) {
42 // Sort by section, address and size
43 if (LHS->getSection().getOrdinal() != RHS->getSection().getOrdinal())
44 return LHS->getSection().getOrdinal() < RHS->getSection().getOrdinal();
45 if (LHS->getAddress() != RHS->getAddress())
46 return LHS->getAddress() < RHS->getAddress();
47 return LHS->getSize() < RHS->getSize();
48 };
49
50 LLVM_DEBUG(dbgs() << "Generated BasicLayout for " << G.getName() << ":\n");
51 for (auto &KV : Segments) {
52 auto &Seg = KV.second;
53
54 llvm::sort(Seg.ContentBlocks, CompareBlocks);
55 llvm::sort(Seg.ZeroFillBlocks, CompareBlocks);
56
57 for (auto *B : Seg.ContentBlocks) {
58 Seg.ContentSize = alignToBlock(Seg.ContentSize, *B);
59 Seg.ContentSize += B->getSize();
60 Seg.Alignment = std::max(Seg.Alignment, Align(B->getAlignment()));
61 }
62
63 uint64_t SegEndOffset = Seg.ContentSize;
64 for (auto *B : Seg.ZeroFillBlocks) {
65 SegEndOffset = alignToBlock(SegEndOffset, *B);
66 SegEndOffset += B->getSize();
67 Seg.Alignment = std::max(Seg.Alignment, Align(B->getAlignment()));
68 }
69 Seg.ZeroFillSize = SegEndOffset - Seg.ContentSize;
70
72 dbgs() << " Seg " << KV.first
73 << ": content-size=" << formatv("{0:x}", Seg.ContentSize)
74 << ", zero-fill-size=" << formatv("{0:x}", Seg.ZeroFillSize)
75 << ", align=" << formatv("{0:x}", Seg.Alignment.value()) << "\n";
76 });
77 }
78}
79
83
84 for (auto &KV : segments()) {
85 auto &AG = KV.first;
86 auto &Seg = KV.second;
87
88 if (Seg.Alignment > PageSize)
89 return make_error<StringError>("Segment alignment greater than page size",
91
92 uint64_t SegSize = alignTo(Seg.ContentSize + Seg.ZeroFillSize, PageSize);
93 if (AG.getMemLifetime() == orc::MemLifetime::Standard)
94 SegsSizes.StandardSegs += SegSize;
95 else
96 SegsSizes.FinalizeSegs += SegSize;
97 }
98
99 return SegsSizes;
100}
101
103 for (auto &KV : Segments) {
104 auto &Seg = KV.second;
105
106 assert(!(Seg.ContentBlocks.empty() && Seg.ZeroFillBlocks.empty()) &&
107 "Empty section recorded?");
108
109 for (auto *B : Seg.ContentBlocks) {
110 // Align addr and working-mem-offset.
111 Seg.Addr = alignToBlock(Seg.Addr, *B);
112 Seg.NextWorkingMemOffset = alignToBlock(Seg.NextWorkingMemOffset, *B);
113
114 // Update block addr.
115 B->setAddress(Seg.Addr);
116 Seg.Addr += B->getSize();
117
118 // Copy content to working memory, then update content to point at working
119 // memory.
120 memcpy(Seg.WorkingMem + Seg.NextWorkingMemOffset, B->getContent().data(),
121 B->getSize());
122 B->setMutableContent(
123 {Seg.WorkingMem + Seg.NextWorkingMemOffset, B->getSize()});
124 Seg.NextWorkingMemOffset += B->getSize();
125 }
126
127 for (auto *B : Seg.ZeroFillBlocks) {
128 // Align addr.
129 Seg.Addr = alignToBlock(Seg.Addr, *B);
130 // Update block addr.
131 B->setAddress(Seg.Addr);
132 Seg.Addr += B->getSize();
133 }
134
135 Seg.ContentBlocks.clear();
136 Seg.ZeroFillBlocks.clear();
137 }
138
139 return Error::success();
140}
141
143 return G.allocActions();
144}
145
147 const JITLinkDylib *JD, SegmentMap Segments,
148 OnCreatedFunction OnCreated) {
149
150 static_assert(orc::AllocGroup::NumGroups == 32,
151 "AllocGroup has changed. Section names below must be updated");
152 StringRef AGSectionNames[] = {
153 "__---.standard", "__R--.standard", "__-W-.standard", "__RW-.standard",
154 "__--X.standard", "__R-X.standard", "__-WX.standard", "__RWX.standard",
155 "__---.finalize", "__R--.finalize", "__-W-.finalize", "__RW-.finalize",
156 "__--X.finalize", "__R-X.finalize", "__-WX.finalize", "__RWX.finalize"};
157
158 auto G = std::make_unique<LinkGraph>("", Triple(), 0,
159 llvm::endianness::native, nullptr);
161
162 orc::ExecutorAddr NextAddr(0x100000);
163 for (auto &KV : Segments) {
164 auto &AG = KV.first;
165 auto &Seg = KV.second;
166
167 assert(AG.getMemLifetime() != orc::MemLifetime::NoAlloc &&
168 "NoAlloc segments are not supported by SimpleSegmentAlloc");
169
170 auto AGSectionName =
171 AGSectionNames[static_cast<unsigned>(AG.getMemProt()) |
172 static_cast<bool>(AG.getMemLifetime()) << 3];
173
174 auto &Sec = G->createSection(AGSectionName, AG.getMemProt());
175 Sec.setMemLifetime(AG.getMemLifetime());
176
177 if (Seg.ContentSize != 0) {
178 NextAddr =
179 orc::ExecutorAddr(alignTo(NextAddr.getValue(), Seg.ContentAlign));
180 auto &B =
181 G->createMutableContentBlock(Sec, G->allocateBuffer(Seg.ContentSize),
182 NextAddr, Seg.ContentAlign.value(), 0);
183 ContentBlocks[AG] = &B;
184 NextAddr += Seg.ContentSize;
185 }
186 }
187
188 // GRef declared separately since order-of-argument-eval isn't specified.
189 auto &GRef = *G;
190 MemMgr.allocate(JD, GRef,
191 [G = std::move(G), ContentBlocks = std::move(ContentBlocks),
192 OnCreated = std::move(OnCreated)](
193 JITLinkMemoryManager::AllocResult Alloc) mutable {
194 if (!Alloc)
195 OnCreated(Alloc.takeError());
196 else
197 OnCreated(SimpleSegmentAlloc(std::move(G),
198 std::move(ContentBlocks),
199 std::move(*Alloc)));
200 });
201}
202
205 SegmentMap Segments) {
206 std::promise<MSVCPExpected<SimpleSegmentAlloc>> AllocP;
207 auto AllocF = AllocP.get_future();
208 Create(MemMgr, JD, std::move(Segments),
209 [&](Expected<SimpleSegmentAlloc> Result) {
210 AllocP.set_value(std::move(Result));
211 });
212 return AllocF.get();
213}
214
219
222 auto I = ContentBlocks.find(AG);
223 if (I != ContentBlocks.end()) {
224 auto &B = *I->second;
225 return {B.getAddress(), B.getAlreadyMutableContent()};
226 }
227 return {};
228}
229
231 std::unique_ptr<LinkGraph> G,
233 std::unique_ptr<JITLinkMemoryManager::InFlightAlloc> Alloc)
234 : G(std::move(G)), ContentBlocks(std::move(ContentBlocks)),
235 Alloc(std::move(Alloc)) {}
236
239public:
241 sys::MemoryBlock StandardSegments,
242 sys::MemoryBlock FinalizationSegments)
243 : MemMgr(MemMgr), G(&G), BL(std::move(BL)),
244 StandardSegments(std::move(StandardSegments)),
245 FinalizationSegments(std::move(FinalizationSegments)) {}
246
248 assert(!G && "InFlight alloc neither abandoned nor finalized");
249 }
250
251 void finalize(OnFinalizedFunction OnFinalized) override {
252
253 // Apply memory protections to all segments.
254 if (auto Err = applyProtections()) {
255 OnFinalized(std::move(Err));
256 return;
257 }
258
259 // Run finalization actions.
260 auto DeallocActions = runFinalizeActions(G->allocActions());
261 if (!DeallocActions) {
262 OnFinalized(DeallocActions.takeError());
263 return;
264 }
265
266 // Release the finalize segments slab.
267 if (auto EC = sys::Memory::releaseMappedMemory(FinalizationSegments)) {
268 OnFinalized(errorCodeToError(EC));
269 return;
270 }
271
272#ifndef NDEBUG
273 // Set 'G' to null to flag that we've been successfully finalized.
274 // This allows us to assert at destruction time that a call has been made
275 // to either finalize or abandon.
276 G = nullptr;
277#endif
278
279 // Continue with finalized allocation.
280 OnFinalized(MemMgr.createFinalizedAlloc(std::move(StandardSegments),
281 std::move(*DeallocActions)));
282 }
283
284 void abandon(OnAbandonedFunction OnAbandoned) override {
285 Error Err = Error::success();
286 if (auto EC = sys::Memory::releaseMappedMemory(FinalizationSegments))
287 Err = joinErrors(std::move(Err), errorCodeToError(EC));
288 if (auto EC = sys::Memory::releaseMappedMemory(StandardSegments))
289 Err = joinErrors(std::move(Err), errorCodeToError(EC));
290
291#ifndef NDEBUG
292 // Set 'G' to null to flag that we've been successfully finalized.
293 // This allows us to assert at destruction time that a call has been made
294 // to either finalize or abandon.
295 G = nullptr;
296#endif
297
298 OnAbandoned(std::move(Err));
299 }
300
301private:
302 Error applyProtections() {
303 for (auto &KV : BL.segments()) {
304 const auto &AG = KV.first;
305 auto &Seg = KV.second;
306
307 auto Prot = toSysMemoryProtectionFlags(AG.getMemProt());
308
309 uint64_t SegSize =
310 alignTo(Seg.ContentSize + Seg.ZeroFillSize, MemMgr.PageSize);
311 sys::MemoryBlock MB(Seg.WorkingMem, SegSize);
312 if (auto EC = sys::Memory::protectMappedMemory(MB, Prot))
313 return errorCodeToError(EC);
314 if (Prot & sys::Memory::MF_EXEC)
315 sys::Memory::InvalidateInstructionCache(MB.base(), MB.allocatedSize());
316 }
317 return Error::success();
318 }
319
320 InProcessMemoryManager &MemMgr;
321 LinkGraph *G;
322 BasicLayout BL;
323 sys::MemoryBlock StandardSegments;
324 sys::MemoryBlock FinalizationSegments;
325};
326
329 if (auto PageSize = sys::Process::getPageSize())
330 return std::make_unique<InProcessMemoryManager>(*PageSize);
331 else
332 return PageSize.takeError();
333}
334
336 OnAllocatedFunction OnAllocated) {
337
338 // FIXME: Just check this once on startup.
339 if (!isPowerOf2_64((uint64_t)PageSize)) {
340 OnAllocated(make_error<StringError>("Page size is not a power of 2",
342 return;
343 }
344
345 BasicLayout BL(G);
346
347 /// Scan the request and calculate the group and total sizes.
348 /// Check that segment size is no larger than a page.
349 auto SegsSizes = BL.getContiguousPageBasedLayoutSizes(PageSize);
350 if (!SegsSizes) {
351 OnAllocated(SegsSizes.takeError());
352 return;
353 }
354
355 /// Check that the total size requested (including zero fill) is not larger
356 /// than a size_t.
357 if (SegsSizes->total() > std::numeric_limits<size_t>::max()) {
358 OnAllocated(make_error<JITLinkError>(
359 "Total requested size " + formatv("{0:x}", SegsSizes->total()) +
360 " for graph " + G.getName() + " exceeds address space"));
361 return;
362 }
363
364 // Allocate one slab for the whole thing (to make sure everything is
365 // in-range), then partition into standard and finalization blocks.
366 //
367 // FIXME: Make two separate allocations in the future to reduce
368 // fragmentation: finalization segments will usually be a single page, and
369 // standard segments are likely to be more than one page. Where multiple
370 // allocations are in-flight at once (likely) the current approach will leave
371 // a lot of single-page holes.
372 sys::MemoryBlock Slab;
373 sys::MemoryBlock StandardSegsMem;
374 sys::MemoryBlock FinalizeSegsMem;
375 {
376 const sys::Memory::ProtectionFlags ReadWrite =
379
380 std::error_code EC;
381 Slab = sys::Memory::allocateMappedMemory(SegsSizes->total(), nullptr,
382 ReadWrite, EC);
383
384 if (EC) {
385 OnAllocated(errorCodeToError(EC));
386 return;
387 }
388
389 // Zero-fill the whole slab up-front.
390 memset(Slab.base(), 0, Slab.allocatedSize());
391
392 StandardSegsMem = {Slab.base(),
393 static_cast<size_t>(SegsSizes->StandardSegs)};
394 FinalizeSegsMem = {(void *)((char *)Slab.base() + SegsSizes->StandardSegs),
395 static_cast<size_t>(SegsSizes->FinalizeSegs)};
396 }
397
398 auto NextStandardSegAddr = orc::ExecutorAddr::fromPtr(StandardSegsMem.base());
399 auto NextFinalizeSegAddr = orc::ExecutorAddr::fromPtr(FinalizeSegsMem.base());
400
401 LLVM_DEBUG({
402 dbgs() << "InProcessMemoryManager allocated:\n";
403 if (SegsSizes->StandardSegs)
404 dbgs() << formatv(" [ {0:x16} -- {1:x16} ]", NextStandardSegAddr,
405 NextStandardSegAddr + StandardSegsMem.allocatedSize())
406 << " to stardard segs\n";
407 else
408 dbgs() << " no standard segs\n";
409 if (SegsSizes->FinalizeSegs)
410 dbgs() << formatv(" [ {0:x16} -- {1:x16} ]", NextFinalizeSegAddr,
411 NextFinalizeSegAddr + FinalizeSegsMem.allocatedSize())
412 << " to finalize segs\n";
413 else
414 dbgs() << " no finalize segs\n";
415 });
416
417 // Build ProtMap, assign addresses.
418 for (auto &KV : BL.segments()) {
419 auto &AG = KV.first;
420 auto &Seg = KV.second;
421
422 auto &SegAddr = (AG.getMemLifetime() == orc::MemLifetime::Standard)
423 ? NextStandardSegAddr
424 : NextFinalizeSegAddr;
425
426 Seg.WorkingMem = SegAddr.toPtr<char *>();
427 Seg.Addr = SegAddr;
428
429 SegAddr += alignTo(Seg.ContentSize + Seg.ZeroFillSize, PageSize);
430 }
431
432 if (auto Err = BL.apply()) {
433 OnAllocated(std::move(Err));
434 return;
435 }
436
437 OnAllocated(std::make_unique<IPInFlightAlloc>(*this, G, std::move(BL),
438 std::move(StandardSegsMem),
439 std::move(FinalizeSegsMem)));
440}
441
442void InProcessMemoryManager::deallocate(std::vector<FinalizedAlloc> Allocs,
443 OnDeallocatedFunction OnDeallocated) {
444 std::vector<sys::MemoryBlock> StandardSegmentsList;
445 std::vector<std::vector<orc::shared::WrapperFunctionCall>> DeallocActionsList;
446
447 {
448 std::lock_guard<std::mutex> Lock(FinalizedAllocsMutex);
449 for (auto &Alloc : Allocs) {
450 auto *FA = Alloc.release().toPtr<FinalizedAllocInfo *>();
451 StandardSegmentsList.push_back(std::move(FA->StandardSegments));
452 DeallocActionsList.push_back(std::move(FA->DeallocActions));
453 FA->~FinalizedAllocInfo();
454 FinalizedAllocInfos.Deallocate(FA);
455 }
456 }
457
458 Error DeallocErr = Error::success();
459
460 while (!DeallocActionsList.empty()) {
461 auto &DeallocActions = DeallocActionsList.back();
462 auto &StandardSegments = StandardSegmentsList.back();
463
464 /// Run any deallocate calls.
465 while (!DeallocActions.empty()) {
466 if (auto Err = DeallocActions.back().runWithSPSRetErrorMerged())
467 DeallocErr = joinErrors(std::move(DeallocErr), std::move(Err));
468 DeallocActions.pop_back();
469 }
470
471 /// Release the standard segments slab.
472 if (auto EC = sys::Memory::releaseMappedMemory(StandardSegments))
473 DeallocErr = joinErrors(std::move(DeallocErr), errorCodeToError(EC));
474
475 DeallocActionsList.pop_back();
476 StandardSegmentsList.pop_back();
477 }
478
479 OnDeallocated(std::move(DeallocErr));
480}
481
483InProcessMemoryManager::createFinalizedAlloc(
484 sys::MemoryBlock StandardSegments,
485 std::vector<orc::shared::WrapperFunctionCall> DeallocActions) {
486 std::lock_guard<std::mutex> Lock(FinalizedAllocsMutex);
487 auto *FA = FinalizedAllocInfos.Allocate<FinalizedAllocInfo>();
488 new (FA) FinalizedAllocInfo(
489 {std::move(StandardSegments), std::move(DeallocActions)});
490 return FinalizedAlloc(orc::ExecutorAddr::fromPtr(FA));
491}
492
493} // end namespace jitlink
494} // end namespace llvm
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
#define LLVM_LIKELY(EXPR)
Definition: Compiler.h:240
#define LLVM_DEBUG(X)
Definition: Debug.h:101
static cl::opt< int > PageSize("imp-null-check-page-size", cl::desc("The page size of the target in bytes"), cl::init(4096), cl::Hidden)
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
Provides a library for accessing information about this process and other processes on the operating ...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Value * RHS
Value * LHS
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:334
Tagged union holding either a T or a Error.
Definition: Error.h:474
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
A pair of memory protections and allocation policies.
Definition: MemoryFlags.h:110
static constexpr unsigned NumGroups
Definition: MemoryFlags.h:120
Represents an address in the executor process.
uint64_t getValue() const
static ExecutorAddr fromPtr(T *Ptr, UnwrapFn &&Unwrap=UnwrapFn())
Create an ExecutorAddr from the given pointer.
This class encapsulates the notion of a memory block which has an address and a size.
Definition: Memory.h:31
void * base() const
Definition: Memory.h:36
size_t allocatedSize() const
The size as it was allocated.
Definition: Memory.h:39
static std::error_code releaseMappedMemory(MemoryBlock &Block)
This method releases a block of memory that was allocated with the allocateMappedMemory method.
static MemoryBlock allocateMappedMemory(size_t NumBytes, const MemoryBlock *const NearBlock, unsigned Flags, std::error_code &EC)
This method allocates a block of memory that is suitable for loading dynamically generated code (e....
static void InvalidateInstructionCache(const void *Addr, size_t Len)
InvalidateInstructionCache - Before the JIT can run a block of code that has been emitted it must inv...
static std::error_code protectMappedMemory(const MemoryBlock &Block, unsigned Flags)
This method sets the protection flags for a block of memory to the state specified by /p Flags.
static Expected< unsigned > getPageSize()
Get the process's page size.
unique_function is a type-erasing functor similar to std::function.
std::vector< AllocActionCallPair > AllocActions
A vector of allocation actions to be run for this allocation.
@ NoAlloc
NoAlloc memory should not be allocated by the JITLinkMemoryManager at all.
@ Standard
Standard memory should be allocated by the allocator and then deallocated when the deallocate method ...
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto formatv(const char *Fmt, Ts &&... Vals) -> formatv_object< decltype(std::make_tuple(detail::build_format_adapter(std::forward< Ts >(Vals))...))>
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:90
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:269
Error joinErrors(Error E1, Error E2)
Concatenate errors.
Definition: Error.h:431
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1656
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:155
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1858
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Definition: Error.cpp:103
Implement std::hash so that hash_code can be used in STL containers.
Definition: BitVector.h:858
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39