Line data Source code
1 : //===- SectionMemoryManager.cpp - Memory manager for MCJIT/RtDyld *- C++ -*-==//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // This file implements the section-based memory manager used by the MCJIT
11 : // execution engine and RuntimeDyld
12 : //
13 : //===----------------------------------------------------------------------===//
14 :
15 : #include "llvm/ExecutionEngine/SectionMemoryManager.h"
16 : #include "llvm/Config/config.h"
17 : #include "llvm/Support/MathExtras.h"
18 : #include "llvm/Support/Process.h"
19 :
20 : namespace llvm {
21 :
22 20345 : uint8_t *SectionMemoryManager::allocateDataSection(uintptr_t Size,
23 : unsigned Alignment,
24 : unsigned SectionID,
25 : StringRef SectionName,
26 : bool IsReadOnly) {
27 20345 : if (IsReadOnly)
28 8603 : return allocateSection(SectionMemoryManager::AllocationPurpose::ROData,
29 8603 : Size, Alignment);
30 11742 : return allocateSection(SectionMemoryManager::AllocationPurpose::RWData, Size,
31 11742 : Alignment);
32 : }
33 :
34 20247 : uint8_t *SectionMemoryManager::allocateCodeSection(uintptr_t Size,
35 : unsigned Alignment,
36 : unsigned SectionID,
37 : StringRef SectionName) {
38 20247 : return allocateSection(SectionMemoryManager::AllocationPurpose::Code, Size,
39 20247 : Alignment);
40 : }
41 :
42 40592 : uint8_t *SectionMemoryManager::allocateSection(
43 : SectionMemoryManager::AllocationPurpose Purpose, uintptr_t Size,
44 : unsigned Alignment) {
45 40592 : if (!Alignment)
46 : Alignment = 16;
47 :
48 : assert(!(Alignment & (Alignment - 1)) && "Alignment must be a power of two.");
49 :
50 40592 : uintptr_t RequiredSize = Alignment * ((Size + Alignment - 1) / Alignment + 1);
51 : uintptr_t Addr = 0;
52 :
53 : MemoryGroup &MemGroup = [&]() -> MemoryGroup & {
54 40592 : switch (Purpose) {
55 20247 : case AllocationPurpose::Code:
56 20247 : return CodeMem;
57 8603 : case AllocationPurpose::ROData:
58 8603 : return RODataMem;
59 11742 : case AllocationPurpose::RWData:
60 11742 : return RWDataMem;
61 : }
62 0 : llvm_unreachable("Unknown SectionMemoryManager::AllocationPurpose");
63 : }();
64 :
65 : // Look in the list of free memory regions and use a block there if one
66 : // is available.
67 1052629 : for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
68 1051887 : if (FreeMB.Free.size() >= RequiredSize) {
69 39850 : Addr = (uintptr_t)FreeMB.Free.base();
70 39850 : uintptr_t EndOfBlock = Addr + FreeMB.Free.size();
71 : // Align the address.
72 39850 : Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
73 :
74 39850 : if (FreeMB.PendingPrefixIndex == (unsigned)-1) {
75 : // The part of the block we're giving out to the user is now pending
76 702 : MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));
77 :
78 : // Remember this pending block, such that future allocations can just
79 : // modify it rather than creating a new one
80 351 : FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1;
81 : } else {
82 : sys::MemoryBlock &PendingMB =
83 39499 : MemGroup.PendingMem[FreeMB.PendingPrefixIndex];
84 39499 : PendingMB = sys::MemoryBlock(PendingMB.base(),
85 39499 : Addr + Size - (uintptr_t)PendingMB.base());
86 : }
87 :
88 : // Remember how much free space is now left in this block
89 39850 : FreeMB.Free =
90 39850 : sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size);
91 39850 : return (uint8_t *)Addr;
92 : }
93 : }
94 :
95 : // No pre-allocated free block was large enough. Allocate a new memory region.
96 : // Note that all sections get allocated as read-write. The permissions will
97 : // be updated later based on memory group.
98 : //
99 : // FIXME: It would be useful to define a default allocation size (or add
100 : // it as a constructor parameter) to minimize the number of allocations.
101 : //
102 : // FIXME: Initialize the Near member for each memory group to avoid
103 : // interleaving.
104 : std::error_code ec;
105 742 : sys::MemoryBlock MB = MMapper.allocateMappedMemory(
106 742 : Purpose, RequiredSize, &MemGroup.Near,
107 742 : sys::Memory::MF_READ | sys::Memory::MF_WRITE, ec);
108 742 : if (ec) {
109 : // FIXME: Add error propagation to the interface.
110 : return nullptr;
111 : }
112 :
113 : // Save this address as the basis for our next request
114 742 : MemGroup.Near = MB;
115 :
116 : // Remember that we allocated this memory
117 742 : MemGroup.AllocatedMem.push_back(MB);
118 742 : Addr = (uintptr_t)MB.base();
119 742 : uintptr_t EndOfBlock = Addr + MB.size();
120 :
121 : // Align the address.
122 742 : Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1);
123 :
124 : // The part of the block we're giving out to the user is now pending
125 1484 : MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size));
126 :
127 : // The allocateMappedMemory may allocate much more memory than we need. In
128 : // this case, we store the unused memory as a free memory block.
129 742 : unsigned FreeSize = EndOfBlock - Addr - Size;
130 742 : if (FreeSize > 16) {
131 : FreeMemBlock FreeMB;
132 742 : FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), FreeSize);
133 742 : FreeMB.PendingPrefixIndex = (unsigned)-1;
134 742 : MemGroup.FreeMem.push_back(FreeMB);
135 : }
136 :
137 : // Return aligned address
138 : return (uint8_t *)Addr;
139 : }
140 :
141 376 : bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) {
142 : // FIXME: Should in-progress permissions be reverted if an error occurs?
143 : std::error_code ec;
144 :
145 : // Make code memory executable.
146 376 : ec = applyMemoryGroupPermissions(CodeMem,
147 376 : sys::Memory::MF_READ | sys::Memory::MF_EXEC);
148 376 : if (ec) {
149 0 : if (ErrMsg) {
150 0 : *ErrMsg = ec.message();
151 : }
152 0 : return true;
153 : }
154 :
155 : // Make read-only data memory read-only.
156 376 : ec = applyMemoryGroupPermissions(RODataMem,
157 376 : sys::Memory::MF_READ | sys::Memory::MF_EXEC);
158 376 : if (ec) {
159 0 : if (ErrMsg) {
160 0 : *ErrMsg = ec.message();
161 : }
162 0 : return true;
163 : }
164 :
165 : // Read-write data memory already has the correct permissions
166 :
167 : // Some platforms with separate data cache and instruction cache require
168 : // explicit cache flush, otherwise JIT code manipulations (like resolved
169 : // relocations) will get to the data cache but not to the instruction cache.
170 376 : invalidateInstructionCache();
171 :
172 376 : return false;
173 : }
174 :
175 537 : static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) {
176 537 : static const size_t PageSize = sys::Process::getPageSize();
177 :
178 : size_t StartOverlap =
179 537 : (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize;
180 :
181 537 : size_t TrimmedSize = M.size();
182 537 : TrimmedSize -= StartOverlap;
183 537 : TrimmedSize -= TrimmedSize % PageSize;
184 :
185 537 : sys::MemoryBlock Trimmed((void *)((uintptr_t)M.base() + StartOverlap),
186 537 : TrimmedSize);
187 :
188 : assert(((uintptr_t)Trimmed.base() % PageSize) == 0);
189 : assert((Trimmed.size() % PageSize) == 0);
190 : assert(M.base() <= Trimmed.base() && Trimmed.size() <= M.size());
191 :
192 537 : return Trimmed;
193 : }
194 :
195 : std::error_code
196 752 : SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup,
197 : unsigned Permissions) {
198 1477 : for (sys::MemoryBlock &MB : MemGroup.PendingMem)
199 725 : if (std::error_code EC = MMapper.protectMappedMemory(MB, Permissions))
200 0 : return EC;
201 :
202 : MemGroup.PendingMem.clear();
203 :
204 : // Now go through free blocks and trim any of them that don't span the entire
205 : // page because one of the pending blocks may have overlapped it.
206 1289 : for (FreeMemBlock &FreeMB : MemGroup.FreeMem) {
207 537 : FreeMB.Free = trimBlockToPageSize(FreeMB.Free);
208 : // We cleared the PendingMem list, so all these pointers are now invalid
209 537 : FreeMB.PendingPrefixIndex = (unsigned)-1;
210 : }
211 :
212 : // Remove all blocks which are now empty
213 : MemGroup.FreeMem.erase(
214 : remove_if(MemGroup.FreeMem,
215 0 : [](FreeMemBlock &FreeMB) { return FreeMB.Free.size() == 0; }),
216 : MemGroup.FreeMem.end());
217 :
218 752 : return std::error_code();
219 : }
220 :
221 512 : void SectionMemoryManager::invalidateInstructionCache() {
222 512 : for (sys::MemoryBlock &Block : CodeMem.PendingMem)
223 0 : sys::Memory::InvalidateInstructionCache(Block.base(), Block.size());
224 512 : }
225 :
226 151 : SectionMemoryManager::~SectionMemoryManager() {
227 340 : for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) {
228 680 : for (sys::MemoryBlock &Block : Group->AllocatedMem)
229 425 : MMapper.releaseMappedMemory(Block);
230 : }
231 151 : }
232 66 :
233 : SectionMemoryManager::MemoryMapper::~MemoryMapper() {}
234 :
235 : void SectionMemoryManager::anchor() {}
236 :
237 66 : namespace {
238 85 : // Trivial implementation of SectionMemoryManager::MemoryMapper that just calls
239 340 : // into sys::Memory.
240 680 : class DefaultMMapper final : public SectionMemoryManager::MemoryMapper {
241 425 : public:
242 : sys::MemoryBlock
243 85 : allocateMappedMemory(SectionMemoryManager::AllocationPurpose Purpose,
244 : size_t NumBytes, const sys::MemoryBlock *const NearBlock,
245 113936 : unsigned Flags, std::error_code &EC) override {
246 : return sys::Memory::allocateMappedMemory(NumBytes, NearBlock, Flags, EC);
247 0 : }
248 :
249 : std::error_code protectMappedMemory(const sys::MemoryBlock &Block,
250 : unsigned Flags) override {
251 : return sys::Memory::protectMappedMemory(Block, Flags);
252 : }
253 :
254 : std::error_code releaseMappedMemory(sys::MemoryBlock &M) override {
255 742 : return sys::Memory::releaseMappedMemory(M);
256 : }
257 : };
258 742 :
259 : DefaultMMapper DefaultMMapperInstance;
260 : } // namespace
261 725 :
262 : SectionMemoryManager::SectionMemoryManager(MemoryMapper *MM)
263 725 : : MMapper(MM ? *MM : DefaultMMapperInstance) {}
264 :
265 : } // namespace llvm
|
