File: | llvm/include/llvm/ADT/FunctionExtras.h |
Warning: | line 198, column 5 Undefined or garbage value returned to caller |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===-- RuntimeDyld.cpp - Run-time dynamic linker for MC-JIT ----*- C++ -*-===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | // Implementation of the MC-JIT runtime dynamic linker. | |||
10 | // | |||
11 | //===----------------------------------------------------------------------===// | |||
12 | ||||
13 | #include "llvm/ExecutionEngine/RuntimeDyld.h" | |||
14 | #include "RuntimeDyldCOFF.h" | |||
15 | #include "RuntimeDyldELF.h" | |||
16 | #include "RuntimeDyldImpl.h" | |||
17 | #include "RuntimeDyldMachO.h" | |||
18 | #include "llvm/Object/COFF.h" | |||
19 | #include "llvm/Object/ELFObjectFile.h" | |||
20 | #include "llvm/Support/Alignment.h" | |||
21 | #include "llvm/Support/MSVCErrorWorkarounds.h" | |||
22 | #include "llvm/Support/ManagedStatic.h" | |||
23 | #include "llvm/Support/MathExtras.h" | |||
24 | #include <mutex> | |||
25 | ||||
26 | #include <future> | |||
27 | ||||
28 | using namespace llvm; | |||
29 | using namespace llvm::object; | |||
30 | ||||
31 | #define DEBUG_TYPE"dyld" "dyld" | |||
32 | ||||
33 | namespace { | |||
34 | ||||
35 | enum RuntimeDyldErrorCode { | |||
36 | GenericRTDyldError = 1 | |||
37 | }; | |||
38 | ||||
39 | // FIXME: This class is only here to support the transition to llvm::Error. It | |||
40 | // will be removed once this transition is complete. Clients should prefer to | |||
41 | // deal with the Error value directly, rather than converting to error_code. | |||
42 | class RuntimeDyldErrorCategory : public std::error_category { | |||
43 | public: | |||
44 | const char *name() const noexcept override { return "runtimedyld"; } | |||
45 | ||||
46 | std::string message(int Condition) const override { | |||
47 | switch (static_cast<RuntimeDyldErrorCode>(Condition)) { | |||
48 | case GenericRTDyldError: return "Generic RuntimeDyld error"; | |||
49 | } | |||
50 | llvm_unreachable("Unrecognized RuntimeDyldErrorCode")::llvm::llvm_unreachable_internal("Unrecognized RuntimeDyldErrorCode" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp" , 50); | |||
51 | } | |||
52 | }; | |||
53 | ||||
54 | static ManagedStatic<RuntimeDyldErrorCategory> RTDyldErrorCategory; | |||
55 | ||||
56 | } | |||
57 | ||||
58 | char RuntimeDyldError::ID = 0; | |||
59 | ||||
60 | void RuntimeDyldError::log(raw_ostream &OS) const { | |||
61 | OS << ErrMsg << "\n"; | |||
62 | } | |||
63 | ||||
64 | std::error_code RuntimeDyldError::convertToErrorCode() const { | |||
65 | return std::error_code(GenericRTDyldError, *RTDyldErrorCategory); | |||
66 | } | |||
67 | ||||
68 | // Empty out-of-line virtual destructor as the key function. | |||
69 | RuntimeDyldImpl::~RuntimeDyldImpl() {} | |||
70 | ||||
71 | // Pin LoadedObjectInfo's vtables to this file. | |||
72 | void RuntimeDyld::LoadedObjectInfo::anchor() {} | |||
73 | ||||
74 | namespace llvm { | |||
75 | ||||
76 | void RuntimeDyldImpl::registerEHFrames() {} | |||
77 | ||||
78 | void RuntimeDyldImpl::deregisterEHFrames() { | |||
79 | MemMgr.deregisterEHFrames(); | |||
80 | } | |||
81 | ||||
82 | #ifndef NDEBUG | |||
83 | static void dumpSectionMemory(const SectionEntry &S, StringRef State) { | |||
84 | dbgs() << "----- Contents of section " << S.getName() << " " << State | |||
85 | << " -----"; | |||
86 | ||||
87 | if (S.getAddress() == nullptr) { | |||
88 | dbgs() << "\n <section not emitted>\n"; | |||
89 | return; | |||
90 | } | |||
91 | ||||
92 | const unsigned ColsPerRow = 16; | |||
93 | ||||
94 | uint8_t *DataAddr = S.getAddress(); | |||
95 | uint64_t LoadAddr = S.getLoadAddress(); | |||
96 | ||||
97 | unsigned StartPadding = LoadAddr & (ColsPerRow - 1); | |||
98 | unsigned BytesRemaining = S.getSize(); | |||
99 | ||||
100 | if (StartPadding) { | |||
101 | dbgs() << "\n" << format("0x%016" PRIx64"l" "x", | |||
102 | LoadAddr & ~(uint64_t)(ColsPerRow - 1)) << ":"; | |||
103 | while (StartPadding--) | |||
104 | dbgs() << " "; | |||
105 | } | |||
106 | ||||
107 | while (BytesRemaining > 0) { | |||
108 | if ((LoadAddr & (ColsPerRow - 1)) == 0) | |||
109 | dbgs() << "\n" << format("0x%016" PRIx64"l" "x", LoadAddr) << ":"; | |||
110 | ||||
111 | dbgs() << " " << format("%02x", *DataAddr); | |||
112 | ||||
113 | ++DataAddr; | |||
114 | ++LoadAddr; | |||
115 | --BytesRemaining; | |||
116 | } | |||
117 | ||||
118 | dbgs() << "\n"; | |||
119 | } | |||
120 | #endif | |||
121 | ||||
122 | // Resolve the relocations for all symbols we currently know about. | |||
123 | void RuntimeDyldImpl::resolveRelocations() { | |||
124 | std::lock_guard<sys::Mutex> locked(lock); | |||
125 | ||||
126 | // Print out the sections prior to relocation. | |||
127 | LLVM_DEBUG(for (int i = 0, e = Sections.size(); i != e; ++i)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { for (int i = 0, e = Sections.size(); i != e; ++i) dumpSectionMemory(Sections[i], "before relocations");; } } while (false) | |||
128 | dumpSectionMemory(Sections[i], "before relocations");)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { for (int i = 0, e = Sections.size(); i != e; ++i) dumpSectionMemory(Sections[i], "before relocations");; } } while (false); | |||
129 | ||||
130 | // First, resolve relocations associated with external symbols. | |||
131 | if (auto Err = resolveExternalSymbols()) { | |||
132 | HasError = true; | |||
133 | ErrorStr = toString(std::move(Err)); | |||
134 | } | |||
135 | ||||
136 | resolveLocalRelocations(); | |||
137 | ||||
138 | // Print out sections after relocation. | |||
139 | LLVM_DEBUG(for (int i = 0, e = Sections.size(); i != e; ++i)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { for (int i = 0, e = Sections.size(); i != e; ++i) dumpSectionMemory(Sections[i], "after relocations");; } } while (false) | |||
140 | dumpSectionMemory(Sections[i], "after relocations");)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { for (int i = 0, e = Sections.size(); i != e; ++i) dumpSectionMemory(Sections[i], "after relocations");; } } while (false); | |||
141 | } | |||
142 | ||||
143 | void RuntimeDyldImpl::resolveLocalRelocations() { | |||
144 | // Iterate over all outstanding relocations | |||
145 | for (auto it = Relocations.begin(), e = Relocations.end(); it != e; ++it) { | |||
146 | // The Section here (Sections[i]) refers to the section in which the | |||
147 | // symbol for the relocation is located. The SectionID in the relocation | |||
148 | // entry provides the section to which the relocation will be applied. | |||
149 | unsigned Idx = it->first; | |||
150 | uint64_t Addr = getSectionLoadAddress(Idx); | |||
151 | LLVM_DEBUG(dbgs() << "Resolving relocations Section #" << Idx << "\t"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "Resolving relocations Section #" << Idx << "\t" << format("%p", (uintptr_t) Addr) << "\n"; } } while (false) | |||
152 | << format("%p", (uintptr_t)Addr) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "Resolving relocations Section #" << Idx << "\t" << format("%p", (uintptr_t) Addr) << "\n"; } } while (false); | |||
153 | resolveRelocationList(it->second, Addr); | |||
154 | } | |||
155 | Relocations.clear(); | |||
156 | } | |||
157 | ||||
158 | void RuntimeDyldImpl::mapSectionAddress(const void *LocalAddress, | |||
159 | uint64_t TargetAddress) { | |||
160 | std::lock_guard<sys::Mutex> locked(lock); | |||
161 | for (unsigned i = 0, e = Sections.size(); i != e; ++i) { | |||
162 | if (Sections[i].getAddress() == LocalAddress) { | |||
163 | reassignSectionAddress(i, TargetAddress); | |||
164 | return; | |||
165 | } | |||
166 | } | |||
167 | llvm_unreachable("Attempting to remap address of unknown section!")::llvm::llvm_unreachable_internal("Attempting to remap address of unknown section!" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp" , 167); | |||
168 | } | |||
169 | ||||
170 | static Error getOffset(const SymbolRef &Sym, SectionRef Sec, | |||
171 | uint64_t &Result) { | |||
172 | Expected<uint64_t> AddressOrErr = Sym.getAddress(); | |||
173 | if (!AddressOrErr) | |||
174 | return AddressOrErr.takeError(); | |||
175 | Result = *AddressOrErr - Sec.getAddress(); | |||
176 | return Error::success(); | |||
177 | } | |||
178 | ||||
179 | Expected<RuntimeDyldImpl::ObjSectionToIDMap> | |||
180 | RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) { | |||
181 | std::lock_guard<sys::Mutex> locked(lock); | |||
182 | ||||
183 | // Save information about our target | |||
184 | Arch = (Triple::ArchType)Obj.getArch(); | |||
185 | IsTargetLittleEndian = Obj.isLittleEndian(); | |||
186 | setMipsABI(Obj); | |||
187 | ||||
188 | // Compute the memory size required to load all sections to be loaded | |||
189 | // and pass this information to the memory manager | |||
190 | if (MemMgr.needsToReserveAllocationSpace()) { | |||
191 | uint64_t CodeSize = 0, RODataSize = 0, RWDataSize = 0; | |||
192 | uint32_t CodeAlign = 1, RODataAlign = 1, RWDataAlign = 1; | |||
193 | if (auto Err = computeTotalAllocSize(Obj, | |||
194 | CodeSize, CodeAlign, | |||
195 | RODataSize, RODataAlign, | |||
196 | RWDataSize, RWDataAlign)) | |||
197 | return std::move(Err); | |||
198 | MemMgr.reserveAllocationSpace(CodeSize, CodeAlign, RODataSize, RODataAlign, | |||
199 | RWDataSize, RWDataAlign); | |||
200 | } | |||
201 | ||||
202 | // Used sections from the object file | |||
203 | ObjSectionToIDMap LocalSections; | |||
204 | ||||
205 | // Common symbols requiring allocation, with their sizes and alignments | |||
206 | CommonSymbolList CommonSymbolsToAllocate; | |||
207 | ||||
208 | uint64_t CommonSize = 0; | |||
209 | uint32_t CommonAlign = 0; | |||
210 | ||||
211 | // First, collect all weak and common symbols. We need to know if stronger | |||
212 | // definitions occur elsewhere. | |||
213 | JITSymbolResolver::LookupSet ResponsibilitySet; | |||
214 | { | |||
215 | JITSymbolResolver::LookupSet Symbols; | |||
216 | for (auto &Sym : Obj.symbols()) { | |||
217 | Expected<uint32_t> FlagsOrErr = Sym.getFlags(); | |||
218 | if (!FlagsOrErr) | |||
219 | // TODO: Test this error. | |||
220 | return FlagsOrErr.takeError(); | |||
221 | if ((*FlagsOrErr & SymbolRef::SF_Common) || | |||
222 | (*FlagsOrErr & SymbolRef::SF_Weak)) { | |||
223 | // Get symbol name. | |||
224 | if (auto NameOrErr = Sym.getName()) | |||
225 | Symbols.insert(*NameOrErr); | |||
226 | else | |||
227 | return NameOrErr.takeError(); | |||
228 | } | |||
229 | } | |||
230 | ||||
231 | if (auto ResultOrErr = Resolver.getResponsibilitySet(Symbols)) | |||
232 | ResponsibilitySet = std::move(*ResultOrErr); | |||
233 | else | |||
234 | return ResultOrErr.takeError(); | |||
235 | } | |||
236 | ||||
237 | // Parse symbols | |||
238 | LLVM_DEBUG(dbgs() << "Parse symbols:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "Parse symbols:\n"; } } while (false ); | |||
239 | for (symbol_iterator I = Obj.symbol_begin(), E = Obj.symbol_end(); I != E; | |||
240 | ++I) { | |||
241 | Expected<uint32_t> FlagsOrErr = I->getFlags(); | |||
242 | if (!FlagsOrErr) | |||
243 | // TODO: Test this error. | |||
244 | return FlagsOrErr.takeError(); | |||
245 | ||||
246 | // Skip undefined symbols. | |||
247 | if (*FlagsOrErr & SymbolRef::SF_Undefined) | |||
248 | continue; | |||
249 | ||||
250 | // Get the symbol type. | |||
251 | object::SymbolRef::Type SymType; | |||
252 | if (auto SymTypeOrErr = I->getType()) | |||
253 | SymType = *SymTypeOrErr; | |||
254 | else | |||
255 | return SymTypeOrErr.takeError(); | |||
256 | ||||
257 | // Get symbol name. | |||
258 | StringRef Name; | |||
259 | if (auto NameOrErr = I->getName()) | |||
260 | Name = *NameOrErr; | |||
261 | else | |||
262 | return NameOrErr.takeError(); | |||
263 | ||||
264 | // Compute JIT symbol flags. | |||
265 | auto JITSymFlags = getJITSymbolFlags(*I); | |||
266 | if (!JITSymFlags) | |||
267 | return JITSymFlags.takeError(); | |||
268 | ||||
269 | // If this is a weak definition, check to see if there's a strong one. | |||
270 | // If there is, skip this symbol (we won't be providing it: the strong | |||
271 | // definition will). If there's no strong definition, make this definition | |||
272 | // strong. | |||
273 | if (JITSymFlags->isWeak() || JITSymFlags->isCommon()) { | |||
274 | // First check whether there's already a definition in this instance. | |||
275 | if (GlobalSymbolTable.count(Name)) | |||
276 | continue; | |||
277 | ||||
278 | // If we're not responsible for this symbol, skip it. | |||
279 | if (!ResponsibilitySet.count(Name)) | |||
280 | continue; | |||
281 | ||||
282 | // Otherwise update the flags on the symbol to make this definition | |||
283 | // strong. | |||
284 | if (JITSymFlags->isWeak()) | |||
285 | *JITSymFlags &= ~JITSymbolFlags::Weak; | |||
286 | if (JITSymFlags->isCommon()) { | |||
287 | *JITSymFlags &= ~JITSymbolFlags::Common; | |||
288 | uint32_t Align = I->getAlignment(); | |||
289 | uint64_t Size = I->getCommonSize(); | |||
290 | if (!CommonAlign) | |||
291 | CommonAlign = Align; | |||
292 | CommonSize = alignTo(CommonSize, Align) + Size; | |||
293 | CommonSymbolsToAllocate.push_back(*I); | |||
294 | } | |||
295 | } | |||
296 | ||||
297 | if (*FlagsOrErr & SymbolRef::SF_Absolute && | |||
298 | SymType != object::SymbolRef::ST_File) { | |||
299 | uint64_t Addr = 0; | |||
300 | if (auto AddrOrErr = I->getAddress()) | |||
301 | Addr = *AddrOrErr; | |||
302 | else | |||
303 | return AddrOrErr.takeError(); | |||
304 | ||||
305 | unsigned SectionID = AbsoluteSymbolSection; | |||
306 | ||||
307 | LLVM_DEBUG(dbgs() << "\tType: " << SymType << " (absolute) Name: " << Namedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "\tType: " << SymType << " (absolute) Name: " << Name << " SID: " << SectionID << " Offset: " << format("%p", (uintptr_t )Addr) << " flags: " << *FlagsOrErr << "\n" ; } } while (false) | |||
308 | << " SID: " << SectionIDdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "\tType: " << SymType << " (absolute) Name: " << Name << " SID: " << SectionID << " Offset: " << format("%p", (uintptr_t )Addr) << " flags: " << *FlagsOrErr << "\n" ; } } while (false) | |||
309 | << " Offset: " << format("%p", (uintptr_t)Addr)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "\tType: " << SymType << " (absolute) Name: " << Name << " SID: " << SectionID << " Offset: " << format("%p", (uintptr_t )Addr) << " flags: " << *FlagsOrErr << "\n" ; } } while (false) | |||
310 | << " flags: " << *FlagsOrErr << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "\tType: " << SymType << " (absolute) Name: " << Name << " SID: " << SectionID << " Offset: " << format("%p", (uintptr_t )Addr) << " flags: " << *FlagsOrErr << "\n" ; } } while (false); | |||
311 | if (!Name.empty()) // Skip absolute symbol relocations. | |||
312 | GlobalSymbolTable[Name] = | |||
313 | SymbolTableEntry(SectionID, Addr, *JITSymFlags); | |||
314 | } else if (SymType == object::SymbolRef::ST_Function || | |||
315 | SymType == object::SymbolRef::ST_Data || | |||
316 | SymType == object::SymbolRef::ST_Unknown || | |||
317 | SymType == object::SymbolRef::ST_Other) { | |||
318 | ||||
319 | section_iterator SI = Obj.section_end(); | |||
320 | if (auto SIOrErr = I->getSection()) | |||
321 | SI = *SIOrErr; | |||
322 | else | |||
323 | return SIOrErr.takeError(); | |||
324 | ||||
325 | if (SI == Obj.section_end()) | |||
326 | continue; | |||
327 | ||||
328 | // Get symbol offset. | |||
329 | uint64_t SectOffset; | |||
330 | if (auto Err = getOffset(*I, *SI, SectOffset)) | |||
331 | return std::move(Err); | |||
332 | ||||
333 | bool IsCode = SI->isText(); | |||
334 | unsigned SectionID; | |||
335 | if (auto SectionIDOrErr = | |||
336 | findOrEmitSection(Obj, *SI, IsCode, LocalSections)) | |||
337 | SectionID = *SectionIDOrErr; | |||
338 | else | |||
339 | return SectionIDOrErr.takeError(); | |||
340 | ||||
341 | LLVM_DEBUG(dbgs() << "\tType: " << SymType << " Name: " << Namedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "\tType: " << SymType << " Name: " << Name << " SID: " << SectionID << " Offset: " << format("%p", (uintptr_t)SectOffset ) << " flags: " << *FlagsOrErr << "\n"; } } while (false) | |||
342 | << " SID: " << SectionIDdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "\tType: " << SymType << " Name: " << Name << " SID: " << SectionID << " Offset: " << format("%p", (uintptr_t)SectOffset ) << " flags: " << *FlagsOrErr << "\n"; } } while (false) | |||
343 | << " Offset: " << format("%p", (uintptr_t)SectOffset)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "\tType: " << SymType << " Name: " << Name << " SID: " << SectionID << " Offset: " << format("%p", (uintptr_t)SectOffset ) << " flags: " << *FlagsOrErr << "\n"; } } while (false) | |||
344 | << " flags: " << *FlagsOrErr << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "\tType: " << SymType << " Name: " << Name << " SID: " << SectionID << " Offset: " << format("%p", (uintptr_t)SectOffset ) << " flags: " << *FlagsOrErr << "\n"; } } while (false); | |||
345 | if (!Name.empty()) // Skip absolute symbol relocations | |||
346 | GlobalSymbolTable[Name] = | |||
347 | SymbolTableEntry(SectionID, SectOffset, *JITSymFlags); | |||
348 | } | |||
349 | } | |||
350 | ||||
351 | // Allocate common symbols | |||
352 | if (auto Err = emitCommonSymbols(Obj, CommonSymbolsToAllocate, CommonSize, | |||
353 | CommonAlign)) | |||
354 | return std::move(Err); | |||
355 | ||||
356 | // Parse and process relocations | |||
357 | LLVM_DEBUG(dbgs() << "Parse relocations:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "Parse relocations:\n"; } } while (false); | |||
358 | for (section_iterator SI = Obj.section_begin(), SE = Obj.section_end(); | |||
359 | SI != SE; ++SI) { | |||
360 | StubMap Stubs; | |||
361 | ||||
362 | Expected<section_iterator> RelSecOrErr = SI->getRelocatedSection(); | |||
363 | if (!RelSecOrErr) | |||
364 | return RelSecOrErr.takeError(); | |||
365 | ||||
366 | section_iterator RelocatedSection = *RelSecOrErr; | |||
367 | if (RelocatedSection == SE) | |||
368 | continue; | |||
369 | ||||
370 | relocation_iterator I = SI->relocation_begin(); | |||
371 | relocation_iterator E = SI->relocation_end(); | |||
372 | ||||
373 | if (I == E && !ProcessAllSections) | |||
374 | continue; | |||
375 | ||||
376 | bool IsCode = RelocatedSection->isText(); | |||
377 | unsigned SectionID = 0; | |||
378 | if (auto SectionIDOrErr = findOrEmitSection(Obj, *RelocatedSection, IsCode, | |||
379 | LocalSections)) | |||
380 | SectionID = *SectionIDOrErr; | |||
381 | else | |||
382 | return SectionIDOrErr.takeError(); | |||
383 | ||||
384 | LLVM_DEBUG(dbgs() << "\tSectionID: " << SectionID << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "\tSectionID: " << SectionID << "\n"; } } while (false); | |||
385 | ||||
386 | for (; I != E;) | |||
387 | if (auto IOrErr = processRelocationRef(SectionID, I, Obj, LocalSections, Stubs)) | |||
388 | I = *IOrErr; | |||
389 | else | |||
390 | return IOrErr.takeError(); | |||
391 | ||||
392 | // If there is a NotifyStubEmitted callback set, call it to register any | |||
393 | // stubs created for this section. | |||
394 | if (NotifyStubEmitted) { | |||
395 | StringRef FileName = Obj.getFileName(); | |||
396 | StringRef SectionName = Sections[SectionID].getName(); | |||
397 | for (auto &KV : Stubs) { | |||
398 | ||||
399 | auto &VR = KV.first; | |||
400 | uint64_t StubAddr = KV.second; | |||
401 | ||||
402 | // If this is a named stub, just call NotifyStubEmitted. | |||
403 | if (VR.SymbolName) { | |||
404 | NotifyStubEmitted(FileName, SectionName, VR.SymbolName, SectionID, | |||
405 | StubAddr); | |||
406 | continue; | |||
407 | } | |||
408 | ||||
409 | // Otherwise we will have to try a reverse lookup on the globla symbol table. | |||
410 | for (auto &GSTMapEntry : GlobalSymbolTable) { | |||
411 | StringRef SymbolName = GSTMapEntry.first(); | |||
412 | auto &GSTEntry = GSTMapEntry.second; | |||
413 | if (GSTEntry.getSectionID() == VR.SectionID && | |||
414 | GSTEntry.getOffset() == VR.Offset) { | |||
415 | NotifyStubEmitted(FileName, SectionName, SymbolName, SectionID, | |||
416 | StubAddr); | |||
417 | break; | |||
418 | } | |||
419 | } | |||
420 | } | |||
421 | } | |||
422 | } | |||
423 | ||||
424 | // Process remaining sections | |||
425 | if (ProcessAllSections) { | |||
426 | LLVM_DEBUG(dbgs() << "Process remaining sections:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "Process remaining sections:\n"; } } while (false); | |||
427 | for (section_iterator SI = Obj.section_begin(), SE = Obj.section_end(); | |||
428 | SI != SE; ++SI) { | |||
429 | ||||
430 | /* Ignore already loaded sections */ | |||
431 | if (LocalSections.find(*SI) != LocalSections.end()) | |||
432 | continue; | |||
433 | ||||
434 | bool IsCode = SI->isText(); | |||
435 | if (auto SectionIDOrErr = | |||
436 | findOrEmitSection(Obj, *SI, IsCode, LocalSections)) | |||
437 | LLVM_DEBUG(dbgs() << "\tSectionID: " << (*SectionIDOrErr) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "\tSectionID: " << (*SectionIDOrErr ) << "\n"; } } while (false); | |||
438 | else | |||
439 | return SectionIDOrErr.takeError(); | |||
440 | } | |||
441 | } | |||
442 | ||||
443 | // Give the subclasses a chance to tie-up any loose ends. | |||
444 | if (auto Err = finalizeLoad(Obj, LocalSections)) | |||
445 | return std::move(Err); | |||
446 | ||||
447 | // for (auto E : LocalSections) | |||
448 | // llvm::dbgs() << "Added: " << E.first.getRawDataRefImpl() << " -> " << E.second << "\n"; | |||
449 | ||||
450 | return LocalSections; | |||
451 | } | |||
452 | ||||
453 | // A helper method for computeTotalAllocSize. | |||
454 | // Computes the memory size required to allocate sections with the given sizes, | |||
455 | // assuming that all sections are allocated with the given alignment | |||
456 | static uint64_t | |||
457 | computeAllocationSizeForSections(std::vector<uint64_t> &SectionSizes, | |||
458 | uint64_t Alignment) { | |||
459 | uint64_t TotalSize = 0; | |||
460 | for (size_t Idx = 0, Cnt = SectionSizes.size(); Idx < Cnt; Idx++) { | |||
461 | uint64_t AlignedSize = | |||
462 | (SectionSizes[Idx] + Alignment - 1) / Alignment * Alignment; | |||
463 | TotalSize += AlignedSize; | |||
464 | } | |||
465 | return TotalSize; | |||
466 | } | |||
467 | ||||
468 | static bool isRequiredForExecution(const SectionRef Section) { | |||
469 | const ObjectFile *Obj = Section.getObject(); | |||
470 | if (isa<object::ELFObjectFileBase>(Obj)) | |||
471 | return ELFSectionRef(Section).getFlags() & ELF::SHF_ALLOC; | |||
472 | if (auto *COFFObj = dyn_cast<object::COFFObjectFile>(Obj)) { | |||
473 | const coff_section *CoffSection = COFFObj->getCOFFSection(Section); | |||
474 | // Avoid loading zero-sized COFF sections. | |||
475 | // In PE files, VirtualSize gives the section size, and SizeOfRawData | |||
476 | // may be zero for sections with content. In Obj files, SizeOfRawData | |||
477 | // gives the section size, and VirtualSize is always zero. Hence | |||
478 | // the need to check for both cases below. | |||
479 | bool HasContent = | |||
480 | (CoffSection->VirtualSize > 0) || (CoffSection->SizeOfRawData > 0); | |||
481 | bool IsDiscardable = | |||
482 | CoffSection->Characteristics & | |||
483 | (COFF::IMAGE_SCN_MEM_DISCARDABLE | COFF::IMAGE_SCN_LNK_INFO); | |||
484 | return HasContent && !IsDiscardable; | |||
485 | } | |||
486 | ||||
487 | assert(isa<MachOObjectFile>(Obj))(static_cast <bool> (isa<MachOObjectFile>(Obj)) ? void (0) : __assert_fail ("isa<MachOObjectFile>(Obj)", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp" , 487, __extension__ __PRETTY_FUNCTION__)); | |||
488 | return true; | |||
489 | } | |||
490 | ||||
491 | static bool isReadOnlyData(const SectionRef Section) { | |||
492 | const ObjectFile *Obj = Section.getObject(); | |||
493 | if (isa<object::ELFObjectFileBase>(Obj)) | |||
494 | return !(ELFSectionRef(Section).getFlags() & | |||
495 | (ELF::SHF_WRITE | ELF::SHF_EXECINSTR)); | |||
496 | if (auto *COFFObj = dyn_cast<object::COFFObjectFile>(Obj)) | |||
497 | return ((COFFObj->getCOFFSection(Section)->Characteristics & | |||
498 | (COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | |||
499 | | COFF::IMAGE_SCN_MEM_READ | |||
500 | | COFF::IMAGE_SCN_MEM_WRITE)) | |||
501 | == | |||
502 | (COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | |||
503 | | COFF::IMAGE_SCN_MEM_READ)); | |||
504 | ||||
505 | assert(isa<MachOObjectFile>(Obj))(static_cast <bool> (isa<MachOObjectFile>(Obj)) ? void (0) : __assert_fail ("isa<MachOObjectFile>(Obj)", "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp" , 505, __extension__ __PRETTY_FUNCTION__)); | |||
506 | return false; | |||
507 | } | |||
508 | ||||
509 | static bool isZeroInit(const SectionRef Section) { | |||
510 | const ObjectFile *Obj = Section.getObject(); | |||
511 | if (isa<object::ELFObjectFileBase>(Obj)) | |||
512 | return ELFSectionRef(Section).getType() == ELF::SHT_NOBITS; | |||
513 | if (auto *COFFObj = dyn_cast<object::COFFObjectFile>(Obj)) | |||
514 | return COFFObj->getCOFFSection(Section)->Characteristics & | |||
515 | COFF::IMAGE_SCN_CNT_UNINITIALIZED_DATA; | |||
516 | ||||
517 | auto *MachO = cast<MachOObjectFile>(Obj); | |||
518 | unsigned SectionType = MachO->getSectionType(Section); | |||
519 | return SectionType == MachO::S_ZEROFILL || | |||
520 | SectionType == MachO::S_GB_ZEROFILL; | |||
521 | } | |||
522 | ||||
523 | // Compute an upper bound of the memory size that is required to load all | |||
524 | // sections | |||
525 | Error RuntimeDyldImpl::computeTotalAllocSize(const ObjectFile &Obj, | |||
526 | uint64_t &CodeSize, | |||
527 | uint32_t &CodeAlign, | |||
528 | uint64_t &RODataSize, | |||
529 | uint32_t &RODataAlign, | |||
530 | uint64_t &RWDataSize, | |||
531 | uint32_t &RWDataAlign) { | |||
532 | // Compute the size of all sections required for execution | |||
533 | std::vector<uint64_t> CodeSectionSizes; | |||
534 | std::vector<uint64_t> ROSectionSizes; | |||
535 | std::vector<uint64_t> RWSectionSizes; | |||
536 | ||||
537 | // Collect sizes of all sections to be loaded; | |||
538 | // also determine the max alignment of all sections | |||
539 | for (section_iterator SI = Obj.section_begin(), SE = Obj.section_end(); | |||
540 | SI != SE; ++SI) { | |||
541 | const SectionRef &Section = *SI; | |||
542 | ||||
543 | bool IsRequired = isRequiredForExecution(Section) || ProcessAllSections; | |||
544 | ||||
545 | // Consider only the sections that are required to be loaded for execution | |||
546 | if (IsRequired) { | |||
547 | uint64_t DataSize = Section.getSize(); | |||
548 | uint64_t Alignment64 = Section.getAlignment(); | |||
549 | unsigned Alignment = (unsigned)Alignment64 & 0xffffffffL; | |||
550 | bool IsCode = Section.isText(); | |||
551 | bool IsReadOnly = isReadOnlyData(Section); | |||
552 | ||||
553 | Expected<StringRef> NameOrErr = Section.getName(); | |||
554 | if (!NameOrErr) | |||
555 | return NameOrErr.takeError(); | |||
556 | StringRef Name = *NameOrErr; | |||
557 | ||||
558 | uint64_t StubBufSize = computeSectionStubBufSize(Obj, Section); | |||
559 | ||||
560 | uint64_t PaddingSize = 0; | |||
561 | if (Name == ".eh_frame") | |||
562 | PaddingSize += 4; | |||
563 | if (StubBufSize != 0) | |||
564 | PaddingSize += getStubAlignment() - 1; | |||
565 | ||||
566 | uint64_t SectionSize = DataSize + PaddingSize + StubBufSize; | |||
567 | ||||
568 | // The .eh_frame section (at least on Linux) needs an extra four bytes | |||
569 | // padded | |||
570 | // with zeroes added at the end. For MachO objects, this section has a | |||
571 | // slightly different name, so this won't have any effect for MachO | |||
572 | // objects. | |||
573 | if (Name == ".eh_frame") | |||
574 | SectionSize += 4; | |||
575 | ||||
576 | if (!SectionSize) | |||
577 | SectionSize = 1; | |||
578 | ||||
579 | if (IsCode) { | |||
580 | CodeAlign = std::max(CodeAlign, Alignment); | |||
581 | CodeSectionSizes.push_back(SectionSize); | |||
582 | } else if (IsReadOnly) { | |||
583 | RODataAlign = std::max(RODataAlign, Alignment); | |||
584 | ROSectionSizes.push_back(SectionSize); | |||
585 | } else { | |||
586 | RWDataAlign = std::max(RWDataAlign, Alignment); | |||
587 | RWSectionSizes.push_back(SectionSize); | |||
588 | } | |||
589 | } | |||
590 | } | |||
591 | ||||
592 | // Compute Global Offset Table size. If it is not zero we | |||
593 | // also update alignment, which is equal to a size of a | |||
594 | // single GOT entry. | |||
595 | if (unsigned GotSize = computeGOTSize(Obj)) { | |||
596 | RWSectionSizes.push_back(GotSize); | |||
597 | RWDataAlign = std::max<uint32_t>(RWDataAlign, getGOTEntrySize()); | |||
598 | } | |||
599 | ||||
600 | // Compute the size of all common symbols | |||
601 | uint64_t CommonSize = 0; | |||
602 | uint32_t CommonAlign = 1; | |||
603 | for (symbol_iterator I = Obj.symbol_begin(), E = Obj.symbol_end(); I != E; | |||
604 | ++I) { | |||
605 | Expected<uint32_t> FlagsOrErr = I->getFlags(); | |||
606 | if (!FlagsOrErr) | |||
607 | // TODO: Test this error. | |||
608 | return FlagsOrErr.takeError(); | |||
609 | if (*FlagsOrErr & SymbolRef::SF_Common) { | |||
610 | // Add the common symbols to a list. We'll allocate them all below. | |||
611 | uint64_t Size = I->getCommonSize(); | |||
612 | uint32_t Align = I->getAlignment(); | |||
613 | // If this is the first common symbol, use its alignment as the alignment | |||
614 | // for the common symbols section. | |||
615 | if (CommonSize == 0) | |||
616 | CommonAlign = Align; | |||
617 | CommonSize = alignTo(CommonSize, Align) + Size; | |||
618 | } | |||
619 | } | |||
620 | if (CommonSize != 0) { | |||
621 | RWSectionSizes.push_back(CommonSize); | |||
622 | RWDataAlign = std::max(RWDataAlign, CommonAlign); | |||
623 | } | |||
624 | ||||
625 | // Compute the required allocation space for each different type of sections | |||
626 | // (code, read-only data, read-write data) assuming that all sections are | |||
627 | // allocated with the max alignment. Note that we cannot compute with the | |||
628 | // individual alignments of the sections, because then the required size | |||
629 | // depends on the order, in which the sections are allocated. | |||
630 | CodeSize = computeAllocationSizeForSections(CodeSectionSizes, CodeAlign); | |||
631 | RODataSize = computeAllocationSizeForSections(ROSectionSizes, RODataAlign); | |||
632 | RWDataSize = computeAllocationSizeForSections(RWSectionSizes, RWDataAlign); | |||
633 | ||||
634 | return Error::success(); | |||
635 | } | |||
636 | ||||
637 | // compute GOT size | |||
638 | unsigned RuntimeDyldImpl::computeGOTSize(const ObjectFile &Obj) { | |||
639 | size_t GotEntrySize = getGOTEntrySize(); | |||
640 | if (!GotEntrySize) | |||
641 | return 0; | |||
642 | ||||
643 | size_t GotSize = 0; | |||
644 | for (section_iterator SI = Obj.section_begin(), SE = Obj.section_end(); | |||
645 | SI != SE; ++SI) { | |||
646 | ||||
647 | for (const RelocationRef &Reloc : SI->relocations()) | |||
648 | if (relocationNeedsGot(Reloc)) | |||
649 | GotSize += GotEntrySize; | |||
650 | } | |||
651 | ||||
652 | return GotSize; | |||
653 | } | |||
654 | ||||
655 | // compute stub buffer size for the given section | |||
656 | unsigned RuntimeDyldImpl::computeSectionStubBufSize(const ObjectFile &Obj, | |||
657 | const SectionRef &Section) { | |||
658 | if (!MemMgr.allowStubAllocation()) { | |||
659 | return 0; | |||
660 | } | |||
661 | ||||
662 | unsigned StubSize = getMaxStubSize(); | |||
663 | if (StubSize == 0) { | |||
664 | return 0; | |||
665 | } | |||
666 | // FIXME: this is an inefficient way to handle this. We should computed the | |||
667 | // necessary section allocation size in loadObject by walking all the sections | |||
668 | // once. | |||
669 | unsigned StubBufSize = 0; | |||
670 | for (section_iterator SI = Obj.section_begin(), SE = Obj.section_end(); | |||
671 | SI != SE; ++SI) { | |||
672 | ||||
673 | Expected<section_iterator> RelSecOrErr = SI->getRelocatedSection(); | |||
674 | if (!RelSecOrErr) | |||
675 | report_fatal_error(toString(RelSecOrErr.takeError())); | |||
676 | ||||
677 | section_iterator RelSecI = *RelSecOrErr; | |||
678 | if (!(RelSecI == Section)) | |||
679 | continue; | |||
680 | ||||
681 | for (const RelocationRef &Reloc : SI->relocations()) | |||
682 | if (relocationNeedsStub(Reloc)) | |||
683 | StubBufSize += StubSize; | |||
684 | } | |||
685 | ||||
686 | // Get section data size and alignment | |||
687 | uint64_t DataSize = Section.getSize(); | |||
688 | uint64_t Alignment64 = Section.getAlignment(); | |||
689 | ||||
690 | // Add stubbuf size alignment | |||
691 | unsigned Alignment = (unsigned)Alignment64 & 0xffffffffL; | |||
692 | unsigned StubAlignment = getStubAlignment(); | |||
693 | unsigned EndAlignment = (DataSize | Alignment) & -(DataSize | Alignment); | |||
694 | if (StubAlignment > EndAlignment) | |||
695 | StubBufSize += StubAlignment - EndAlignment; | |||
696 | return StubBufSize; | |||
697 | } | |||
698 | ||||
699 | uint64_t RuntimeDyldImpl::readBytesUnaligned(uint8_t *Src, | |||
700 | unsigned Size) const { | |||
701 | uint64_t Result = 0; | |||
702 | if (IsTargetLittleEndian) { | |||
703 | Src += Size - 1; | |||
704 | while (Size--) | |||
705 | Result = (Result << 8) | *Src--; | |||
706 | } else | |||
707 | while (Size--) | |||
708 | Result = (Result << 8) | *Src++; | |||
709 | ||||
710 | return Result; | |||
711 | } | |||
712 | ||||
713 | void RuntimeDyldImpl::writeBytesUnaligned(uint64_t Value, uint8_t *Dst, | |||
714 | unsigned Size) const { | |||
715 | if (IsTargetLittleEndian) { | |||
716 | while (Size--) { | |||
717 | *Dst++ = Value & 0xFF; | |||
718 | Value >>= 8; | |||
719 | } | |||
720 | } else { | |||
721 | Dst += Size - 1; | |||
722 | while (Size--) { | |||
723 | *Dst-- = Value & 0xFF; | |||
724 | Value >>= 8; | |||
725 | } | |||
726 | } | |||
727 | } | |||
728 | ||||
729 | Expected<JITSymbolFlags> | |||
730 | RuntimeDyldImpl::getJITSymbolFlags(const SymbolRef &SR) { | |||
731 | return JITSymbolFlags::fromObjectSymbol(SR); | |||
732 | } | |||
733 | ||||
734 | Error RuntimeDyldImpl::emitCommonSymbols(const ObjectFile &Obj, | |||
735 | CommonSymbolList &SymbolsToAllocate, | |||
736 | uint64_t CommonSize, | |||
737 | uint32_t CommonAlign) { | |||
738 | if (SymbolsToAllocate.empty()) | |||
739 | return Error::success(); | |||
740 | ||||
741 | // Allocate memory for the section | |||
742 | unsigned SectionID = Sections.size(); | |||
743 | uint8_t *Addr = MemMgr.allocateDataSection(CommonSize, CommonAlign, SectionID, | |||
744 | "<common symbols>", false); | |||
745 | if (!Addr) | |||
746 | report_fatal_error("Unable to allocate memory for common symbols!"); | |||
747 | uint64_t Offset = 0; | |||
748 | Sections.push_back( | |||
749 | SectionEntry("<common symbols>", Addr, CommonSize, CommonSize, 0)); | |||
750 | memset(Addr, 0, CommonSize); | |||
751 | ||||
752 | LLVM_DEBUG(dbgs() << "emitCommonSection SectionID: " << SectionIDdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "emitCommonSection SectionID: " << SectionID << " new addr: " << format("%p", Addr) << " DataSize: " << CommonSize << "\n"; } } while (false) | |||
753 | << " new addr: " << format("%p", Addr)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "emitCommonSection SectionID: " << SectionID << " new addr: " << format("%p", Addr) << " DataSize: " << CommonSize << "\n"; } } while (false) | |||
754 | << " DataSize: " << CommonSize << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "emitCommonSection SectionID: " << SectionID << " new addr: " << format("%p", Addr) << " DataSize: " << CommonSize << "\n"; } } while (false); | |||
755 | ||||
756 | // Assign the address of each symbol | |||
757 | for (auto &Sym : SymbolsToAllocate) { | |||
758 | uint32_t Alignment = Sym.getAlignment(); | |||
759 | uint64_t Size = Sym.getCommonSize(); | |||
760 | StringRef Name; | |||
761 | if (auto NameOrErr = Sym.getName()) | |||
762 | Name = *NameOrErr; | |||
763 | else | |||
764 | return NameOrErr.takeError(); | |||
765 | if (Alignment) { | |||
766 | // This symbol has an alignment requirement. | |||
767 | uint64_t AlignOffset = | |||
768 | offsetToAlignment((uint64_t)Addr, Align(Alignment)); | |||
769 | Addr += AlignOffset; | |||
770 | Offset += AlignOffset; | |||
771 | } | |||
772 | auto JITSymFlags = getJITSymbolFlags(Sym); | |||
773 | ||||
774 | if (!JITSymFlags) | |||
775 | return JITSymFlags.takeError(); | |||
776 | ||||
777 | LLVM_DEBUG(dbgs() << "Allocating common symbol " << Name << " address "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "Allocating common symbol " << Name << " address " << format("%p", Addr) << "\n"; } } while (false) | |||
778 | << format("%p", Addr) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "Allocating common symbol " << Name << " address " << format("%p", Addr) << "\n"; } } while (false); | |||
779 | if (!Name.empty()) // Skip absolute symbol relocations. | |||
780 | GlobalSymbolTable[Name] = | |||
781 | SymbolTableEntry(SectionID, Offset, std::move(*JITSymFlags)); | |||
782 | Offset += Size; | |||
783 | Addr += Size; | |||
784 | } | |||
785 | ||||
786 | return Error::success(); | |||
787 | } | |||
788 | ||||
789 | Expected<unsigned> | |||
790 | RuntimeDyldImpl::emitSection(const ObjectFile &Obj, | |||
791 | const SectionRef &Section, | |||
792 | bool IsCode) { | |||
793 | StringRef data; | |||
794 | uint64_t Alignment64 = Section.getAlignment(); | |||
795 | ||||
796 | unsigned Alignment = (unsigned)Alignment64 & 0xffffffffL; | |||
797 | unsigned PaddingSize = 0; | |||
798 | unsigned StubBufSize = 0; | |||
799 | bool IsRequired = isRequiredForExecution(Section); | |||
800 | bool IsVirtual = Section.isVirtual(); | |||
801 | bool IsZeroInit = isZeroInit(Section); | |||
802 | bool IsReadOnly = isReadOnlyData(Section); | |||
803 | uint64_t DataSize = Section.getSize(); | |||
804 | ||||
805 | // An alignment of 0 (at least with ELF) is identical to an alignment of 1, | |||
806 | // while being more "polite". Other formats do not support 0-aligned sections | |||
807 | // anyway, so we should guarantee that the alignment is always at least 1. | |||
808 | Alignment = std::max(1u, Alignment); | |||
809 | ||||
810 | Expected<StringRef> NameOrErr = Section.getName(); | |||
811 | if (!NameOrErr) | |||
812 | return NameOrErr.takeError(); | |||
813 | StringRef Name = *NameOrErr; | |||
814 | ||||
815 | StubBufSize = computeSectionStubBufSize(Obj, Section); | |||
816 | ||||
817 | // The .eh_frame section (at least on Linux) needs an extra four bytes padded | |||
818 | // with zeroes added at the end. For MachO objects, this section has a | |||
819 | // slightly different name, so this won't have any effect for MachO objects. | |||
820 | if (Name == ".eh_frame") | |||
821 | PaddingSize = 4; | |||
822 | ||||
823 | uintptr_t Allocate; | |||
824 | unsigned SectionID = Sections.size(); | |||
825 | uint8_t *Addr; | |||
826 | const char *pData = nullptr; | |||
827 | ||||
828 | // If this section contains any bits (i.e. isn't a virtual or bss section), | |||
829 | // grab a reference to them. | |||
830 | if (!IsVirtual && !IsZeroInit) { | |||
831 | // In either case, set the location of the unrelocated section in memory, | |||
832 | // since we still process relocations for it even if we're not applying them. | |||
833 | if (Expected<StringRef> E = Section.getContents()) | |||
834 | data = *E; | |||
835 | else | |||
836 | return E.takeError(); | |||
837 | pData = data.data(); | |||
838 | } | |||
839 | ||||
840 | // If there are any stubs then the section alignment needs to be at least as | |||
841 | // high as stub alignment or padding calculations may by incorrect when the | |||
842 | // section is remapped. | |||
843 | if (StubBufSize != 0) { | |||
844 | Alignment = std::max(Alignment, getStubAlignment()); | |||
845 | PaddingSize += getStubAlignment() - 1; | |||
846 | } | |||
847 | ||||
848 | // Some sections, such as debug info, don't need to be loaded for execution. | |||
849 | // Process those only if explicitly requested. | |||
850 | if (IsRequired || ProcessAllSections) { | |||
851 | Allocate = DataSize + PaddingSize + StubBufSize; | |||
852 | if (!Allocate) | |||
853 | Allocate = 1; | |||
854 | Addr = IsCode ? MemMgr.allocateCodeSection(Allocate, Alignment, SectionID, | |||
855 | Name) | |||
856 | : MemMgr.allocateDataSection(Allocate, Alignment, SectionID, | |||
857 | Name, IsReadOnly); | |||
858 | if (!Addr) | |||
859 | report_fatal_error("Unable to allocate section memory!"); | |||
860 | ||||
861 | // Zero-initialize or copy the data from the image | |||
862 | if (IsZeroInit || IsVirtual) | |||
863 | memset(Addr, 0, DataSize); | |||
864 | else | |||
865 | memcpy(Addr, pData, DataSize); | |||
866 | ||||
867 | // Fill in any extra bytes we allocated for padding | |||
868 | if (PaddingSize != 0) { | |||
869 | memset(Addr + DataSize, 0, PaddingSize); | |||
870 | // Update the DataSize variable to include padding. | |||
871 | DataSize += PaddingSize; | |||
872 | ||||
873 | // Align DataSize to stub alignment if we have any stubs (PaddingSize will | |||
874 | // have been increased above to account for this). | |||
875 | if (StubBufSize > 0) | |||
876 | DataSize &= -(uint64_t)getStubAlignment(); | |||
877 | } | |||
878 | ||||
879 | LLVM_DEBUG(dbgs() << "emitSection SectionID: " << SectionID << " Name: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "emitSection SectionID: " << SectionID << " Name: " << Name << " obj addr: " << format("%p", pData) << " new addr: " << format("%p", Addr) << " DataSize: " << DataSize << " StubBufSize: " << StubBufSize << " Allocate: " << Allocate << "\n"; } } while (false) | |||
880 | << Name << " obj addr: " << format("%p", pData)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "emitSection SectionID: " << SectionID << " Name: " << Name << " obj addr: " << format("%p", pData) << " new addr: " << format("%p", Addr) << " DataSize: " << DataSize << " StubBufSize: " << StubBufSize << " Allocate: " << Allocate << "\n"; } } while (false) | |||
881 | << " new addr: " << format("%p", Addr) << " DataSize: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "emitSection SectionID: " << SectionID << " Name: " << Name << " obj addr: " << format("%p", pData) << " new addr: " << format("%p", Addr) << " DataSize: " << DataSize << " StubBufSize: " << StubBufSize << " Allocate: " << Allocate << "\n"; } } while (false) | |||
882 | << DataSize << " StubBufSize: " << StubBufSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "emitSection SectionID: " << SectionID << " Name: " << Name << " obj addr: " << format("%p", pData) << " new addr: " << format("%p", Addr) << " DataSize: " << DataSize << " StubBufSize: " << StubBufSize << " Allocate: " << Allocate << "\n"; } } while (false) | |||
883 | << " Allocate: " << Allocate << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "emitSection SectionID: " << SectionID << " Name: " << Name << " obj addr: " << format("%p", pData) << " new addr: " << format("%p", Addr) << " DataSize: " << DataSize << " StubBufSize: " << StubBufSize << " Allocate: " << Allocate << "\n"; } } while (false); | |||
884 | } else { | |||
885 | // Even if we didn't load the section, we need to record an entry for it | |||
886 | // to handle later processing (and by 'handle' I mean don't do anything | |||
887 | // with these sections). | |||
888 | Allocate = 0; | |||
889 | Addr = nullptr; | |||
890 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "emitSection SectionID: " << SectionID << " Name: " << Name << " obj addr: " << format("%p", data.data()) << " new addr: 0" << " DataSize: " << DataSize << " StubBufSize: " << StubBufSize << " Allocate: " << Allocate << "\n"; } } while (false) | |||
891 | dbgs() << "emitSection SectionID: " << SectionID << " Name: " << Namedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "emitSection SectionID: " << SectionID << " Name: " << Name << " obj addr: " << format("%p", data.data()) << " new addr: 0" << " DataSize: " << DataSize << " StubBufSize: " << StubBufSize << " Allocate: " << Allocate << "\n"; } } while (false) | |||
892 | << " obj addr: " << format("%p", data.data()) << " new addr: 0"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "emitSection SectionID: " << SectionID << " Name: " << Name << " obj addr: " << format("%p", data.data()) << " new addr: 0" << " DataSize: " << DataSize << " StubBufSize: " << StubBufSize << " Allocate: " << Allocate << "\n"; } } while (false) | |||
893 | << " DataSize: " << DataSize << " StubBufSize: " << StubBufSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "emitSection SectionID: " << SectionID << " Name: " << Name << " obj addr: " << format("%p", data.data()) << " new addr: 0" << " DataSize: " << DataSize << " StubBufSize: " << StubBufSize << " Allocate: " << Allocate << "\n"; } } while (false) | |||
894 | << " Allocate: " << Allocate << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "emitSection SectionID: " << SectionID << " Name: " << Name << " obj addr: " << format("%p", data.data()) << " new addr: 0" << " DataSize: " << DataSize << " StubBufSize: " << StubBufSize << " Allocate: " << Allocate << "\n"; } } while (false); | |||
895 | } | |||
896 | ||||
897 | Sections.push_back( | |||
898 | SectionEntry(Name, Addr, DataSize, Allocate, (uintptr_t)pData)); | |||
899 | ||||
900 | // Debug info sections are linked as if their load address was zero | |||
901 | if (!IsRequired) | |||
902 | Sections.back().setLoadAddress(0); | |||
903 | ||||
904 | return SectionID; | |||
905 | } | |||
906 | ||||
907 | Expected<unsigned> | |||
908 | RuntimeDyldImpl::findOrEmitSection(const ObjectFile &Obj, | |||
909 | const SectionRef &Section, | |||
910 | bool IsCode, | |||
911 | ObjSectionToIDMap &LocalSections) { | |||
912 | ||||
913 | unsigned SectionID = 0; | |||
914 | ObjSectionToIDMap::iterator i = LocalSections.find(Section); | |||
915 | if (i != LocalSections.end()) | |||
916 | SectionID = i->second; | |||
917 | else { | |||
918 | if (auto SectionIDOrErr = emitSection(Obj, Section, IsCode)) | |||
919 | SectionID = *SectionIDOrErr; | |||
920 | else | |||
921 | return SectionIDOrErr.takeError(); | |||
922 | LocalSections[Section] = SectionID; | |||
923 | } | |||
924 | return SectionID; | |||
925 | } | |||
926 | ||||
927 | void RuntimeDyldImpl::addRelocationForSection(const RelocationEntry &RE, | |||
928 | unsigned SectionID) { | |||
929 | Relocations[SectionID].push_back(RE); | |||
930 | } | |||
931 | ||||
932 | void RuntimeDyldImpl::addRelocationForSymbol(const RelocationEntry &RE, | |||
933 | StringRef SymbolName) { | |||
934 | // Relocation by symbol. If the symbol is found in the global symbol table, | |||
935 | // create an appropriate section relocation. Otherwise, add it to | |||
936 | // ExternalSymbolRelocations. | |||
937 | RTDyldSymbolTable::const_iterator Loc = GlobalSymbolTable.find(SymbolName); | |||
938 | if (Loc == GlobalSymbolTable.end()) { | |||
939 | ExternalSymbolRelocations[SymbolName].push_back(RE); | |||
940 | } else { | |||
941 | assert(!SymbolName.empty() &&(static_cast <bool> (!SymbolName.empty() && "Empty symbol should not be in GlobalSymbolTable" ) ? void (0) : __assert_fail ("!SymbolName.empty() && \"Empty symbol should not be in GlobalSymbolTable\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp" , 942, __extension__ __PRETTY_FUNCTION__)) | |||
942 | "Empty symbol should not be in GlobalSymbolTable")(static_cast <bool> (!SymbolName.empty() && "Empty symbol should not be in GlobalSymbolTable" ) ? void (0) : __assert_fail ("!SymbolName.empty() && \"Empty symbol should not be in GlobalSymbolTable\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp" , 942, __extension__ __PRETTY_FUNCTION__)); | |||
943 | // Copy the RE since we want to modify its addend. | |||
944 | RelocationEntry RECopy = RE; | |||
945 | const auto &SymInfo = Loc->second; | |||
946 | RECopy.Addend += SymInfo.getOffset(); | |||
947 | Relocations[SymInfo.getSectionID()].push_back(RECopy); | |||
948 | } | |||
949 | } | |||
950 | ||||
951 | uint8_t *RuntimeDyldImpl::createStubFunction(uint8_t *Addr, | |||
952 | unsigned AbiVariant) { | |||
953 | if (Arch == Triple::aarch64 || Arch == Triple::aarch64_be || | |||
954 | Arch == Triple::aarch64_32) { | |||
955 | // This stub has to be able to access the full address space, | |||
956 | // since symbol lookup won't necessarily find a handy, in-range, | |||
957 | // PLT stub for functions which could be anywhere. | |||
958 | // Stub can use ip0 (== x16) to calculate address | |||
959 | writeBytesUnaligned(0xd2e00010, Addr, 4); // movz ip0, #:abs_g3:<addr> | |||
960 | writeBytesUnaligned(0xf2c00010, Addr+4, 4); // movk ip0, #:abs_g2_nc:<addr> | |||
961 | writeBytesUnaligned(0xf2a00010, Addr+8, 4); // movk ip0, #:abs_g1_nc:<addr> | |||
962 | writeBytesUnaligned(0xf2800010, Addr+12, 4); // movk ip0, #:abs_g0_nc:<addr> | |||
963 | writeBytesUnaligned(0xd61f0200, Addr+16, 4); // br ip0 | |||
964 | ||||
965 | return Addr; | |||
966 | } else if (Arch == Triple::arm || Arch == Triple::armeb) { | |||
967 | // TODO: There is only ARM far stub now. We should add the Thumb stub, | |||
968 | // and stubs for branches Thumb - ARM and ARM - Thumb. | |||
969 | writeBytesUnaligned(0xe51ff004, Addr, 4); // ldr pc, [pc, #-4] | |||
970 | return Addr + 4; | |||
971 | } else if (IsMipsO32ABI || IsMipsN32ABI) { | |||
972 | // 0: 3c190000 lui t9,%hi(addr). | |||
973 | // 4: 27390000 addiu t9,t9,%lo(addr). | |||
974 | // 8: 03200008 jr t9. | |||
975 | // c: 00000000 nop. | |||
976 | const unsigned LuiT9Instr = 0x3c190000, AdduiT9Instr = 0x27390000; | |||
977 | const unsigned NopInstr = 0x0; | |||
978 | unsigned JrT9Instr = 0x03200008; | |||
979 | if ((AbiVariant & ELF::EF_MIPS_ARCH) == ELF::EF_MIPS_ARCH_32R6 || | |||
980 | (AbiVariant & ELF::EF_MIPS_ARCH) == ELF::EF_MIPS_ARCH_64R6) | |||
981 | JrT9Instr = 0x03200009; | |||
982 | ||||
983 | writeBytesUnaligned(LuiT9Instr, Addr, 4); | |||
984 | writeBytesUnaligned(AdduiT9Instr, Addr + 4, 4); | |||
985 | writeBytesUnaligned(JrT9Instr, Addr + 8, 4); | |||
986 | writeBytesUnaligned(NopInstr, Addr + 12, 4); | |||
987 | return Addr; | |||
988 | } else if (IsMipsN64ABI) { | |||
989 | // 0: 3c190000 lui t9,%highest(addr). | |||
990 | // 4: 67390000 daddiu t9,t9,%higher(addr). | |||
991 | // 8: 0019CC38 dsll t9,t9,16. | |||
992 | // c: 67390000 daddiu t9,t9,%hi(addr). | |||
993 | // 10: 0019CC38 dsll t9,t9,16. | |||
994 | // 14: 67390000 daddiu t9,t9,%lo(addr). | |||
995 | // 18: 03200008 jr t9. | |||
996 | // 1c: 00000000 nop. | |||
997 | const unsigned LuiT9Instr = 0x3c190000, DaddiuT9Instr = 0x67390000, | |||
998 | DsllT9Instr = 0x19CC38; | |||
999 | const unsigned NopInstr = 0x0; | |||
1000 | unsigned JrT9Instr = 0x03200008; | |||
1001 | if ((AbiVariant & ELF::EF_MIPS_ARCH) == ELF::EF_MIPS_ARCH_64R6) | |||
1002 | JrT9Instr = 0x03200009; | |||
1003 | ||||
1004 | writeBytesUnaligned(LuiT9Instr, Addr, 4); | |||
1005 | writeBytesUnaligned(DaddiuT9Instr, Addr + 4, 4); | |||
1006 | writeBytesUnaligned(DsllT9Instr, Addr + 8, 4); | |||
1007 | writeBytesUnaligned(DaddiuT9Instr, Addr + 12, 4); | |||
1008 | writeBytesUnaligned(DsllT9Instr, Addr + 16, 4); | |||
1009 | writeBytesUnaligned(DaddiuT9Instr, Addr + 20, 4); | |||
1010 | writeBytesUnaligned(JrT9Instr, Addr + 24, 4); | |||
1011 | writeBytesUnaligned(NopInstr, Addr + 28, 4); | |||
1012 | return Addr; | |||
1013 | } else if (Arch == Triple::ppc64 || Arch == Triple::ppc64le) { | |||
1014 | // Depending on which version of the ELF ABI is in use, we need to | |||
1015 | // generate one of two variants of the stub. They both start with | |||
1016 | // the same sequence to load the target address into r12. | |||
1017 | writeInt32BE(Addr, 0x3D800000); // lis r12, highest(addr) | |||
1018 | writeInt32BE(Addr+4, 0x618C0000); // ori r12, higher(addr) | |||
1019 | writeInt32BE(Addr+8, 0x798C07C6); // sldi r12, r12, 32 | |||
1020 | writeInt32BE(Addr+12, 0x658C0000); // oris r12, r12, h(addr) | |||
1021 | writeInt32BE(Addr+16, 0x618C0000); // ori r12, r12, l(addr) | |||
1022 | if (AbiVariant == 2) { | |||
1023 | // PowerPC64 stub ELFv2 ABI: The address points to the function itself. | |||
1024 | // The address is already in r12 as required by the ABI. Branch to it. | |||
1025 | writeInt32BE(Addr+20, 0xF8410018); // std r2, 24(r1) | |||
1026 | writeInt32BE(Addr+24, 0x7D8903A6); // mtctr r12 | |||
1027 | writeInt32BE(Addr+28, 0x4E800420); // bctr | |||
1028 | } else { | |||
1029 | // PowerPC64 stub ELFv1 ABI: The address points to a function descriptor. | |||
1030 | // Load the function address on r11 and sets it to control register. Also | |||
1031 | // loads the function TOC in r2 and environment pointer to r11. | |||
1032 | writeInt32BE(Addr+20, 0xF8410028); // std r2, 40(r1) | |||
1033 | writeInt32BE(Addr+24, 0xE96C0000); // ld r11, 0(r12) | |||
1034 | writeInt32BE(Addr+28, 0xE84C0008); // ld r2, 0(r12) | |||
1035 | writeInt32BE(Addr+32, 0x7D6903A6); // mtctr r11 | |||
1036 | writeInt32BE(Addr+36, 0xE96C0010); // ld r11, 16(r2) | |||
1037 | writeInt32BE(Addr+40, 0x4E800420); // bctr | |||
1038 | } | |||
1039 | return Addr; | |||
1040 | } else if (Arch == Triple::systemz) { | |||
1041 | writeInt16BE(Addr, 0xC418); // lgrl %r1,.+8 | |||
1042 | writeInt16BE(Addr+2, 0x0000); | |||
1043 | writeInt16BE(Addr+4, 0x0004); | |||
1044 | writeInt16BE(Addr+6, 0x07F1); // brc 15,%r1 | |||
1045 | // 8-byte address stored at Addr + 8 | |||
1046 | return Addr; | |||
1047 | } else if (Arch == Triple::x86_64) { | |||
1048 | *Addr = 0xFF; // jmp | |||
1049 | *(Addr+1) = 0x25; // rip | |||
1050 | // 32-bit PC-relative address of the GOT entry will be stored at Addr+2 | |||
1051 | } else if (Arch == Triple::x86) { | |||
1052 | *Addr = 0xE9; // 32-bit pc-relative jump. | |||
1053 | } | |||
1054 | return Addr; | |||
1055 | } | |||
1056 | ||||
1057 | // Assign an address to a symbol name and resolve all the relocations | |||
1058 | // associated with it. | |||
1059 | void RuntimeDyldImpl::reassignSectionAddress(unsigned SectionID, | |||
1060 | uint64_t Addr) { | |||
1061 | // The address to use for relocation resolution is not | |||
1062 | // the address of the local section buffer. We must be doing | |||
1063 | // a remote execution environment of some sort. Relocations can't | |||
1064 | // be applied until all the sections have been moved. The client must | |||
1065 | // trigger this with a call to MCJIT::finalize() or | |||
1066 | // RuntimeDyld::resolveRelocations(). | |||
1067 | // | |||
1068 | // Addr is a uint64_t because we can't assume the pointer width | |||
1069 | // of the target is the same as that of the host. Just use a generic | |||
1070 | // "big enough" type. | |||
1071 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "Reassigning address for section " << SectionID << " (" << Sections[SectionID ].getName() << "): " << format("0x%016" "l" "x", Sections [SectionID].getLoadAddress()) << " -> " << format ("0x%016" "l" "x", Addr) << "\n"; } } while (false) | |||
1072 | dbgs() << "Reassigning address for section " << SectionID << " ("do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "Reassigning address for section " << SectionID << " (" << Sections[SectionID ].getName() << "): " << format("0x%016" "l" "x", Sections [SectionID].getLoadAddress()) << " -> " << format ("0x%016" "l" "x", Addr) << "\n"; } } while (false) | |||
1073 | << Sections[SectionID].getName() << "): "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "Reassigning address for section " << SectionID << " (" << Sections[SectionID ].getName() << "): " << format("0x%016" "l" "x", Sections [SectionID].getLoadAddress()) << " -> " << format ("0x%016" "l" "x", Addr) << "\n"; } } while (false) | |||
1074 | << format("0x%016" PRIx64, Sections[SectionID].getLoadAddress())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "Reassigning address for section " << SectionID << " (" << Sections[SectionID ].getName() << "): " << format("0x%016" "l" "x", Sections [SectionID].getLoadAddress()) << " -> " << format ("0x%016" "l" "x", Addr) << "\n"; } } while (false) | |||
1075 | << " -> " << format("0x%016" PRIx64, Addr) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "Reassigning address for section " << SectionID << " (" << Sections[SectionID ].getName() << "): " << format("0x%016" "l" "x", Sections [SectionID].getLoadAddress()) << " -> " << format ("0x%016" "l" "x", Addr) << "\n"; } } while (false); | |||
1076 | Sections[SectionID].setLoadAddress(Addr); | |||
1077 | } | |||
1078 | ||||
1079 | void RuntimeDyldImpl::resolveRelocationList(const RelocationList &Relocs, | |||
1080 | uint64_t Value) { | |||
1081 | for (unsigned i = 0, e = Relocs.size(); i != e; ++i) { | |||
1082 | const RelocationEntry &RE = Relocs[i]; | |||
1083 | // Ignore relocations for sections that were not loaded | |||
1084 | if (RE.SectionID != AbsoluteSymbolSection && | |||
1085 | Sections[RE.SectionID].getAddress() == nullptr) | |||
1086 | continue; | |||
1087 | resolveRelocation(RE, Value); | |||
1088 | } | |||
1089 | } | |||
1090 | ||||
1091 | void RuntimeDyldImpl::applyExternalSymbolRelocations( | |||
1092 | const StringMap<JITEvaluatedSymbol> ExternalSymbolMap) { | |||
1093 | for (auto &RelocKV : ExternalSymbolRelocations) { | |||
1094 | StringRef Name = RelocKV.first(); | |||
1095 | RelocationList &Relocs = RelocKV.second; | |||
1096 | if (Name.size() == 0) { | |||
1097 | // This is an absolute symbol, use an address of zero. | |||
1098 | LLVM_DEBUG(dbgs() << "Resolving absolute relocations."do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "Resolving absolute relocations." << "\n"; } } while (false) | |||
1099 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "Resolving absolute relocations." << "\n"; } } while (false); | |||
1100 | resolveRelocationList(Relocs, 0); | |||
1101 | } else { | |||
1102 | uint64_t Addr = 0; | |||
1103 | JITSymbolFlags Flags; | |||
1104 | RTDyldSymbolTable::const_iterator Loc = GlobalSymbolTable.find(Name); | |||
1105 | if (Loc == GlobalSymbolTable.end()) { | |||
1106 | auto RRI = ExternalSymbolMap.find(Name); | |||
1107 | assert(RRI != ExternalSymbolMap.end() && "No result for symbol")(static_cast <bool> (RRI != ExternalSymbolMap.end() && "No result for symbol") ? void (0) : __assert_fail ("RRI != ExternalSymbolMap.end() && \"No result for symbol\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp" , 1107, __extension__ __PRETTY_FUNCTION__)); | |||
1108 | Addr = RRI->second.getAddress(); | |||
1109 | Flags = RRI->second.getFlags(); | |||
1110 | } else { | |||
1111 | // We found the symbol in our global table. It was probably in a | |||
1112 | // Module that we loaded previously. | |||
1113 | const auto &SymInfo = Loc->second; | |||
1114 | Addr = getSectionLoadAddress(SymInfo.getSectionID()) + | |||
1115 | SymInfo.getOffset(); | |||
1116 | Flags = SymInfo.getFlags(); | |||
1117 | } | |||
1118 | ||||
1119 | // FIXME: Implement error handling that doesn't kill the host program! | |||
1120 | if (!Addr && !Resolver.allowsZeroSymbols()) | |||
1121 | report_fatal_error("Program used external function '" + Name + | |||
1122 | "' which could not be resolved!"); | |||
1123 | ||||
1124 | // If Resolver returned UINT64_MAX, the client wants to handle this symbol | |||
1125 | // manually and we shouldn't resolve its relocations. | |||
1126 | if (Addr != UINT64_MAX(18446744073709551615UL)) { | |||
1127 | ||||
1128 | // Tweak the address based on the symbol flags if necessary. | |||
1129 | // For example, this is used by RuntimeDyldMachOARM to toggle the low bit | |||
1130 | // if the target symbol is Thumb. | |||
1131 | Addr = modifyAddressBasedOnFlags(Addr, Flags); | |||
1132 | ||||
1133 | LLVM_DEBUG(dbgs() << "Resolving relocations Name: " << Name << "\t"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "Resolving relocations Name: " << Name << "\t" << format("0x%lx", Addr) << "\n" ; } } while (false) | |||
1134 | << format("0x%lx", Addr) << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("dyld")) { dbgs() << "Resolving relocations Name: " << Name << "\t" << format("0x%lx", Addr) << "\n" ; } } while (false); | |||
1135 | resolveRelocationList(Relocs, Addr); | |||
1136 | } | |||
1137 | } | |||
1138 | } | |||
1139 | ExternalSymbolRelocations.clear(); | |||
1140 | } | |||
1141 | ||||
1142 | Error RuntimeDyldImpl::resolveExternalSymbols() { | |||
1143 | StringMap<JITEvaluatedSymbol> ExternalSymbolMap; | |||
1144 | ||||
1145 | // Resolution can trigger emission of more symbols, so iterate until | |||
1146 | // we've resolved *everything*. | |||
1147 | { | |||
1148 | JITSymbolResolver::LookupSet ResolvedSymbols; | |||
1149 | ||||
1150 | while (true) { | |||
1151 | JITSymbolResolver::LookupSet NewSymbols; | |||
1152 | ||||
1153 | for (auto &RelocKV : ExternalSymbolRelocations) { | |||
1154 | StringRef Name = RelocKV.first(); | |||
1155 | if (!Name.empty() && !GlobalSymbolTable.count(Name) && | |||
1156 | !ResolvedSymbols.count(Name)) | |||
1157 | NewSymbols.insert(Name); | |||
1158 | } | |||
1159 | ||||
1160 | if (NewSymbols.empty()) | |||
1161 | break; | |||
1162 | ||||
1163 | #ifdef _MSC_VER | |||
1164 | using ExpectedLookupResult = | |||
1165 | MSVCPExpected<JITSymbolResolver::LookupResult>; | |||
1166 | #else | |||
1167 | using ExpectedLookupResult = Expected<JITSymbolResolver::LookupResult>; | |||
1168 | #endif | |||
1169 | ||||
1170 | auto NewSymbolsP = std::make_shared<std::promise<ExpectedLookupResult>>(); | |||
1171 | auto NewSymbolsF = NewSymbolsP->get_future(); | |||
1172 | Resolver.lookup(NewSymbols, | |||
1173 | [=](Expected<JITSymbolResolver::LookupResult> Result) { | |||
1174 | NewSymbolsP->set_value(std::move(Result)); | |||
1175 | }); | |||
1176 | ||||
1177 | auto NewResolverResults = NewSymbolsF.get(); | |||
1178 | ||||
1179 | if (!NewResolverResults) | |||
1180 | return NewResolverResults.takeError(); | |||
1181 | ||||
1182 | assert(NewResolverResults->size() == NewSymbols.size() &&(static_cast <bool> (NewResolverResults->size() == NewSymbols .size() && "Should have errored on unresolved symbols" ) ? void (0) : __assert_fail ("NewResolverResults->size() == NewSymbols.size() && \"Should have errored on unresolved symbols\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp" , 1183, __extension__ __PRETTY_FUNCTION__)) | |||
1183 | "Should have errored on unresolved symbols")(static_cast <bool> (NewResolverResults->size() == NewSymbols .size() && "Should have errored on unresolved symbols" ) ? void (0) : __assert_fail ("NewResolverResults->size() == NewSymbols.size() && \"Should have errored on unresolved symbols\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp" , 1183, __extension__ __PRETTY_FUNCTION__)); | |||
1184 | ||||
1185 | for (auto &RRKV : *NewResolverResults) { | |||
1186 | assert(!ResolvedSymbols.count(RRKV.first) && "Redundant resolution?")(static_cast <bool> (!ResolvedSymbols.count(RRKV.first) && "Redundant resolution?") ? void (0) : __assert_fail ("!ResolvedSymbols.count(RRKV.first) && \"Redundant resolution?\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp" , 1186, __extension__ __PRETTY_FUNCTION__)); | |||
1187 | ExternalSymbolMap.insert(RRKV); | |||
1188 | ResolvedSymbols.insert(RRKV.first); | |||
1189 | } | |||
1190 | } | |||
1191 | } | |||
1192 | ||||
1193 | applyExternalSymbolRelocations(ExternalSymbolMap); | |||
1194 | ||||
1195 | return Error::success(); | |||
1196 | } | |||
1197 | ||||
1198 | void RuntimeDyldImpl::finalizeAsync( | |||
1199 | std::unique_ptr<RuntimeDyldImpl> This, | |||
1200 | unique_function<void(object::OwningBinary<object::ObjectFile>, | |||
1201 | std::unique_ptr<RuntimeDyld::LoadedObjectInfo>, Error)> | |||
1202 | OnEmitted, | |||
1203 | object::OwningBinary<object::ObjectFile> O, | |||
1204 | std::unique_ptr<RuntimeDyld::LoadedObjectInfo> Info) { | |||
1205 | ||||
1206 | auto SharedThis = std::shared_ptr<RuntimeDyldImpl>(std::move(This)); | |||
1207 | auto PostResolveContinuation = | |||
1208 | [SharedThis, OnEmitted = std::move(OnEmitted), O = std::move(O), | |||
1209 | Info = std::move(Info)]( | |||
1210 | Expected<JITSymbolResolver::LookupResult> Result) mutable { | |||
1211 | if (!Result) { | |||
1212 | OnEmitted(std::move(O), std::move(Info), Result.takeError()); | |||
1213 | return; | |||
1214 | } | |||
1215 | ||||
1216 | /// Copy the result into a StringMap, where the keys are held by value. | |||
1217 | StringMap<JITEvaluatedSymbol> Resolved; | |||
1218 | for (auto &KV : *Result) | |||
1219 | Resolved[KV.first] = KV.second; | |||
1220 | ||||
1221 | SharedThis->applyExternalSymbolRelocations(Resolved); | |||
1222 | SharedThis->resolveLocalRelocations(); | |||
1223 | SharedThis->registerEHFrames(); | |||
1224 | std::string ErrMsg; | |||
1225 | if (SharedThis->MemMgr.finalizeMemory(&ErrMsg)) | |||
1226 | OnEmitted(std::move(O), std::move(Info), | |||
1227 | make_error<StringError>(std::move(ErrMsg), | |||
1228 | inconvertibleErrorCode())); | |||
1229 | else | |||
1230 | OnEmitted(std::move(O), std::move(Info), Error::success()); | |||
1231 | }; | |||
1232 | ||||
1233 | JITSymbolResolver::LookupSet Symbols; | |||
1234 | ||||
1235 | for (auto &RelocKV : SharedThis->ExternalSymbolRelocations) { | |||
1236 | StringRef Name = RelocKV.first(); | |||
1237 | if (Name.empty()) // Skip absolute symbol relocations. | |||
1238 | continue; | |||
1239 | assert(!SharedThis->GlobalSymbolTable.count(Name) &&(static_cast <bool> (!SharedThis->GlobalSymbolTable. count(Name) && "Name already processed. RuntimeDyld instances can not be re-used " "when finalizing with finalizeAsync.") ? void (0) : __assert_fail ("!SharedThis->GlobalSymbolTable.count(Name) && \"Name already processed. RuntimeDyld instances can not be re-used \" \"when finalizing with finalizeAsync.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp" , 1241, __extension__ __PRETTY_FUNCTION__)) | |||
1240 | "Name already processed. RuntimeDyld instances can not be re-used "(static_cast <bool> (!SharedThis->GlobalSymbolTable. count(Name) && "Name already processed. RuntimeDyld instances can not be re-used " "when finalizing with finalizeAsync.") ? void (0) : __assert_fail ("!SharedThis->GlobalSymbolTable.count(Name) && \"Name already processed. RuntimeDyld instances can not be re-used \" \"when finalizing with finalizeAsync.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp" , 1241, __extension__ __PRETTY_FUNCTION__)) | |||
1241 | "when finalizing with finalizeAsync.")(static_cast <bool> (!SharedThis->GlobalSymbolTable. count(Name) && "Name already processed. RuntimeDyld instances can not be re-used " "when finalizing with finalizeAsync.") ? void (0) : __assert_fail ("!SharedThis->GlobalSymbolTable.count(Name) && \"Name already processed. RuntimeDyld instances can not be re-used \" \"when finalizing with finalizeAsync.\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp" , 1241, __extension__ __PRETTY_FUNCTION__)); | |||
1242 | Symbols.insert(Name); | |||
1243 | } | |||
1244 | ||||
1245 | if (!Symbols.empty()) { | |||
1246 | SharedThis->Resolver.lookup(Symbols, std::move(PostResolveContinuation)); | |||
1247 | } else | |||
1248 | PostResolveContinuation(std::map<StringRef, JITEvaluatedSymbol>()); | |||
1249 | } | |||
1250 | ||||
1251 | //===----------------------------------------------------------------------===// | |||
1252 | // RuntimeDyld class implementation | |||
1253 | ||||
1254 | uint64_t RuntimeDyld::LoadedObjectInfo::getSectionLoadAddress( | |||
1255 | const object::SectionRef &Sec) const { | |||
1256 | ||||
1257 | auto I = ObjSecToIDMap.find(Sec); | |||
1258 | if (I != ObjSecToIDMap.end()) | |||
1259 | return RTDyld.Sections[I->second].getLoadAddress(); | |||
1260 | ||||
1261 | return 0; | |||
1262 | } | |||
1263 | ||||
1264 | void RuntimeDyld::MemoryManager::anchor() {} | |||
1265 | void JITSymbolResolver::anchor() {} | |||
1266 | void LegacyJITSymbolResolver::anchor() {} | |||
1267 | ||||
1268 | RuntimeDyld::RuntimeDyld(RuntimeDyld::MemoryManager &MemMgr, | |||
1269 | JITSymbolResolver &Resolver) | |||
1270 | : MemMgr(MemMgr), Resolver(Resolver) { | |||
1271 | // FIXME: There's a potential issue lurking here if a single instance of | |||
1272 | // RuntimeDyld is used to load multiple objects. The current implementation | |||
1273 | // associates a single memory manager with a RuntimeDyld instance. Even | |||
1274 | // though the public class spawns a new 'impl' instance for each load, | |||
1275 | // they share a single memory manager. This can become a problem when page | |||
1276 | // permissions are applied. | |||
1277 | Dyld = nullptr; | |||
1278 | ProcessAllSections = false; | |||
1279 | } | |||
1280 | ||||
1281 | RuntimeDyld::~RuntimeDyld() {} | |||
1282 | ||||
1283 | static std::unique_ptr<RuntimeDyldCOFF> | |||
1284 | createRuntimeDyldCOFF( | |||
1285 | Triple::ArchType Arch, RuntimeDyld::MemoryManager &MM, | |||
1286 | JITSymbolResolver &Resolver, bool ProcessAllSections, | |||
1287 | RuntimeDyld::NotifyStubEmittedFunction NotifyStubEmitted) { | |||
1288 | std::unique_ptr<RuntimeDyldCOFF> Dyld = | |||
1289 | RuntimeDyldCOFF::create(Arch, MM, Resolver); | |||
1290 | Dyld->setProcessAllSections(ProcessAllSections); | |||
1291 | Dyld->setNotifyStubEmitted(std::move(NotifyStubEmitted)); | |||
1292 | return Dyld; | |||
1293 | } | |||
1294 | ||||
1295 | static std::unique_ptr<RuntimeDyldELF> | |||
1296 | createRuntimeDyldELF(Triple::ArchType Arch, RuntimeDyld::MemoryManager &MM, | |||
1297 | JITSymbolResolver &Resolver, bool ProcessAllSections, | |||
1298 | RuntimeDyld::NotifyStubEmittedFunction NotifyStubEmitted) { | |||
1299 | std::unique_ptr<RuntimeDyldELF> Dyld = | |||
1300 | RuntimeDyldELF::create(Arch, MM, Resolver); | |||
1301 | Dyld->setProcessAllSections(ProcessAllSections); | |||
1302 | Dyld->setNotifyStubEmitted(std::move(NotifyStubEmitted)); | |||
1303 | return Dyld; | |||
1304 | } | |||
1305 | ||||
1306 | static std::unique_ptr<RuntimeDyldMachO> | |||
1307 | createRuntimeDyldMachO( | |||
1308 | Triple::ArchType Arch, RuntimeDyld::MemoryManager &MM, | |||
1309 | JITSymbolResolver &Resolver, | |||
1310 | bool ProcessAllSections, | |||
1311 | RuntimeDyld::NotifyStubEmittedFunction NotifyStubEmitted) { | |||
1312 | std::unique_ptr<RuntimeDyldMachO> Dyld = | |||
1313 | RuntimeDyldMachO::create(Arch, MM, Resolver); | |||
1314 | Dyld->setProcessAllSections(ProcessAllSections); | |||
1315 | Dyld->setNotifyStubEmitted(std::move(NotifyStubEmitted)); | |||
1316 | return Dyld; | |||
1317 | } | |||
1318 | ||||
1319 | std::unique_ptr<RuntimeDyld::LoadedObjectInfo> | |||
1320 | RuntimeDyld::loadObject(const ObjectFile &Obj) { | |||
1321 | if (!Dyld) { | |||
1322 | if (Obj.isELF()) | |||
1323 | Dyld = | |||
1324 | createRuntimeDyldELF(static_cast<Triple::ArchType>(Obj.getArch()), | |||
1325 | MemMgr, Resolver, ProcessAllSections, | |||
1326 | std::move(NotifyStubEmitted)); | |||
1327 | else if (Obj.isMachO()) | |||
1328 | Dyld = createRuntimeDyldMachO( | |||
1329 | static_cast<Triple::ArchType>(Obj.getArch()), MemMgr, Resolver, | |||
1330 | ProcessAllSections, std::move(NotifyStubEmitted)); | |||
1331 | else if (Obj.isCOFF()) | |||
1332 | Dyld = createRuntimeDyldCOFF( | |||
1333 | static_cast<Triple::ArchType>(Obj.getArch()), MemMgr, Resolver, | |||
1334 | ProcessAllSections, std::move(NotifyStubEmitted)); | |||
1335 | else | |||
1336 | report_fatal_error("Incompatible object format!"); | |||
1337 | } | |||
1338 | ||||
1339 | if (!Dyld->isCompatibleFile(Obj)) | |||
1340 | report_fatal_error("Incompatible object format!"); | |||
1341 | ||||
1342 | auto LoadedObjInfo = Dyld->loadObject(Obj); | |||
1343 | MemMgr.notifyObjectLoaded(*this, Obj); | |||
1344 | return LoadedObjInfo; | |||
1345 | } | |||
1346 | ||||
1347 | void *RuntimeDyld::getSymbolLocalAddress(StringRef Name) const { | |||
1348 | if (!Dyld) | |||
1349 | return nullptr; | |||
1350 | return Dyld->getSymbolLocalAddress(Name); | |||
1351 | } | |||
1352 | ||||
1353 | unsigned RuntimeDyld::getSymbolSectionID(StringRef Name) const { | |||
1354 | assert(Dyld && "No RuntimeDyld instance attached")(static_cast <bool> (Dyld && "No RuntimeDyld instance attached" ) ? void (0) : __assert_fail ("Dyld && \"No RuntimeDyld instance attached\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp" , 1354, __extension__ __PRETTY_FUNCTION__)); | |||
1355 | return Dyld->getSymbolSectionID(Name); | |||
1356 | } | |||
1357 | ||||
1358 | JITEvaluatedSymbol RuntimeDyld::getSymbol(StringRef Name) const { | |||
1359 | if (!Dyld) | |||
1360 | return nullptr; | |||
1361 | return Dyld->getSymbol(Name); | |||
1362 | } | |||
1363 | ||||
1364 | std::map<StringRef, JITEvaluatedSymbol> RuntimeDyld::getSymbolTable() const { | |||
1365 | if (!Dyld) | |||
1366 | return std::map<StringRef, JITEvaluatedSymbol>(); | |||
1367 | return Dyld->getSymbolTable(); | |||
1368 | } | |||
1369 | ||||
1370 | void RuntimeDyld::resolveRelocations() { Dyld->resolveRelocations(); } | |||
1371 | ||||
1372 | void RuntimeDyld::reassignSectionAddress(unsigned SectionID, uint64_t Addr) { | |||
1373 | Dyld->reassignSectionAddress(SectionID, Addr); | |||
1374 | } | |||
1375 | ||||
1376 | void RuntimeDyld::mapSectionAddress(const void *LocalAddress, | |||
1377 | uint64_t TargetAddress) { | |||
1378 | Dyld->mapSectionAddress(LocalAddress, TargetAddress); | |||
1379 | } | |||
1380 | ||||
1381 | bool RuntimeDyld::hasError() { return Dyld->hasError(); } | |||
1382 | ||||
1383 | StringRef RuntimeDyld::getErrorString() { return Dyld->getErrorString(); } | |||
1384 | ||||
1385 | void RuntimeDyld::finalizeWithMemoryManagerLocking() { | |||
1386 | bool MemoryFinalizationLocked = MemMgr.FinalizationLocked; | |||
1387 | MemMgr.FinalizationLocked = true; | |||
1388 | resolveRelocations(); | |||
1389 | registerEHFrames(); | |||
1390 | if (!MemoryFinalizationLocked) { | |||
1391 | MemMgr.finalizeMemory(); | |||
1392 | MemMgr.FinalizationLocked = false; | |||
1393 | } | |||
1394 | } | |||
1395 | ||||
1396 | StringRef RuntimeDyld::getSectionContent(unsigned SectionID) const { | |||
1397 | assert(Dyld && "No Dyld instance attached")(static_cast <bool> (Dyld && "No Dyld instance attached" ) ? void (0) : __assert_fail ("Dyld && \"No Dyld instance attached\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp" , 1397, __extension__ __PRETTY_FUNCTION__)); | |||
1398 | return Dyld->getSectionContent(SectionID); | |||
1399 | } | |||
1400 | ||||
1401 | uint64_t RuntimeDyld::getSectionLoadAddress(unsigned SectionID) const { | |||
1402 | assert(Dyld && "No Dyld instance attached")(static_cast <bool> (Dyld && "No Dyld instance attached" ) ? void (0) : __assert_fail ("Dyld && \"No Dyld instance attached\"" , "/build/llvm-toolchain-snapshot-13~++20210726100616+dead50d4427c/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp" , 1402, __extension__ __PRETTY_FUNCTION__)); | |||
1403 | return Dyld->getSectionLoadAddress(SectionID); | |||
1404 | } | |||
1405 | ||||
1406 | void RuntimeDyld::registerEHFrames() { | |||
1407 | if (Dyld) | |||
1408 | Dyld->registerEHFrames(); | |||
1409 | } | |||
1410 | ||||
1411 | void RuntimeDyld::deregisterEHFrames() { | |||
1412 | if (Dyld) | |||
1413 | Dyld->deregisterEHFrames(); | |||
1414 | } | |||
1415 | // FIXME: Kill this with fire once we have a new JIT linker: this is only here | |||
1416 | // so that we can re-use RuntimeDyld's implementation without twisting the | |||
1417 | // interface any further for ORC's purposes. | |||
1418 | void jitLinkForORC( | |||
1419 | object::OwningBinary<object::ObjectFile> O, | |||
1420 | RuntimeDyld::MemoryManager &MemMgr, JITSymbolResolver &Resolver, | |||
1421 | bool ProcessAllSections, | |||
1422 | unique_function<Error(const object::ObjectFile &Obj, | |||
1423 | RuntimeDyld::LoadedObjectInfo &LoadedObj, | |||
1424 | std::map<StringRef, JITEvaluatedSymbol>)> | |||
1425 | OnLoaded, | |||
1426 | unique_function<void(object::OwningBinary<object::ObjectFile>, | |||
1427 | std::unique_ptr<RuntimeDyld::LoadedObjectInfo>, Error)> | |||
1428 | OnEmitted) { | |||
1429 | ||||
1430 | RuntimeDyld RTDyld(MemMgr, Resolver); | |||
1431 | RTDyld.setProcessAllSections(ProcessAllSections); | |||
1432 | ||||
1433 | auto Info = RTDyld.loadObject(*O.getBinary()); | |||
1434 | ||||
1435 | if (RTDyld.hasError()) { | |||
| ||||
1436 | OnEmitted(std::move(O), std::move(Info), | |||
1437 | make_error<StringError>(RTDyld.getErrorString(), | |||
1438 | inconvertibleErrorCode())); | |||
1439 | return; | |||
1440 | } | |||
1441 | ||||
1442 | if (auto Err = OnLoaded(*O.getBinary(), *Info, RTDyld.getSymbolTable())) | |||
1443 | OnEmitted(std::move(O), std::move(Info), std::move(Err)); | |||
1444 | ||||
1445 | RuntimeDyldImpl::finalizeAsync(std::move(RTDyld.Dyld), std::move(OnEmitted), | |||
1446 | std::move(O), std::move(Info)); | |||
1447 | } | |||
1448 | ||||
1449 | } // end namespace llvm |
1 | //===- FunctionExtras.h - Function type erasure utilities -------*- C++ -*-===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | /// \file | |||
9 | /// This file provides a collection of function (or more generally, callable) | |||
10 | /// type erasure utilities supplementing those provided by the standard library | |||
11 | /// in `<function>`. | |||
12 | /// | |||
13 | /// It provides `unique_function`, which works like `std::function` but supports | |||
14 | /// move-only callable objects and const-qualification. | |||
15 | /// | |||
16 | /// Future plans: | |||
17 | /// - Add a `function` that provides ref-qualified support, which doesn't work | |||
18 | /// with `std::function`. | |||
19 | /// - Provide support for specifying multiple signatures to type erase callable | |||
20 | /// objects with an overload set, such as those produced by generic lambdas. | |||
21 | /// - Expand to include a copyable utility that directly replaces std::function | |||
22 | /// but brings the above improvements. | |||
23 | /// | |||
24 | /// Note that LLVM's utilities are greatly simplified by not supporting | |||
25 | /// allocators. | |||
26 | /// | |||
27 | /// If the standard library ever begins to provide comparable facilities we can | |||
28 | /// consider switching to those. | |||
29 | /// | |||
30 | //===----------------------------------------------------------------------===// | |||
31 | ||||
32 | #ifndef LLVM_ADT_FUNCTIONEXTRAS_H | |||
33 | #define LLVM_ADT_FUNCTIONEXTRAS_H | |||
34 | ||||
35 | #include "llvm/ADT/PointerIntPair.h" | |||
36 | #include "llvm/ADT/PointerUnion.h" | |||
37 | #include "llvm/ADT/STLForwardCompat.h" | |||
38 | #include "llvm/Support/MemAlloc.h" | |||
39 | #include "llvm/Support/type_traits.h" | |||
40 | #include <memory> | |||
41 | #include <type_traits> | |||
42 | ||||
43 | namespace llvm { | |||
44 | ||||
45 | /// unique_function is a type-erasing functor similar to std::function. | |||
46 | /// | |||
47 | /// It can hold move-only function objects, like lambdas capturing unique_ptrs. | |||
48 | /// Accordingly, it is movable but not copyable. | |||
49 | /// | |||
50 | /// It supports const-qualification: | |||
51 | /// - unique_function<int() const> has a const operator(). | |||
52 | /// It can only hold functions which themselves have a const operator(). | |||
53 | /// - unique_function<int()> has a non-const operator(). | |||
54 | /// It can hold functions with a non-const operator(), like mutable lambdas. | |||
55 | template <typename FunctionT> class unique_function; | |||
56 | ||||
57 | namespace detail { | |||
58 | ||||
59 | template <typename T> | |||
60 | using EnableIfTrivial = | |||
61 | std::enable_if_t<llvm::is_trivially_move_constructible<T>::value && | |||
62 | std::is_trivially_destructible<T>::value>; | |||
63 | template <typename CallableT, typename ThisT> | |||
64 | using EnableUnlessSameType = | |||
65 | std::enable_if_t<!std::is_same<remove_cvref_t<CallableT>, ThisT>::value>; | |||
66 | template <typename CallableT, typename Ret, typename... Params> | |||
67 | using EnableIfCallable = | |||
68 | std::enable_if_t<std::is_void<Ret>::value || | |||
69 | std::is_convertible<decltype(std::declval<CallableT>()( | |||
70 | std::declval<Params>()...)), | |||
71 | Ret>::value>; | |||
72 | ||||
73 | template <typename ReturnT, typename... ParamTs> class UniqueFunctionBase { | |||
74 | protected: | |||
75 | static constexpr size_t InlineStorageSize = sizeof(void *) * 3; | |||
76 | ||||
77 | template <typename T, class = void> | |||
78 | struct IsSizeLessThanThresholdT : std::false_type {}; | |||
79 | ||||
80 | template <typename T> | |||
81 | struct IsSizeLessThanThresholdT< | |||
82 | T, std::enable_if_t<sizeof(T) <= 2 * sizeof(void *)>> : std::true_type {}; | |||
83 | ||||
84 | // Provide a type function to map parameters that won't observe extra copies | |||
85 | // or moves and which are small enough to likely pass in register to values | |||
86 | // and all other types to l-value reference types. We use this to compute the | |||
87 | // types used in our erased call utility to minimize copies and moves unless | |||
88 | // doing so would force things unnecessarily into memory. | |||
89 | // | |||
90 | // The heuristic used is related to common ABI register passing conventions. | |||
91 | // It doesn't have to be exact though, and in one way it is more strict | |||
92 | // because we want to still be able to observe either moves *or* copies. | |||
93 | template <typename T> struct AdjustedParamTBase { | |||
94 | static_assert(!std::is_reference<T>::value, | |||
95 | "references should be handled by template specialization"); | |||
96 | using type = typename std::conditional< | |||
97 | llvm::is_trivially_copy_constructible<T>::value && | |||
98 | llvm::is_trivially_move_constructible<T>::value && | |||
99 | IsSizeLessThanThresholdT<T>::value, | |||
100 | T, T &>::type; | |||
101 | }; | |||
102 | ||||
103 | // This specialization ensures that 'AdjustedParam<V<T>&>' or | |||
104 | // 'AdjustedParam<V<T>&&>' does not trigger a compile-time error when 'T' is | |||
105 | // an incomplete type and V a templated type. | |||
106 | template <typename T> struct AdjustedParamTBase<T &> { using type = T &; }; | |||
107 | template <typename T> struct AdjustedParamTBase<T &&> { using type = T &; }; | |||
108 | ||||
109 | template <typename T> | |||
110 | using AdjustedParamT = typename AdjustedParamTBase<T>::type; | |||
111 | ||||
112 | // The type of the erased function pointer we use as a callback to dispatch to | |||
113 | // the stored callable when it is trivial to move and destroy. | |||
114 | using CallPtrT = ReturnT (*)(void *CallableAddr, | |||
115 | AdjustedParamT<ParamTs>... Params); | |||
116 | using MovePtrT = void (*)(void *LHSCallableAddr, void *RHSCallableAddr); | |||
117 | using DestroyPtrT = void (*)(void *CallableAddr); | |||
118 | ||||
119 | /// A struct to hold a single trivial callback with sufficient alignment for | |||
120 | /// our bitpacking. | |||
121 | struct alignas(8) TrivialCallback { | |||
122 | CallPtrT CallPtr; | |||
123 | }; | |||
124 | ||||
125 | /// A struct we use to aggregate three callbacks when we need full set of | |||
126 | /// operations. | |||
127 | struct alignas(8) NonTrivialCallbacks { | |||
128 | CallPtrT CallPtr; | |||
129 | MovePtrT MovePtr; | |||
130 | DestroyPtrT DestroyPtr; | |||
131 | }; | |||
132 | ||||
133 | // Create a pointer union between either a pointer to a static trivial call | |||
134 | // pointer in a struct or a pointer to a static struct of the call, move, and | |||
135 | // destroy pointers. | |||
136 | using CallbackPointerUnionT = | |||
137 | PointerUnion<TrivialCallback *, NonTrivialCallbacks *>; | |||
138 | ||||
139 | // The main storage buffer. This will either have a pointer to out-of-line | |||
140 | // storage or an inline buffer storing the callable. | |||
141 | union StorageUnionT { | |||
142 | // For out-of-line storage we keep a pointer to the underlying storage and | |||
143 | // the size. This is enough to deallocate the memory. | |||
144 | struct OutOfLineStorageT { | |||
145 | void *StoragePtr; | |||
146 | size_t Size; | |||
147 | size_t Alignment; | |||
148 | } OutOfLineStorage; | |||
149 | static_assert( | |||
150 | sizeof(OutOfLineStorageT) <= InlineStorageSize, | |||
151 | "Should always use all of the out-of-line storage for inline storage!"); | |||
152 | ||||
153 | // For in-line storage, we just provide an aligned character buffer. We | |||
154 | // provide three pointers worth of storage here. | |||
155 | // This is mutable as an inlined `const unique_function<void() const>` may | |||
156 | // still modify its own mutable members. | |||
157 | mutable | |||
158 | typename std::aligned_storage<InlineStorageSize, alignof(void *)>::type | |||
159 | InlineStorage; | |||
160 | } StorageUnion; | |||
161 | ||||
162 | // A compressed pointer to either our dispatching callback or our table of | |||
163 | // dispatching callbacks and the flag for whether the callable itself is | |||
164 | // stored inline or not. | |||
165 | PointerIntPair<CallbackPointerUnionT, 1, bool> CallbackAndInlineFlag; | |||
166 | ||||
167 | bool isInlineStorage() const { return CallbackAndInlineFlag.getInt(); } | |||
168 | ||||
169 | bool isTrivialCallback() const { | |||
170 | return CallbackAndInlineFlag.getPointer().template is<TrivialCallback *>(); | |||
171 | } | |||
172 | ||||
173 | CallPtrT getTrivialCallback() const { | |||
174 | return CallbackAndInlineFlag.getPointer().template get<TrivialCallback *>()->CallPtr; | |||
175 | } | |||
176 | ||||
177 | NonTrivialCallbacks *getNonTrivialCallbacks() const { | |||
178 | return CallbackAndInlineFlag.getPointer() | |||
179 | .template get<NonTrivialCallbacks *>(); | |||
180 | } | |||
181 | ||||
182 | CallPtrT getCallPtr() const { | |||
183 | return isTrivialCallback() ? getTrivialCallback() | |||
184 | : getNonTrivialCallbacks()->CallPtr; | |||
185 | } | |||
186 | ||||
187 | // These three functions are only const in the narrow sense. They return | |||
188 | // mutable pointers to function state. | |||
189 | // This allows unique_function<T const>::operator() to be const, even if the | |||
190 | // underlying functor may be internally mutable. | |||
191 | // | |||
192 | // const callers must ensure they're only used in const-correct ways. | |||
193 | void *getCalleePtr() const { | |||
194 | return isInlineStorage() ? getInlineStorage() : getOutOfLineStorage(); | |||
195 | } | |||
196 | void *getInlineStorage() const { return &StorageUnion.InlineStorage; } | |||
197 | void *getOutOfLineStorage() const { | |||
198 | return StorageUnion.OutOfLineStorage.StoragePtr; | |||
| ||||
199 | } | |||
200 | ||||
201 | size_t getOutOfLineStorageSize() const { | |||
202 | return StorageUnion.OutOfLineStorage.Size; | |||
203 | } | |||
204 | size_t getOutOfLineStorageAlignment() const { | |||
205 | return StorageUnion.OutOfLineStorage.Alignment; | |||
206 | } | |||
207 | ||||
208 | void setOutOfLineStorage(void *Ptr, size_t Size, size_t Alignment) { | |||
209 | StorageUnion.OutOfLineStorage = {Ptr, Size, Alignment}; | |||
210 | } | |||
211 | ||||
212 | template <typename CalledAsT> | |||
213 | static ReturnT CallImpl(void *CallableAddr, | |||
214 | AdjustedParamT<ParamTs>... Params) { | |||
215 | auto &Func = *reinterpret_cast<CalledAsT *>(CallableAddr); | |||
216 | return Func(std::forward<ParamTs>(Params)...); | |||
217 | } | |||
218 | ||||
219 | template <typename CallableT> | |||
220 | static void MoveImpl(void *LHSCallableAddr, void *RHSCallableAddr) noexcept { | |||
221 | new (LHSCallableAddr) | |||
222 | CallableT(std::move(*reinterpret_cast<CallableT *>(RHSCallableAddr))); | |||
223 | } | |||
224 | ||||
225 | template <typename CallableT> | |||
226 | static void DestroyImpl(void *CallableAddr) noexcept { | |||
227 | reinterpret_cast<CallableT *>(CallableAddr)->~CallableT(); | |||
228 | } | |||
229 | ||||
230 | // The pointers to call/move/destroy functions are determined for each | |||
231 | // callable type (and called-as type, which determines the overload chosen). | |||
232 | // (definitions are out-of-line). | |||
233 | ||||
234 | // By default, we need an object that contains all the different | |||
235 | // type erased behaviors needed. Create a static instance of the struct type | |||
236 | // here and each instance will contain a pointer to it. | |||
237 | // Wrap in a struct to avoid https://gcc.gnu.org/PR71954 | |||
238 | template <typename CallableT, typename CalledAs, typename Enable = void> | |||
239 | struct CallbacksHolder { | |||
240 | static NonTrivialCallbacks Callbacks; | |||
241 | }; | |||
242 | // See if we can create a trivial callback. We need the callable to be | |||
243 | // trivially moved and trivially destroyed so that we don't have to store | |||
244 | // type erased callbacks for those operations. | |||
245 | template <typename CallableT, typename CalledAs> | |||
246 | struct CallbacksHolder<CallableT, CalledAs, EnableIfTrivial<CallableT>> { | |||
247 | static TrivialCallback Callbacks; | |||
248 | }; | |||
249 | ||||
250 | // A simple tag type so the call-as type to be passed to the constructor. | |||
251 | template <typename T> struct CalledAs {}; | |||
252 | ||||
253 | // Essentially the "main" unique_function constructor, but subclasses | |||
254 | // provide the qualified type to be used for the call. | |||
255 | // (We always store a T, even if the call will use a pointer to const T). | |||
256 | template <typename CallableT, typename CalledAsT> | |||
257 | UniqueFunctionBase(CallableT Callable, CalledAs<CalledAsT>) { | |||
258 | bool IsInlineStorage = true; | |||
259 | void *CallableAddr = getInlineStorage(); | |||
260 | if (sizeof(CallableT) > InlineStorageSize || | |||
261 | alignof(CallableT) > alignof(decltype(StorageUnion.InlineStorage))) { | |||
262 | IsInlineStorage = false; | |||
263 | // Allocate out-of-line storage. FIXME: Use an explicit alignment | |||
264 | // parameter in C++17 mode. | |||
265 | auto Size = sizeof(CallableT); | |||
266 | auto Alignment = alignof(CallableT); | |||
267 | CallableAddr = allocate_buffer(Size, Alignment); | |||
268 | setOutOfLineStorage(CallableAddr, Size, Alignment); | |||
269 | } | |||
270 | ||||
271 | // Now move into the storage. | |||
272 | new (CallableAddr) CallableT(std::move(Callable)); | |||
273 | CallbackAndInlineFlag.setPointerAndInt( | |||
274 | &CallbacksHolder<CallableT, CalledAsT>::Callbacks, IsInlineStorage); | |||
275 | } | |||
276 | ||||
277 | ~UniqueFunctionBase() { | |||
278 | if (!CallbackAndInlineFlag.getPointer()) | |||
279 | return; | |||
280 | ||||
281 | // Cache this value so we don't re-check it after type-erased operations. | |||
282 | bool IsInlineStorage = isInlineStorage(); | |||
283 | ||||
284 | if (!isTrivialCallback()) | |||
285 | getNonTrivialCallbacks()->DestroyPtr( | |||
286 | IsInlineStorage ? getInlineStorage() : getOutOfLineStorage()); | |||
287 | ||||
288 | if (!IsInlineStorage) | |||
289 | deallocate_buffer(getOutOfLineStorage(), getOutOfLineStorageSize(), | |||
290 | getOutOfLineStorageAlignment()); | |||
291 | } | |||
292 | ||||
293 | UniqueFunctionBase(UniqueFunctionBase &&RHS) noexcept { | |||
294 | // Copy the callback and inline flag. | |||
295 | CallbackAndInlineFlag = RHS.CallbackAndInlineFlag; | |||
296 | ||||
297 | // If the RHS is empty, just copying the above is sufficient. | |||
298 | if (!RHS) | |||
299 | return; | |||
300 | ||||
301 | if (!isInlineStorage()) { | |||
302 | // The out-of-line case is easiest to move. | |||
303 | StorageUnion.OutOfLineStorage = RHS.StorageUnion.OutOfLineStorage; | |||
304 | } else if (isTrivialCallback()) { | |||
305 | // Move is trivial, just memcpy the bytes across. | |||
306 | memcpy(getInlineStorage(), RHS.getInlineStorage(), InlineStorageSize); | |||
307 | } else { | |||
308 | // Non-trivial move, so dispatch to a type-erased implementation. | |||
309 | getNonTrivialCallbacks()->MovePtr(getInlineStorage(), | |||
310 | RHS.getInlineStorage()); | |||
311 | } | |||
312 | ||||
313 | // Clear the old callback and inline flag to get back to as-if-null. | |||
314 | RHS.CallbackAndInlineFlag = {}; | |||
315 | ||||
316 | #ifndef NDEBUG | |||
317 | // In debug builds, we also scribble across the rest of the storage. | |||
318 | memset(RHS.getInlineStorage(), 0xAD, InlineStorageSize); | |||
319 | #endif | |||
320 | } | |||
321 | ||||
322 | UniqueFunctionBase &operator=(UniqueFunctionBase &&RHS) noexcept { | |||
323 | if (this == &RHS) | |||
324 | return *this; | |||
325 | ||||
326 | // Because we don't try to provide any exception safety guarantees we can | |||
327 | // implement move assignment very simply by first destroying the current | |||
328 | // object and then move-constructing over top of it. | |||
329 | this->~UniqueFunctionBase(); | |||
330 | new (this) UniqueFunctionBase(std::move(RHS)); | |||
331 | return *this; | |||
332 | } | |||
333 | ||||
334 | UniqueFunctionBase() = default; | |||
335 | ||||
336 | public: | |||
337 | explicit operator bool() const { | |||
338 | return (bool)CallbackAndInlineFlag.getPointer(); | |||
339 | } | |||
340 | }; | |||
341 | ||||
342 | template <typename R, typename... P> | |||
343 | template <typename CallableT, typename CalledAsT, typename Enable> | |||
344 | typename UniqueFunctionBase<R, P...>::NonTrivialCallbacks UniqueFunctionBase< | |||
345 | R, P...>::CallbacksHolder<CallableT, CalledAsT, Enable>::Callbacks = { | |||
346 | &CallImpl<CalledAsT>, &MoveImpl<CallableT>, &DestroyImpl<CallableT>}; | |||
347 | ||||
348 | template <typename R, typename... P> | |||
349 | template <typename CallableT, typename CalledAsT> | |||
350 | typename UniqueFunctionBase<R, P...>::TrivialCallback | |||
351 | UniqueFunctionBase<R, P...>::CallbacksHolder< | |||
352 | CallableT, CalledAsT, EnableIfTrivial<CallableT>>::Callbacks{ | |||
353 | &CallImpl<CalledAsT>}; | |||
354 | ||||
355 | } // namespace detail | |||
356 | ||||
357 | template <typename R, typename... P> | |||
358 | class unique_function<R(P...)> : public detail::UniqueFunctionBase<R, P...> { | |||
359 | using Base = detail::UniqueFunctionBase<R, P...>; | |||
360 | ||||
361 | public: | |||
362 | unique_function() = default; | |||
363 | unique_function(std::nullptr_t) {} | |||
364 | unique_function(unique_function &&) = default; | |||
365 | unique_function(const unique_function &) = delete; | |||
366 | unique_function &operator=(unique_function &&) = default; | |||
367 | unique_function &operator=(const unique_function &) = delete; | |||
368 | ||||
369 | template <typename CallableT> | |||
370 | unique_function( | |||
371 | CallableT Callable, | |||
372 | detail::EnableUnlessSameType<CallableT, unique_function> * = nullptr, | |||
373 | detail::EnableIfCallable<CallableT, R, P...> * = nullptr) | |||
374 | : Base(std::forward<CallableT>(Callable), | |||
375 | typename Base::template CalledAs<CallableT>{}) {} | |||
376 | ||||
377 | R operator()(P... Params) { | |||
378 | return this->getCallPtr()(this->getCalleePtr(), Params...); | |||
379 | } | |||
380 | }; | |||
381 | ||||
382 | template <typename R, typename... P> | |||
383 | class unique_function<R(P...) const> | |||
384 | : public detail::UniqueFunctionBase<R, P...> { | |||
385 | using Base = detail::UniqueFunctionBase<R, P...>; | |||
386 | ||||
387 | public: | |||
388 | unique_function() = default; | |||
389 | unique_function(std::nullptr_t) {} | |||
390 | unique_function(unique_function &&) = default; | |||
391 | unique_function(const unique_function &) = delete; | |||
392 | unique_function &operator=(unique_function &&) = default; | |||
393 | unique_function &operator=(const unique_function &) = delete; | |||
394 | ||||
395 | template <typename CallableT> | |||
396 | unique_function( | |||
397 | CallableT Callable, | |||
398 | detail::EnableUnlessSameType<CallableT, unique_function> * = nullptr, | |||
399 | detail::EnableIfCallable<const CallableT, R, P...> * = nullptr) | |||
400 | : Base(std::forward<CallableT>(Callable), | |||
401 | typename Base::template CalledAs<const CallableT>{}) {} | |||
402 | ||||
403 | R operator()(P... Params) const { | |||
404 | return this->getCallPtr()(this->getCalleePtr(), Params...); | |||
405 | } | |||
406 | }; | |||
407 | ||||
408 | } // end namespace llvm | |||
409 | ||||
410 | #endif // LLVM_ADT_FUNCTIONEXTRAS_H |