31 #define DEBUG_TYPE "dyld"
35 enum RuntimeDyldErrorCode {
36 GenericRTDyldError = 1
44 const char *
name()
const noexcept
override {
return "runtimedyld"; }
46 std::string
message(
int Condition)
const override {
47 switch (
static_cast<RuntimeDyldErrorCode
>(Condition)) {
48 case GenericRTDyldError:
return "Generic RuntimeDyld error";
65 return std::error_code(GenericRTDyldError, *RTDyldErrorCategory);
79 MemMgr.deregisterEHFrames();
84 dbgs() <<
"----- Contents of section " <<
S.getName() <<
" " << State
87 if (
S.getAddress() ==
nullptr) {
88 dbgs() <<
"\n <section not emitted>\n";
92 const unsigned ColsPerRow = 16;
94 uint8_t *DataAddr =
S.getAddress();
97 unsigned StartPadding = LoadAddr & (ColsPerRow - 1);
98 unsigned BytesRemaining =
S.getSize();
102 LoadAddr & ~(
uint64_t)(ColsPerRow - 1)) <<
":";
103 while (StartPadding--)
107 while (BytesRemaining > 0) {
108 if ((LoadAddr & (ColsPerRow - 1)) == 0)
109 dbgs() <<
"\n" <<
format(
"0x%016" PRIx64, LoadAddr) <<
":";
124 std::lock_guard<sys::Mutex> locked(lock);
133 if (
auto Err = resolveExternalSymbols()) {
138 resolveLocalRelocations();
149 for (
const auto &Rel : Relocations) {
153 unsigned Idx = Rel.first;
155 LLVM_DEBUG(
dbgs() <<
"Resolving relocations Section #" << Idx <<
"\t"
157 resolveRelocationList(Rel.second,
Addr);
164 std::lock_guard<sys::Mutex> locked(lock);
165 for (
unsigned i = 0,
e = Sections.size();
i !=
e; ++
i) {
166 if (Sections[
i].getAddress() == LocalAddress) {
167 reassignSectionAddress(
i, TargetAddress);
185 std::lock_guard<sys::Mutex> locked(lock);
194 if (MemMgr.needsToReserveAllocationSpace()) {
195 uint64_t CodeSize = 0, RODataSize = 0, RWDataSize = 0;
196 uint32_t CodeAlign = 1, RODataAlign = 1, RWDataAlign = 1;
197 if (
auto Err = computeTotalAllocSize(Obj,
199 RODataSize, RODataAlign,
200 RWDataSize, RWDataAlign))
202 MemMgr.reserveAllocationSpace(CodeSize, CodeAlign, RODataSize, RODataAlign,
203 RWDataSize, RWDataAlign);
220 for (
auto &Sym : Obj.
symbols()) {
228 if (
auto NameOrErr = Sym.getName())
229 Symbols.insert(*NameOrErr);
231 return NameOrErr.takeError();
235 if (
auto ResultOrErr =
Resolver.getResponsibilitySet(Symbols))
236 ResponsibilitySet =
std::move(*ResultOrErr);
238 return ResultOrErr.takeError();
256 if (
auto SymTypeOrErr =
I->getType())
257 SymType = *SymTypeOrErr;
259 return SymTypeOrErr.takeError();
263 if (
auto NameOrErr =
I->getName())
266 return NameOrErr.takeError();
269 auto JITSymFlags = getJITSymbolFlags(*
I);
271 return JITSymFlags.takeError();
277 if (JITSymFlags->isWeak() || JITSymFlags->isCommon()) {
279 if (GlobalSymbolTable.count(
Name))
283 if (!ResponsibilitySet.count(
Name))
288 if (JITSymFlags->isWeak())
290 if (JITSymFlags->isCommon()) {
297 CommonSymbolsToAllocate.push_back(*
I);
304 if (
auto AddrOrErr =
I->getAddress())
307 return AddrOrErr.takeError();
309 unsigned SectionID = AbsoluteSymbolSection;
312 <<
" SID: " << SectionID
313 <<
" Offset: " <<
format(
"%p", (uintptr_t)
Addr)
314 <<
" flags: " << *FlagsOrErr <<
"\n");
316 GlobalSymbolTable[
Name] =
324 if (
auto SIOrErr =
I->getSection())
327 return SIOrErr.takeError();
337 bool IsCode =
SI->isText();
339 if (
auto SectionIDOrErr =
340 findOrEmitSection(Obj, *
SI, IsCode, LocalSections))
341 SectionID = *SectionIDOrErr;
343 return SectionIDOrErr.takeError();
346 <<
" SID: " << SectionID
347 <<
" Offset: " <<
format(
"%p", (uintptr_t)SectOffset)
348 <<
" flags: " << *FlagsOrErr <<
"\n");
350 GlobalSymbolTable[
Name] =
356 if (
auto Err = emitCommonSymbols(Obj, CommonSymbolsToAllocate, CommonSize,
371 if (RelocatedSection == SE)
377 if (
I ==
E && !ProcessAllSections)
380 bool IsCode = RelocatedSection->isText();
381 unsigned SectionID = 0;
382 if (
auto SectionIDOrErr = findOrEmitSection(Obj, *RelocatedSection, IsCode,
384 SectionID = *SectionIDOrErr;
386 return SectionIDOrErr.takeError();
391 if (
auto IOrErr = processRelocationRef(SectionID,
I, Obj, LocalSections, Stubs))
394 return IOrErr.takeError();
398 if (NotifyStubEmitted) {
401 for (
auto &KV : Stubs) {
408 NotifyStubEmitted(FileName,
SectionName, VR.SymbolName, SectionID,
414 for (
auto &GSTMapEntry : GlobalSymbolTable) {
416 auto &GSTEntry = GSTMapEntry.second;
417 if (GSTEntry.getSectionID() == VR.SectionID &&
418 GSTEntry.getOffset() == VR.Offset) {
429 if (ProcessAllSections) {
435 if (LocalSections.find(*
SI) != LocalSections.end())
438 bool IsCode =
SI->isText();
439 if (
auto SectionIDOrErr =
440 findOrEmitSection(Obj, *
SI, IsCode, LocalSections))
441 LLVM_DEBUG(
dbgs() <<
"\tSectionID: " << (*SectionIDOrErr) <<
"\n");
443 return SectionIDOrErr.takeError();
448 if (
auto Err = finalizeLoad(Obj, LocalSections))
454 return LocalSections;
466 (
SectionSize + Alignment - 1) / Alignment * Alignment;
467 TotalSize += AlignedSize;
474 if (isa<object::ELFObjectFileBase>(Obj))
476 if (
auto *COFFObj = dyn_cast<object::COFFObjectFile>(Obj)) {
477 const coff_section *CoffSection = COFFObj->getCOFFSection(Section);
488 return HasContent && !IsDiscardable;
491 assert(isa<MachOObjectFile>(Obj));
497 if (isa<object::ELFObjectFileBase>(Obj))
500 if (
auto *COFFObj = dyn_cast<object::COFFObjectFile>(Obj))
501 return ((COFFObj->getCOFFSection(Section)->Characteristics &
509 assert(isa<MachOObjectFile>(Obj));
515 if (isa<object::ELFObjectFileBase>(Obj))
517 if (
auto *COFFObj = dyn_cast<object::COFFObjectFile>(Obj))
518 return COFFObj->getCOFFSection(Section)->Characteristics &
521 auto *MachO = cast<MachOObjectFile>(Obj);
522 unsigned SectionType = MachO->getSectionType(Section);
529 if (isa<object::ELFObjectFileBase>(Obj))
544 std::vector<uint64_t> CodeSectionSizes;
545 std::vector<uint64_t> ROSectionSizes;
546 std::vector<uint64_t> RWSectionSizes;
558 uint64_t DataSize = Section.getSize();
559 uint64_t Alignment64 = Section.getAlignment();
560 unsigned Alignment = (unsigned)Alignment64 & 0xffffffffL;
561 bool IsCode = Section.isText();
563 bool IsTLS =
isTLS(Section);
570 uint64_t StubBufSize = computeSectionStubBufSize(Obj, Section);
573 if (
Name ==
".eh_frame")
575 if (StubBufSize != 0)
576 PaddingSize += getStubAlignment() - 1;
585 if (
Name ==
".eh_frame")
592 CodeAlign =
std::max(CodeAlign, Alignment);
594 }
else if (IsReadOnly) {
595 RODataAlign =
std::max(RODataAlign, Alignment);
598 RWDataAlign =
std::max(RWDataAlign, Alignment);
607 if (
unsigned GotSize = computeGOTSize(Obj)) {
608 RWSectionSizes.push_back(GotSize);
609 RWDataAlign = std::max<uint32_t>(RWDataAlign, getGOTEntrySize());
632 if (CommonSize != 0) {
633 RWSectionSizes.push_back(CommonSize);
634 RWDataAlign =
std::max(RWDataAlign, CommonAlign);
651 size_t GotEntrySize = getGOTEntrySize();
660 if (relocationNeedsGot(Reloc))
661 GotSize += GotEntrySize;
670 if (!MemMgr.allowStubAllocation()) {
674 unsigned StubSize = getMaxStubSize();
681 unsigned StubBufSize = 0;
690 if (!(RelSecI == Section))
694 if (relocationNeedsStub(Reloc))
695 StubBufSize += StubSize;
699 uint64_t DataSize = Section.getSize();
700 uint64_t Alignment64 = Section.getAlignment();
703 unsigned Alignment = (unsigned)Alignment64 & 0xffffffffL;
704 unsigned StubAlignment = getStubAlignment();
705 unsigned EndAlignment = (DataSize | Alignment) & -(DataSize | Alignment);
706 if (StubAlignment > EndAlignment)
707 StubBufSize += StubAlignment - EndAlignment;
712 unsigned Size)
const {
714 if (IsTargetLittleEndian) {
717 Result = (Result << 8) | *Src--;
720 Result = (Result << 8) | *Src++;
726 unsigned Size)
const {
727 if (IsTargetLittleEndian) {
729 *Dst++ =
Value & 0xFF;
735 *Dst-- =
Value & 0xFF;
750 if (SymbolsToAllocate.empty())
754 unsigned SectionID = Sections.size();
755 uint8_t *
Addr = MemMgr.allocateDataSection(CommonSize, CommonAlign, SectionID,
756 "<common symbols>",
false);
762 memset(
Addr, 0, CommonSize);
766 <<
" DataSize: " << CommonSize <<
"\n");
769 for (
auto &Sym : SymbolsToAllocate) {
770 uint32_t Alignment = Sym.getAlignment();
771 uint64_t Size = Sym.getCommonSize();
773 if (
auto NameOrErr = Sym.getName())
776 return NameOrErr.takeError();
782 Offset += AlignOffset;
784 auto JITSymFlags = getJITSymbolFlags(Sym);
787 return JITSymFlags.takeError();
792 GlobalSymbolTable[
Name] =
806 uint64_t Alignment64 = Section.getAlignment();
808 unsigned Alignment = (unsigned)Alignment64 & 0xffffffffL;
809 unsigned PaddingSize = 0;
810 unsigned StubBufSize = 0;
812 bool IsVirtual = Section.isVirtual();
815 bool IsTLS =
isTLS(Section);
816 uint64_t DataSize = Section.getSize();
821 Alignment =
std::max(1u, Alignment);
828 StubBufSize = computeSectionStubBufSize(Obj, Section);
833 if (
Name ==
".eh_frame")
837 unsigned SectionID = Sections.size();
840 const char *pData =
nullptr;
844 if (!IsVirtual && !IsZeroInit) {
850 return E.takeError();
857 if (StubBufSize != 0) {
858 Alignment =
std::max(Alignment, getStubAlignment());
859 PaddingSize += getStubAlignment() - 1;
864 if (IsRequired || ProcessAllSections) {
865 Allocate = DataSize + PaddingSize + StubBufSize;
870 MemMgr.allocateTLSSection(Allocate, Alignment, SectionID,
Name);
871 Addr = TLSSection.InitializationImage;
872 LoadAddress = TLSSection.Offset;
874 Addr = MemMgr.allocateCodeSection(Allocate, Alignment, SectionID,
Name);
876 Addr = MemMgr.allocateDataSection(Allocate, Alignment, SectionID,
Name,
883 if (IsZeroInit || IsVirtual)
884 memset(
Addr, 0, DataSize);
889 if (PaddingSize != 0) {
890 memset(
Addr + DataSize, 0, PaddingSize);
892 DataSize += PaddingSize;
897 DataSize &= -(
uint64_t)getStubAlignment();
900 LLVM_DEBUG(
dbgs() <<
"emitSection SectionID: " << SectionID <<
" Name: "
901 <<
Name <<
" obj addr: " <<
format(
"%p", pData)
902 <<
" new addr: " <<
format(
"%p",
Addr) <<
" DataSize: "
903 << DataSize <<
" StubBufSize: " << StubBufSize
904 <<
" Allocate: " << Allocate <<
"\n");
912 dbgs() <<
"emitSection SectionID: " << SectionID <<
" Name: " <<
Name
913 <<
" obj addr: " <<
format(
"%p", data.
data()) <<
" new addr: 0"
914 <<
" DataSize: " << DataSize <<
" StubBufSize: " << StubBufSize
915 <<
" Allocate: " << Allocate <<
"\n");
924 Sections.back().setLoadAddress(LoadAddress);
927 Sections.back().setLoadAddress(0);
938 unsigned SectionID = 0;
939 ObjSectionToIDMap::iterator
i = LocalSections.find(Section);
940 if (
i != LocalSections.end())
941 SectionID =
i->second;
943 if (
auto SectionIDOrErr = emitSection(Obj, Section, IsCode))
944 SectionID = *SectionIDOrErr;
946 return SectionIDOrErr.takeError();
947 LocalSections[Section] = SectionID;
953 unsigned SectionID) {
954 Relocations[SectionID].push_back(RE);
963 if (Loc == GlobalSymbolTable.end()) {
964 ExternalSymbolRelocations[
SymbolName].push_back(RE);
967 "Empty symbol should not be in GlobalSymbolTable");
970 const auto &
SymInfo = Loc->second;
972 Relocations[
SymInfo.getSectionID()].push_back(RECopy);
977 unsigned AbiVariant) {
984 writeBytesUnaligned(0xd2e00010,
Addr, 4);
985 writeBytesUnaligned(0xf2c00010,
Addr+4, 4);
986 writeBytesUnaligned(0xf2a00010,
Addr+8, 4);
987 writeBytesUnaligned(0xf2800010,
Addr+12, 4);
988 writeBytesUnaligned(0xd61f0200,
Addr+16, 4);
994 writeBytesUnaligned(0xe51ff004,
Addr, 4);
996 }
else if (IsMipsO32ABI || IsMipsN32ABI) {
1001 const unsigned LuiT9Instr = 0x3c190000, AdduiT9Instr = 0x27390000;
1002 const unsigned NopInstr = 0x0;
1003 unsigned JrT9Instr = 0x03200008;
1006 JrT9Instr = 0x03200009;
1008 writeBytesUnaligned(LuiT9Instr,
Addr, 4);
1009 writeBytesUnaligned(AdduiT9Instr,
Addr + 4, 4);
1010 writeBytesUnaligned(JrT9Instr,
Addr + 8, 4);
1011 writeBytesUnaligned(NopInstr,
Addr + 12, 4);
1013 }
else if (IsMipsN64ABI) {
1022 const unsigned LuiT9Instr = 0x3c190000, DaddiuT9Instr = 0x67390000,
1023 DsllT9Instr = 0x19CC38;
1024 const unsigned NopInstr = 0x0;
1025 unsigned JrT9Instr = 0x03200008;
1027 JrT9Instr = 0x03200009;
1029 writeBytesUnaligned(LuiT9Instr,
Addr, 4);
1030 writeBytesUnaligned(DaddiuT9Instr,
Addr + 4, 4);
1031 writeBytesUnaligned(DsllT9Instr,
Addr + 8, 4);
1032 writeBytesUnaligned(DaddiuT9Instr,
Addr + 12, 4);
1033 writeBytesUnaligned(DsllT9Instr,
Addr + 16, 4);
1034 writeBytesUnaligned(DaddiuT9Instr,
Addr + 20, 4);
1035 writeBytesUnaligned(JrT9Instr,
Addr + 24, 4);
1036 writeBytesUnaligned(NopInstr,
Addr + 28, 4);
1042 writeInt32BE(
Addr, 0x3D800000);
1043 writeInt32BE(
Addr+4, 0x618C0000);
1044 writeInt32BE(
Addr+8, 0x798C07C6);
1045 writeInt32BE(
Addr+12, 0x658C0000);
1046 writeInt32BE(
Addr+16, 0x618C0000);
1047 if (AbiVariant == 2) {
1050 writeInt32BE(
Addr+20, 0xF8410018);
1051 writeInt32BE(
Addr+24, 0x7D8903A6);
1052 writeInt32BE(
Addr+28, 0x4E800420);
1057 writeInt32BE(
Addr+20, 0xF8410028);
1058 writeInt32BE(
Addr+24, 0xE96C0000);
1059 writeInt32BE(
Addr+28, 0xE84C0008);
1060 writeInt32BE(
Addr+32, 0x7D6903A6);
1061 writeInt32BE(
Addr+36, 0xE96C0010);
1062 writeInt32BE(
Addr+40, 0x4E800420);
1066 writeInt16BE(
Addr, 0xC418);
1067 writeInt16BE(
Addr+2, 0x0000);
1068 writeInt16BE(
Addr+4, 0x0004);
1069 writeInt16BE(
Addr+6, 0x07F1);
1097 dbgs() <<
"Reassigning address for section " << SectionID <<
" ("
1098 << Sections[SectionID].
getName() <<
"): "
1099 <<
format(
"0x%016" PRIx64, Sections[SectionID].getLoadAddress())
1100 <<
" -> " <<
format(
"0x%016" PRIx64,
Addr) <<
"\n");
1101 Sections[SectionID].setLoadAddress(
Addr);
1106 for (
unsigned i = 0,
e = Relocs.size();
i !=
e; ++
i) {
1109 if (RE.
SectionID != AbsoluteSymbolSection &&
1110 Sections[RE.
SectionID].getAddress() ==
nullptr)
1118 for (
auto &RelocKV : ExternalSymbolRelocations) {
1121 if (
Name.size() == 0) {
1125 resolveRelocationList(Relocs, 0);
1130 if (Loc == GlobalSymbolTable.end()) {
1131 auto RRI = ExternalSymbolMap.
find(
Name);
1132 assert(RRI != ExternalSymbolMap.
end() &&
"No result for symbol");
1133 Addr = RRI->second.getAddress();
1134 Flags = RRI->second.getFlags();
1138 const auto &
SymInfo = Loc->second;
1139 Addr = getSectionLoadAddress(
SymInfo.getSectionID()) +
1147 "' which could not be resolved!");
1156 Addr = modifyAddressBasedOnFlags(
Addr, Flags);
1160 resolveRelocationList(Relocs,
Addr);
1164 ExternalSymbolRelocations.clear();
1178 for (
auto &RelocKV : ExternalSymbolRelocations) {
1180 if (!
Name.empty() && !GlobalSymbolTable.count(
Name) &&
1181 !ResolvedSymbols.count(
Name))
1182 NewSymbols.insert(
Name);
1185 if (NewSymbols.empty())
1189 using ExpectedLookupResult =
1195 auto NewSymbolsP = std::make_shared<std::promise<ExpectedLookupResult>>();
1196 auto NewSymbolsF = NewSymbolsP->get_future();
1199 NewSymbolsP->set_value(
std::move(Result));
1202 auto NewResolverResults = NewSymbolsF.get();
1204 if (!NewResolverResults)
1205 return NewResolverResults.takeError();
1207 assert(NewResolverResults->size() == NewSymbols.size() &&
1208 "Should have errored on unresolved symbols");
1210 for (
auto &RRKV : *NewResolverResults) {
1211 assert(!ResolvedSymbols.count(RRKV.first) &&
"Redundant resolution?");
1212 ExternalSymbolMap.
insert(RRKV);
1213 ResolvedSymbols.insert(RRKV.first);
1218 applyExternalSymbolRelocations(ExternalSymbolMap);
1224 std::unique_ptr<RuntimeDyldImpl> This,
1226 std::unique_ptr<RuntimeDyld::LoadedObjectInfo>,
Error)>
1229 std::unique_ptr<RuntimeDyld::LoadedObjectInfo>
Info) {
1231 auto SharedThis = std::shared_ptr<RuntimeDyldImpl>(
std::move(
This));
1232 auto PostResolveContinuation =
1243 for (
auto &KV : *Result)
1244 Resolved[KV.first] = KV.second;
1246 SharedThis->applyExternalSymbolRelocations(Resolved);
1247 SharedThis->resolveLocalRelocations();
1248 SharedThis->registerEHFrames();
1250 if (SharedThis->MemMgr.finalizeMemory(&ErrMsg))
1252 make_error<StringError>(
std::move(ErrMsg),
1260 for (
auto &RelocKV : SharedThis->ExternalSymbolRelocations) {
1264 assert(!SharedThis->GlobalSymbolTable.count(
Name) &&
1265 "Name already processed. RuntimeDyld instances can not be re-used "
1266 "when finalizing with finalizeAsync.");
1267 Symbols.insert(
Name);
1270 if (!Symbols.empty()) {
1271 SharedThis->Resolver.lookup(Symbols,
std::move(PostResolveContinuation));
1273 PostResolveContinuation(std::map<StringRef, JITEvaluatedSymbol>());
1282 auto I = ObjSecToIDMap.find(Sec);
1283 if (
I != ObjSecToIDMap.end())
1284 return RTDyld.Sections[
I->second].getLoadAddress();
1297 void RuntimeDyld::MemoryManager::anchor() {}
1298 void JITSymbolResolver::anchor() {}
1299 void LegacyJITSymbolResolver::anchor() {}
1311 ProcessAllSections =
false;
1316 static std::unique_ptr<RuntimeDyldCOFF>
1321 std::unique_ptr<RuntimeDyldCOFF> Dyld =
1323 Dyld->setProcessAllSections(ProcessAllSections);
1324 Dyld->setNotifyStubEmitted(
std::move(NotifyStubEmitted));
1328 static std::unique_ptr<RuntimeDyldELF>
1332 std::unique_ptr<RuntimeDyldELF> Dyld =
1334 Dyld->setProcessAllSections(ProcessAllSections);
1335 Dyld->setNotifyStubEmitted(
std::move(NotifyStubEmitted));
1339 static std::unique_ptr<RuntimeDyldMachO>
1343 bool ProcessAllSections,
1345 std::unique_ptr<RuntimeDyldMachO> Dyld =
1347 Dyld->setProcessAllSections(ProcessAllSections);
1348 Dyld->setNotifyStubEmitted(
std::move(NotifyStubEmitted));
1352 std::unique_ptr<RuntimeDyld::LoadedObjectInfo>
1358 MemMgr,
Resolver, ProcessAllSections,
1363 ProcessAllSections,
std::move(NotifyStubEmitted));
1367 ProcessAllSections,
std::move(NotifyStubEmitted));
1372 if (!Dyld->isCompatibleFile(Obj))
1375 auto LoadedObjInfo = Dyld->loadObject(Obj);
1377 return LoadedObjInfo;
1383 return Dyld->getSymbolLocalAddress(
Name);
1387 assert(Dyld &&
"No RuntimeDyld instance attached");
1388 return Dyld->getSymbolSectionID(
Name);
1394 return Dyld->getSymbol(
Name);
1399 return std::map<StringRef, JITEvaluatedSymbol>();
1400 return Dyld->getSymbolTable();
1406 Dyld->reassignSectionAddress(SectionID,
Addr);
1411 Dyld->mapSectionAddress(LocalAddress, TargetAddress);
1419 bool MemoryFinalizationLocked = MemMgr.FinalizationLocked;
1420 MemMgr.FinalizationLocked =
true;
1423 if (!MemoryFinalizationLocked) {
1425 MemMgr.FinalizationLocked =
false;
1430 assert(Dyld &&
"No Dyld instance attached");
1431 return Dyld->getSectionContent(SectionID);
1435 assert(Dyld &&
"No Dyld instance attached");
1436 return Dyld->getSectionLoadAddress(SectionID);
1441 Dyld->registerEHFrames();
1446 Dyld->deregisterEHFrames();
1454 bool ProcessAllSections,
1457 std::map<StringRef, JITEvaluatedSymbol>)>
1460 std::unique_ptr<RuntimeDyld::LoadedObjectInfo>,
Error)>