30 #define DEBUG_TYPE "dyld"
34 enum RuntimeDyldErrorCode {
35 GenericRTDyldError = 1
41 class RuntimeDyldErrorCategory :
public std::error_category {
43 const char *
name()
const noexcept
override {
return "runtimedyld"; }
45 std::string
message(
int Condition)
const override {
46 switch (
static_cast<RuntimeDyldErrorCode
>(Condition)) {
47 case GenericRTDyldError:
return "Generic RuntimeDyld error";
62 static RuntimeDyldErrorCategory RTDyldErrorCategory;
63 return std::error_code(GenericRTDyldError, RTDyldErrorCategory);
77 MemMgr.deregisterEHFrames();
82 dbgs() <<
"----- Contents of section " <<
S.getName() <<
" " << State
85 if (
S.getAddress() ==
nullptr) {
86 dbgs() <<
"\n <section not emitted>\n";
90 const unsigned ColsPerRow = 16;
92 uint8_t *DataAddr =
S.getAddress();
95 unsigned StartPadding = LoadAddr & (ColsPerRow - 1);
96 unsigned BytesRemaining =
S.getSize();
100 LoadAddr & ~(
uint64_t)(ColsPerRow - 1)) <<
":";
101 while (StartPadding--)
105 while (BytesRemaining > 0) {
106 if ((LoadAddr & (ColsPerRow - 1)) == 0)
107 dbgs() <<
"\n" <<
format(
"0x%016" PRIx64, LoadAddr) <<
":";
122 std::lock_guard<sys::Mutex> locked(lock);
131 if (
auto Err = resolveExternalSymbols()) {
136 resolveLocalRelocations();
147 for (
const auto &Rel : Relocations) {
151 unsigned Idx = Rel.first;
153 LLVM_DEBUG(
dbgs() <<
"Resolving relocations Section #" << Idx <<
"\t"
155 resolveRelocationList(Rel.second,
Addr);
162 std::lock_guard<sys::Mutex> locked(lock);
163 for (
unsigned i = 0,
e = Sections.size();
i !=
e; ++
i) {
164 if (Sections[
i].getAddress() == LocalAddress) {
165 reassignSectionAddress(
i, TargetAddress);
183 std::lock_guard<sys::Mutex> locked(lock);
192 if (MemMgr.needsToReserveAllocationSpace()) {
193 uint64_t CodeSize = 0, RODataSize = 0, RWDataSize = 0;
194 Align CodeAlign, RODataAlign, RWDataAlign;
195 if (
auto Err = computeTotalAllocSize(Obj, CodeSize, CodeAlign, RODataSize,
196 RODataAlign, RWDataSize, RWDataAlign))
198 MemMgr.reserveAllocationSpace(CodeSize, CodeAlign, RODataSize, RODataAlign,
199 RWDataSize, RWDataAlign);
216 for (
auto &Sym : Obj.
symbols()) {
224 if (
auto NameOrErr = Sym.getName())
225 Symbols.insert(*NameOrErr);
227 return NameOrErr.takeError();
231 if (
auto ResultOrErr =
Resolver.getResponsibilitySet(Symbols))
232 ResponsibilitySet =
std::move(*ResultOrErr);
234 return ResultOrErr.takeError();
252 if (
auto SymTypeOrErr =
I->getType())
253 SymType = *SymTypeOrErr;
255 return SymTypeOrErr.takeError();
259 if (
auto NameOrErr =
I->getName())
262 return NameOrErr.takeError();
265 auto JITSymFlags = getJITSymbolFlags(*
I);
267 return JITSymFlags.takeError();
273 if (JITSymFlags->isWeak() || JITSymFlags->isCommon()) {
275 if (GlobalSymbolTable.count(Name))
279 if (!ResponsibilitySet.count(Name))
284 if (JITSymFlags->isWeak())
286 if (JITSymFlags->isCommon()) {
293 CommonSymbolsToAllocate.push_back(*
I);
300 if (
auto AddrOrErr =
I->getAddress())
303 return AddrOrErr.takeError();
305 unsigned SectionID = AbsoluteSymbolSection;
307 LLVM_DEBUG(
dbgs() <<
"\tType: " << SymType <<
" (absolute) Name: " << Name
308 <<
" SID: " << SectionID
309 <<
" Offset: " <<
format(
"%p", (uintptr_t)
Addr)
310 <<
" flags: " << *FlagsOrErr <<
"\n");
313 auto Result = GlobalSymbolTable.insert_or_assign(
315 processNewSymbol(*
I, Result.first->getValue());
323 if (
auto SIOrErr =
I->getSection())
326 return SIOrErr.takeError();
336 bool IsCode =
SI->isText();
338 if (
auto SectionIDOrErr =
339 findOrEmitSection(Obj, *
SI, IsCode, LocalSections))
340 SectionID = *SectionIDOrErr;
342 return SectionIDOrErr.takeError();
345 <<
" SID: " << SectionID
346 <<
" Offset: " <<
format(
"%p", (uintptr_t)SectOffset)
347 <<
" flags: " << *FlagsOrErr <<
"\n");
350 auto Result = GlobalSymbolTable.insert_or_assign(
352 processNewSymbol(*
I, Result.first->getValue());
358 if (
auto Err = emitCommonSymbols(Obj, CommonSymbolsToAllocate, CommonSize,
373 if (RelocatedSection == SE)
379 if (
I ==
E && !ProcessAllSections)
382 bool IsCode = RelocatedSection->isText();
383 unsigned SectionID = 0;
384 if (
auto SectionIDOrErr = findOrEmitSection(Obj, *RelocatedSection, IsCode,
386 SectionID = *SectionIDOrErr;
388 return SectionIDOrErr.takeError();
393 if (
auto IOrErr = processRelocationRef(SectionID,
I, Obj, LocalSections, Stubs))
396 return IOrErr.takeError();
400 if (NotifyStubEmitted) {
403 for (
auto &KV : Stubs) {
410 NotifyStubEmitted(FileName,
SectionName, VR.SymbolName, SectionID,
416 for (
auto &GSTMapEntry : GlobalSymbolTable) {
418 auto &GSTEntry = GSTMapEntry.second;
419 if (GSTEntry.getSectionID() == VR.SectionID &&
420 GSTEntry.getOffset() == VR.Offset) {
431 if (ProcessAllSections) {
437 if (LocalSections.find(*
SI) != LocalSections.end())
440 bool IsCode =
SI->isText();
441 if (
auto SectionIDOrErr =
442 findOrEmitSection(Obj, *
SI, IsCode, LocalSections))
443 LLVM_DEBUG(
dbgs() <<
"\tSectionID: " << (*SectionIDOrErr) <<
"\n");
445 return SectionIDOrErr.takeError();
450 if (
auto Err = finalizeLoad(Obj, LocalSections))
456 return LocalSections;
473 if (isa<object::ELFObjectFileBase>(Obj))
475 if (
auto *COFFObj = dyn_cast<object::COFFObjectFile>(Obj)) {
476 const coff_section *CoffSection = COFFObj->getCOFFSection(Section);
487 return HasContent && !IsDiscardable;
490 assert(isa<MachOObjectFile>(Obj));
496 if (isa<object::ELFObjectFileBase>(Obj))
499 if (
auto *COFFObj = dyn_cast<object::COFFObjectFile>(Obj))
500 return ((COFFObj->getCOFFSection(Section)->Characteristics &
508 assert(isa<MachOObjectFile>(Obj));
514 if (isa<object::ELFObjectFileBase>(Obj))
516 if (
auto *COFFObj = dyn_cast<object::COFFObjectFile>(Obj))
517 return COFFObj->getCOFFSection(Section)->Characteristics &
520 auto *MachO = cast<MachOObjectFile>(Obj);
521 unsigned SectionType = MachO->getSectionType(Section);
528 if (isa<object::ELFObjectFileBase>(Obj))
538 Align &RWDataAlign) {
540 std::vector<uint64_t> CodeSectionSizes;
541 std::vector<uint64_t> ROSectionSizes;
542 std::vector<uint64_t> RWSectionSizes;
554 uint64_t DataSize = Section.getSize();
555 Align Alignment = Section.getAlignment();
556 bool IsCode = Section.isText();
558 bool IsTLS =
isTLS(Section);
565 uint64_t StubBufSize = computeSectionStubBufSize(Obj, Section);
568 if (Name ==
".eh_frame")
570 if (StubBufSize != 0)
571 PaddingSize += getStubAlignment().value() - 1;
580 if (Name ==
".eh_frame")
587 CodeAlign =
std::max(CodeAlign, Alignment);
589 }
else if (IsReadOnly) {
590 RODataAlign =
std::max(RODataAlign, Alignment);
593 RWDataAlign =
std::max(RWDataAlign, Alignment);
602 if (
unsigned GotSize = computeGOTSize(Obj)) {
603 RWSectionSizes.push_back(GotSize);
604 RWDataAlign =
std::max(RWDataAlign,
Align(getGOTEntrySize()));
623 CommonAlign = Alignment;
624 CommonSize =
alignTo(CommonSize, Alignment) + Size;
627 if (CommonSize != 0) {
628 RWSectionSizes.push_back(CommonSize);
629 RWDataAlign =
std::max(RWDataAlign, CommonAlign);
632 if (!CodeSectionSizes.empty()) {
634 CodeSectionSizes.push_back(64);
651 size_t GotEntrySize = getGOTEntrySize();
660 if (relocationNeedsGot(Reloc))
661 GotSize += GotEntrySize;
670 if (!MemMgr.allowStubAllocation()) {
674 unsigned StubSize = getMaxStubSize();
681 unsigned StubBufSize = 0;
690 if (!(RelSecI == Section))
694 if (relocationNeedsStub(Reloc))
695 StubBufSize += StubSize;
699 uint64_t DataSize = Section.getSize();
700 Align Alignment = Section.getAlignment();
703 Align StubAlignment = getStubAlignment();
705 if (StubAlignment > EndAlignment)
706 StubBufSize += StubAlignment.
value() - EndAlignment.
value();
711 unsigned Size)
const {
713 if (IsTargetLittleEndian) {
716 Result = (Result << 8) | *Src--;
719 Result = (Result << 8) | *Src++;
725 unsigned Size)
const {
726 if (IsTargetLittleEndian) {
728 *Dst++ =
Value & 0xFF;
734 *Dst-- =
Value & 0xFF;
749 if (SymbolsToAllocate.empty())
753 unsigned SectionID = Sections.size();
754 uint8_t *
Addr = MemMgr.allocateDataSection(CommonSize, CommonAlign, SectionID,
755 "<common symbols>",
false);
761 memset(
Addr, 0, CommonSize);
765 <<
" DataSize: " << CommonSize <<
"\n");
768 for (
auto &Sym : SymbolsToAllocate) {
769 uint32_t Alignment = Sym.getAlignment();
770 uint64_t Size = Sym.getCommonSize();
772 if (
auto NameOrErr = Sym.getName())
775 return NameOrErr.takeError();
781 Offset += AlignOffset;
783 auto JITSymFlags = getJITSymbolFlags(Sym);
786 return JITSymFlags.takeError();
788 LLVM_DEBUG(
dbgs() <<
"Allocating common symbol " << Name <<
" address "
791 GlobalSymbolTable[Name] =
805 Align Alignment = Section.getAlignment();
807 unsigned PaddingSize = 0;
808 unsigned StubBufSize = 0;
810 bool IsVirtual = Section.isVirtual();
813 bool IsTLS =
isTLS(Section);
814 uint64_t DataSize = Section.getSize();
821 StubBufSize = computeSectionStubBufSize(Obj, Section);
826 if (Name ==
".eh_frame")
830 unsigned SectionID = Sections.size();
833 const char *pData =
nullptr;
837 if (!IsVirtual && !IsZeroInit) {
843 return E.takeError();
850 if (StubBufSize != 0) {
851 Alignment =
std::max(Alignment, getStubAlignment());
852 PaddingSize += getStubAlignment().value() - 1;
857 if (IsRequired || ProcessAllSections) {
858 Allocate = DataSize + PaddingSize + StubBufSize;
862 auto TLSSection = MemMgr.allocateTLSSection(Allocate, Alignment.
value(),
864 Addr = TLSSection.InitializationImage;
865 LoadAddress = TLSSection.Offset;
867 Addr = MemMgr.allocateCodeSection(Allocate, Alignment.
value(), SectionID,
870 Addr = MemMgr.allocateDataSection(Allocate, Alignment.
value(), SectionID,
877 if (IsZeroInit || IsVirtual)
878 memset(
Addr, 0, DataSize);
883 if (PaddingSize != 0) {
884 memset(
Addr + DataSize, 0, PaddingSize);
886 DataSize += PaddingSize;
891 DataSize &= -(
uint64_t)getStubAlignment().value();
894 LLVM_DEBUG(
dbgs() <<
"emitSection SectionID: " << SectionID <<
" Name: "
895 << Name <<
" obj addr: " <<
format(
"%p", pData)
896 <<
" new addr: " <<
format(
"%p",
Addr) <<
" DataSize: "
897 << DataSize <<
" StubBufSize: " << StubBufSize
898 <<
" Allocate: " << Allocate <<
"\n");
906 dbgs() <<
"emitSection SectionID: " << SectionID <<
" Name: " << Name
907 <<
" obj addr: " <<
format(
"%p", data.
data()) <<
" new addr: 0"
908 <<
" DataSize: " << DataSize <<
" StubBufSize: " << StubBufSize
909 <<
" Allocate: " << Allocate <<
"\n");
918 Sections.back().setLoadAddress(LoadAddress);
921 Sections.back().setLoadAddress(0);
932 unsigned SectionID = 0;
933 ObjSectionToIDMap::iterator
i = LocalSections.find(Section);
934 if (
i != LocalSections.end())
935 SectionID =
i->second;
937 if (
auto SectionIDOrErr = emitSection(Obj, Section, IsCode))
938 SectionID = *SectionIDOrErr;
940 return SectionIDOrErr.takeError();
941 LocalSections[Section] = SectionID;
947 unsigned SectionID) {
948 Relocations[SectionID].push_back(RE);
957 if (Loc == GlobalSymbolTable.end()) {
958 ExternalSymbolRelocations[
SymbolName].push_back(RE);
961 "Empty symbol should not be in GlobalSymbolTable");
964 const auto &
SymInfo = Loc->second;
966 Relocations[
SymInfo.getSectionID()].push_back(RECopy);
971 unsigned AbiVariant) {
978 writeBytesUnaligned(0xd2e00010,
Addr, 4);
979 writeBytesUnaligned(0xf2c00010,
Addr+4, 4);
980 writeBytesUnaligned(0xf2a00010,
Addr+8, 4);
981 writeBytesUnaligned(0xf2800010,
Addr+12, 4);
982 writeBytesUnaligned(0xd61f0200,
Addr+16, 4);
988 writeBytesUnaligned(0xe51ff004,
Addr, 4);
990 }
else if (IsMipsO32ABI || IsMipsN32ABI) {
995 const unsigned LuiT9Instr = 0x3c190000, AdduiT9Instr = 0x27390000;
996 const unsigned NopInstr = 0x0;
997 unsigned JrT9Instr = 0x03200008;
1000 JrT9Instr = 0x03200009;
1002 writeBytesUnaligned(LuiT9Instr,
Addr, 4);
1003 writeBytesUnaligned(AdduiT9Instr,
Addr + 4, 4);
1004 writeBytesUnaligned(JrT9Instr,
Addr + 8, 4);
1005 writeBytesUnaligned(NopInstr,
Addr + 12, 4);
1007 }
else if (IsMipsN64ABI) {
1016 const unsigned LuiT9Instr = 0x3c190000, DaddiuT9Instr = 0x67390000,
1017 DsllT9Instr = 0x19CC38;
1018 const unsigned NopInstr = 0x0;
1019 unsigned JrT9Instr = 0x03200008;
1021 JrT9Instr = 0x03200009;
1023 writeBytesUnaligned(LuiT9Instr,
Addr, 4);
1024 writeBytesUnaligned(DaddiuT9Instr,
Addr + 4, 4);
1025 writeBytesUnaligned(DsllT9Instr,
Addr + 8, 4);
1026 writeBytesUnaligned(DaddiuT9Instr,
Addr + 12, 4);
1027 writeBytesUnaligned(DsllT9Instr,
Addr + 16, 4);
1028 writeBytesUnaligned(DaddiuT9Instr,
Addr + 20, 4);
1029 writeBytesUnaligned(JrT9Instr,
Addr + 24, 4);
1030 writeBytesUnaligned(NopInstr,
Addr + 28, 4);
1036 writeInt32BE(
Addr, 0x3D800000);
1037 writeInt32BE(
Addr+4, 0x618C0000);
1038 writeInt32BE(
Addr+8, 0x798C07C6);
1039 writeInt32BE(
Addr+12, 0x658C0000);
1040 writeInt32BE(
Addr+16, 0x618C0000);
1041 if (AbiVariant == 2) {
1044 writeInt32BE(
Addr+20, 0xF8410018);
1045 writeInt32BE(
Addr+24, 0x7D8903A6);
1046 writeInt32BE(
Addr+28, 0x4E800420);
1051 writeInt32BE(
Addr+20, 0xF8410028);
1052 writeInt32BE(
Addr+24, 0xE96C0000);
1053 writeInt32BE(
Addr+28, 0xE84C0008);
1054 writeInt32BE(
Addr+32, 0x7D6903A6);
1055 writeInt32BE(
Addr+36, 0xE96C0010);
1056 writeInt32BE(
Addr+40, 0x4E800420);
1060 writeInt16BE(
Addr, 0xC418);
1061 writeInt16BE(
Addr+2, 0x0000);
1062 writeInt16BE(
Addr+4, 0x0004);
1063 writeInt16BE(
Addr+6, 0x07F1);
1091 dbgs() <<
"Reassigning address for section " << SectionID <<
" ("
1092 << Sections[SectionID].
getName() <<
"): "
1093 <<
format(
"0x%016" PRIx64, Sections[SectionID].getLoadAddress())
1094 <<
" -> " <<
format(
"0x%016" PRIx64,
Addr) <<
"\n");
1095 Sections[SectionID].setLoadAddress(
Addr);
1100 for (
unsigned i = 0,
e = Relocs.size();
i !=
e; ++
i) {
1103 if (RE.
SectionID != AbsoluteSymbolSection &&
1104 Sections[RE.
SectionID].getAddress() ==
nullptr)
1112 for (
auto &RelocKV : ExternalSymbolRelocations) {
1115 if (Name.size() == 0) {
1119 resolveRelocationList(Relocs, 0);
1124 if (Loc == GlobalSymbolTable.end()) {
1125 auto RRI = ExternalSymbolMap.
find(Name);
1126 assert(RRI != ExternalSymbolMap.
end() &&
"No result for symbol");
1127 Addr = RRI->second.getAddress();
1128 Flags = RRI->second.getFlags();
1132 const auto &
SymInfo = Loc->second;
1133 Addr = getSectionLoadAddress(
SymInfo.getSectionID()) +
1141 "' which could not be resolved!");
1150 Addr = modifyAddressBasedOnFlags(
Addr, Flags);
1152 LLVM_DEBUG(
dbgs() <<
"Resolving relocations Name: " << Name <<
"\t"
1154 resolveRelocationList(Relocs,
Addr);
1158 ExternalSymbolRelocations.clear();
1172 for (
auto &RelocKV : ExternalSymbolRelocations) {
1174 if (!Name.empty() && !GlobalSymbolTable.count(Name) &&
1175 !ResolvedSymbols.count(Name))
1176 NewSymbols.insert(Name);
1179 if (NewSymbols.empty())
1183 using ExpectedLookupResult =
1189 auto NewSymbolsP = std::make_shared<std::promise<ExpectedLookupResult>>();
1190 auto NewSymbolsF = NewSymbolsP->get_future();
1193 NewSymbolsP->set_value(
std::move(Result));
1196 auto NewResolverResults = NewSymbolsF.get();
1198 if (!NewResolverResults)
1199 return NewResolverResults.takeError();
1201 assert(NewResolverResults->size() == NewSymbols.size() &&
1202 "Should have errored on unresolved symbols");
1204 for (
auto &RRKV : *NewResolverResults) {
1205 assert(!ResolvedSymbols.count(RRKV.first) &&
"Redundant resolution?");
1206 ExternalSymbolMap.
insert(RRKV);
1207 ResolvedSymbols.insert(RRKV.first);
1212 applyExternalSymbolRelocations(ExternalSymbolMap);
1218 std::unique_ptr<RuntimeDyldImpl> This,
1220 std::unique_ptr<RuntimeDyld::LoadedObjectInfo>,
Error)>
1223 std::unique_ptr<RuntimeDyld::LoadedObjectInfo>
Info) {
1225 auto SharedThis = std::shared_ptr<RuntimeDyldImpl>(
std::move(
This));
1226 auto PostResolveContinuation =
1237 for (
auto &KV : *Result)
1238 Resolved[KV.first] = KV.second;
1240 SharedThis->applyExternalSymbolRelocations(Resolved);
1241 SharedThis->resolveLocalRelocations();
1242 SharedThis->registerEHFrames();
1244 if (SharedThis->MemMgr.finalizeMemory(&ErrMsg))
1246 make_error<StringError>(
std::move(ErrMsg),
1254 for (
auto &RelocKV : SharedThis->ExternalSymbolRelocations) {
1258 assert(!SharedThis->GlobalSymbolTable.count(Name) &&
1259 "Name already processed. RuntimeDyld instances can not be re-used "
1260 "when finalizing with finalizeAsync.");
1261 Symbols.insert(Name);
1264 if (!Symbols.empty()) {
1265 SharedThis->Resolver.lookup(Symbols,
std::move(PostResolveContinuation));
1267 PostResolveContinuation(std::map<StringRef, JITEvaluatedSymbol>());
1276 auto I = ObjSecToIDMap.find(Sec);
1277 if (
I != ObjSecToIDMap.end())
1278 return RTDyld.Sections[
I->second].getLoadAddress();
1291 void RuntimeDyld::MemoryManager::anchor() {}
1292 void JITSymbolResolver::anchor() {}
1293 void LegacyJITSymbolResolver::anchor() {}
1305 ProcessAllSections =
false;
1310 static std::unique_ptr<RuntimeDyldCOFF>
1315 std::unique_ptr<RuntimeDyldCOFF> Dyld =
1317 Dyld->setProcessAllSections(ProcessAllSections);
1318 Dyld->setNotifyStubEmitted(
std::move(NotifyStubEmitted));
1322 static std::unique_ptr<RuntimeDyldELF>
1326 std::unique_ptr<RuntimeDyldELF> Dyld =
1328 Dyld->setProcessAllSections(ProcessAllSections);
1329 Dyld->setNotifyStubEmitted(
std::move(NotifyStubEmitted));
1333 static std::unique_ptr<RuntimeDyldMachO>
1337 bool ProcessAllSections,
1339 std::unique_ptr<RuntimeDyldMachO> Dyld =
1341 Dyld->setProcessAllSections(ProcessAllSections);
1342 Dyld->setNotifyStubEmitted(
std::move(NotifyStubEmitted));
1346 std::unique_ptr<RuntimeDyld::LoadedObjectInfo>
1352 MemMgr,
Resolver, ProcessAllSections,
1357 ProcessAllSections,
std::move(NotifyStubEmitted));
1361 ProcessAllSections,
std::move(NotifyStubEmitted));
1366 if (!Dyld->isCompatibleFile(Obj))
1369 auto LoadedObjInfo = Dyld->loadObject(Obj);
1371 return LoadedObjInfo;
1377 return Dyld->getSymbolLocalAddress(Name);
1381 assert(Dyld &&
"No RuntimeDyld instance attached");
1382 return Dyld->getSymbolSectionID(Name);
1388 return Dyld->getSymbol(Name);
1393 return std::map<StringRef, JITEvaluatedSymbol>();
1394 return Dyld->getSymbolTable();
1400 Dyld->reassignSectionAddress(SectionID,
Addr);
1405 Dyld->mapSectionAddress(LocalAddress, TargetAddress);
1413 bool MemoryFinalizationLocked = MemMgr.FinalizationLocked;
1414 MemMgr.FinalizationLocked =
true;
1417 if (!MemoryFinalizationLocked) {
1419 MemMgr.FinalizationLocked =
false;
1424 assert(Dyld &&
"No Dyld instance attached");
1425 return Dyld->getSectionContent(SectionID);
1429 assert(Dyld &&
"No Dyld instance attached");
1430 return Dyld->getSectionLoadAddress(SectionID);
1435 Dyld->registerEHFrames();
1440 Dyld->deregisterEHFrames();
1448 bool ProcessAllSections,
1451 std::map<StringRef, JITEvaluatedSymbol>)>
1454 std::unique_ptr<RuntimeDyld::LoadedObjectInfo>,
Error)>