19 #define DEBUG_TYPE "orc"
43 return SPSMachOJITDylibDepInfo::AsArgList::serialize(
OB, DDI.
Sealed,
49 return SPSMachOJITDylibDepInfo::AsArgList::deserialize(IB, DDI.
Sealed,
69 void materialize(std::unique_ptr<MaterializationResponsibility> R)
override {
73 MOP.getExecutionSession().getExecutorProcessControl().getTargetTriple();
75 switch (
TT.getArch()) {
85 auto G = std::make_unique<jitlink::LinkGraph>(
89 auto &HeaderBlock = createHeaderBlock(*
G, HeaderSection);
92 G->addDefinedSymbol(HeaderBlock, 0, *
R->getInitializerSymbol(),
95 for (
auto &
HS : AdditionalHeaderSymbols)
96 G->addDefinedSymbol(HeaderBlock,
HS.Offset,
HS.Name,
106 struct HeaderSymbol {
111 static constexpr HeaderSymbol AdditionalHeaderSymbols[] = {
112 {
"___mh_executable_header", 0}};
118 switch (
G.getTargetTriple().getArch()) {
139 auto HeaderContent =
G.allocateString(
140 StringRef(
reinterpret_cast<const char *
>(&Hdr),
sizeof(Hdr)));
142 return G.createContentBlock(HeaderSection, HeaderContent,
ExecutorAddr(), 8,
152 for (
auto &
HS : AdditionalHeaderSymbols)
163 constexpr MachOHeaderMaterializationUnit::HeaderSymbol
164 MachOHeaderMaterializationUnit::AdditionalHeaderSymbols[];
166 StringRef EHFrameSectionName =
"__TEXT,__eh_frame";
167 StringRef ModInitFuncSectionName =
"__DATA,__mod_init_func";
168 StringRef ObjCClassListSectionName =
"__DATA,__objc_classlist";
169 StringRef ObjCImageInfoSectionName =
"__DATA,__objc_image_info";
170 StringRef ObjCSelRefsSectionName =
"__DATA,__objc_selrefs";
171 StringRef Swift5ProtoSectionName =
"__TEXT,__swift5_proto";
172 StringRef Swift5ProtosSectionName =
"__TEXT,__swift5_protos";
173 StringRef Swift5TypesSectionName =
"__TEXT,__swift5_types";
174 StringRef ThreadBSSSectionName =
"__DATA,__thread_bss";
175 StringRef ThreadDataSectionName =
"__DATA,__thread_data";
176 StringRef ThreadVarsSectionName =
"__DATA,__thread_vars";
179 ModInitFuncSectionName, ObjCSelRefsSectionName, ObjCClassListSectionName,
180 Swift5ProtosSectionName, Swift5ProtoSectionName, Swift5TypesSectionName};
189 JITDylib &PlatformJD,
const char *OrcRuntimePath,
195 if (!supportedTarget(EPC.getTargetTriple()))
196 return make_error<StringError>(
"Unsupported MachOPlatform triple: " +
197 EPC.getTargetTriple().str(),
202 RuntimeAliases = standardPlatformAliases(ES);
210 {{ES.
intern(
"___orc_rt_jit_dispatch"),
211 {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(),
213 {ES.
intern(
"___orc_rt_jit_dispatch_ctx"),
214 {EPC.getJITDispatchInfo().JITDispatchContext.getValue(),
220 ObjLinkingLayer, OrcRuntimePath, EPC.getTargetTriple());
221 if (!OrcRuntimeArchiveGenerator)
222 return OrcRuntimeArchiveGenerator.takeError();
226 auto P = std::unique_ptr<MachOPlatform>(
228 std::move(*OrcRuntimeArchiveGenerator), Err));
235 if (
auto Err = JD.
define(std::make_unique<MachOHeaderMaterializationUnit>(
236 *
this, MachOHeaderStartSymbol)))
239 return ES.lookup({&JD}, MachOHeaderStartSymbol).takeError();
243 std::lock_guard<std::mutex> Lock(PlatformMutex);
244 auto I = JITDylibToHeaderAddr.find(&JD);
245 if (
I != JITDylibToHeaderAddr.end()) {
246 assert(HeaderAddrToJITDylib.count(
I->second) &&
247 "HeaderAddrToJITDylib missing entry");
248 HeaderAddrToJITDylib.erase(
I->second);
249 JITDylibToHeaderAddr.erase(
I);
251 JITDylibToPThreadKey.erase(&JD);
262 RegisteredInitSymbols[&JD].add(InitSym,
265 dbgs() <<
"MachOPlatform: Registered init symbol " << *InitSym <<
" for MU "
276 ArrayRef<std::pair<const char *, const char *>>
AL) {
277 for (
auto &KV :
AL) {
278 auto AliasName = ES.
intern(KV.first);
279 assert(!Aliases.
count(AliasName) &&
"Duplicate symbol name in alias map");
287 addAliases(ES, Aliases, requiredCXXAliases());
288 addAliases(ES, Aliases, standardRuntimeUtilityAliases());
294 static const std::pair<const char *, const char *> RequiredCXXAliases[] = {
295 {
"___cxa_atexit",
"___orc_rt_macho_cxa_atexit"}};
302 static const std::pair<const char *, const char *>
303 StandardRuntimeUtilityAliases[] = {
304 {
"___orc_rt_run_program",
"___orc_rt_macho_run_program"},
305 {
"___orc_rt_jit_dlerror",
"___orc_rt_macho_jit_dlerror"},
306 {
"___orc_rt_jit_dlopen",
"___orc_rt_macho_jit_dlopen"},
307 {
"___orc_rt_jit_dlclose",
"___orc_rt_macho_jit_dlclose"},
308 {
"___orc_rt_jit_dlsym",
"___orc_rt_macho_jit_dlsym"},
309 {
"___orc_rt_log_error",
"___orc_rt_log_error_to_stderr"}};
312 StandardRuntimeUtilityAliases);
317 for (
auto &
Name : InitSectionNames) {
318 if (
Name.startswith(SegName) &&
Name.substr(7) == SectName)
324 bool MachOPlatform::supportedTarget(
const Triple &TT) {
325 switch (TT.getArch()) {
334 MachOPlatform::MachOPlatform(
337 std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator,
Error &Err)
338 : ES(ES), ObjLinkingLayer(ObjLinkingLayer),
339 MachOHeaderStartSymbol(ES.intern(
"___dso_handle")) {
342 ObjLinkingLayer.
addPlugin(std::make_unique<MachOPlatformPlugin>(*
this));
349 {{ES.
intern(
"___orc_rt_macho_register_ehframe_section"),
350 &orc_rt_macho_register_ehframe_section},
351 {ES.
intern(
"___orc_rt_macho_deregister_ehframe_section"),
352 &orc_rt_macho_deregister_ehframe_section}})) {
357 State = BootstrapPhase2;
360 if (
auto E2 = associateRuntimeSupportFunctions(PlatformJD)) {
368 if (
auto E2 = bootstrapMachORuntime(PlatformJD)) {
383 Error MachOPlatform::associateRuntimeSupportFunctions(
JITDylib &PlatformJD) {
386 using PushInitializersSPSSig =
388 WFs[ES.
intern(
"___orc_rt_macho_push_initializers_tag")] =
390 this, &MachOPlatform::rt_pushInitializers);
392 using LookupSymbolSPSSig =
394 WFs[ES.
intern(
"___orc_rt_macho_symbol_lookup_tag")] =
396 &MachOPlatform::rt_lookupSymbol);
401 void MachOPlatform::pushInitializersLoop(
402 PushInitializersSendResultFn SendResult,
JITDylibSP JD) {
408 while (!Worklist.empty()) {
411 auto DepJD = Worklist.back();
415 if (JDDepMap.count(DepJD))
419 auto &DM = JDDepMap[DepJD];
420 DepJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) {
422 if (KV.first == DepJD)
424 DM.push_back(KV.first);
425 Worklist.push_back(KV.first);
430 auto RISItr = RegisteredInitSymbols.find(DepJD);
431 if (RISItr != RegisteredInitSymbols.end()) {
432 NewInitSymbols[DepJD] =
std::move(RISItr->second);
433 RegisteredInitSymbols.erase(RISItr);
440 if (NewInitSymbols.
empty()) {
447 std::lock_guard<std::mutex> Lock(PlatformMutex);
448 for (
auto &KV : JDDepMap) {
449 auto I = JITDylibToHeaderAddr.find(KV.first);
450 if (
I == JITDylibToHeaderAddr.end()) {
455 make_error<StringError>(
"JITDylib " + KV.first->getName() +
456 " has no registered header address",
460 HeaderAddrs[KV.first] =
I->second;
465 MachOJITDylibDepInfoMap DIM;
466 DIM.reserve(JDDepMap.size());
467 for (
auto &KV : JDDepMap) {
468 assert(HeaderAddrs.
count(KV.first) &&
"Missing header addr");
469 auto H = HeaderAddrs[KV.first];
470 MachOJITDylibDepInfo DepInfo;
471 for (
auto &Dep : KV.second) {
472 assert(HeaderAddrs.
count(Dep) &&
"Missing header addr");
473 DepInfo.DepHeaders.push_back(HeaderAddrs[Dep]);
475 DIM.push_back(std::make_pair(
H,
std::move(DepInfo)));
482 lookupInitSymbolsAsync(
483 [
this, SendResult =
std::move(SendResult), &JD](
Error Err)
mutable {
487 pushInitializersLoop(
std::move(SendResult), JD);
492 void MachOPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult,
496 std::lock_guard<std::mutex> Lock(PlatformMutex);
497 auto I = HeaderAddrToJITDylib.find(JDHeaderAddr);
498 if (
I != HeaderAddrToJITDylib.end())
503 dbgs() <<
"MachOPlatform::rt_pushInitializers(" << JDHeaderAddr <<
") ";
505 dbgs() <<
"pushing initializers for " << JD->getName() <<
"\n";
507 dbgs() <<
"No JITDylib for header address.\n";
512 make_error<StringError>(
"No JITDylib with header addr " +
518 pushInitializersLoop(
std::move(SendResult), JD);
521 void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult,
524 dbgs() <<
"MachOPlatform::rt_lookupSymbol(\""
531 std::lock_guard<std::mutex> Lock(PlatformMutex);
532 auto I = HeaderAddrToJITDylib.find(Handle);
533 if (
I != HeaderAddrToJITDylib.end())
539 dbgs() <<
" No JITDylib for handle "
542 SendResult(make_error<StringError>(
"No JITDylib associated with handle " +
549 class RtLookupNotifyComplete {
551 RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult)
552 : SendResult(
std::
move(SendResult)) {}
555 assert(
Result->size() == 1 &&
"Unexpected result map count");
558 SendResult(
Result.takeError());
563 SendSymbolAddressFn SendResult;
569 LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}},
574 Error MachOPlatform::bootstrapMachORuntime(
JITDylib &PlatformJD) {
577 {{ES.
intern(
"___orc_rt_macho_platform_bootstrap"),
578 &orc_rt_macho_platform_bootstrap},
579 {ES.
intern(
"___orc_rt_macho_platform_shutdown"),
580 &orc_rt_macho_platform_shutdown},
581 {ES.
intern(
"___orc_rt_macho_register_jitdylib"),
582 &orc_rt_macho_register_jitdylib},
583 {ES.
intern(
"___orc_rt_macho_deregister_jitdylib"),
584 &orc_rt_macho_deregister_jitdylib},
585 {ES.
intern(
"___orc_rt_macho_register_object_platform_sections"),
586 &orc_rt_macho_register_object_platform_sections},
587 {ES.
intern(
"___orc_rt_macho_deregister_object_platform_sections"),
588 &orc_rt_macho_deregister_object_platform_sections},
589 {ES.
intern(
"___orc_rt_macho_create_pthread_key"),
590 &orc_rt_macho_create_pthread_key}}))
593 return ES.
callSPSWrapper<void()>(orc_rt_macho_platform_bootstrap);
597 if (!orc_rt_macho_create_pthread_key)
598 return make_error<StringError>(
599 "Attempting to create pthread key in target, but runtime support has "
600 "not been loaded yet",
605 orc_rt_macho_create_pthread_key, Result))
610 void MachOPlatform::MachOPlatformPlugin::modifyPassConfig(
614 auto PS = MP.State.load();
622 if (InitSymbol == MP.MachOHeaderStartSymbol) {
624 return associateJITDylibHeaderSymbol(
G, MR);
633 if (
auto Err = preserveInitSections(
G, MR))
635 return processObjCImageInfo(
G, MR);
640 if (PS == MachOPlatform::BootstrapPhase1) {
651 return fixTLVSectionsAndEdges(
G, JD);
658 return registerObjectPlatformSections(
G, JD);
663 MachOPlatform::MachOPlatformPlugin::getSyntheticSymbolDependencies(
665 std::lock_guard<std::mutex> Lock(PluginMutex);
666 auto I = InitSymbolDeps.find(&MR);
667 if (
I != InitSymbolDeps.end()) {
668 SyntheticSymbolDependenciesMap
Result;
670 InitSymbolDeps.erase(&MR);
673 return SyntheticSymbolDependenciesMap();
676 Error MachOPlatform::MachOPlatformPlugin::associateJITDylibHeaderSymbol(
679 return Sym->getName() == *MP.MachOHeaderStartSymbol;
681 assert(
I !=
G.defined_symbols().end() &&
"Missing MachO header start symbol");
684 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
685 auto HeaderAddr = (*I)->getAddress();
686 MP.JITDylibToHeaderAddr[&JD] = HeaderAddr;
687 MP.HeaderAddrToJITDylib[HeaderAddr] = &JD;
688 G.allocActions().push_back(
691 MP.orc_rt_macho_register_jitdylib, JD.getName(), HeaderAddr)),
693 MP.orc_rt_macho_deregister_jitdylib, HeaderAddr))});
697 Error MachOPlatform::MachOPlatformPlugin::preserveInitSections(
700 JITLinkSymbolSet InitSectionSymbols;
701 for (
auto &InitSectionName : InitSectionNames) {
703 auto *InitSection =
G.findSectionByName(InitSectionName);
710 for (
auto &Sym : InitSection->symbols()) {
711 auto &
B = Sym->getBlock();
712 if (Sym->isLive() && Sym->getOffset() == 0 &&
713 Sym->getSize() ==
B.getSize() && !AlreadyLiveBlocks.
count(&
B)) {
714 InitSectionSymbols.insert(Sym);
720 for (
auto *
B : InitSection->blocks())
721 if (!AlreadyLiveBlocks.
count(
B))
722 InitSectionSymbols.insert(
723 &
G.addAnonymousSymbol(*
B, 0,
B->getSize(),
false,
true));
726 if (!InitSectionSymbols.empty()) {
727 std::lock_guard<std::mutex> Lock(PluginMutex);
728 InitSymbolDeps[&MR] =
std::move(InitSectionSymbols);
734 Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo(
743 auto *ObjCImageInfo =
G.findSectionByName(ObjCImageInfoSectionName);
747 auto ObjCImageInfoBlocks = ObjCImageInfo->blocks();
751 return make_error<StringError>(
"Empty " + ObjCImageInfoSectionName +
752 " section in " +
G.getName(),
756 if (std::next(ObjCImageInfoBlocks.begin()) != ObjCImageInfoBlocks.end())
757 return make_error<StringError>(
"Multiple blocks in " +
758 ObjCImageInfoSectionName +
759 " section in " +
G.getName(),
764 for (
auto &Sec :
G.sections()) {
765 if (&Sec != ObjCImageInfo)
766 for (
auto *
B : Sec.blocks())
767 for (
auto &
E :
B->edges())
768 if (
E.getTarget().isDefined() &&
769 &
E.getTarget().getBlock().getSection() == ObjCImageInfo)
770 return make_error<StringError>(ObjCImageInfoSectionName +
771 " is referenced within file " +
776 auto &ObjCImageInfoBlock = **ObjCImageInfoBlocks.begin();
777 auto *ObjCImageInfoData = ObjCImageInfoBlock.getContent().data();
783 std::lock_guard<std::mutex> Lock(PluginMutex);
786 if (ObjCImageInfoItr != ObjCImageInfos.end()) {
789 if (ObjCImageInfoItr->second.first !=
Version)
790 return make_error<StringError>(
791 "ObjC version in " +
G.getName() +
792 " does not match first registered version",
794 if (ObjCImageInfoItr->second.second != Flags)
795 return make_error<StringError>(
"ObjC flags in " +
G.getName() +
796 " do not match first registered flags",
800 for (
auto *
S : ObjCImageInfo->symbols())
801 G.removeDefinedSymbol(*
S);
802 G.removeBlock(ObjCImageInfoBlock);
812 Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges(
816 for (
auto *Sym :
G.external_symbols())
817 if (Sym->getName() ==
"__tlv_bootstrap") {
818 Sym->setName(
"___orc_rt_macho_tlv_get_addr");
823 if (
auto *ThreadDataSec =
G.findSectionByName(ThreadVarsSectionName)) {
826 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
827 auto I = MP.JITDylibToPThreadKey.find(&JD);
828 if (
I != MP.JITDylibToPThreadKey.end())
833 if (
auto KeyOrErr = MP.createPThreadKey())
836 return KeyOrErr.takeError();
842 for (
auto *
B : ThreadDataSec->blocks()) {
843 if (
B->getSize() != 3 *
G.getPointerSize())
844 return make_error<StringError>(
"__thread_vars block at " +
846 " has unexpected size",
849 auto NewBlockContent =
G.allocateBuffer(
B->getSize());
850 llvm::copy(
B->getContent(), NewBlockContent.data());
851 memcpy(NewBlockContent.data() +
G.getPointerSize(), &PlatformKeyBits,
853 B->setContent(NewBlockContent);
858 for (
auto *
B :
G.blocks())
859 for (
auto &
E :
B->edges())
862 E.setKind(jitlink::x86_64::
868 Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections(
872 if (
auto *EHFrameSection =
G.findSectionByName(EHFrameSectionName)) {
875 G.allocActions().push_back(
878 MP.orc_rt_macho_register_ehframe_section,
R.getRange())),
881 MP.orc_rt_macho_deregister_ehframe_section,
R.getRange()))});
887 G.findSectionByName(ThreadDataSectionName);
890 if (
auto *ThreadBSSSection =
G.findSectionByName(ThreadBSSSectionName)) {
894 if (ThreadDataSection)
895 G.mergeSections(*ThreadDataSection, *ThreadBSSSection);
897 ThreadDataSection = ThreadBSSSection;
904 if (ThreadDataSection) {
907 if (MP.State != MachOPlatform::Initialized)
908 return make_error<StringError>(
"__thread_data section encountered, but "
909 "MachOPlatform has not finished booting",
912 MachOPlatformSecs.push_back({ThreadDataSectionName,
R.getRange()});
919 ModInitFuncSectionName, ObjCClassListSectionName,
920 ObjCImageInfoSectionName, ObjCSelRefsSectionName,
921 Swift5ProtoSectionName, Swift5ProtosSectionName,
922 Swift5TypesSectionName,
925 for (
auto &SecName : PlatformSections) {
926 auto *Sec =
G.findSectionByName(SecName);
933 MachOPlatformSecs.push_back({SecName,
R.getRange()});
936 if (!MachOPlatformSecs.empty()) {
939 std::lock_guard<std::mutex> Lock(MP.PlatformMutex);
940 auto I = MP.JITDylibToHeaderAddr.find(&JD);
941 if (
I != MP.JITDylibToHeaderAddr.end())
942 HeaderAddr =
I->second;
946 return make_error<StringError>(
"Missing header for " + JD.
getName(),
951 dbgs() <<
"MachOPlatform: Scraped " <<
G.getName() <<
" init sections:\n";
952 for (
auto &KV : MachOPlatformSecs)
953 dbgs() <<
" " << KV.first <<
": " << KV.second <<
"\n";
956 using SPSRegisterObjectPlatformSectionsArgs =
959 G.allocActions().push_back(
961 WrapperFunctionCall::Create<SPSRegisterObjectPlatformSectionsArgs>(
962 MP.orc_rt_macho_register_object_platform_sections, *HeaderAddr,
965 WrapperFunctionCall::Create<SPSRegisterObjectPlatformSectionsArgs>(
966 MP.orc_rt_macho_deregister_object_platform_sections,
967 *HeaderAddr, MachOPlatformSecs))});
973 Error MachOPlatform::MachOPlatformPlugin::registerEHSectionsPhase1(
977 auto *EHFrameSection =
G.findSectionByName(EHFrameSectionName);
991 for (
auto *Sym :
G.defined_symbols()) {
994 if (Sym->getName() ==
"___orc_rt_macho_register_ehframe_section")
995 orc_rt_macho_register_ehframe_section =
ExecutorAddr(Sym->getAddress());
996 else if (Sym->getName() ==
"___orc_rt_macho_deregister_ehframe_section")
997 orc_rt_macho_deregister_ehframe_section =
ExecutorAddr(Sym->getAddress());
999 if (orc_rt_macho_register_ehframe_section &&
1000 orc_rt_macho_deregister_ehframe_section)
1005 if (!orc_rt_macho_register_ehframe_section ||
1006 !orc_rt_macho_deregister_ehframe_section)
1007 return make_error<StringError>(
"Could not find eh-frame registration "
1008 "functions during platform bootstrap",
1013 G.allocActions().push_back(
1015 orc_rt_macho_register_ehframe_section,
R.getRange())),
1017 orc_rt_macho_deregister_ehframe_section,
R.getRange()))});