17 #define DEBUG_TYPE "orc"
27 static std::unique_ptr<LinkGraphMaterializationUnit>
29 auto LGI = scanLinkGraph(ObjLinkingLayer.getExecutionSession(), *
G);
30 return std::unique_ptr<LinkGraphMaterializationUnit>(
31 new LinkGraphMaterializationUnit(ObjLinkingLayer,
std::move(
G),
36 void materialize(std::unique_ptr<MaterializationResponsibility> MR)
override {
45 for (
auto *Sym :
G.defined_symbols()) {
49 assert(Sym->hasName() &&
"Anonymous non-local symbol?");
55 if (Sym->isCallable())
58 LGI.SymbolFlags[ES.
intern(Sym->getName())] = Flags;
61 if ((
G.getTargetTriple().isOSBinFormatMachO() && hasMachOInitSection(
G)) ||
62 (
G.getTargetTriple().isOSBinFormatELF() && hasELFInitSection(
G)))
63 LGI.InitSymbol = makeInitSymbol(ES,
G);
68 static bool hasMachOInitSection(
LinkGraph &
G) {
69 for (
auto &Sec :
G.sections())
70 if (Sec.getName() ==
"__DATA,__obj_selrefs" ||
71 Sec.getName() ==
"__DATA,__objc_classlist" ||
72 Sec.getName() ==
"__TEXT,__swift5_protos" ||
73 Sec.getName() ==
"__TEXT,__swift5_proto" ||
74 Sec.getName() ==
"__TEXT,__swift5_types" ||
75 Sec.getName() ==
"__DATA,__mod_init_func")
81 for (
auto &Sec :
G.sections()) {
82 auto SecName = Sec.getName();
83 if (SecName.consume_front(
".init_array") &&
84 (SecName.empty() || SecName[0] ==
'.'))
91 std::string InitSymString;
93 <<
"$." <<
G.getName() <<
".__inits" << Counter++;
94 return ES.
intern(InitSymString);
98 std::unique_ptr<LinkGraph>
G, Interface LGI)
103 for (
auto *Sym :
G->defined_symbols())
104 if (Sym->getName() == *
Name) {
106 "Discarding non-weak definition");
107 G->makeExternal(*Sym);
113 std::unique_ptr<LinkGraph>
G;
114 static std::atomic<uint64_t> Counter;
117 std::atomic<uint64_t> LinkGraphMaterializationUnit::Counter{0};
128 std::unique_ptr<MaterializationResponsibility> MR,
129 std::unique_ptr<MemoryBuffer> ObjBuffer)
136 if (Layer.ReturnObjectBuffer && ObjBuffer)
137 Layer.ReturnObjectBuffer(
std::move(ObjBuffer));
143 for (
auto &
P : Layer.Plugins)
144 P->notifyMaterializing(*MR,
G, *
this,
145 ObjBuffer ? ObjBuffer->getMemBufferRef()
150 for (
auto &
P : Layer.Plugins)
152 Layer.getExecutionSession().reportError(
std::move(Err));
153 MR->failMaterialization();
157 std::unique_ptr<JITLinkAsyncLookupContinuation> LC)
override {
160 MR->getTargetJITDylib().withLinkOrderDo(
163 auto &ES = Layer.getExecutionSession();
166 for (
auto &KV : Symbols) {
176 LookupSet.
add(ES.intern(KV.first), LookupFlags);
180 auto OnResolve = [LookupContinuation =
183 LookupContinuation->run(Result.takeError());
186 for (
auto &KV : *Result)
187 LR[*KV.first] = KV.second;
192 for (
auto &KV : InternalNamedSymbolDeps) {
194 InternalDeps[&MR->getTargetJITDylib()] =
std::move(KV.second);
195 MR->addDependencies(KV.first, InternalDeps);
199 SymbolState::Resolved,
std::move(OnResolve),
201 registerDependencies(Deps);
206 auto &ES = Layer.getExecutionSession();
209 bool AutoClaim = Layer.AutoClaimObjectSymbols;
212 for (
auto *Sym :
G.defined_symbols())
213 if (Sym->hasName() && Sym->getScope() !=
Scope::Local) {
214 auto InternedName = ES.intern(Sym->getName());
217 if (Sym->isCallable())
222 InternedResult[InternedName] =
224 if (AutoClaim && !MR->getSymbols().count(InternedName)) {
226 "Duplicate symbol to claim?");
227 ExtraSymbolsToClaim[InternedName] = Flags;
231 for (
auto *Sym :
G.absolute_symbols())
232 if (Sym->hasName() && Sym->getScope() !=
Scope::Local) {
233 auto InternedName = ES.intern(Sym->getName());
235 if (Sym->isCallable())
241 InternedResult[InternedName] =
243 if (AutoClaim && !MR->getSymbols().count(InternedName)) {
245 "Duplicate symbol to claim?");
246 ExtraSymbolsToClaim[InternedName] = Flags;
250 if (!ExtraSymbolsToClaim.
empty())
251 if (
auto Err = MR->defineMaterializing(ExtraSymbolsToClaim))
261 size_t NumMaterializationSideEffectsOnlySymbols = 0;
264 for (
auto &KV : MR->getSymbols()) {
266 auto I = InternedResult.
find(KV.first);
271 if (KV.second.hasMaterializationSideEffectsOnly()) {
272 ++NumMaterializationSideEffectsOnlySymbols;
273 if (
I != InternedResult.
end())
274 ExtraSymbols.push_back(KV.first);
276 }
else if (
I == InternedResult.
end())
277 MissingSymbols.push_back(KV.first);
278 else if (Layer.OverrideObjectFlags)
279 I->second.setFlags(KV.second);
283 if (!MissingSymbols.empty())
284 return make_error<MissingSymbolDefinitions>(
285 Layer.getExecutionSession().getSymbolStringPool(),
G.getName(),
290 if (InternedResult.
size() >
291 MR->getSymbols().size() - NumMaterializationSideEffectsOnlySymbols) {
292 for (
auto &KV : InternedResult)
293 if (!MR->getSymbols().count(KV.first))
294 ExtraSymbols.push_back(KV.first);
298 if (!ExtraSymbols.empty())
299 return make_error<UnexpectedSymbolDefinitions>(
300 Layer.getExecutionSession().getSymbolStringPool(),
G.getName(),
304 if (
auto Err = MR->notifyResolved(InternedResult))
307 Layer.notifyLoaded(*MR);
312 if (
auto Err = Layer.notifyEmitted(*MR,
std::move(A))) {
313 Layer.getExecutionSession().reportError(
std::move(Err));
314 MR->failMaterialization();
317 if (
auto Err = MR->notifyEmitted()) {
318 Layer.getExecutionSession().reportError(
std::move(Err));
319 MR->failMaterialization();
324 return [
this](
LinkGraph &
G) {
return markResponsibilitySymbolsLive(
G); };
331 return claimOrExternalizeWeakAndCommonSymbols(
G);
334 Layer.modifyPassConfig(*MR, LG, Config);
337 [
this](
LinkGraph &
G) {
return computeNamedSymbolDependencies(
G); });
346 struct BlockSymbolDependencies {
351 class BlockDependenciesMap {
355 : ES(ES), BlockDeps(
std::
move(BlockDeps)) {}
357 const BlockSymbolDependencies &operator[](
const Block &
B) {
359 auto I = BlockTransitiveDepsCache.find(&
B);
360 if (
I != BlockTransitiveDepsCache.end())
364 BlockSymbolDependencies BTDCacheVal;
365 auto BDI = BlockDeps.find(&
B);
366 assert(BDI != BlockDeps.end() &&
"No block dependencies");
368 for (
auto *BDep : BDI->second) {
369 auto &BID = getBlockImmediateDeps(*BDep);
370 for (
auto &ExternalDep : BID.External)
371 BTDCacheVal.External.insert(ExternalDep);
372 for (
auto &InternalDep : BID.Internal)
373 BTDCacheVal.Internal.insert(InternalDep);
376 return BlockTransitiveDepsCache
377 .insert(std::make_pair(&
B,
std::move(BTDCacheVal)))
382 auto I = NameCache.find(&Sym);
383 if (
I != NameCache.end())
386 return NameCache.insert(std::make_pair(&Sym, ES.intern(Sym.
getName())))
391 BlockSymbolDependencies &getBlockImmediateDeps(
Block &
B) {
393 auto I = BlockImmediateDepsCache.find(&
B);
394 if (
I != BlockImmediateDepsCache.end())
397 BlockSymbolDependencies BIDCacheVal;
398 for (
auto &
E :
B.edges()) {
399 auto &Tgt =
E.getTarget();
401 if (Tgt.isExternal())
402 BIDCacheVal.External.insert(getInternedName(Tgt));
404 BIDCacheVal.Internal.insert(getInternedName(Tgt));
408 return BlockImmediateDepsCache
409 .insert(std::make_pair(&
B,
std::move(BIDCacheVal)))
421 auto &ES =
Layer.getExecutionSession();
424 std::vector<std::pair<SymbolStringPtr, Symbol *>> NameToSym;
426 auto ProcessSymbol = [&](
Symbol *Sym) {
430 if (!MR->getSymbols().count(ES.intern(Sym->
getName()))) {
434 NewSymbolsToClaim[
Name] = SF;
440 for (
auto *Sym :
G.defined_symbols())
442 for (
auto *Sym :
G.absolute_symbols())
450 for (
auto &KV : NameToSym)
451 if (!MR->getSymbols().count(KV.first))
452 G.makeExternal(*KV.second);
458 auto &ES =
Layer.getExecutionSession();
459 for (
auto *Sym :
G.defined_symbols())
460 if (Sym->
hasName() && MR->getSymbols().count(ES.intern(Sym->
getName())))
466 auto &ES = MR->getTargetJITDylib().getExecutionSession();
467 auto BlockDeps = computeBlockNonLocalDeps(
G);
470 for (
auto *Sym :
G.defined_symbols()) {
476 "Defined non-local jitlink::Symbol should have a name");
478 auto &SymDeps = BlockDeps[Sym->
getBlock()];
479 if (SymDeps.External.empty() && SymDeps.Internal.empty())
482 auto SymName = ES.intern(Sym->
getName());
483 if (!SymDeps.External.empty())
484 ExternalNamedSymbolDeps[SymName] = SymDeps.External;
485 if (!SymDeps.Internal.empty())
486 InternalNamedSymbolDeps[SymName] = SymDeps.Internal;
489 for (
auto &
P :
Layer.Plugins) {
490 auto SynthDeps =
P->getSyntheticSymbolDependencies(*MR);
491 if (SynthDeps.empty())
495 for (
auto &KV : SynthDeps) {
496 auto &
Name = KV.first;
497 auto &DepsForName = KV.second;
498 for (
auto *Sym : DepsForName) {
500 auto &BDeps = BlockDeps[Sym->
getBlock()];
501 for (
auto &
S : BDeps.Internal)
502 InternalNamedSymbolDeps[
Name].insert(
S);
503 for (
auto &
S : BDeps.External)
504 ExternalNamedSymbolDeps[
Name].insert(
S);
507 ExternalNamedSymbolDeps[
Name].insert(
508 BlockDeps.getInternedName(*Sym));
510 InternalNamedSymbolDeps[
Name].insert(
511 BlockDeps.getInternedName(*Sym));
520 BlockDependenciesMap computeBlockNonLocalDeps(
LinkGraph &
G) {
525 bool DependenciesChanged =
true;
532 for (
auto *
B :
G.blocks())
537 for (
auto *
B :
G.blocks()) {
538 auto &BI = BlockInfos[
B];
539 for (
auto &
E :
B->edges()) {
541 auto &TgtB =
E.getTarget().getBlock();
543 BI.Dependencies.insert(&TgtB);
544 BlockInfos[&TgtB].Dependants.
insert(
B);
551 if (!BI.Dependants.empty() && !BI.Dependencies.empty())
552 WorkList.push_back(
B);
556 while (!WorkList.empty()) {
559 auto &BI = BlockInfos[
B];
560 assert(BI.DependenciesChanged &&
561 "Block in worklist has unchanged dependencies");
562 BI.DependenciesChanged =
false;
563 for (
auto *Dependant : BI.Dependants) {
564 auto &DependantBI = BlockInfos[Dependant];
565 for (
auto *Dependency : BI.Dependencies) {
566 if (Dependant != Dependency &&
567 DependantBI.Dependencies.insert(Dependency).second)
568 if (!DependantBI.DependenciesChanged) {
569 DependantBI.DependenciesChanged =
true;
570 WorkList.push_back(Dependant);
577 for (
auto &KV : BlockInfos)
578 BlockDeps[KV.first] =
std::move(KV.second.Dependencies);
580 return BlockDependenciesMap(
Layer.getExecutionSession(),
585 for (
auto &NamedDepsEntry : ExternalNamedSymbolDeps) {
586 auto &
Name = NamedDepsEntry.first;
587 auto &NameDeps = NamedDepsEntry.second;
590 for (
const auto &QueryDepsEntry : QueryDeps) {
591 JITDylib &SourceJD = *QueryDepsEntry.first;
593 auto &DepsForJD = SymbolDeps[&SourceJD];
595 for (
const auto &
S : Symbols)
596 if (NameDeps.count(
S))
599 if (DepsForJD.empty())
600 SymbolDeps.
erase(&SourceJD);
603 MR->addDependencies(
Name, SymbolDeps);
608 std::unique_ptr<MaterializationResponsibility> MR;
609 std::unique_ptr<MemoryBuffer> ObjBuffer;
614 ObjectLinkingLayer::Plugin::~Plugin() =
default;
621 :
BaseT(ES), MemMgr(ES.getExecutorProcessControl().getMemMgr()) {
627 :
BaseT(ES), MemMgr(MemMgr) {
633 :
BaseT(ES), MemMgr(*MemMgr), MemMgrOwnership(
std::
move(MemMgr)) {
638 assert(Allocs.empty() &&
"Layer destroyed with resources still attached");
639 getExecutionSession().deregisterResourceManager(*
this);
643 std::unique_ptr<LinkGraph>
G) {
644 auto &JD = RT->getJITDylib();
645 return JD.define(LinkGraphMaterializationUnit::Create(*
this,
std::move(
G)),
650 std::unique_ptr<MemoryBuffer>
O) {
651 assert(
O &&
"Object must not be null");
654 auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>(
657 Ctx->notifyMaterializing(**
G);
660 Ctx->notifyFailed(
G.takeError());
665 std::unique_ptr<LinkGraph>
G) {
666 auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>(
668 Ctx->notifyMaterializing(*
G);
675 for (
auto &
P : Plugins)
676 P->modifyPassConfig(MR,
G, PassConfig);
680 for (
auto &
P : Plugins)
687 for (
auto &
P : Plugins)
701 for (
auto &
P : Plugins)
707 std::vector<FinalizedAlloc> AllocsToRemove;
708 getExecutionSession().runSessionLocked([&] {
709 auto I = Allocs.find(K);
710 if (
I != Allocs.end()) {
711 std::swap(AllocsToRemove, I->second);
716 if (AllocsToRemove.empty())
719 return MemMgr.deallocate(
std::move(AllocsToRemove));
722 void ObjectLinkingLayer::handleTransferResources(
ResourceKey DstKey,
724 auto I = Allocs.find(SrcKey);
725 if (
I != Allocs.end()) {
726 auto &SrcAllocs =
I->second;
727 auto &DstAllocs = Allocs[DstKey];
728 DstAllocs.reserve(DstAllocs.size() + SrcAllocs.size());
729 for (
auto &Alloc : SrcAllocs)
734 Allocs.erase(SrcKey);
737 for (
auto &
P : Plugins)
738 P->notifyTransferringResources(DstKey, SrcKey);
743 : ES(ES), Registrar(
std::
move(Registrar)) {}
752 std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
753 assert(!InProcessLinks.count(&MR) &&
754 "Link for MR already being tracked?");
755 InProcessLinks[&MR] = {Addr, Size};
760 Error EHFrameRegistrationPlugin::notifyEmitted(
765 std::lock_guard<std::mutex>
Lock(EHFramePluginMutex);
767 auto EHFrameRangeItr = InProcessLinks.find(&MR);
768 if (EHFrameRangeItr == InProcessLinks.end())
771 EmittedRange = EHFrameRangeItr->second;
772 assert(EmittedRange.
Start &&
"eh-frame addr to register can not be null");
773 InProcessLinks.erase(EHFrameRangeItr);
777 [&](
ResourceKey K) { EHFrameRanges[K].push_back(EmittedRange); }))
780 return Registrar->registerEHFrames(EmittedRange);
783 Error EHFrameRegistrationPlugin::notifyFailed(
785 std::lock_guard<std::mutex>
Lock(EHFramePluginMutex);
786 InProcessLinks.erase(&MR);
791 std::vector<ExecutorAddrRange> RangesToRemove;
794 auto I = EHFrameRanges.find(K);
795 if (
I != EHFrameRanges.end()) {
796 RangesToRemove = std::move(I->second);
797 EHFrameRanges.erase(I);
802 while (!RangesToRemove.empty()) {
803 auto RangeToRemove = RangesToRemove.back();
804 RangesToRemove.pop_back();
805 assert(RangeToRemove.Start &&
"Untracked eh-frame range must not be null");
807 Registrar->deregisterEHFrames(RangeToRemove));
813 void EHFrameRegistrationPlugin::notifyTransferringResources(
815 auto SI = EHFrameRanges.find(SrcKey);
816 if (
SI == EHFrameRanges.end())
819 auto DI = EHFrameRanges.find(DstKey);
820 if (DI != EHFrameRanges.end()) {
821 auto &SrcRanges =
SI->second;
822 auto &DstRanges = DI->second;
823 DstRanges.reserve(DstRanges.size() + SrcRanges.size());
824 for (
auto &SrcRange : SrcRanges)
825 DstRanges.push_back(
std::move(SrcRange));
826 EHFrameRanges.erase(
SI);
831 EHFrameRanges.erase(
SI);