16#define DEBUG_TYPE "orc"
25 switch (TT.getArch()) {
50 std::unique_ptr<MaterializationResponsibility> MR,
51 std::unique_ptr<MemoryBuffer> ObjBuffer)
54 std::lock_guard<std::mutex> Lock(Layer.LayerMutex);
55 Plugins = Layer.Plugins;
68 for (
auto &
P : Plugins)
69 P->notifyMaterializing(*MR,
G, *
this,
70 ObjBuffer ? ObjBuffer->getMemBufferRef()
75 for (
auto &
P : Plugins)
76 Err =
joinErrors(std::move(Err),
P->notifyFailed(*MR));
78 MR->failMaterialization();
82 std::unique_ptr<JITLinkAsyncLookupContinuation> LC)
override {
85 MR->getTargetJITDylib().withLinkOrderDo(
91 for (
auto &KV : Symbols) {
101 LookupSet.
add(KV.first, LookupFlags);
105 auto OnResolve = [LookupContinuation =
108 LookupContinuation->run(
Result.takeError());
112 LR[KV.first] = KV.second;
113 LookupContinuation->run(std::move(LR));
121 for (
auto &[DepJD, Deps] : Deps)
122 for (
auto &DepSym : Deps)
130 bool AutoClaim = Layer.AutoClaimObjectSymbols;
133 for (
auto *
Sym :
G.defined_symbols())
135 auto Ptr = getJITSymbolPtrForSymbol(*
Sym,
G.getTargetTriple());
138 if (AutoClaim && !MR->getSymbols().count(
Sym->
getName())) {
140 "Duplicate symbol to claim?");
145 for (
auto *
Sym :
G.absolute_symbols())
147 auto Ptr = getJITSymbolPtrForSymbol(*
Sym,
G.getTargetTriple());
150 if (AutoClaim && !MR->getSymbols().count(
Sym->
getName())) {
152 "Duplicate symbol to claim?");
157 if (!ExtraSymbolsToClaim.
empty())
158 if (
auto Err = MR->defineMaterializing(ExtraSymbolsToClaim))
168 size_t NumMaterializationSideEffectsOnlySymbols = 0;
170 for (
auto &[
Sym, Flags] : MR->getSymbols()) {
177 if (Flags.hasMaterializationSideEffectsOnly())
178 ++NumMaterializationSideEffectsOnlySymbols;
179 else if (
I == InternedResult.
end())
180 MissingSymbols.push_back(
Sym);
181 else if (Layer.OverrideObjectFlags)
182 I->second.setFlags(Flags);
186 if (!MissingSymbols.empty())
187 return make_error<MissingSymbolDefinitions>(
189 std::move(MissingSymbols));
194 if (InternedResult.
size() >
195 MR->getSymbols().size() - NumMaterializationSideEffectsOnlySymbols) {
196 for (
auto &KV : InternedResult)
197 if (!MR->getSymbols().count(KV.first))
198 ExtraSymbols.push_back(KV.first);
202 if (!ExtraSymbols.empty())
203 return make_error<UnexpectedSymbolDefinitions>(
205 std::move(ExtraSymbols));
208 if (
auto Err = MR->notifyResolved(InternedResult))
218 MR->failMaterialization();
222 if (
auto Err = MR->notifyEmitted(SymbolDepGroups)) {
224 MR->failMaterialization();
229 return [
this](
LinkGraph &
G) {
return markResponsibilitySymbolsLive(
G); };
236 return claimOrExternalizeWeakAndCommonSymbols(
G);
239 for (
auto &
P : Plugins)
240 P->modifyPassConfig(*MR, LG,
Config);
242 Config.PreFixupPasses.push_back(
243 [
this](
LinkGraph &
G) {
return registerDependencies(
G); });
249 for (
auto &
P : Plugins)
250 P->notifyLoaded(*MR);
255 for (
auto &
P : Plugins)
256 Err =
joinErrors(std::move(Err),
P->notifyEmitted(*MR));
266 return Layer.recordFinalizedAlloc(*MR, std::move(FA));
274 std::vector<std::pair<SymbolStringPtr, Symbol *>> NameToSym;
279 if (!MR->getSymbols().count(
Sym->
getName())) {
287 for (
auto *
Sym :
G.defined_symbols())
289 for (
auto *
Sym :
G.absolute_symbols())
295 if (
auto Err = MR->defineMaterializing(std::move(NewSymbolsToClaim)))
301 for (
auto &KV : NameToSym) {
302 if (MR->getSymbols().count(KV.first))
303 KV.second->setLive(
true);
305 G.makeExternal(*KV.second);
312 for (
auto *
Sym :
G.defined_symbols())
321 bool InWorklist =
false;
332 size_t NumBlocks = 0;
333 for (
auto &Sec :
G.sections())
334 NumBlocks += Sec.blocks_size();
339 for (
auto *
Sym :
G.defined_symbols()) {
345 for (
auto *
B :
G.blocks()) {
346 auto &BI = BlockInfos[
B];
348 for (
auto &E :
B->edges()) {
351 if (E.getTarget().isExternal()) {
352 BI.SymbolDeps.
insert(&E.getTarget());
358 if (E.getTarget().isAbsolute())
363 auto &TgtBI = BlockInfos[&E.getTarget().getBlock()];
369 if (!TgtBI.Defs.empty()) {
370 BI.SymbolDeps.
insert(*TgtBI.Defs.begin());
376 BI.AnonEdges.insert(&E.getTarget().getBlock());
377 TgtBI.AnonBackEdges.insert(
B);
383 std::vector<Block *> BlocksToRemove;
384 for (
auto &[
B, BI] : BlockInfos) {
386 if (!BI.Defs.empty())
389 BlocksToRemove.push_back(
B);
391 for (
auto *FB : BI.AnonEdges)
392 BlockInfos[FB].AnonBackEdges.erase(
B);
394 for (
auto *BB : BI.AnonBackEdges)
395 BlockInfos[BB].AnonEdges.erase(
B);
397 for (
auto *FB : BI.AnonEdges) {
398 auto &FBI = BlockInfos[FB];
399 for (
auto *BB : BI.AnonBackEdges)
400 FBI.AnonBackEdges.insert(BB);
403 for (
auto *BB : BI.AnonBackEdges) {
404 auto &BBI = BlockInfos[BB];
405 for (
auto *SD : BI.SymbolDeps)
406 BBI.SymbolDeps.insert(SD);
407 for (
auto *FB : BI.AnonEdges)
408 BBI.AnonEdges.insert(FB);
412 for (
auto *
B : BlocksToRemove)
417 std::deque<Block *> Worklist;
418 for (
auto &[
B, BI] : BlockInfos) {
419 if (!BI.SymbolDeps.empty() && !BI.AnonBackEdges.empty()) {
420 Worklist.push_back(
B);
421 BI.InWorklist =
true;
426 while (!Worklist.empty()) {
427 auto *
B = Worklist.front();
428 Worklist.pop_front();
430 auto &BI = BlockInfos[
B];
431 BI.InWorklist =
false;
433 for (
auto *DB : BI.AnonBackEdges) {
434 auto &DBI = BlockInfos[
DB];
435 for (
auto *
Sym : BI.SymbolDeps) {
436 if (DBI.SymbolDeps.insert(
Sym).second && !DBI.InWorklist) {
437 Worklist.push_back(DB);
438 DBI.InWorklist =
true;
447 auto &TargetJD = MR->getTargetJITDylib();
449 for (
auto &[
B, BI] : BlockInfos) {
450 if (!BI.Defs.empty()) {
452 auto &SDG = SymbolDepGroups.back();
454 for (
auto *Def : BI.Defs)
455 SDG.Symbols.insert(
Def->getName());
457 for (
auto *Dep : BI.SymbolDeps) {
458 auto DepName = Dep->getName();
459 if (Dep->isDefined())
460 SDG.Dependencies[&TargetJD].insert(std::move(DepName));
464 if (SourceJDItr != SymbolSourceJDs.end())
465 SDG.Dependencies[SourceJDItr->second].insert(std::move(DepName));
475 std::vector<std::shared_ptr<LinkGraphLinkingLayer::Plugin>> Plugins;
476 std::unique_ptr<MaterializationResponsibility> MR;
477 std::unique_ptr<MemoryBuffer> ObjBuffer;
479 std::vector<SymbolDependenceGroup> SymbolDepGroups;
485 :
LinkGraphLayer(ES), MemMgr(ES.getExecutorProcessControl().getMemMgr()) {
502 assert(Allocs.empty() &&
"Layer destroyed with resources still attached");
507 std::unique_ptr<MaterializationResponsibility> R,
508 std::unique_ptr<LinkGraph>
G) {
509 assert(R &&
"R must not be null");
510 assert(
G &&
"G must not be null");
511 auto Ctx = std::make_unique<JITLinkCtx>(*
this, std::move(R),
nullptr);
512 Ctx->notifyMaterializing(*
G);
513 link(std::move(
G), std::move(Ctx));
517 std::unique_ptr<MaterializationResponsibility> R,
518 std::unique_ptr<LinkGraph>
G, std::unique_ptr<MemoryBuffer> ObjBuf) {
519 assert(R &&
"R must not be null");
520 assert(
G &&
"G must not be null");
521 assert(ObjBuf &&
"Object must not be null");
523 std::make_unique<JITLinkCtx>(*
this, std::move(R), std::move(ObjBuf));
524 Ctx->notifyMaterializing(*
G);
525 link(std::move(
G), std::move(Ctx));
528Error LinkGraphLinkingLayer::recordFinalizedAlloc(
531 [&](
ResourceKey K) { Allocs[K].push_back(std::move(FA)); });
534 Err =
joinErrors(std::move(Err), MemMgr.deallocate(std::move(FA)));
539Error LinkGraphLinkingLayer::handleRemoveResources(
JITDylib &JD,
544 for (
auto &
P : Plugins)
545 Err =
joinErrors(std::move(Err),
P->notifyRemovingResources(JD, K));
550 std::vector<FinalizedAlloc> AllocsToRemove;
552 auto I = Allocs.find(K);
553 if (
I != Allocs.end()) {
554 std::swap(AllocsToRemove, I->second);
559 if (AllocsToRemove.empty())
562 return MemMgr.deallocate(std::move(AllocsToRemove));
565void LinkGraphLinkingLayer::handleTransferResources(
JITDylib &JD,
568 if (Allocs.contains(SrcKey)) {
571 auto &DstAllocs = Allocs[DstKey];
572 auto &SrcAllocs = Allocs[SrcKey];
573 DstAllocs.reserve(DstAllocs.size() + SrcAllocs.size());
574 for (
auto &
Alloc : SrcAllocs)
575 DstAllocs.push_back(std::move(
Alloc));
577 Allocs.erase(SrcKey);
580 for (
auto &
P : Plugins)
581 P->notifyTransferringResources(JD, DstKey, SrcKey);
586 : ES(ES), Registrar(
std::
move(Registrar)) {}
595 std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
596 assert(!InProcessLinks.count(&MR) &&
597 "Link for MR already being tracked?");
598 InProcessLinks[&MR] = {Addr, Size};
608 std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
610 auto EHFrameRangeItr = InProcessLinks.find(&MR);
611 if (EHFrameRangeItr == InProcessLinks.end())
614 EmittedRange = EHFrameRangeItr->second;
615 assert(EmittedRange.
Start &&
"eh-frame addr to register can not be null");
616 InProcessLinks.erase(EHFrameRangeItr);
620 [&](
ResourceKey K) { EHFrameRanges[K].push_back(EmittedRange); }))
623 return Registrar->registerEHFrames(EmittedRange);
628 std::lock_guard<std::mutex> Lock(EHFramePluginMutex);
629 InProcessLinks.erase(&MR);
635 std::vector<ExecutorAddrRange> RangesToRemove;
638 auto I = EHFrameRanges.find(K);
639 if (
I != EHFrameRanges.end()) {
640 RangesToRemove = std::move(I->second);
641 EHFrameRanges.erase(I);
646 while (!RangesToRemove.empty()) {
647 auto RangeToRemove = RangesToRemove.back();
648 RangesToRemove.pop_back();
649 assert(RangeToRemove.Start &&
"Untracked eh-frame range must not be null");
651 Registrar->deregisterEHFrames(RangeToRemove));
659 auto SI = EHFrameRanges.find(SrcKey);
660 if (SI == EHFrameRanges.end())
663 auto DI = EHFrameRanges.find(DstKey);
664 if (DI != EHFrameRanges.end()) {
665 auto &SrcRanges = SI->second;
666 auto &DstRanges = DI->second;
667 DstRanges.reserve(DstRanges.size() + SrcRanges.size());
668 for (
auto &SrcRange : SrcRanges)
669 DstRanges.push_back(std::move(SrcRange));
670 EHFrameRanges.erase(SI);
674 auto Tmp = std::move(SI->second);
675 EHFrameRanges.erase(SI);
676 EHFrameRanges[DstKey] = std::move(Tmp);
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
iterator find(const_arg_type_t< KeyT > Val)
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
void reserve(size_type NumEntries)
Grow the densemap so that it can contain at least NumEntries items before resizing again.
Implements a dense probed hash-table based set.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
Triple - Helper class for working with autoconf configuration names.
Holds context for a single jitLink invocation.
Represents a finalized allocation.
Manages allocations of JIT memory.
virtual void deallocate(std::vector< FinalizedAlloc > Allocs, OnDeallocatedFunction OnDeallocated)=0
Deallocate a list of allocation objects.
bool isCallable() const
Returns true is this symbol is callable.
const orc::SymbolStringPtr & getName() const
Returns the name of this symbol (empty if the symbol is anonymous).
Scope getScope() const
Get the visibility for this Symbol.
Linkage getLinkage() const
Get the linkage for this Symbol.
orc::ExecutorAddr getAddress() const
Returns the address of this symbol.
void setLive(bool IsLive)
Set this symbol's live bit.
Block & getBlock()
Return the Block for this Symbol (Symbol must be defined).
bool hasName() const
Returns true if this symbol has a name.
void notifyTransferringResources(JITDylib &JD, ResourceKey DstKey, ResourceKey SrcKey) override
Error notifyEmitted(MaterializationResponsibility &MR) override
Error notifyFailed(MaterializationResponsibility &MR) override
EHFrameRegistrationPlugin(ExecutionSession &ES, std::unique_ptr< jitlink::EHFrameRegistrar > Registrar)
Error notifyRemovingResources(JITDylib &JD, ResourceKey K) override
void modifyPassConfig(MaterializationResponsibility &MR, jitlink::LinkGraph &G, jitlink::PassConfiguration &PassConfig) override
An ExecutionSession represents a running JIT program.
void reportError(Error Err)
Report a error for this execution session.
std::shared_ptr< SymbolStringPool > getSymbolStringPool()
Get the SymbolStringPool for this instance.
void registerResourceManager(ResourceManager &RM)
Register the given ResourceManager with this ExecutionSession.
void deregisterResourceManager(ResourceManager &RM)
Deregister the given ResourceManager with this ExecutionSession.
decltype(auto) runSessionLocked(Func &&F)
Run the given lambda with the session mutex locked.
Represents an address in the executor process.
uint64_t getValue() const
Represents a JIT'd dynamic library.
ExecutionSession & getExecutionSession()
static JITSymbolFlags getJITSymbolFlagsForSymbol(jitlink::Symbol &Sym)
Get the JITSymbolFlags for the given symbol.
JITLinkCtx(LinkGraphLinkingLayer &Layer, std::unique_ptr< MaterializationResponsibility > MR, std::unique_ptr< MemoryBuffer > ObjBuffer)
void notifyFailed(Error Err) override
Notify this context that linking failed.
void notifyFinalized(JITLinkMemoryManager::FinalizedAlloc A) override
Called by JITLink to notify the context that the object has been finalized (i.e.
JITLinkMemoryManager & getMemoryManager() override
Return the MemoryManager to be used for this link.
Error notifyResolved(LinkGraph &G) override
Called by JITLink once all defined symbols in the graph have been assigned their final memory locatio...
void lookup(const LookupMap &Symbols, std::unique_ptr< JITLinkAsyncLookupContinuation > LC) override
Called by JITLink to resolve external symbols.
LinkGraphPassFunction getMarkLivePass(const Triple &TT) const override
Returns the mark-live pass to be used for this link.
Error modifyPassConfig(LinkGraph &LG, PassConfiguration &Config) override
Called by JITLink to modify the pass pipeline prior to linking.
void notifyMaterializing(LinkGraph &G)
Error notifyEmitted(jitlink::JITLinkMemoryManager::FinalizedAlloc FA)
LinkGraphLinkingLayer links LinkGraphs into the Executor using JITLink.
void emit(std::unique_ptr< MaterializationResponsibility > R, std::unique_ptr< jitlink::LinkGraph > G) override
Emit a LinkGraph.
std::function< void(std::unique_ptr< MemoryBuffer >)> ReturnObjectBuffer
~LinkGraphLinkingLayer()
Destroy the LinkGraphLinkingLayer.
LinkGraphLinkingLayer(ExecutionSession &ES)
Construct a LinkGraphLinkingLayer using the ExecutorProcessControl instance's memory manager.
Tracks responsibility for materialization, and mediates interactions between MaterializationUnits and...
Error withResourceKeyDo(Func &&F) const
Runs the given callback under the session lock, passing in the associated ResourceKey.
Non-owning SymbolStringPool entry pointer.
A set of symbols to look up, each associated with a SymbolLookupFlags value.
SymbolLookupSet & add(SymbolStringPtr Name, SymbolLookupFlags Flags=SymbolLookupFlags::RequiredSymbol)
Add an element to the set.
unique_function is a type-erasing functor similar to std::function.
LinkGraphPassFunction createEHFrameRecorderPass(const Triple &TT, StoreFrameRangeFunction StoreFrameRange)
Creates a pass that records the address and size of the EH frame section.
void link(std::unique_ptr< LinkGraph > G, std::unique_ptr< JITLinkContext > Ctx)
Link the given graph.
std::vector< std::pair< JITDylib *, JITDylibLookupFlags > > JITDylibSearchOrder
A list of (JITDylib*, JITDylibLookupFlags) pairs to be used as a search order during symbol lookup.
SymbolLookupFlags
Lookup flags that apply to each symbol in a lookup.
std::vector< SymbolStringPtr > SymbolNameVector
A vector of symbol names.
@ Resolved
Queried, materialization begun.
NodeAddr< DefNode * > Def
This is an optimization pass for GlobalISel generic memory operations.
Error joinErrors(Error E1, Error E2)
Concatenate errors.
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Implement std::hash so that hash_code can be used in STL containers.
An LinkGraph pass configuration, consisting of a list of pre-prune, post-prune, and post-fixup passes...
LinkGraphPassList PostFixupPasses
Post-fixup passes.
Represents an address range in the exceutor process.
A set of symbols and the their dependencies.