16#define DEBUG_TYPE "orc"
49 : Parent(Parent), I(
std::
move(I)) {}
53 return LHS.I == RHS.I;
57 return LHS.I != RHS.I;
70 assert(Parent->Graph &&
"No pointer to BlockDepInfoMap");
71 return &(*Parent->Graph)[*I];
94 switch (
TT.getArch()) {
119 std::unique_ptr<MaterializationResponsibility> MR,
120 std::unique_ptr<MemoryBuffer> ObjBuffer)
123 std::lock_guard<std::mutex> Lock(Layer.LayerMutex);
124 Plugins = Layer.Plugins;
130 if (Layer.ReturnObjectBuffer && ObjBuffer)
131 Layer.ReturnObjectBuffer(std::move(ObjBuffer));
137 for (
auto &
P : Plugins)
138 P->notifyMaterializing(*MR,
G, *
this,
139 ObjBuffer ? ObjBuffer->getMemBufferRef()
144 for (
auto &
P : Plugins)
145 Err =
joinErrors(std::move(Err),
P->notifyFailed(*MR));
146 Layer.getExecutionSession().reportError(std::move(Err));
147 MR->failMaterialization();
151 std::unique_ptr<JITLinkAsyncLookupContinuation> LC)
override {
154 MR->getTargetJITDylib().withLinkOrderDo(
157 auto &ES = Layer.getExecutionSession();
160 for (
auto &[Name, JLFlags] : Symbols)
161 LookupSet.
add(std::move(Name),
167 auto OnResolve = [LookupContinuation =
170 LookupContinuation->run(
Result.takeError());
174 LookupContinuation->run(std::move(LR));
182 for (
auto &[DepJD, Deps] : Deps)
183 for (
auto &DepSym : Deps)
191 bool AutoClaim = Layer.AutoClaimObjectSymbols;
194 for (
auto *Sym :
G.defined_symbols())
196 auto Ptr = getJITSymbolPtrForSymbol(*Sym,
G.getTargetTriple());
198 InternedResult[Sym->
getName()] = {Ptr, Flags};
199 if (AutoClaim && !MR->getSymbols().count(Sym->
getName())) {
201 "Duplicate symbol to claim?");
202 ExtraSymbolsToClaim[Sym->
getName()] = Flags;
206 for (
auto *Sym :
G.absolute_symbols())
208 auto Ptr = getJITSymbolPtrForSymbol(*Sym,
G.getTargetTriple());
210 InternedResult[Sym->
getName()] = {Ptr, Flags};
211 if (AutoClaim && !MR->getSymbols().count(Sym->
getName())) {
213 "Duplicate symbol to claim?");
214 ExtraSymbolsToClaim[Sym->
getName()] = Flags;
218 if (!ExtraSymbolsToClaim.
empty())
219 if (
auto Err = MR->defineMaterializing(ExtraSymbolsToClaim))
229 size_t NumMaterializationSideEffectsOnlySymbols = 0;
231 for (
auto &[Sym, Flags] : MR->getSymbols()) {
233 auto I = InternedResult.
find(Sym);
238 if (Flags.hasMaterializationSideEffectsOnly())
239 ++NumMaterializationSideEffectsOnlySymbols;
240 else if (
I == InternedResult.
end())
241 MissingSymbols.push_back(Sym);
242 else if (Layer.OverrideObjectFlags)
243 I->second.setFlags(Flags);
247 if (!MissingSymbols.empty())
249 Layer.getExecutionSession().getSymbolStringPool(),
G.getName(),
250 std::move(MissingSymbols));
255 if (InternedResult.
size() >
256 MR->getSymbols().size() - NumMaterializationSideEffectsOnlySymbols) {
257 for (
auto &KV : InternedResult)
258 if (!MR->getSymbols().count(KV.first))
259 ExtraSymbols.push_back(KV.first);
263 if (!ExtraSymbols.empty())
265 Layer.getExecutionSession().getSymbolStringPool(),
G.getName(),
266 std::move(ExtraSymbols));
269 if (
auto Err = MR->notifyResolved(InternedResult))
277 Layer.getExecutionSession().reportError(std::move(Err));
278 MR->failMaterialization();
282 if (
auto Err = MR->notifyEmitted(SymbolDepGroups)) {
283 Layer.getExecutionSession().reportError(std::move(Err));
284 MR->failMaterialization();
289 return [
this](
LinkGraph &
G) {
return markResponsibilitySymbolsLive(
G); };
296 return claimOrExternalizeWeakAndCommonSymbols(
G);
299 for (
auto &
P : Plugins)
300 P->modifyPassConfig(*MR, LG, Config);
303 [
this](
LinkGraph &
G) {
return registerDependencies(
G); });
310 for (
auto &
P : Plugins)
311 Err =
joinErrors(std::move(Err),
P->notifyEmitted(*MR));
316 joinErrors(std::move(Err), Layer.MemMgr.deallocate(std::move(FA)));
321 return Layer.recordFinalizedAlloc(*MR, std::move(FA));
329 std::vector<std::pair<SymbolStringPtr, Symbol *>> NameToSym;
331 auto ProcessSymbol = [&](
Symbol *Sym) {
334 if (!MR->getSymbols().count(Sym->
getName())) {
335 NewSymbolsToClaim[Sym->
getName()] =
337 NameToSym.push_back(std::make_pair(Sym->
getName(), Sym));
342 for (
auto *Sym :
G.defined_symbols())
344 for (
auto *Sym :
G.absolute_symbols())
350 if (
auto Err = MR->defineMaterializing(std::move(NewSymbolsToClaim)))
356 for (
auto &KV : NameToSym) {
357 if (MR->getSymbols().count(KV.first))
358 KV.second->setLive(
true);
360 G.makeExternal(*KV.second);
366 Error markResponsibilitySymbolsLive(LinkGraph &
G)
const {
367 for (
auto *Sym :
G.defined_symbols())
373 Error registerDependencies(LinkGraph &
G) {
374 auto &TargetJD = MR->getTargetJITDylib();
375 for (
auto &[Defs, Deps] : calculateDepGroups(
G)) {
376 SymbolDepGroups.push_back(SymbolDependenceGroup());
377 auto &SDG = SymbolDepGroups.back();
378 for (
auto *Def : Defs)
379 SDG.Symbols.insert(
Def->getName());
380 for (
auto *Dep : Deps) {
381 if (Dep->isDefined())
382 SDG.Dependencies[&TargetJD].insert(Dep->getName());
385 SymbolSourceJDs.find(NonOwningSymbolStringPtr(Dep->getName()));
386 if (
I != SymbolSourceJDs.end()) {
387 auto &SymJD = *
I->second;
388 SDG.Dependencies[&SymJD].insert(Dep->getName());
397 std::vector<std::shared_ptr<LinkGraphLinkingLayer::Plugin>> Plugins;
398 std::unique_ptr<MaterializationResponsibility> MR;
399 std::unique_ptr<MemoryBuffer> ObjBuffer;
400 DenseMap<NonOwningSymbolStringPtr, JITDylib *> SymbolSourceJDs;
401 std::vector<SymbolDependenceGroup> SymbolDepGroups;
409 ES.registerResourceManager(*
this);
420 "Layer destroyed with resources still attached "
421 "(ExecutionSession::endSession() must be called prior to "
427 std::unique_ptr<MaterializationResponsibility> R,
428 std::unique_ptr<LinkGraph>
G) {
429 assert(R &&
"R must not be null");
430 assert(
G &&
"G must not be null");
431 auto Ctx = std::make_unique<JITLinkCtx>(*
this, std::move(R),
nullptr);
432 Ctx->notifyMaterializing(*
G);
433 link(std::move(
G), std::move(Ctx));
437 std::unique_ptr<MaterializationResponsibility> R,
438 std::unique_ptr<LinkGraph>
G, std::unique_ptr<MemoryBuffer> ObjBuf) {
439 assert(R &&
"R must not be null");
440 assert(
G &&
"G must not be null");
441 assert(ObjBuf &&
"Object must not be null");
443 std::make_unique<JITLinkCtx>(*
this, std::move(R), std::move(ObjBuf));
444 Ctx->notifyMaterializing(*
G);
445 link(std::move(
G), std::move(Ctx));
449LinkGraphLinkingLayer::calculateDepGroups(
LinkGraph &
G) {
454 for (
auto *Sym :
G.defined_symbols())
456 BlockDepInfos[&Sym->
getBlock()].SymbolDefs.push_back(Sym);
468 for (
auto &[
B, BDInfo] : BlockDepInfos) {
469 BDInfo.Graph = &BlockDepInfos;
481 while (!Worklist.
empty()) {
483 BlockDepInfo *BDInfo =
nullptr;
485 for (
auto &
E :
B->edges()) {
486 if (
E.getTarget().isAbsolute())
490 BDInfo = &BlockDepInfos[
B];
492 if (
E.getTarget().isExternal()) {
498 auto *TgtB = &
E.getTarget().getBlock();
499 auto I = BlockDepInfos.find(TgtB);
501 if (
I != BlockDepInfos.end()) {
504 auto &TgtBInfo =
I->second;
505 if (!TgtBInfo.SymbolDefs.empty())
516 BlockDepInfos[TgtB].Graph = &BlockDepInfos;
527 for (
auto &[
B, BDInfo] : BlockDepInfos) {
530 auto &SCCRootInfo = *
SCC.front();
535 if (SCCRootInfo.SCCRoot)
537 SCCRootInfo.SCCRoot = &SCCRootInfo;
541 auto SCCSymbolDefs = std::move(SCCRootInfo.SymbolDefs);
542 auto SCCSymbolDeps = std::move(SCCRootInfo.SymbolDeps);
543 auto SCCAnonBlockDeps = std::move(SCCRootInfo.AnonBlockDeps);
545 SCCBInfo->SCCRoot = &SCCRootInfo;
546 SCCSymbolDefs.append(SCCBInfo->SymbolDefs);
547 SCCBInfo->SymbolDefs.clear();
548 SCCSymbolDeps.insert(SCCBInfo->SymbolDeps.begin(),
549 SCCBInfo->SymbolDeps.end());
550 SCCBInfo->SymbolDeps.clear();
551 SCCAnonBlockDeps.insert(SCCBInfo->AnonBlockDeps.begin(),
552 SCCBInfo->AnonBlockDeps.end());
553 SCCBInfo->AnonBlockDeps.clear();
558 DenseSet<size_t> SrcDepGroups;
559 for (
auto *DepB : SCCAnonBlockDeps) {
560 assert(BlockDepInfos.count(DepB) &&
"Unrecognized block");
561 auto &DepBRootInfo = *BlockDepInfos[DepB].SCCRoot;
562 if (DepBRootInfo.DepGroupIndex)
563 SrcDepGroups.
insert(*DepBRootInfo.DepGroupIndex);
568 if (SrcDepGroups.
empty()) {
572 if (!SCCSymbolDeps.empty()) {
573 SCCRootInfo.DepGroupIndex = DGs.
size();
575 DGs.
back().Defs = std::move(SCCSymbolDefs);
576 DGs.
back().Deps = std::move(SCCSymbolDeps);
585 if (SrcDepGroups.
size() == 1 && SCCSymbolDeps.empty()) {
586 SCCRootInfo.DepGroupIndex = *SrcDepGroups.
begin();
587 DGs[*SCCRootInfo.DepGroupIndex].Defs.
append(SCCSymbolDefs);
593 SCCRootInfo.DepGroupIndex = DGs.
size();
595 auto &DG = DGs.
back();
596 DG.Defs = std::move(SCCSymbolDefs);
597 for (
auto &DGIndex : SrcDepGroups)
598 DG.Deps.insert(DGs[DGIndex].Deps.begin(), DGs[DGIndex].Deps.end());
599 DG.Deps.insert(SCCSymbolDeps.begin(), SCCSymbolDeps.end());
605 for (
size_t I = 0;
I != DGs.
size();) {
609 for (
auto &Def : DG.Defs)
613 if (DG.Defs.empty() || DG.Deps.empty()) {
623Error LinkGraphLinkingLayer::recordFinalizedAlloc(
626 [&](
ResourceKey K) { Allocs[
K].push_back(std::move(FA)); });
629 Err =
joinErrors(std::move(Err), MemMgr.deallocate(std::move(FA)));
634Error LinkGraphLinkingLayer::handleRemoveResources(
JITDylib &JD,
639 for (
auto &
P : Plugins)
640 Err =
joinErrors(std::move(Err),
P->notifyRemovingResources(JD, K));
645 std::vector<FinalizedAlloc> AllocsToRemove;
647 auto I = Allocs.find(K);
648 if (
I != Allocs.end()) {
654 if (AllocsToRemove.empty())
657 return MemMgr.deallocate(std::move(AllocsToRemove));
660void LinkGraphLinkingLayer::handleTransferResources(
JITDylib &JD,
663 if (Allocs.contains(SrcKey)) {
666 auto &DstAllocs = Allocs[DstKey];
667 auto &SrcAllocs = Allocs[SrcKey];
668 DstAllocs.reserve(DstAllocs.size() + SrcAllocs.size());
669 for (
auto &
Alloc : SrcAllocs)
670 DstAllocs.push_back(std::move(
Alloc));
672 Allocs.erase(SrcKey);
675 for (
auto &
P : Plugins)
676 P->notifyTransferringResources(JD, DstKey, SrcKey);
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
This builds on the llvm/ADT/GraphTraits.h file to find the strongly connected components (SCCs) of a ...
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
iterator find(const_arg_type_t< KeyT > Val)
void insert_range(Range &&R)
Inserts a range of 'std::pair<KeyT, ValueT>' values into the map.
Implements a dense probed hash-table based set.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
friend bool operator!=(const ChildIteratorType &LHS, const ChildIteratorType &RHS)
ChildIteratorType(NodeRef Parent, impl_iterator I)
friend bool operator==(const ChildIteratorType &LHS, const ChildIteratorType &RHS)
ChildIteratorType & operator++()
ChildIteratorType operator++(int)
void reserve(size_type N)
iterator erase(const_iterator CI)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Triple - Helper class for working with autoconf configuration names.
std::pair< iterator, bool > insert(const ValueT &V)
DenseSetIterator< false > iterator
JITLinkContext(const JITLinkDylib *JD)
Create a JITLinkContext.
DenseMap< orc::SymbolStringPtr, SymbolLookupFlags > LookupMap
Represents a finalized allocation.
Manages allocations of JIT memory.
bool isCallable() const
Returns true is this symbol is callable.
const orc::SymbolStringPtr & getName() const
Returns the name of this symbol (empty if the symbol is anonymous).
Scope getScope() const
Get the visibility for this Symbol.
Linkage getLinkage() const
Get the linkage for this Symbol.
orc::ExecutorAddr getAddress() const
Returns the address of this symbol.
void setLive(bool IsLive)
Set this symbol's live bit.
Block & getBlock()
Return the Block for this Symbol (Symbol must be defined).
bool hasName() const
Returns true if this symbol has a name.
An ExecutionSession represents a running JIT program.
LLVM_ABI void registerResourceManager(ResourceManager &RM)
Register the given ResourceManager with this ExecutionSession.
LLVM_ABI void deregisterResourceManager(ResourceManager &RM)
Deregister the given ResourceManager with this ExecutionSession.
decltype(auto) runSessionLocked(Func &&F)
Run the given lambda with the session mutex locked.
Represents an address in the executor process.
uint64_t getValue() const
Represents a JIT'd dynamic library.
LinkGraphLayer(ExecutionSession &ES)
ExecutionSession & getExecutionSession()
static JITSymbolFlags getJITSymbolFlagsForSymbol(jitlink::Symbol &Sym)
Get the JITSymbolFlags for the given symbol.
JITLinkCtx(LinkGraphLinkingLayer &Layer, std::unique_ptr< MaterializationResponsibility > MR, std::unique_ptr< MemoryBuffer > ObjBuffer)
void notifyFailed(Error Err) override
Notify this context that linking failed.
void notifyFinalized(JITLinkMemoryManager::FinalizedAlloc A) override
Called by JITLink to notify the context that the object has been finalized (i.e.
JITLinkMemoryManager & getMemoryManager() override
Return the MemoryManager to be used for this link.
void lookup(LookupMap Symbols, std::unique_ptr< JITLinkAsyncLookupContinuation > LC) override
Called by JITLink to resolve external symbols.
Error notifyResolved(LinkGraph &G) override
Called by JITLink once all defined symbols in the graph have been assigned their final memory locatio...
LinkGraphPassFunction getMarkLivePass(const Triple &TT) const override
Returns the mark-live pass to be used for this link.
Error modifyPassConfig(LinkGraph &LG, PassConfiguration &Config) override
Called by JITLink to modify the pass pipeline prior to linking.
void notifyMaterializing(LinkGraph &G)
Error notifyEmitted(jitlink::JITLinkMemoryManager::FinalizedAlloc FA)
~LinkGraphLinkingLayer() override
Destroy the LinkGraphLinkingLayer.
void emit(std::unique_ptr< MaterializationResponsibility > R, std::unique_ptr< jitlink::LinkGraph > G) override
Emit a LinkGraph.
LinkGraphLinkingLayer(ExecutionSession &ES, jitlink::JITLinkMemoryManager &MemMgr)
Construct a LinkGraphLinkingLayer.
Tracks responsibility for materialization, and mediates interactions between MaterializationUnits and...
Error withResourceKeyDo(Func &&F) const
Runs the given callback under the session lock, passing in the associated ResourceKey.
Non-owning SymbolStringPool entry pointer.
A set of symbols to look up, each associated with a SymbolLookupFlags value.
SymbolLookupSet & add(SymbolStringPtr Name, SymbolLookupFlags Flags=SymbolLookupFlags::RequiredSymbol)
Add an element to the set.
LLVM_ABI bool hasTargetFlags(Symbol &Sym, TargetFlagsType Flags)
Check whether the given target flags are set for this Symbol.
unique_function< Error(LinkGraph &)> LinkGraphPassFunction
A function for mutating LinkGraphs.
LLVM_ABI void link(std::unique_ptr< LinkGraph > G, std::unique_ptr< JITLinkContext > Ctx)
Link the given graph.
DenseMap< orc::SymbolStringPtr, orc::ExecutorSymbolDef > AsyncLookupResult
A map of symbol names to resolved addresses.
std::vector< std::pair< JITDylib *, JITDylibLookupFlags > > JITDylibSearchOrder
A list of (JITDylib*, JITDylibLookupFlags) pairs to be used as a search order during symbol lookup.
DenseMap< SymbolStringPtr, ExecutorSymbolDef > SymbolMap
A map from symbol names (as SymbolStringPtrs) to JITSymbols (address/flags pairs).
std::vector< SymbolStringPtr > SymbolNameVector
A vector of symbol names.
DenseMap< JITDylib *, SymbolNameSet > SymbolDependenceMap
A map from JITDylibs to sets of symbols.
@ Resolved
Queried, materialization begun.
DenseMap< SymbolStringPtr, JITSymbolFlags > SymbolFlagsMap
A map from symbol names (as SymbolStringPtrs) to JITSymbolFlags.
NodeAddr< DefNode * > Def
This is an optimization pass for GlobalISel generic memory operations.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
scc_iterator< T > scc_begin(const T &G)
Construct the begin iterator for a deduced graph type T.
DenseMap< jitlink::Block *, BlockDepInfo > BlockDepInfoMap
Error joinErrors(Error E1, Error E2)
Concatenate errors.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
Error make_error(ArgTs &&... Args)
Make a Error instance representing failure using the given error info type.
scc_iterator< T > scc_end(const T &G)
Construct the end iterator for a deduced graph type T.
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Implement std::hash so that hash_code can be used in STL containers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
SmallVector< jitlink::Symbol * > SymbolDefList
AnonBlockDepSet AnonBlockDeps
DenseSet< jitlink::Symbol * > SymbolDepSet
std::optional< size_t > DepGroupIndex
DenseSet< jitlink::Block * > AnonBlockDepSet
static NodeRef getEntryNode(NodeRef N)
static ChildIteratorType child_end(NodeRef N)
static ChildIteratorType child_begin(NodeRef N)
An LinkGraph pass configuration, consisting of a list of pre-prune, post-prune, and post-fixup passes...
LinkGraphPassList PreFixupPasses
Pre-fixup passes.
LinkGraphPassList PrePrunePasses
Pre-prune passes.