LLVM 20.0.0git
Core.cpp
Go to the documentation of this file.
1//===--- Core.cpp - Core ORC APIs (MaterializationUnit, JITDylib, etc.) ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
10
11#include "llvm/ADT/STLExtras.h"
12#include "llvm/Config/llvm-config.h"
17
18#include <condition_variable>
19#include <future>
20#include <optional>
21
22#define DEBUG_TYPE "orc"
23
24namespace llvm {
25namespace orc {
26
29char SymbolsNotFound::ID = 0;
35char LookupTask::ID = 0;
36
39
40void MaterializationUnit::anchor() {}
41
43 assert((reinterpret_cast<uintptr_t>(JD.get()) & 0x1) == 0 &&
44 "JITDylib must be two byte aligned");
45 JD->Retain();
46 JDAndFlag.store(reinterpret_cast<uintptr_t>(JD.get()));
47}
48
50 getJITDylib().getExecutionSession().destroyResourceTracker(*this);
52}
53
55 return getJITDylib().getExecutionSession().removeResourceTracker(*this);
56}
57
59 getJITDylib().getExecutionSession().transferResourceTracker(DstRT, *this);
60}
61
62void ResourceTracker::makeDefunct() {
63 uintptr_t Val = JDAndFlag.load();
64 Val |= 0x1U;
65 JDAndFlag.store(Val);
66}
67
69
71 : RT(std::move(RT)) {}
72
75}
76
78 OS << "Resource tracker " << (void *)RT.get() << " became defunct";
79}
80
82 std::shared_ptr<SymbolStringPool> SSP,
83 std::shared_ptr<SymbolDependenceMap> Symbols)
84 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
85 assert(this->SSP && "String pool cannot be null");
86 assert(!this->Symbols->empty() && "Can not fail to resolve an empty set");
87
88 // FIXME: Use a new dep-map type for FailedToMaterialize errors so that we
89 // don't have to manually retain/release.
90 for (auto &[JD, Syms] : *this->Symbols)
91 JD->Retain();
92}
93
95 for (auto &[JD, Syms] : *Symbols)
96 JD->Release();
97}
98
101}
102
104 OS << "Failed to materialize symbols: " << *Symbols;
105}
106
108 std::shared_ptr<SymbolStringPool> SSP, JITDylibSP JD,
109 SymbolNameSet FailedSymbols, SymbolDependenceMap BadDeps,
110 std::string Explanation)
111 : SSP(std::move(SSP)), JD(std::move(JD)),
112 FailedSymbols(std::move(FailedSymbols)), BadDeps(std::move(BadDeps)),
113 Explanation(std::move(Explanation)) {}
114
117}
118
120 OS << "In " << JD->getName() << ", failed to materialize " << FailedSymbols
121 << ", due to unsatisfied dependencies " << BadDeps;
122 if (!Explanation.empty())
123 OS << " (" << Explanation << ")";
124}
125
126SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
127 SymbolNameSet Symbols)
128 : SSP(std::move(SSP)) {
129 for (auto &Sym : Symbols)
130 this->Symbols.push_back(Sym);
131 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
132}
133
134SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
135 SymbolNameVector Symbols)
136 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
137 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
138}
139
140std::error_code SymbolsNotFound::convertToErrorCode() const {
142}
143
145 OS << "Symbols not found: " << Symbols;
146}
147
149 std::shared_ptr<SymbolStringPool> SSP, SymbolNameSet Symbols)
150 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
151 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
152}
153
156}
157
159 OS << "Symbols could not be removed: " << Symbols;
160}
161
164}
165
167 OS << "Missing definitions in module " << ModuleName
168 << ": " << Symbols;
169}
170
173}
174
176 OS << "Unexpected definitions in module " << ModuleName
177 << ": " << Symbols;
178}
179
181 const SymbolLookupSet &Symbols, SymbolState RequiredState,
182 SymbolsResolvedCallback NotifyComplete)
183 : NotifyComplete(std::move(NotifyComplete)), RequiredState(RequiredState) {
184 assert(RequiredState >= SymbolState::Resolved &&
185 "Cannot query for a symbols that have not reached the resolve state "
186 "yet");
187
188 OutstandingSymbolsCount = Symbols.size();
189
190 for (auto &[Name, Flags] : Symbols)
191 ResolvedSymbols[Name] = ExecutorSymbolDef();
192}
193
196 auto I = ResolvedSymbols.find(Name);
197 assert(I != ResolvedSymbols.end() &&
198 "Resolving symbol outside the requested set");
199 assert(I->second == ExecutorSymbolDef() &&
200 "Redundantly resolving symbol Name");
201
202 // If this is a materialization-side-effects-only symbol then drop it,
203 // otherwise update its map entry with its resolved address.
204 if (Sym.getFlags().hasMaterializationSideEffectsOnly())
205 ResolvedSymbols.erase(I);
206 else
207 I->second = std::move(Sym);
208 --OutstandingSymbolsCount;
209}
210
211void AsynchronousSymbolQuery::handleComplete(ExecutionSession &ES) {
212 assert(OutstandingSymbolsCount == 0 &&
213 "Symbols remain, handleComplete called prematurely");
214
215 class RunQueryCompleteTask : public Task {
216 public:
217 RunQueryCompleteTask(SymbolMap ResolvedSymbols,
218 SymbolsResolvedCallback NotifyComplete)
219 : ResolvedSymbols(std::move(ResolvedSymbols)),
220 NotifyComplete(std::move(NotifyComplete)) {}
221 void printDescription(raw_ostream &OS) override {
222 OS << "Execute query complete callback for " << ResolvedSymbols;
223 }
224 void run() override { NotifyComplete(std::move(ResolvedSymbols)); }
225
226 private:
227 SymbolMap ResolvedSymbols;
228 SymbolsResolvedCallback NotifyComplete;
229 };
230
231 auto T = std::make_unique<RunQueryCompleteTask>(std::move(ResolvedSymbols),
232 std::move(NotifyComplete));
233 NotifyComplete = SymbolsResolvedCallback();
234 ES.dispatchTask(std::move(T));
235}
236
237void AsynchronousSymbolQuery::handleFailed(Error Err) {
238 assert(QueryRegistrations.empty() && ResolvedSymbols.empty() &&
239 OutstandingSymbolsCount == 0 &&
240 "Query should already have been abandoned");
241 NotifyComplete(std::move(Err));
242 NotifyComplete = SymbolsResolvedCallback();
243}
244
245void AsynchronousSymbolQuery::addQueryDependence(JITDylib &JD,
246 SymbolStringPtr Name) {
247 bool Added = QueryRegistrations[&JD].insert(std::move(Name)).second;
248 (void)Added;
249 assert(Added && "Duplicate dependence notification?");
250}
251
252void AsynchronousSymbolQuery::removeQueryDependence(
253 JITDylib &JD, const SymbolStringPtr &Name) {
254 auto QRI = QueryRegistrations.find(&JD);
255 assert(QRI != QueryRegistrations.end() &&
256 "No dependencies registered for JD");
257 assert(QRI->second.count(Name) && "No dependency on Name in JD");
258 QRI->second.erase(Name);
259 if (QRI->second.empty())
260 QueryRegistrations.erase(QRI);
261}
262
263void AsynchronousSymbolQuery::dropSymbol(const SymbolStringPtr &Name) {
264 auto I = ResolvedSymbols.find(Name);
265 assert(I != ResolvedSymbols.end() &&
266 "Redundant removal of weakly-referenced symbol");
267 ResolvedSymbols.erase(I);
268 --OutstandingSymbolsCount;
269}
270
271void AsynchronousSymbolQuery::detach() {
272 ResolvedSymbols.clear();
273 OutstandingSymbolsCount = 0;
274 for (auto &[JD, Syms] : QueryRegistrations)
275 JD->detachQueryHelper(*this, Syms);
276 QueryRegistrations.clear();
277}
278
280 SymbolMap Symbols)
281 : MaterializationUnit(extractFlags(Symbols)), Symbols(std::move(Symbols)) {}
282
284 return "<Absolute Symbols>";
285}
286
287void AbsoluteSymbolsMaterializationUnit::materialize(
288 std::unique_ptr<MaterializationResponsibility> R) {
289 // Even though these are just absolute symbols we need to check for failure
290 // to resolve/emit: the tracker for these symbols may have been removed while
291 // the materialization was in flight (e.g. due to a failure in some action
292 // triggered by the queries attached to the resolution/emission of these
293 // symbols).
294 if (auto Err = R->notifyResolved(Symbols)) {
295 R->getExecutionSession().reportError(std::move(Err));
296 R->failMaterialization();
297 return;
298 }
299 if (auto Err = R->notifyEmitted({})) {
300 R->getExecutionSession().reportError(std::move(Err));
301 R->failMaterialization();
302 return;
303 }
304}
305
306void AbsoluteSymbolsMaterializationUnit::discard(const JITDylib &JD,
307 const SymbolStringPtr &Name) {
308 assert(Symbols.count(Name) && "Symbol is not part of this MU");
309 Symbols.erase(Name);
310}
311
312MaterializationUnit::Interface
313AbsoluteSymbolsMaterializationUnit::extractFlags(const SymbolMap &Symbols) {
315 for (const auto &[Name, Def] : Symbols)
316 Flags[Name] = Def.getFlags();
317 return MaterializationUnit::Interface(std::move(Flags), nullptr);
318}
319
321 JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags,
322 SymbolAliasMap Aliases)
323 : MaterializationUnit(extractFlags(Aliases)), SourceJD(SourceJD),
324 SourceJDLookupFlags(SourceJDLookupFlags), Aliases(std::move(Aliases)) {}
325
327 return "<Reexports>";
328}
329
330void ReExportsMaterializationUnit::materialize(
331 std::unique_ptr<MaterializationResponsibility> R) {
332
333 auto &ES = R->getTargetJITDylib().getExecutionSession();
334 JITDylib &TgtJD = R->getTargetJITDylib();
335 JITDylib &SrcJD = SourceJD ? *SourceJD : TgtJD;
336
337 // Find the set of requested aliases and aliasees. Return any unrequested
338 // aliases back to the JITDylib so as to not prematurely materialize any
339 // aliasees.
340 auto RequestedSymbols = R->getRequestedSymbols();
341 SymbolAliasMap RequestedAliases;
342
343 for (auto &Name : RequestedSymbols) {
344 auto I = Aliases.find(Name);
345 assert(I != Aliases.end() && "Symbol not found in aliases map?");
346 RequestedAliases[Name] = std::move(I->second);
347 Aliases.erase(I);
348 }
349
350 LLVM_DEBUG({
351 ES.runSessionLocked([&]() {
352 dbgs() << "materializing reexports: target = " << TgtJD.getName()
353 << ", source = " << SrcJD.getName() << " " << RequestedAliases
354 << "\n";
355 });
356 });
357
358 if (!Aliases.empty()) {
359 auto Err = SourceJD ? R->replace(reexports(*SourceJD, std::move(Aliases),
360 SourceJDLookupFlags))
361 : R->replace(symbolAliases(std::move(Aliases)));
362
363 if (Err) {
364 // FIXME: Should this be reported / treated as failure to materialize?
365 // Or should this be treated as a sanctioned bailing-out?
366 ES.reportError(std::move(Err));
367 R->failMaterialization();
368 return;
369 }
370 }
371
372 // The OnResolveInfo struct will hold the aliases and responsibility for each
373 // query in the list.
374 struct OnResolveInfo {
375 OnResolveInfo(std::unique_ptr<MaterializationResponsibility> R,
376 SymbolAliasMap Aliases)
377 : R(std::move(R)), Aliases(std::move(Aliases)) {}
378
379 std::unique_ptr<MaterializationResponsibility> R;
380 SymbolAliasMap Aliases;
381 std::vector<SymbolDependenceGroup> SDGs;
382 };
383
384 // Build a list of queries to issue. In each round we build a query for the
385 // largest set of aliases that we can resolve without encountering a chain of
386 // aliases (e.g. Foo -> Bar, Bar -> Baz). Such a chain would deadlock as the
387 // query would be waiting on a symbol that it itself had to resolve. Creating
388 // a new query for each link in such a chain eliminates the possibility of
389 // deadlock. In practice chains are likely to be rare, and this algorithm will
390 // usually result in a single query to issue.
391
392 std::vector<std::pair<SymbolLookupSet, std::shared_ptr<OnResolveInfo>>>
393 QueryInfos;
394 while (!RequestedAliases.empty()) {
395 SymbolNameSet ResponsibilitySymbols;
396 SymbolLookupSet QuerySymbols;
397 SymbolAliasMap QueryAliases;
398
399 // Collect as many aliases as we can without including a chain.
400 for (auto &KV : RequestedAliases) {
401 // Chain detected. Skip this symbol for this round.
402 if (&SrcJD == &TgtJD && (QueryAliases.count(KV.second.Aliasee) ||
403 RequestedAliases.count(KV.second.Aliasee)))
404 continue;
405
406 ResponsibilitySymbols.insert(KV.first);
407 QuerySymbols.add(KV.second.Aliasee,
408 KV.second.AliasFlags.hasMaterializationSideEffectsOnly()
411 QueryAliases[KV.first] = std::move(KV.second);
412 }
413
414 // Remove the aliases collected this round from the RequestedAliases map.
415 for (auto &KV : QueryAliases)
416 RequestedAliases.erase(KV.first);
417
418 assert(!QuerySymbols.empty() && "Alias cycle detected!");
419
420 auto NewR = R->delegate(ResponsibilitySymbols);
421 if (!NewR) {
422 ES.reportError(NewR.takeError());
423 R->failMaterialization();
424 return;
425 }
426
427 auto QueryInfo = std::make_shared<OnResolveInfo>(std::move(*NewR),
428 std::move(QueryAliases));
429 QueryInfos.push_back(
430 make_pair(std::move(QuerySymbols), std::move(QueryInfo)));
431 }
432
433 // Issue the queries.
434 while (!QueryInfos.empty()) {
435 auto QuerySymbols = std::move(QueryInfos.back().first);
436 auto QueryInfo = std::move(QueryInfos.back().second);
437
438 QueryInfos.pop_back();
439
440 auto RegisterDependencies = [QueryInfo,
441 &SrcJD](const SymbolDependenceMap &Deps) {
442 // If there were no materializing symbols, just bail out.
443 if (Deps.empty())
444 return;
445
446 // Otherwise the only deps should be on SrcJD.
447 assert(Deps.size() == 1 && Deps.count(&SrcJD) &&
448 "Unexpected dependencies for reexports");
449
450 auto &SrcJDDeps = Deps.find(&SrcJD)->second;
451
452 for (auto &[Alias, AliasInfo] : QueryInfo->Aliases)
453 if (SrcJDDeps.count(AliasInfo.Aliasee))
454 QueryInfo->SDGs.push_back({{Alias}, {{&SrcJD, {AliasInfo.Aliasee}}}});
455 };
456
457 auto OnComplete = [QueryInfo](Expected<SymbolMap> Result) {
458 auto &ES = QueryInfo->R->getTargetJITDylib().getExecutionSession();
459 if (Result) {
460 SymbolMap ResolutionMap;
461 for (auto &KV : QueryInfo->Aliases) {
462 assert((KV.second.AliasFlags.hasMaterializationSideEffectsOnly() ||
463 Result->count(KV.second.Aliasee)) &&
464 "Result map missing entry?");
465 // Don't try to resolve materialization-side-effects-only symbols.
466 if (KV.second.AliasFlags.hasMaterializationSideEffectsOnly())
467 continue;
468
469 ResolutionMap[KV.first] = {(*Result)[KV.second.Aliasee].getAddress(),
470 KV.second.AliasFlags};
471 }
472 if (auto Err = QueryInfo->R->notifyResolved(ResolutionMap)) {
473 ES.reportError(std::move(Err));
474 QueryInfo->R->failMaterialization();
475 return;
476 }
477 if (auto Err = QueryInfo->R->notifyEmitted(QueryInfo->SDGs)) {
478 ES.reportError(std::move(Err));
479 QueryInfo->R->failMaterialization();
480 return;
481 }
482 } else {
483 ES.reportError(Result.takeError());
484 QueryInfo->R->failMaterialization();
485 }
486 };
487
489 JITDylibSearchOrder({{&SrcJD, SourceJDLookupFlags}}),
490 QuerySymbols, SymbolState::Resolved, std::move(OnComplete),
491 std::move(RegisterDependencies));
492 }
493}
494
495void ReExportsMaterializationUnit::discard(const JITDylib &JD,
496 const SymbolStringPtr &Name) {
497 assert(Aliases.count(Name) &&
498 "Symbol not covered by this MaterializationUnit");
499 Aliases.erase(Name);
500}
501
502MaterializationUnit::Interface
503ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) {
505 for (auto &KV : Aliases)
506 SymbolFlags[KV.first] = KV.second.AliasFlags;
507
508 return MaterializationUnit::Interface(std::move(SymbolFlags), nullptr);
509}
510
512 SymbolNameSet Symbols) {
513 SymbolLookupSet LookupSet(Symbols);
514 auto Flags = SourceJD.getExecutionSession().lookupFlags(
516 SymbolLookupSet(std::move(Symbols)));
517
518 if (!Flags)
519 return Flags.takeError();
520
522 for (auto &Name : Symbols) {
523 assert(Flags->count(Name) && "Missing entry in flags map");
524 Result[Name] = SymbolAliasMapEntry(Name, (*Flags)[Name]);
525 }
526
527 return Result;
528}
529
531public:
532 // FIXME: Reduce the number of SymbolStringPtrs here. See
533 // https://github.com/llvm/llvm-project/issues/55576.
534
540 }
541 virtual ~InProgressLookupState() = default;
542 virtual void complete(std::unique_ptr<InProgressLookupState> IPLS) = 0;
543 virtual void fail(Error Err) = 0;
544
549
551 bool NewJITDylib = true;
554
555 enum {
556 NotInGenerator, // Not currently using a generator.
557 ResumedForGenerator, // Resumed after being auto-suspended before generator.
558 InGenerator // Currently using generator.
560 std::vector<std::weak_ptr<DefinitionGenerator>> CurDefGeneratorStack;
561};
562
564public:
570 OnComplete(std::move(OnComplete)) {}
571
572 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
573 auto &ES = SearchOrder.front().first->getExecutionSession();
574 ES.OL_completeLookupFlags(std::move(IPLS), std::move(OnComplete));
575 }
576
577 void fail(Error Err) override { OnComplete(std::move(Err)); }
578
579private:
581};
582
584public:
588 std::shared_ptr<AsynchronousSymbolQuery> Q,
589 RegisterDependenciesFunction RegisterDependencies)
592 Q(std::move(Q)), RegisterDependencies(std::move(RegisterDependencies)) {
593 }
594
595 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
596 auto &ES = SearchOrder.front().first->getExecutionSession();
597 ES.OL_completeLookup(std::move(IPLS), std::move(Q),
598 std::move(RegisterDependencies));
599 }
600
601 void fail(Error Err) override {
602 Q->detach();
603 Q->handleFailed(std::move(Err));
604 }
605
606private:
607 std::shared_ptr<AsynchronousSymbolQuery> Q;
608 RegisterDependenciesFunction RegisterDependencies;
609};
610
612 JITDylibLookupFlags SourceJDLookupFlags,
613 SymbolPredicate Allow)
614 : SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags),
615 Allow(std::move(Allow)) {}
616
618 JITDylib &JD,
619 JITDylibLookupFlags JDLookupFlags,
620 const SymbolLookupSet &LookupSet) {
621 assert(&JD != &SourceJD && "Cannot re-export from the same dylib");
622
623 // Use lookupFlags to find the subset of symbols that match our lookup.
624 auto Flags = JD.getExecutionSession().lookupFlags(
625 K, {{&SourceJD, JDLookupFlags}}, LookupSet);
626 if (!Flags)
627 return Flags.takeError();
628
629 // Create an alias map.
630 orc::SymbolAliasMap AliasMap;
631 for (auto &KV : *Flags)
632 if (!Allow || Allow(KV.first))
633 AliasMap[KV.first] = SymbolAliasMapEntry(KV.first, KV.second);
634
635 if (AliasMap.empty())
636 return Error::success();
637
638 // Define the re-exports.
639 return JD.define(reexports(SourceJD, AliasMap, SourceJDLookupFlags));
640}
641
642LookupState::LookupState(std::unique_ptr<InProgressLookupState> IPLS)
643 : IPLS(std::move(IPLS)) {}
644
645void LookupState::reset(InProgressLookupState *IPLS) { this->IPLS.reset(IPLS); }
646
647LookupState::LookupState() = default;
648LookupState::LookupState(LookupState &&) = default;
649LookupState &LookupState::operator=(LookupState &&) = default;
650LookupState::~LookupState() = default;
651
653 assert(IPLS && "Cannot call continueLookup on empty LookupState");
654 auto &ES = IPLS->SearchOrder.begin()->first->getExecutionSession();
655 ES.OL_applyQueryPhase1(std::move(IPLS), std::move(Err));
656}
657
659 std::deque<LookupState> LookupsToFail;
660 {
661 std::lock_guard<std::mutex> Lock(M);
662 std::swap(PendingLookups, LookupsToFail);
663 InUse = false;
664 }
665
666 for (auto &LS : LookupsToFail)
667 LS.continueLookup(make_error<StringError>(
668 "Query waiting on DefinitionGenerator that was destroyed",
670}
671
673 LLVM_DEBUG(dbgs() << "Destroying JITDylib " << getName() << "\n");
674}
675
677 std::vector<ResourceTrackerSP> TrackersToRemove;
678 ES.runSessionLocked([&]() {
679 assert(State != Closed && "JD is defunct");
680 for (auto &KV : TrackerSymbols)
681 TrackersToRemove.push_back(KV.first);
682 TrackersToRemove.push_back(getDefaultResourceTracker());
683 });
684
685 Error Err = Error::success();
686 for (auto &RT : TrackersToRemove)
687 Err = joinErrors(std::move(Err), RT->remove());
688 return Err;
689}
690
692 return ES.runSessionLocked([this] {
693 assert(State != Closed && "JD is defunct");
694 if (!DefaultTracker)
695 DefaultTracker = new ResourceTracker(this);
696 return DefaultTracker;
697 });
698}
699
701 return ES.runSessionLocked([this] {
702 assert(State == Open && "JD is defunct");
703 ResourceTrackerSP RT = new ResourceTracker(this);
704 return RT;
705 });
706}
707
709 // DefGenerator moved into TmpDG to ensure that it's destroyed outside the
710 // session lock (since it may have to send errors to pending queries).
711 std::shared_ptr<DefinitionGenerator> TmpDG;
712
713 ES.runSessionLocked([&] {
714 assert(State == Open && "JD is defunct");
715 auto I = llvm::find_if(DefGenerators,
716 [&](const std::shared_ptr<DefinitionGenerator> &H) {
717 return H.get() == &G;
718 });
719 assert(I != DefGenerators.end() && "Generator not found");
720 TmpDG = std::move(*I);
721 DefGenerators.erase(I);
722 });
723}
724
726JITDylib::defineMaterializing(MaterializationResponsibility &FromMR,
727 SymbolFlagsMap SymbolFlags) {
728
729 return ES.runSessionLocked([&]() -> Expected<SymbolFlagsMap> {
730 if (FromMR.RT->isDefunct())
731 return make_error<ResourceTrackerDefunct>(FromMR.RT);
732
733 std::vector<NonOwningSymbolStringPtr> AddedSyms;
734 std::vector<NonOwningSymbolStringPtr> RejectedWeakDefs;
735
736 for (auto SFItr = SymbolFlags.begin(), SFEnd = SymbolFlags.end();
737 SFItr != SFEnd; ++SFItr) {
738
739 auto &Name = SFItr->first;
740 auto &Flags = SFItr->second;
741
742 auto EntryItr = Symbols.find(Name);
743
744 // If the entry already exists...
745 if (EntryItr != Symbols.end()) {
746
747 // If this is a strong definition then error out.
748 if (!Flags.isWeak()) {
749 // Remove any symbols already added.
750 for (auto &S : AddedSyms)
751 Symbols.erase(Symbols.find_as(S));
752
753 // FIXME: Return all duplicates.
754 return make_error<DuplicateDefinition>(std::string(*Name));
755 }
756
757 // Otherwise just make a note to discard this symbol after the loop.
758 RejectedWeakDefs.push_back(NonOwningSymbolStringPtr(Name));
759 continue;
760 } else
761 EntryItr =
762 Symbols.insert(std::make_pair(Name, SymbolTableEntry(Flags))).first;
763
764 AddedSyms.push_back(NonOwningSymbolStringPtr(Name));
765 EntryItr->second.setState(SymbolState::Materializing);
766 }
767
768 // Remove any rejected weak definitions from the SymbolFlags map.
769 while (!RejectedWeakDefs.empty()) {
770 SymbolFlags.erase(SymbolFlags.find_as(RejectedWeakDefs.back()));
771 RejectedWeakDefs.pop_back();
772 }
773
774 return SymbolFlags;
775 });
776}
777
778Error JITDylib::replace(MaterializationResponsibility &FromMR,
779 std::unique_ptr<MaterializationUnit> MU) {
780 assert(MU != nullptr && "Can not replace with a null MaterializationUnit");
781 std::unique_ptr<MaterializationUnit> MustRunMU;
782 std::unique_ptr<MaterializationResponsibility> MustRunMR;
783
784 auto Err =
785 ES.runSessionLocked([&, this]() -> Error {
786 if (FromMR.RT->isDefunct())
787 return make_error<ResourceTrackerDefunct>(std::move(FromMR.RT));
788
789#ifndef NDEBUG
790 for (auto &KV : MU->getSymbols()) {
791 auto SymI = Symbols.find(KV.first);
792 assert(SymI != Symbols.end() && "Replacing unknown symbol");
793 assert(SymI->second.getState() == SymbolState::Materializing &&
794 "Can not replace a symbol that ha is not materializing");
795 assert(!SymI->second.hasMaterializerAttached() &&
796 "Symbol should not have materializer attached already");
797 assert(UnmaterializedInfos.count(KV.first) == 0 &&
798 "Symbol being replaced should have no UnmaterializedInfo");
799 }
800#endif // NDEBUG
801
802 // If the tracker is defunct we need to bail out immediately.
803
804 // If any symbol has pending queries against it then we need to
805 // materialize MU immediately.
806 for (auto &KV : MU->getSymbols()) {
807 auto MII = MaterializingInfos.find(KV.first);
808 if (MII != MaterializingInfos.end()) {
809 if (MII->second.hasQueriesPending()) {
810 MustRunMR = ES.createMaterializationResponsibility(
811 *FromMR.RT, std::move(MU->SymbolFlags),
812 std::move(MU->InitSymbol));
813 MustRunMU = std::move(MU);
814 return Error::success();
815 }
816 }
817 }
818
819 // Otherwise, make MU responsible for all the symbols.
820 auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU),
821 FromMR.RT.get());
822 for (auto &KV : UMI->MU->getSymbols()) {
823 auto SymI = Symbols.find(KV.first);
824 assert(SymI->second.getState() == SymbolState::Materializing &&
825 "Can not replace a symbol that is not materializing");
826 assert(!SymI->second.hasMaterializerAttached() &&
827 "Can not replace a symbol that has a materializer attached");
828 assert(UnmaterializedInfos.count(KV.first) == 0 &&
829 "Unexpected materializer entry in map");
830 SymI->second.setAddress(SymI->second.getAddress());
831 SymI->second.setMaterializerAttached(true);
832
833 auto &UMIEntry = UnmaterializedInfos[KV.first];
834 assert((!UMIEntry || !UMIEntry->MU) &&
835 "Replacing symbol with materializer still attached");
836 UMIEntry = UMI;
837 }
838
839 return Error::success();
840 });
841
842 if (Err)
843 return Err;
844
845 if (MustRunMU) {
846 assert(MustRunMR && "MustRunMU set implies MustRunMR set");
847 ES.dispatchTask(std::make_unique<MaterializationTask>(
848 std::move(MustRunMU), std::move(MustRunMR)));
849 } else {
850 assert(!MustRunMR && "MustRunMU unset implies MustRunMR unset");
851 }
852
853 return Error::success();
854}
855
856Expected<std::unique_ptr<MaterializationResponsibility>>
857JITDylib::delegate(MaterializationResponsibility &FromMR,
858 SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol) {
859
860 return ES.runSessionLocked(
861 [&]() -> Expected<std::unique_ptr<MaterializationResponsibility>> {
862 if (FromMR.RT->isDefunct())
863 return make_error<ResourceTrackerDefunct>(std::move(FromMR.RT));
864
865 return ES.createMaterializationResponsibility(
866 *FromMR.RT, std::move(SymbolFlags), std::move(InitSymbol));
867 });
868}
869
871JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const {
872 return ES.runSessionLocked([&]() {
873 SymbolNameSet RequestedSymbols;
874
875 for (auto &KV : SymbolFlags) {
876 assert(Symbols.count(KV.first) && "JITDylib does not cover this symbol?");
877 assert(Symbols.find(KV.first)->second.getState() !=
878 SymbolState::NeverSearched &&
879 Symbols.find(KV.first)->second.getState() != SymbolState::Ready &&
880 "getRequestedSymbols can only be called for symbols that have "
881 "started materializing");
882 auto I = MaterializingInfos.find(KV.first);
883 if (I == MaterializingInfos.end())
884 continue;
885
886 if (I->second.hasQueriesPending())
887 RequestedSymbols.insert(KV.first);
888 }
889
890 return RequestedSymbols;
891 });
892}
893
894Error JITDylib::resolve(MaterializationResponsibility &MR,
895 const SymbolMap &Resolved) {
896 AsynchronousSymbolQuerySet CompletedQueries;
897
898 if (auto Err = ES.runSessionLocked([&, this]() -> Error {
899 if (MR.RT->isDefunct())
900 return make_error<ResourceTrackerDefunct>(MR.RT);
901
902 if (State != Open)
903 return make_error<StringError>("JITDylib " + getName() +
904 " is defunct",
905 inconvertibleErrorCode());
906
907 struct WorklistEntry {
908 SymbolTable::iterator SymI;
909 ExecutorSymbolDef ResolvedSym;
910 };
911
912 SymbolNameSet SymbolsInErrorState;
913 std::vector<WorklistEntry> Worklist;
914 Worklist.reserve(Resolved.size());
915
916 // Build worklist and check for any symbols in the error state.
917 for (const auto &KV : Resolved) {
918
919 assert(!KV.second.getFlags().hasError() &&
920 "Resolution result can not have error flag set");
921
922 auto SymI = Symbols.find(KV.first);
923
924 assert(SymI != Symbols.end() && "Symbol not found");
925 assert(!SymI->second.hasMaterializerAttached() &&
926 "Resolving symbol with materializer attached?");
927 assert(SymI->second.getState() == SymbolState::Materializing &&
928 "Symbol should be materializing");
929 assert(SymI->second.getAddress() == ExecutorAddr() &&
930 "Symbol has already been resolved");
931
932 if (SymI->second.getFlags().hasError())
933 SymbolsInErrorState.insert(KV.first);
934 else {
935 if (SymI->second.getFlags() & JITSymbolFlags::Common) {
936 [[maybe_unused]] auto WeakOrCommon =
937 JITSymbolFlags::Weak | JITSymbolFlags::Common;
938 assert((KV.second.getFlags() & WeakOrCommon) &&
939 "Common symbols must be resolved as common or weak");
940 assert((KV.second.getFlags() & ~WeakOrCommon) ==
941 (SymI->second.getFlags() & ~JITSymbolFlags::Common) &&
942 "Resolving symbol with incorrect flags");
943
944 } else
945 assert(KV.second.getFlags() == SymI->second.getFlags() &&
946 "Resolved flags should match the declared flags");
947
948 Worklist.push_back(
949 {SymI, {KV.second.getAddress(), SymI->second.getFlags()}});
950 }
951 }
952
953 // If any symbols were in the error state then bail out.
954 if (!SymbolsInErrorState.empty()) {
955 auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>();
956 (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState);
957 return make_error<FailedToMaterialize>(
958 getExecutionSession().getSymbolStringPool(),
959 std::move(FailedSymbolsDepMap));
960 }
961
962 while (!Worklist.empty()) {
963 auto SymI = Worklist.back().SymI;
964 auto ResolvedSym = Worklist.back().ResolvedSym;
965 Worklist.pop_back();
966
967 auto &Name = SymI->first;
968
969 // Resolved symbols can not be weak: discard the weak flag.
970 JITSymbolFlags ResolvedFlags = ResolvedSym.getFlags();
971 SymI->second.setAddress(ResolvedSym.getAddress());
972 SymI->second.setFlags(ResolvedFlags);
973 SymI->second.setState(SymbolState::Resolved);
974
975 auto MII = MaterializingInfos.find(Name);
976 if (MII == MaterializingInfos.end())
977 continue;
978
979 auto &MI = MII->second;
980 for (auto &Q : MI.takeQueriesMeeting(SymbolState::Resolved)) {
981 Q->notifySymbolMetRequiredState(Name, ResolvedSym);
982 Q->removeQueryDependence(*this, Name);
983 if (Q->isComplete())
984 CompletedQueries.insert(std::move(Q));
985 }
986 }
987
988 return Error::success();
989 }))
990 return Err;
991
992 // Otherwise notify all the completed queries.
993 for (auto &Q : CompletedQueries) {
994 assert(Q->isComplete() && "Q not completed");
995 Q->handleComplete(ES);
996 }
997
998 return Error::success();
999}
1000
1001void JITDylib::unlinkMaterializationResponsibility(
1002 MaterializationResponsibility &MR) {
1003 ES.runSessionLocked([&]() {
1004 auto I = TrackerMRs.find(MR.RT.get());
1005 assert(I != TrackerMRs.end() && "No MRs in TrackerMRs list for RT");
1006 assert(I->second.count(&MR) && "MR not in TrackerMRs list for RT");
1007 I->second.erase(&MR);
1008 if (I->second.empty())
1009 TrackerMRs.erase(MR.RT.get());
1010 });
1011}
1012
1013void JITDylib::shrinkMaterializationInfoMemory() {
1014 // DenseMap::erase never shrinks its storage; use clear to heuristically free
1015 // memory since we may have long-lived JDs after linking is done.
1016
1017 if (UnmaterializedInfos.empty())
1018 UnmaterializedInfos.clear();
1019
1020 if (MaterializingInfos.empty())
1021 MaterializingInfos.clear();
1022}
1023
1024void JITDylib::setLinkOrder(JITDylibSearchOrder NewLinkOrder,
1025 bool LinkAgainstThisJITDylibFirst) {
1026 ES.runSessionLocked([&]() {
1027 assert(State == Open && "JD is defunct");
1028 if (LinkAgainstThisJITDylibFirst) {
1029 LinkOrder.clear();
1030 if (NewLinkOrder.empty() || NewLinkOrder.front().first != this)
1031 LinkOrder.push_back(
1032 std::make_pair(this, JITDylibLookupFlags::MatchAllSymbols));
1033 llvm::append_range(LinkOrder, NewLinkOrder);
1034 } else
1035 LinkOrder = std::move(NewLinkOrder);
1036 });
1037}
1038
1039void JITDylib::addToLinkOrder(const JITDylibSearchOrder &NewLinks) {
1040 ES.runSessionLocked([&]() {
1041 for (auto &KV : NewLinks) {
1042 // Skip elements of NewLinks that are already in the link order.
1043 if (llvm::is_contained(LinkOrder, KV))
1044 continue;
1045
1046 LinkOrder.push_back(std::move(KV));
1047 }
1048 });
1049}
1050
1051void JITDylib::addToLinkOrder(JITDylib &JD, JITDylibLookupFlags JDLookupFlags) {
1052 ES.runSessionLocked([&]() { LinkOrder.push_back({&JD, JDLookupFlags}); });
1053}
1054
1055void JITDylib::replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD,
1056 JITDylibLookupFlags JDLookupFlags) {
1057 ES.runSessionLocked([&]() {
1058 assert(State == Open && "JD is defunct");
1059 for (auto &KV : LinkOrder)
1060 if (KV.first == &OldJD) {
1061 KV = {&NewJD, JDLookupFlags};
1062 break;
1063 }
1064 });
1065}
1066
1067void JITDylib::removeFromLinkOrder(JITDylib &JD) {
1068 ES.runSessionLocked([&]() {
1069 assert(State == Open && "JD is defunct");
1070 auto I = llvm::find_if(LinkOrder,
1071 [&](const JITDylibSearchOrder::value_type &KV) {
1072 return KV.first == &JD;
1073 });
1074 if (I != LinkOrder.end())
1075 LinkOrder.erase(I);
1076 });
1077}
1078
1079Error JITDylib::remove(const SymbolNameSet &Names) {
1080 return ES.runSessionLocked([&]() -> Error {
1081 assert(State == Open && "JD is defunct");
1082 using SymbolMaterializerItrPair =
1083 std::pair<SymbolTable::iterator, UnmaterializedInfosMap::iterator>;
1084 std::vector<SymbolMaterializerItrPair> SymbolsToRemove;
1085 SymbolNameSet Missing;
1087
1088 for (auto &Name : Names) {
1089 auto I = Symbols.find(Name);
1090
1091 // Note symbol missing.
1092 if (I == Symbols.end()) {
1093 Missing.insert(Name);
1094 continue;
1095 }
1096
1097 // Note symbol materializing.
1098 if (I->second.getState() != SymbolState::NeverSearched &&
1099 I->second.getState() != SymbolState::Ready) {
1100 Materializing.insert(Name);
1101 continue;
1102 }
1103
1104 auto UMII = I->second.hasMaterializerAttached()
1105 ? UnmaterializedInfos.find(Name)
1106 : UnmaterializedInfos.end();
1107 SymbolsToRemove.push_back(std::make_pair(I, UMII));
1108 }
1109
1110 // If any of the symbols are not defined, return an error.
1111 if (!Missing.empty())
1112 return make_error<SymbolsNotFound>(ES.getSymbolStringPool(),
1113 std::move(Missing));
1114
1115 // If any of the symbols are currently materializing, return an error.
1116 if (!Materializing.empty())
1117 return make_error<SymbolsCouldNotBeRemoved>(ES.getSymbolStringPool(),
1118 std::move(Materializing));
1119
1120 // Remove the symbols.
1121 for (auto &SymbolMaterializerItrPair : SymbolsToRemove) {
1122 auto UMII = SymbolMaterializerItrPair.second;
1123
1124 // If there is a materializer attached, call discard.
1125 if (UMII != UnmaterializedInfos.end()) {
1126 UMII->second->MU->doDiscard(*this, UMII->first);
1127 UnmaterializedInfos.erase(UMII);
1128 }
1129
1130 auto SymI = SymbolMaterializerItrPair.first;
1131 Symbols.erase(SymI);
1132 }
1133
1134 shrinkMaterializationInfoMemory();
1135
1136 return Error::success();
1137 });
1138}
1139
1140void JITDylib::dump(raw_ostream &OS) {
1141 ES.runSessionLocked([&, this]() {
1142 OS << "JITDylib \"" << getName() << "\" (ES: "
1143 << format("0x%016" PRIx64, reinterpret_cast<uintptr_t>(&ES))
1144 << ", State = ";
1145 switch (State) {
1146 case Open:
1147 OS << "Open";
1148 break;
1149 case Closing:
1150 OS << "Closing";
1151 break;
1152 case Closed:
1153 OS << "Closed";
1154 break;
1155 }
1156 OS << ")\n";
1157 if (State == Closed)
1158 return;
1159 OS << "Link order: " << LinkOrder << "\n"
1160 << "Symbol table:\n";
1161
1162 // Sort symbols so we get a deterministic order and can check them in tests.
1163 std::vector<std::pair<SymbolStringPtr, SymbolTableEntry *>> SymbolsSorted;
1164 for (auto &KV : Symbols)
1165 SymbolsSorted.emplace_back(KV.first, &KV.second);
1166 std::sort(SymbolsSorted.begin(), SymbolsSorted.end(),
1167 [](const auto &L, const auto &R) { return *L.first < *R.first; });
1168
1169 for (auto &KV : SymbolsSorted) {
1170 OS << " \"" << *KV.first << "\": ";
1171 if (auto Addr = KV.second->getAddress())
1172 OS << Addr;
1173 else
1174 OS << "<not resolved> ";
1175
1176 OS << " " << KV.second->getFlags() << " " << KV.second->getState();
1177
1178 if (KV.second->hasMaterializerAttached()) {
1179 OS << " (Materializer ";
1180 auto I = UnmaterializedInfos.find(KV.first);
1181 assert(I != UnmaterializedInfos.end() &&
1182 "Lazy symbol should have UnmaterializedInfo");
1183 OS << I->second->MU.get() << ", " << I->second->MU->getName() << ")\n";
1184 } else
1185 OS << "\n";
1186 }
1187
1188 if (!MaterializingInfos.empty())
1189 OS << " MaterializingInfos entries:\n";
1190 for (auto &KV : MaterializingInfos) {
1191 OS << " \"" << *KV.first << "\":\n"
1192 << " " << KV.second.pendingQueries().size()
1193 << " pending queries: { ";
1194 for (const auto &Q : KV.second.pendingQueries())
1195 OS << Q.get() << " (" << Q->getRequiredState() << ") ";
1196 OS << "}\n Defining EDU: ";
1197 if (KV.second.DefiningEDU) {
1198 OS << KV.second.DefiningEDU.get() << " { ";
1199 for (auto &[Name, Flags] : KV.second.DefiningEDU->Symbols)
1200 OS << Name << " ";
1201 OS << "}\n";
1202 OS << " Dependencies:\n";
1203 if (!KV.second.DefiningEDU->Dependencies.empty()) {
1204 for (auto &[DepJD, Deps] : KV.second.DefiningEDU->Dependencies) {
1205 OS << " " << DepJD->getName() << ": [ ";
1206 for (auto &Dep : Deps)
1207 OS << Dep << " ";
1208 OS << "]\n";
1209 }
1210 } else
1211 OS << " none\n";
1212 } else
1213 OS << "none\n";
1214 OS << " Dependant EDUs:\n";
1215 if (!KV.second.DependantEDUs.empty()) {
1216 for (auto &DependantEDU : KV.second.DependantEDUs) {
1217 OS << " " << DependantEDU << ": "
1218 << DependantEDU->JD->getName() << " { ";
1219 for (auto &[Name, Flags] : DependantEDU->Symbols)
1220 OS << Name << " ";
1221 OS << "}\n";
1222 }
1223 } else
1224 OS << " none\n";
1225 assert((Symbols[KV.first].getState() != SymbolState::Ready ||
1226 (KV.second.pendingQueries().empty() && !KV.second.DefiningEDU &&
1227 !KV.second.DependantEDUs.empty())) &&
1228 "Stale materializing info entry");
1229 }
1230 });
1231}
1232
1233void JITDylib::MaterializingInfo::addQuery(
1234 std::shared_ptr<AsynchronousSymbolQuery> Q) {
1235
1236 auto I = llvm::lower_bound(
1237 llvm::reverse(PendingQueries), Q->getRequiredState(),
1238 [](const std::shared_ptr<AsynchronousSymbolQuery> &V, SymbolState S) {
1239 return V->getRequiredState() <= S;
1240 });
1241 PendingQueries.insert(I.base(), std::move(Q));
1242}
1243
1244void JITDylib::MaterializingInfo::removeQuery(
1245 const AsynchronousSymbolQuery &Q) {
1246 // FIXME: Implement 'find_as' for shared_ptr<T>/T*.
1247 auto I = llvm::find_if(
1248 PendingQueries, [&Q](const std::shared_ptr<AsynchronousSymbolQuery> &V) {
1249 return V.get() == &Q;
1250 });
1251 assert(I != PendingQueries.end() &&
1252 "Query is not attached to this MaterializingInfo");
1253 PendingQueries.erase(I);
1254}
1255
1256JITDylib::AsynchronousSymbolQueryList
1257JITDylib::MaterializingInfo::takeQueriesMeeting(SymbolState RequiredState) {
1258 AsynchronousSymbolQueryList Result;
1259 while (!PendingQueries.empty()) {
1260 if (PendingQueries.back()->getRequiredState() > RequiredState)
1261 break;
1262
1263 Result.push_back(std::move(PendingQueries.back()));
1264 PendingQueries.pop_back();
1265 }
1266
1267 return Result;
1268}
1269
1270JITDylib::JITDylib(ExecutionSession &ES, std::string Name)
1271 : JITLinkDylib(std::move(Name)), ES(ES) {
1272 LinkOrder.push_back({this, JITDylibLookupFlags::MatchAllSymbols});
1273}
1274
1275std::pair<JITDylib::AsynchronousSymbolQuerySet,
1276 std::shared_ptr<SymbolDependenceMap>>
1277JITDylib::IL_removeTracker(ResourceTracker &RT) {
1278 // Note: Should be called under the session lock.
1279 assert(State != Closed && "JD is defunct");
1280
1281 SymbolNameVector SymbolsToRemove;
1282 SymbolNameVector SymbolsToFail;
1283
1284 if (&RT == DefaultTracker.get()) {
1285 SymbolNameSet TrackedSymbols;
1286 for (auto &KV : TrackerSymbols)
1287 for (auto &Sym : KV.second)
1288 TrackedSymbols.insert(Sym);
1289
1290 for (auto &KV : Symbols) {
1291 auto &Sym = KV.first;
1292 if (!TrackedSymbols.count(Sym))
1293 SymbolsToRemove.push_back(Sym);
1294 }
1295
1296 DefaultTracker.reset();
1297 } else {
1298 /// Check for a non-default tracker.
1299 auto I = TrackerSymbols.find(&RT);
1300 if (I != TrackerSymbols.end()) {
1301 SymbolsToRemove = std::move(I->second);
1302 TrackerSymbols.erase(I);
1303 }
1304 // ... if not found this tracker was already defunct. Nothing to do.
1305 }
1306
1307 for (auto &Sym : SymbolsToRemove) {
1308 assert(Symbols.count(Sym) && "Symbol not in symbol table");
1309
1310 // If there is a MaterializingInfo then collect any queries to fail.
1311 auto MII = MaterializingInfos.find(Sym);
1312 if (MII != MaterializingInfos.end())
1313 SymbolsToFail.push_back(Sym);
1314 }
1315
1316 auto Result = ES.IL_failSymbols(*this, std::move(SymbolsToFail));
1317
1318 // Removed symbols should be taken out of the table altogether.
1319 for (auto &Sym : SymbolsToRemove) {
1320 auto I = Symbols.find(Sym);
1321 assert(I != Symbols.end() && "Symbol not present in table");
1322
1323 // Remove Materializer if present.
1324 if (I->second.hasMaterializerAttached()) {
1325 // FIXME: Should this discard the symbols?
1326 UnmaterializedInfos.erase(Sym);
1327 } else {
1328 assert(!UnmaterializedInfos.count(Sym) &&
1329 "Symbol has materializer attached");
1330 }
1331
1332 Symbols.erase(I);
1333 }
1334
1335 shrinkMaterializationInfoMemory();
1336
1337 return Result;
1338}
1339
1340void JITDylib::transferTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT) {
1341 assert(State != Closed && "JD is defunct");
1342 assert(&DstRT != &SrcRT && "No-op transfers shouldn't call transferTracker");
1343 assert(&DstRT.getJITDylib() == this && "DstRT is not for this JITDylib");
1344 assert(&SrcRT.getJITDylib() == this && "SrcRT is not for this JITDylib");
1345
1346 // Update trackers for any not-yet materialized units.
1347 for (auto &KV : UnmaterializedInfos) {
1348 if (KV.second->RT == &SrcRT)
1349 KV.second->RT = &DstRT;
1350 }
1351
1352 // Update trackers for any active materialization responsibilities.
1353 {
1354 auto I = TrackerMRs.find(&SrcRT);
1355 if (I != TrackerMRs.end()) {
1356 auto &SrcMRs = I->second;
1357 auto &DstMRs = TrackerMRs[&DstRT];
1358 for (auto *MR : SrcMRs)
1359 MR->RT = &DstRT;
1360 if (DstMRs.empty())
1361 DstMRs = std::move(SrcMRs);
1362 else
1363 for (auto *MR : SrcMRs)
1364 DstMRs.insert(MR);
1365 // Erase SrcRT entry in TrackerMRs. Use &SrcRT key rather than iterator I
1366 // for this, since I may have been invalidated by 'TrackerMRs[&DstRT]'.
1367 TrackerMRs.erase(&SrcRT);
1368 }
1369 }
1370
1371 // If we're transfering to the default tracker we just need to delete the
1372 // tracked symbols for the source tracker.
1373 if (&DstRT == DefaultTracker.get()) {
1374 TrackerSymbols.erase(&SrcRT);
1375 return;
1376 }
1377
1378 // If we're transferring from the default tracker we need to find all
1379 // currently untracked symbols.
1380 if (&SrcRT == DefaultTracker.get()) {
1381 assert(!TrackerSymbols.count(&SrcRT) &&
1382 "Default tracker should not appear in TrackerSymbols");
1383
1384 SymbolNameVector SymbolsToTrack;
1385
1386 SymbolNameSet CurrentlyTrackedSymbols;
1387 for (auto &KV : TrackerSymbols)
1388 for (auto &Sym : KV.second)
1389 CurrentlyTrackedSymbols.insert(Sym);
1390
1391 for (auto &KV : Symbols) {
1392 auto &Sym = KV.first;
1393 if (!CurrentlyTrackedSymbols.count(Sym))
1394 SymbolsToTrack.push_back(Sym);
1395 }
1396
1397 TrackerSymbols[&DstRT] = std::move(SymbolsToTrack);
1398 return;
1399 }
1400
1401 auto &DstTrackedSymbols = TrackerSymbols[&DstRT];
1402
1403 // Finally if neither SrtRT or DstRT are the default tracker then
1404 // just append DstRT's tracked symbols to SrtRT's.
1405 auto SI = TrackerSymbols.find(&SrcRT);
1406 if (SI == TrackerSymbols.end())
1407 return;
1408
1409 DstTrackedSymbols.reserve(DstTrackedSymbols.size() + SI->second.size());
1410 for (auto &Sym : SI->second)
1411 DstTrackedSymbols.push_back(std::move(Sym));
1412 TrackerSymbols.erase(SI);
1413}
1414
1415Error JITDylib::defineImpl(MaterializationUnit &MU) {
1416 LLVM_DEBUG({ dbgs() << " " << MU.getSymbols() << "\n"; });
1417
1418 SymbolNameSet Duplicates;
1419 std::vector<SymbolStringPtr> ExistingDefsOverridden;
1420 std::vector<SymbolStringPtr> MUDefsOverridden;
1421
1422 for (const auto &KV : MU.getSymbols()) {
1423 auto I = Symbols.find(KV.first);
1424
1425 if (I != Symbols.end()) {
1426 if (KV.second.isStrong()) {
1427 if (I->second.getFlags().isStrong() ||
1428 I->second.getState() > SymbolState::NeverSearched)
1429 Duplicates.insert(KV.first);
1430 else {
1431 assert(I->second.getState() == SymbolState::NeverSearched &&
1432 "Overridden existing def should be in the never-searched "
1433 "state");
1434 ExistingDefsOverridden.push_back(KV.first);
1435 }
1436 } else
1437 MUDefsOverridden.push_back(KV.first);
1438 }
1439 }
1440
1441 // If there were any duplicate definitions then bail out.
1442 if (!Duplicates.empty()) {
1443 LLVM_DEBUG(
1444 { dbgs() << " Error: Duplicate symbols " << Duplicates << "\n"; });
1445 return make_error<DuplicateDefinition>(std::string(**Duplicates.begin()));
1446 }
1447
1448 // Discard any overridden defs in this MU.
1449 LLVM_DEBUG({
1450 if (!MUDefsOverridden.empty())
1451 dbgs() << " Defs in this MU overridden: " << MUDefsOverridden << "\n";
1452 });
1453 for (auto &S : MUDefsOverridden)
1454 MU.doDiscard(*this, S);
1455
1456 // Discard existing overridden defs.
1457 LLVM_DEBUG({
1458 if (!ExistingDefsOverridden.empty())
1459 dbgs() << " Existing defs overridden by this MU: " << MUDefsOverridden
1460 << "\n";
1461 });
1462 for (auto &S : ExistingDefsOverridden) {
1463
1464 auto UMII = UnmaterializedInfos.find(S);
1465 assert(UMII != UnmaterializedInfos.end() &&
1466 "Overridden existing def should have an UnmaterializedInfo");
1467 UMII->second->MU->doDiscard(*this, S);
1468 }
1469
1470 // Finally, add the defs from this MU.
1471 for (auto &KV : MU.getSymbols()) {
1472 auto &SymEntry = Symbols[KV.first];
1473 SymEntry.setFlags(KV.second);
1474 SymEntry.setState(SymbolState::NeverSearched);
1475 SymEntry.setMaterializerAttached(true);
1476 }
1477
1478 return Error::success();
1479}
1480
1481void JITDylib::installMaterializationUnit(
1482 std::unique_ptr<MaterializationUnit> MU, ResourceTracker &RT) {
1483
1484 /// defineImpl succeeded.
1485 if (&RT != DefaultTracker.get()) {
1486 auto &TS = TrackerSymbols[&RT];
1487 TS.reserve(TS.size() + MU->getSymbols().size());
1488 for (auto &KV : MU->getSymbols())
1489 TS.push_back(KV.first);
1490 }
1491
1492 auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU), &RT);
1493 for (auto &KV : UMI->MU->getSymbols())
1494 UnmaterializedInfos[KV.first] = UMI;
1495}
1496
1497void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q,
1498 const SymbolNameSet &QuerySymbols) {
1499 for (auto &QuerySymbol : QuerySymbols) {
1500 assert(MaterializingInfos.count(QuerySymbol) &&
1501 "QuerySymbol does not have MaterializingInfo");
1502 auto &MI = MaterializingInfos[QuerySymbol];
1503 MI.removeQuery(Q);
1504 }
1505}
1506
1507Platform::~Platform() = default;
1508
1510 ExecutionSession &ES,
1511 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1512
1513 DenseMap<JITDylib *, SymbolMap> CompoundResult;
1514 Error CompoundErr = Error::success();
1515 std::mutex LookupMutex;
1516 std::condition_variable CV;
1517 uint64_t Count = InitSyms.size();
1518
1519 LLVM_DEBUG({
1520 dbgs() << "Issuing init-symbol lookup:\n";
1521 for (auto &KV : InitSyms)
1522 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1523 });
1524
1525 for (auto &KV : InitSyms) {
1526 auto *JD = KV.first;
1527 auto Names = std::move(KV.second);
1528 ES.lookup(
1531 std::move(Names), SymbolState::Ready,
1532 [&, JD](Expected<SymbolMap> Result) {
1533 {
1534 std::lock_guard<std::mutex> Lock(LookupMutex);
1535 --Count;
1536 if (Result) {
1537 assert(!CompoundResult.count(JD) &&
1538 "Duplicate JITDylib in lookup?");
1539 CompoundResult[JD] = std::move(*Result);
1540 } else
1541 CompoundErr =
1542 joinErrors(std::move(CompoundErr), Result.takeError());
1543 }
1544 CV.notify_one();
1545 },
1547 }
1548
1549 std::unique_lock<std::mutex> Lock(LookupMutex);
1550 CV.wait(Lock, [&] { return Count == 0 || CompoundErr; });
1551
1552 if (CompoundErr)
1553 return std::move(CompoundErr);
1554
1555 return std::move(CompoundResult);
1556}
1557
1559 unique_function<void(Error)> OnComplete, ExecutionSession &ES,
1560 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1561
1562 class TriggerOnComplete {
1563 public:
1564 using OnCompleteFn = unique_function<void(Error)>;
1565 TriggerOnComplete(OnCompleteFn OnComplete)
1566 : OnComplete(std::move(OnComplete)) {}
1567 ~TriggerOnComplete() { OnComplete(std::move(LookupResult)); }
1568 void reportResult(Error Err) {
1569 std::lock_guard<std::mutex> Lock(ResultMutex);
1570 LookupResult = joinErrors(std::move(LookupResult), std::move(Err));
1571 }
1572
1573 private:
1574 std::mutex ResultMutex;
1575 Error LookupResult{Error::success()};
1576 OnCompleteFn OnComplete;
1577 };
1578
1579 LLVM_DEBUG({
1580 dbgs() << "Issuing init-symbol lookup:\n";
1581 for (auto &KV : InitSyms)
1582 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1583 });
1584
1585 auto TOC = std::make_shared<TriggerOnComplete>(std::move(OnComplete));
1586
1587 for (auto &KV : InitSyms) {
1588 auto *JD = KV.first;
1589 auto Names = std::move(KV.second);
1590 ES.lookup(
1593 std::move(Names), SymbolState::Ready,
1595 TOC->reportResult(Result.takeError());
1596 },
1598 }
1599}
1600
1602 OS << "Materialization task: " << MU->getName() << " in "
1603 << MR->getTargetJITDylib().getName();
1604}
1605
1606void MaterializationTask::run() { MU->materialize(std::move(MR)); }
1607
1608void LookupTask::printDescription(raw_ostream &OS) { OS << "Lookup task"; }
1609
1611
1612ExecutionSession::ExecutionSession(std::unique_ptr<ExecutorProcessControl> EPC)
1613 : EPC(std::move(EPC)) {
1614 // Associated EPC and this.
1615 this->EPC->ES = this;
1616}
1617
1619 // You must call endSession prior to destroying the session.
1620 assert(!SessionOpen &&
1621 "Session still open. Did you forget to call endSession?");
1622}
1623
1625 LLVM_DEBUG(dbgs() << "Ending ExecutionSession " << this << "\n");
1626
1627 auto JDsToRemove = runSessionLocked([&] {
1628
1629#ifdef EXPENSIVE_CHECKS
1630 verifySessionState("Entering ExecutionSession::endSession");
1631#endif
1632
1633 SessionOpen = false;
1634 return JDs;
1635 });
1636
1637 std::reverse(JDsToRemove.begin(), JDsToRemove.end());
1638
1639 auto Err = removeJITDylibs(std::move(JDsToRemove));
1640
1641 Err = joinErrors(std::move(Err), EPC->disconnect());
1642
1643 return Err;
1644}
1645
1647 runSessionLocked([&] { ResourceManagers.push_back(&RM); });
1648}
1649
1651 runSessionLocked([&] {
1652 assert(!ResourceManagers.empty() && "No managers registered");
1653 if (ResourceManagers.back() == &RM)
1654 ResourceManagers.pop_back();
1655 else {
1656 auto I = llvm::find(ResourceManagers, &RM);
1657 assert(I != ResourceManagers.end() && "RM not registered");
1658 ResourceManagers.erase(I);
1659 }
1660 });
1661}
1662
1664 return runSessionLocked([&, this]() -> JITDylib * {
1665 for (auto &JD : JDs)
1666 if (JD->getName() == Name)
1667 return JD.get();
1668 return nullptr;
1669 });
1670}
1671
1673 assert(!getJITDylibByName(Name) && "JITDylib with that name already exists");
1674 return runSessionLocked([&, this]() -> JITDylib & {
1675 assert(SessionOpen && "Cannot create JITDylib after session is closed");
1676 JDs.push_back(new JITDylib(*this, std::move(Name)));
1677 return *JDs.back();
1678 });
1679}
1680
1682 auto &JD = createBareJITDylib(Name);
1683 if (P)
1684 if (auto Err = P->setupJITDylib(JD))
1685 return std::move(Err);
1686 return JD;
1687}
1688
1689Error ExecutionSession::removeJITDylibs(std::vector<JITDylibSP> JDsToRemove) {
1690 // Set JD to 'Closing' state and remove JD from the ExecutionSession.
1691 runSessionLocked([&] {
1692 for (auto &JD : JDsToRemove) {
1693 assert(JD->State == JITDylib::Open && "JD already closed");
1694 JD->State = JITDylib::Closing;
1695 auto I = llvm::find(JDs, JD);
1696 assert(I != JDs.end() && "JD does not appear in session JDs");
1697 JDs.erase(I);
1698 }
1699 });
1700
1701 // Clear JITDylibs and notify the platform.
1702 Error Err = Error::success();
1703 for (auto JD : JDsToRemove) {
1704 Err = joinErrors(std::move(Err), JD->clear());
1705 if (P)
1706 Err = joinErrors(std::move(Err), P->teardownJITDylib(*JD));
1707 }
1708
1709 // Set JD to closed state. Clear remaining data structures.
1710 runSessionLocked([&] {
1711 for (auto &JD : JDsToRemove) {
1712 assert(JD->State == JITDylib::Closing && "JD should be closing");
1713 JD->State = JITDylib::Closed;
1714 assert(JD->Symbols.empty() && "JD.Symbols is not empty after clear");
1715 assert(JD->UnmaterializedInfos.empty() &&
1716 "JD.UnmaterializedInfos is not empty after clear");
1717 assert(JD->MaterializingInfos.empty() &&
1718 "JD.MaterializingInfos is not empty after clear");
1719 assert(JD->TrackerSymbols.empty() &&
1720 "TrackerSymbols is not empty after clear");
1721 JD->DefGenerators.clear();
1722 JD->LinkOrder.clear();
1723 }
1724 });
1725
1726 return Err;
1727}
1728
1731 if (JDs.empty())
1732 return std::vector<JITDylibSP>();
1733
1734 auto &ES = JDs.front()->getExecutionSession();
1735 return ES.runSessionLocked([&]() -> Expected<std::vector<JITDylibSP>> {
1736 DenseSet<JITDylib *> Visited;
1737 std::vector<JITDylibSP> Result;
1738
1739 for (auto &JD : JDs) {
1740
1741 if (JD->State != Open)
1742 return make_error<StringError>(
1743 "Error building link order: " + JD->getName() + " is defunct",
1745 if (Visited.count(JD.get()))
1746 continue;
1747
1749 WorkStack.push_back(JD);
1750 Visited.insert(JD.get());
1751
1752 while (!WorkStack.empty()) {
1753 Result.push_back(std::move(WorkStack.back()));
1754 WorkStack.pop_back();
1755
1756 for (auto &KV : llvm::reverse(Result.back()->LinkOrder)) {
1757 auto &JD = *KV.first;
1758 if (!Visited.insert(&JD).second)
1759 continue;
1760 WorkStack.push_back(&JD);
1761 }
1762 }
1763 }
1764 return Result;
1765 });
1766}
1767
1770 auto Result = getDFSLinkOrder(JDs);
1771 if (Result)
1772 std::reverse(Result->begin(), Result->end());
1773 return Result;
1774}
1775
1777 return getDFSLinkOrder({this});
1778}
1779
1781 return getReverseDFSLinkOrder({this});
1782}
1783
1785 LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet,
1786 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
1787
1788 OL_applyQueryPhase1(std::make_unique<InProgressLookupFlagsState>(
1789 K, std::move(SearchOrder), std::move(LookupSet),
1790 std::move(OnComplete)),
1791 Error::success());
1792}
1793
1796 SymbolLookupSet LookupSet) {
1797
1798 std::promise<MSVCPExpected<SymbolFlagsMap>> ResultP;
1799 OL_applyQueryPhase1(std::make_unique<InProgressLookupFlagsState>(
1800 K, std::move(SearchOrder), std::move(LookupSet),
1801 [&ResultP](Expected<SymbolFlagsMap> Result) {
1802 ResultP.set_value(std::move(Result));
1803 }),
1804 Error::success());
1805
1806 auto ResultF = ResultP.get_future();
1807 return ResultF.get();
1808}
1809
1811 LookupKind K, const JITDylibSearchOrder &SearchOrder,
1812 SymbolLookupSet Symbols, SymbolState RequiredState,
1813 SymbolsResolvedCallback NotifyComplete,
1814 RegisterDependenciesFunction RegisterDependencies) {
1815
1816 LLVM_DEBUG({
1817 runSessionLocked([&]() {
1818 dbgs() << "Looking up " << Symbols << " in " << SearchOrder
1819 << " (required state: " << RequiredState << ")\n";
1820 });
1821 });
1822
1823 // lookup can be re-entered recursively if running on a single thread. Run any
1824 // outstanding MUs in case this query depends on them, otherwise this lookup
1825 // will starve waiting for a result from an MU that is stuck in the queue.
1826 dispatchOutstandingMUs();
1827
1828 auto Unresolved = std::move(Symbols);
1829 auto Q = std::make_shared<AsynchronousSymbolQuery>(Unresolved, RequiredState,
1830 std::move(NotifyComplete));
1831
1832 auto IPLS = std::make_unique<InProgressFullLookupState>(
1833 K, SearchOrder, std::move(Unresolved), RequiredState, std::move(Q),
1834 std::move(RegisterDependencies));
1835
1836 OL_applyQueryPhase1(std::move(IPLS), Error::success());
1837}
1838
1841 SymbolLookupSet Symbols, LookupKind K,
1842 SymbolState RequiredState,
1843 RegisterDependenciesFunction RegisterDependencies) {
1844#if LLVM_ENABLE_THREADS
1845 // In the threaded case we use promises to return the results.
1846 std::promise<SymbolMap> PromisedResult;
1847 Error ResolutionError = Error::success();
1848
1849 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1850 if (R)
1851 PromisedResult.set_value(std::move(*R));
1852 else {
1853 ErrorAsOutParameter _(&ResolutionError);
1854 ResolutionError = R.takeError();
1855 PromisedResult.set_value(SymbolMap());
1856 }
1857 };
1858
1859#else
1861 Error ResolutionError = Error::success();
1862
1863 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1864 ErrorAsOutParameter _(&ResolutionError);
1865 if (R)
1866 Result = std::move(*R);
1867 else
1868 ResolutionError = R.takeError();
1869 };
1870#endif
1871
1872 // Perform the asynchronous lookup.
1873 lookup(K, SearchOrder, std::move(Symbols), RequiredState, NotifyComplete,
1874 RegisterDependencies);
1875
1876#if LLVM_ENABLE_THREADS
1877 auto ResultFuture = PromisedResult.get_future();
1878 auto Result = ResultFuture.get();
1879
1880 if (ResolutionError)
1881 return std::move(ResolutionError);
1882
1883 return std::move(Result);
1884
1885#else
1886 if (ResolutionError)
1887 return std::move(ResolutionError);
1888
1889 return Result;
1890#endif
1891}
1892
1895 SymbolStringPtr Name, SymbolState RequiredState) {
1896 SymbolLookupSet Names({Name});
1897
1898 if (auto ResultMap = lookup(SearchOrder, std::move(Names), LookupKind::Static,
1899 RequiredState, NoDependenciesToRegister)) {
1900 assert(ResultMap->size() == 1 && "Unexpected number of results");
1901 assert(ResultMap->count(Name) && "Missing result for symbol");
1902 return std::move(ResultMap->begin()->second);
1903 } else
1904 return ResultMap.takeError();
1905}
1906
1909 SymbolState RequiredState) {
1910 return lookup(makeJITDylibSearchOrder(SearchOrder), Name, RequiredState);
1911}
1912
1915 SymbolState RequiredState) {
1916 return lookup(SearchOrder, intern(Name), RequiredState);
1917}
1918
1921
1922 auto TagAddrs = lookup({{&JD, JITDylibLookupFlags::MatchAllSymbols}},
1925 if (!TagAddrs)
1926 return TagAddrs.takeError();
1927
1928 // Associate tag addresses with implementations.
1929 std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex);
1930 for (auto &KV : *TagAddrs) {
1931 auto TagAddr = KV.second.getAddress();
1932 if (JITDispatchHandlers.count(TagAddr))
1933 return make_error<StringError>("Tag " + formatv("{0:x16}", TagAddr) +
1934 " (for " + *KV.first +
1935 ") already registered",
1937 auto I = WFs.find(KV.first);
1938 assert(I != WFs.end() && I->second &&
1939 "JITDispatchHandler implementation missing");
1940 JITDispatchHandlers[KV.second.getAddress()] =
1941 std::make_shared<JITDispatchHandlerFunction>(std::move(I->second));
1942 LLVM_DEBUG({
1943 dbgs() << "Associated function tag \"" << *KV.first << "\" ("
1944 << formatv("{0:x}", KV.second.getAddress()) << ") with handler\n";
1945 });
1946 }
1947 return Error::success();
1948}
1949
1951 ExecutorAddr HandlerFnTagAddr,
1952 ArrayRef<char> ArgBuffer) {
1953
1954 std::shared_ptr<JITDispatchHandlerFunction> F;
1955 {
1956 std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex);
1957 auto I = JITDispatchHandlers.find(HandlerFnTagAddr);
1958 if (I != JITDispatchHandlers.end())
1959 F = I->second;
1960 }
1961
1962 if (F)
1963 (*F)(std::move(SendResult), ArgBuffer.data(), ArgBuffer.size());
1964 else
1966 ("No function registered for tag " +
1967 formatv("{0:x16}", HandlerFnTagAddr))
1968 .str()));
1969}
1970
1972 runSessionLocked([this, &OS]() {
1973 for (auto &JD : JDs)
1974 JD->dump(OS);
1975 });
1976}
1977
1978#ifdef EXPENSIVE_CHECKS
1979bool ExecutionSession::verifySessionState(Twine Phase) {
1980 return runSessionLocked([&]() {
1981 bool AllOk = true;
1982
1983 // We'll collect these and verify them later to avoid redundant checks.
1985
1986 for (auto &JD : JDs) {
1987
1988 auto LogFailure = [&]() -> raw_fd_ostream & {
1989 auto &Stream = errs();
1990 if (AllOk)
1991 Stream << "ERROR: Bad ExecutionSession state detected " << Phase
1992 << "\n";
1993 Stream << " In JITDylib " << JD->getName() << ", ";
1994 AllOk = false;
1995 return Stream;
1996 };
1997
1998 if (JD->State != JITDylib::Open) {
1999 LogFailure()
2000 << "state is not Open, but JD is in ExecutionSession list.";
2001 }
2002
2003 // Check symbol table.
2004 // 1. If the entry state isn't resolved then check that no address has
2005 // been set.
2006 // 2. Check that if the hasMaterializerAttached flag is set then there is
2007 // an UnmaterializedInfo entry, and vice-versa.
2008 for (auto &[Sym, Entry] : JD->Symbols) {
2009 // Check that unresolved symbols have null addresses.
2010 if (Entry.getState() < SymbolState::Resolved) {
2011 if (Entry.getAddress()) {
2012 LogFailure() << "symbol " << Sym << " has state "
2013 << Entry.getState()
2014 << " (not-yet-resolved) but non-null address "
2015 << Entry.getAddress() << ".\n";
2016 }
2017 }
2018
2019 // Check that the hasMaterializerAttached flag is correct.
2020 auto UMIItr = JD->UnmaterializedInfos.find(Sym);
2021 if (Entry.hasMaterializerAttached()) {
2022 if (UMIItr == JD->UnmaterializedInfos.end()) {
2023 LogFailure() << "symbol " << Sym
2024 << " entry claims materializer attached, but "
2025 "UnmaterializedInfos has no corresponding entry.\n";
2026 }
2027 } else if (UMIItr != JD->UnmaterializedInfos.end()) {
2028 LogFailure()
2029 << "symbol " << Sym
2030 << " entry claims no materializer attached, but "
2031 "UnmaterializedInfos has an unexpected entry for it.\n";
2032 }
2033 }
2034
2035 // Check that every UnmaterializedInfo entry has a corresponding entry
2036 // in the Symbols table.
2037 for (auto &[Sym, UMI] : JD->UnmaterializedInfos) {
2038 auto SymItr = JD->Symbols.find(Sym);
2039 if (SymItr == JD->Symbols.end()) {
2040 LogFailure()
2041 << "symbol " << Sym
2042 << " has UnmaterializedInfos entry, but no Symbols entry.\n";
2043 }
2044 }
2045
2046 // Check consistency of the MaterializingInfos table.
2047 for (auto &[Sym, MII] : JD->MaterializingInfos) {
2048
2049 auto SymItr = JD->Symbols.find(Sym);
2050 if (SymItr == JD->Symbols.end()) {
2051 // If there's no Symbols entry for this MaterializingInfos entry then
2052 // report that.
2053 LogFailure()
2054 << "symbol " << Sym
2055 << " has MaterializingInfos entry, but no Symbols entry.\n";
2056 } else {
2057 // Otherwise check consistency between Symbols and MaterializingInfos.
2058
2059 // Ready symbols should not have MaterializingInfos.
2060 if (SymItr->second.getState() == SymbolState::Ready) {
2061 LogFailure()
2062 << "symbol " << Sym
2063 << " is in Ready state, should not have MaterializingInfo.\n";
2064 }
2065
2066 // Pending queries should be for subsequent states.
2067 auto CurState = static_cast<SymbolState>(
2068 static_cast<std::underlying_type_t<SymbolState>>(
2069 SymItr->second.getState()) + 1);
2070 for (auto &Q : MII.PendingQueries) {
2071 if (Q->getRequiredState() != CurState) {
2072 if (Q->getRequiredState() > CurState)
2073 CurState = Q->getRequiredState();
2074 else
2075 LogFailure() << "symbol " << Sym
2076 << " has stale or misordered queries.\n";
2077 }
2078 }
2079
2080 // If there's a DefiningEDU then check that...
2081 // 1. The JD matches.
2082 // 2. The symbol is in the EDU's Symbols map.
2083 // 3. The symbol table entry is in the Emitted state.
2084 if (MII.DefiningEDU) {
2085
2086 EDUsToCheck.insert(MII.DefiningEDU.get());
2087
2088 if (MII.DefiningEDU->JD != JD.get()) {
2089 LogFailure() << "symbol " << Sym
2090 << " has DefiningEDU with incorrect JD"
2091 << (llvm::is_contained(JDs, MII.DefiningEDU->JD)
2092 ? " (JD not currently in ExecutionSession"
2093 : "")
2094 << "\n";
2095 }
2096
2097 if (SymItr->second.getState() != SymbolState::Emitted) {
2098 LogFailure()
2099 << "symbol " << Sym
2100 << " has DefiningEDU, but is not in Emitted state.\n";
2101 }
2102 }
2103
2104 // Check that JDs for any DependantEDUs are also in the session --
2105 // that guarantees that we'll also visit them during this loop.
2106 for (auto &DepEDU : MII.DependantEDUs) {
2107 if (!llvm::is_contained(JDs, DepEDU->JD)) {
2108 LogFailure() << "symbol " << Sym << " has DependantEDU "
2109 << (void *)DepEDU << " with JD (" << DepEDU->JD
2110 << ") that isn't in ExecutionSession.\n";
2111 }
2112 }
2113 }
2114 }
2115 }
2116
2117 // Check EDUs.
2118 for (auto *EDU : EDUsToCheck) {
2119 assert(EDU->JD->State == JITDylib::Open && "EDU->JD is not Open");
2120
2121 auto LogFailure = [&]() -> raw_fd_ostream & {
2122 AllOk = false;
2123 auto &Stream = errs();
2124 Stream << "In EDU defining " << EDU->JD->getName() << ": { ";
2125 for (auto &[Sym, Flags] : EDU->Symbols)
2126 Stream << Sym << " ";
2127 Stream << "}, ";
2128 return Stream;
2129 };
2130
2131 if (EDU->Symbols.empty())
2132 LogFailure() << "no symbols defined.\n";
2133 else {
2134 for (auto &[Sym, Flags] : EDU->Symbols) {
2135 if (!Sym)
2136 LogFailure() << "null symbol defined.\n";
2137 else {
2138 if (!EDU->JD->Symbols.count(SymbolStringPtr(Sym))) {
2139 LogFailure() << "symbol " << Sym
2140 << " isn't present in JD's symbol table.\n";
2141 }
2142 }
2143 }
2144 }
2145
2146 for (auto &[DepJD, Symbols] : EDU->Dependencies) {
2147 if (!llvm::is_contained(JDs, DepJD)) {
2148 LogFailure() << "dependant symbols listed for JD that isn't in "
2149 "ExecutionSession.\n";
2150 } else {
2151 for (auto &DepSym : Symbols) {
2152 if (!DepJD->Symbols.count(SymbolStringPtr(DepSym))) {
2153 LogFailure()
2154 << "dependant symbol " << DepSym
2155 << " does not appear in symbol table for dependant JD "
2156 << DepJD->getName() << ".\n";
2157 }
2158 }
2159 }
2160 }
2161 }
2162
2163 return AllOk;
2164 });
2165}
2166#endif // EXPENSIVE_CHECKS
2167
2168void ExecutionSession::dispatchOutstandingMUs() {
2169 LLVM_DEBUG(dbgs() << "Dispatching MaterializationUnits...\n");
2170 while (true) {
2171 std::optional<std::pair<std::unique_ptr<MaterializationUnit>,
2172 std::unique_ptr<MaterializationResponsibility>>>
2173 JMU;
2174
2175 {
2176 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2177 if (!OutstandingMUs.empty()) {
2178 JMU.emplace(std::move(OutstandingMUs.back()));
2179 OutstandingMUs.pop_back();
2180 }
2181 }
2182
2183 if (!JMU)
2184 break;
2185
2186 assert(JMU->first && "No MU?");
2187 LLVM_DEBUG(dbgs() << " Dispatching \"" << JMU->first->getName() << "\"\n");
2188 dispatchTask(std::make_unique<MaterializationTask>(std::move(JMU->first),
2189 std::move(JMU->second)));
2190 }
2191 LLVM_DEBUG(dbgs() << "Done dispatching MaterializationUnits.\n");
2192}
2193
2194Error ExecutionSession::removeResourceTracker(ResourceTracker &RT) {
2195 LLVM_DEBUG({
2196 dbgs() << "In " << RT.getJITDylib().getName() << " removing tracker "
2197 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2198 });
2199 std::vector<ResourceManager *> CurrentResourceManagers;
2200
2201 JITDylib::AsynchronousSymbolQuerySet QueriesToFail;
2202 std::shared_ptr<SymbolDependenceMap> FailedSymbols;
2203
2204 runSessionLocked([&] {
2205 CurrentResourceManagers = ResourceManagers;
2206 RT.makeDefunct();
2207 std::tie(QueriesToFail, FailedSymbols) =
2208 RT.getJITDylib().IL_removeTracker(RT);
2209 });
2210
2211 Error Err = Error::success();
2212
2213 auto &JD = RT.getJITDylib();
2214 for (auto *L : reverse(CurrentResourceManagers))
2215 Err = joinErrors(std::move(Err),
2216 L->handleRemoveResources(JD, RT.getKeyUnsafe()));
2217
2218 for (auto &Q : QueriesToFail)
2219 Q->handleFailed(
2220 make_error<FailedToMaterialize>(getSymbolStringPool(), FailedSymbols));
2221
2222 return Err;
2223}
2224
2225void ExecutionSession::transferResourceTracker(ResourceTracker &DstRT,
2226 ResourceTracker &SrcRT) {
2227 LLVM_DEBUG({
2228 dbgs() << "In " << SrcRT.getJITDylib().getName()
2229 << " transfering resources from tracker "
2230 << formatv("{0:x}", SrcRT.getKeyUnsafe()) << " to tracker "
2231 << formatv("{0:x}", DstRT.getKeyUnsafe()) << "\n";
2232 });
2233
2234 // No-op transfers are allowed and do not invalidate the source.
2235 if (&DstRT == &SrcRT)
2236 return;
2237
2238 assert(&DstRT.getJITDylib() == &SrcRT.getJITDylib() &&
2239 "Can't transfer resources between JITDylibs");
2240 runSessionLocked([&]() {
2241 SrcRT.makeDefunct();
2242 auto &JD = DstRT.getJITDylib();
2243 JD.transferTracker(DstRT, SrcRT);
2244 for (auto *L : reverse(ResourceManagers))
2245 L->handleTransferResources(JD, DstRT.getKeyUnsafe(),
2246 SrcRT.getKeyUnsafe());
2247 });
2248}
2249
2250void ExecutionSession::destroyResourceTracker(ResourceTracker &RT) {
2251 runSessionLocked([&]() {
2252 LLVM_DEBUG({
2253 dbgs() << "In " << RT.getJITDylib().getName() << " destroying tracker "
2254 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2255 });
2256 if (!RT.isDefunct())
2257 transferResourceTracker(*RT.getJITDylib().getDefaultResourceTracker(),
2258 RT);
2259 });
2260}
2261
2262Error ExecutionSession::IL_updateCandidatesFor(
2263 JITDylib &JD, JITDylibLookupFlags JDLookupFlags,
2264 SymbolLookupSet &Candidates, SymbolLookupSet *NonCandidates) {
2265 return Candidates.forEachWithRemoval(
2266 [&](const SymbolStringPtr &Name,
2267 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2268 /// Search for the symbol. If not found then continue without
2269 /// removal.
2270 auto SymI = JD.Symbols.find(Name);
2271 if (SymI == JD.Symbols.end())
2272 return false;
2273
2274 // If this is a non-exported symbol and we're matching exported
2275 // symbols only then remove this symbol from the candidates list.
2276 //
2277 // If we're tracking non-candidates then add this to the non-candidate
2278 // list.
2279 if (!SymI->second.getFlags().isExported() &&
2281 if (NonCandidates)
2282 NonCandidates->add(Name, SymLookupFlags);
2283 return true;
2284 }
2285
2286 // If we match against a materialization-side-effects only symbol
2287 // then make sure it is weakly-referenced. Otherwise bail out with
2288 // an error.
2289 // FIXME: Use a "materialization-side-effects-only symbols must be
2290 // weakly referenced" specific error here to reduce confusion.
2291 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2293 return make_error<SymbolsNotFound>(getSymbolStringPool(),
2295
2296 // If we matched against this symbol but it is in the error state
2297 // then bail out and treat it as a failure to materialize.
2298 if (SymI->second.getFlags().hasError()) {
2299 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2300 (*FailedSymbolsMap)[&JD] = {Name};
2301 return make_error<FailedToMaterialize>(getSymbolStringPool(),
2302 std::move(FailedSymbolsMap));
2303 }
2304
2305 // Otherwise this is a match. Remove it from the candidate set.
2306 return true;
2307 });
2308}
2309
2310void ExecutionSession::OL_resumeLookupAfterGeneration(
2311 InProgressLookupState &IPLS) {
2312
2314 "Should not be called for not-in-generator lookups");
2316
2318
2319 if (auto DG = IPLS.CurDefGeneratorStack.back().lock()) {
2320 IPLS.CurDefGeneratorStack.pop_back();
2321 std::lock_guard<std::mutex> Lock(DG->M);
2322
2323 // If there are no pending lookups then mark the generator as free and
2324 // return.
2325 if (DG->PendingLookups.empty()) {
2326 DG->InUse = false;
2327 return;
2328 }
2329
2330 // Otherwise resume the next lookup.
2331 LS = std::move(DG->PendingLookups.front());
2332 DG->PendingLookups.pop_front();
2333 }
2334
2335 if (LS.IPLS) {
2337 dispatchTask(std::make_unique<LookupTask>(std::move(LS)));
2338 }
2339}
2340
2341void ExecutionSession::OL_applyQueryPhase1(
2342 std::unique_ptr<InProgressLookupState> IPLS, Error Err) {
2343
2344 LLVM_DEBUG({
2345 dbgs() << "Entering OL_applyQueryPhase1:\n"
2346 << " Lookup kind: " << IPLS->K << "\n"
2347 << " Search order: " << IPLS->SearchOrder
2348 << ", Current index = " << IPLS->CurSearchOrderIndex
2349 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2350 << " Lookup set: " << IPLS->LookupSet << "\n"
2351 << " Definition generator candidates: "
2352 << IPLS->DefGeneratorCandidates << "\n"
2353 << " Definition generator non-candidates: "
2354 << IPLS->DefGeneratorNonCandidates << "\n";
2355 });
2356
2357 if (IPLS->GenState == InProgressLookupState::InGenerator)
2358 OL_resumeLookupAfterGeneration(*IPLS);
2359
2360 assert(IPLS->GenState != InProgressLookupState::InGenerator &&
2361 "Lookup should not be in InGenerator state here");
2362
2363 // FIXME: We should attach the query as we go: This provides a result in a
2364 // single pass in the common case where all symbols have already reached the
2365 // required state. The query could be detached again in the 'fail' method on
2366 // IPLS. Phase 2 would be reduced to collecting and dispatching the MUs.
2367
2368 while (IPLS->CurSearchOrderIndex != IPLS->SearchOrder.size()) {
2369
2370 // If we've been handed an error or received one back from a generator then
2371 // fail the query. We don't need to unlink: At this stage the query hasn't
2372 // actually been lodged.
2373 if (Err)
2374 return IPLS->fail(std::move(Err));
2375
2376 // Get the next JITDylib and lookup flags.
2377 auto &KV = IPLS->SearchOrder[IPLS->CurSearchOrderIndex];
2378 auto &JD = *KV.first;
2379 auto JDLookupFlags = KV.second;
2380
2381 LLVM_DEBUG({
2382 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2383 << ") with lookup set " << IPLS->LookupSet << ":\n";
2384 });
2385
2386 // If we've just reached a new JITDylib then perform some setup.
2387 if (IPLS->NewJITDylib) {
2388 // Add any non-candidates from the last JITDylib (if any) back on to the
2389 // list of definition candidates for this JITDylib, reset definition
2390 // non-candidates to the empty set.
2391 SymbolLookupSet Tmp;
2392 std::swap(IPLS->DefGeneratorNonCandidates, Tmp);
2393 IPLS->DefGeneratorCandidates.append(std::move(Tmp));
2394
2395 LLVM_DEBUG({
2396 dbgs() << " First time visiting " << JD.getName()
2397 << ", resetting candidate sets and building generator stack\n";
2398 });
2399
2400 // Build the definition generator stack for this JITDylib.
2401 runSessionLocked([&] {
2402 IPLS->CurDefGeneratorStack.reserve(JD.DefGenerators.size());
2403 for (auto &DG : reverse(JD.DefGenerators))
2404 IPLS->CurDefGeneratorStack.push_back(DG);
2405 });
2406
2407 // Flag that we've done our initialization.
2408 IPLS->NewJITDylib = false;
2409 }
2410
2411 // Remove any generation candidates that are already defined (and match) in
2412 // this JITDylib.
2413 runSessionLocked([&] {
2414 // Update the list of candidates (and non-candidates) for definition
2415 // generation.
2416 LLVM_DEBUG(dbgs() << " Updating candidate set...\n");
2417 Err = IL_updateCandidatesFor(
2418 JD, JDLookupFlags, IPLS->DefGeneratorCandidates,
2419 JD.DefGenerators.empty() ? nullptr
2420 : &IPLS->DefGeneratorNonCandidates);
2421 LLVM_DEBUG({
2422 dbgs() << " Remaining candidates = " << IPLS->DefGeneratorCandidates
2423 << "\n";
2424 });
2425
2426 // If this lookup was resumed after auto-suspension but all candidates
2427 // have already been generated (by some previous call to the generator)
2428 // treat the lookup as if it had completed generation.
2429 if (IPLS->GenState == InProgressLookupState::ResumedForGenerator &&
2430 IPLS->DefGeneratorCandidates.empty())
2431 OL_resumeLookupAfterGeneration(*IPLS);
2432 });
2433
2434 // If we encountered an error while filtering generation candidates then
2435 // bail out.
2436 if (Err)
2437 return IPLS->fail(std::move(Err));
2438
2439 /// Apply any definition generators on the stack.
2440 LLVM_DEBUG({
2441 if (IPLS->CurDefGeneratorStack.empty())
2442 LLVM_DEBUG(dbgs() << " No generators to run for this JITDylib.\n");
2443 else if (IPLS->DefGeneratorCandidates.empty())
2444 LLVM_DEBUG(dbgs() << " No candidates to generate.\n");
2445 else
2446 dbgs() << " Running " << IPLS->CurDefGeneratorStack.size()
2447 << " remaining generators for "
2448 << IPLS->DefGeneratorCandidates.size() << " candidates\n";
2449 });
2450 while (!IPLS->CurDefGeneratorStack.empty() &&
2451 !IPLS->DefGeneratorCandidates.empty()) {
2452 auto DG = IPLS->CurDefGeneratorStack.back().lock();
2453
2454 if (!DG)
2455 return IPLS->fail(make_error<StringError>(
2456 "DefinitionGenerator removed while lookup in progress",
2458
2459 // At this point the lookup is in either the NotInGenerator state, or in
2460 // the ResumedForGenerator state.
2461 // If this lookup is in the NotInGenerator state then check whether the
2462 // generator is in use. If the generator is not in use then move the
2463 // lookup to the InGenerator state and continue. If the generator is
2464 // already in use then just add this lookup to the pending lookups list
2465 // and bail out.
2466 // If this lookup is in the ResumedForGenerator state then just move it
2467 // to InGenerator and continue.
2468 if (IPLS->GenState == InProgressLookupState::NotInGenerator) {
2469 std::lock_guard<std::mutex> Lock(DG->M);
2470 if (DG->InUse) {
2471 DG->PendingLookups.push_back(std::move(IPLS));
2472 return;
2473 }
2474 DG->InUse = true;
2475 }
2476
2477 IPLS->GenState = InProgressLookupState::InGenerator;
2478
2479 auto K = IPLS->K;
2480 auto &LookupSet = IPLS->DefGeneratorCandidates;
2481
2482 // Run the generator. If the generator takes ownership of QA then this
2483 // will break the loop.
2484 {
2485 LLVM_DEBUG(dbgs() << " Attempting to generate " << LookupSet << "\n");
2486 LookupState LS(std::move(IPLS));
2487 Err = DG->tryToGenerate(LS, K, JD, JDLookupFlags, LookupSet);
2488 IPLS = std::move(LS.IPLS);
2489 }
2490
2491 // If the lookup returned then pop the generator stack and unblock the
2492 // next lookup on this generator (if any).
2493 if (IPLS)
2494 OL_resumeLookupAfterGeneration(*IPLS);
2495
2496 // If there was an error then fail the query.
2497 if (Err) {
2498 LLVM_DEBUG({
2499 dbgs() << " Error attempting to generate " << LookupSet << "\n";
2500 });
2501 assert(IPLS && "LS cannot be retained if error is returned");
2502 return IPLS->fail(std::move(Err));
2503 }
2504
2505 // Otherwise if QA was captured then break the loop.
2506 if (!IPLS) {
2507 LLVM_DEBUG(
2508 { dbgs() << " LookupState captured. Exiting phase1 for now.\n"; });
2509 return;
2510 }
2511
2512 // Otherwise if we're continuing around the loop then update candidates
2513 // for the next round.
2514 runSessionLocked([&] {
2515 LLVM_DEBUG(dbgs() << " Updating candidate set post-generation\n");
2516 Err = IL_updateCandidatesFor(
2517 JD, JDLookupFlags, IPLS->DefGeneratorCandidates,
2518 JD.DefGenerators.empty() ? nullptr
2519 : &IPLS->DefGeneratorNonCandidates);
2520 });
2521
2522 // If updating candidates failed then fail the query.
2523 if (Err) {
2524 LLVM_DEBUG(dbgs() << " Error encountered while updating candidates\n");
2525 return IPLS->fail(std::move(Err));
2526 }
2527 }
2528
2529 if (IPLS->DefGeneratorCandidates.empty() &&
2530 IPLS->DefGeneratorNonCandidates.empty()) {
2531 // Early out if there are no remaining symbols.
2532 LLVM_DEBUG(dbgs() << "All symbols matched.\n");
2533 IPLS->CurSearchOrderIndex = IPLS->SearchOrder.size();
2534 break;
2535 } else {
2536 // If we get here then we've moved on to the next JITDylib with candidates
2537 // remaining.
2538 LLVM_DEBUG(dbgs() << "Phase 1 moving to next JITDylib.\n");
2539 ++IPLS->CurSearchOrderIndex;
2540 IPLS->NewJITDylib = true;
2541 }
2542 }
2543
2544 // Remove any weakly referenced candidates that could not be found/generated.
2545 IPLS->DefGeneratorCandidates.remove_if(
2546 [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2547 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2548 });
2549
2550 // If we get here then we've finished searching all JITDylibs.
2551 // If we matched all symbols then move to phase 2, otherwise fail the query
2552 // with a SymbolsNotFound error.
2553 if (IPLS->DefGeneratorCandidates.empty()) {
2554 LLVM_DEBUG(dbgs() << "Phase 1 succeeded.\n");
2555 IPLS->complete(std::move(IPLS));
2556 } else {
2557 LLVM_DEBUG(dbgs() << "Phase 1 failed with unresolved symbols.\n");
2558 IPLS->fail(make_error<SymbolsNotFound>(
2559 getSymbolStringPool(), IPLS->DefGeneratorCandidates.getSymbolNames()));
2560 }
2561}
2562
2563void ExecutionSession::OL_completeLookup(
2564 std::unique_ptr<InProgressLookupState> IPLS,
2565 std::shared_ptr<AsynchronousSymbolQuery> Q,
2566 RegisterDependenciesFunction RegisterDependencies) {
2567
2568 LLVM_DEBUG({
2569 dbgs() << "Entering OL_completeLookup:\n"
2570 << " Lookup kind: " << IPLS->K << "\n"
2571 << " Search order: " << IPLS->SearchOrder
2572 << ", Current index = " << IPLS->CurSearchOrderIndex
2573 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2574 << " Lookup set: " << IPLS->LookupSet << "\n"
2575 << " Definition generator candidates: "
2576 << IPLS->DefGeneratorCandidates << "\n"
2577 << " Definition generator non-candidates: "
2578 << IPLS->DefGeneratorNonCandidates << "\n";
2579 });
2580
2581 bool QueryComplete = false;
2582 DenseMap<JITDylib *, JITDylib::UnmaterializedInfosList> CollectedUMIs;
2583
2584 auto LodgingErr = runSessionLocked([&]() -> Error {
2585 for (auto &KV : IPLS->SearchOrder) {
2586 auto &JD = *KV.first;
2587 auto JDLookupFlags = KV.second;
2588 LLVM_DEBUG({
2589 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2590 << ") with lookup set " << IPLS->LookupSet << ":\n";
2591 });
2592
2593 auto Err = IPLS->LookupSet.forEachWithRemoval(
2594 [&](const SymbolStringPtr &Name,
2595 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2596 LLVM_DEBUG({
2597 dbgs() << " Attempting to match \"" << Name << "\" ("
2598 << SymLookupFlags << ")... ";
2599 });
2600
2601 /// Search for the symbol. If not found then continue without
2602 /// removal.
2603 auto SymI = JD.Symbols.find(Name);
2604 if (SymI == JD.Symbols.end()) {
2605 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2606 return false;
2607 }
2608
2609 // If this is a non-exported symbol and we're matching exported
2610 // symbols only then skip this symbol without removal.
2611 if (!SymI->second.getFlags().isExported() &&
2612 JDLookupFlags ==
2614 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2615 return false;
2616 }
2617
2618 // If we match against a materialization-side-effects only symbol
2619 // then make sure it is weakly-referenced. Otherwise bail out with
2620 // an error.
2621 // FIXME: Use a "materialization-side-effects-only symbols must be
2622 // weakly referenced" specific error here to reduce confusion.
2623 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2625 LLVM_DEBUG({
2626 dbgs() << "error: "
2627 "required, but symbol is has-side-effects-only\n";
2628 });
2629 return make_error<SymbolsNotFound>(getSymbolStringPool(),
2631 }
2632
2633 // If we matched against this symbol but it is in the error state
2634 // then bail out and treat it as a failure to materialize.
2635 if (SymI->second.getFlags().hasError()) {
2636 LLVM_DEBUG(dbgs() << "error: symbol is in error state\n");
2637 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2638 (*FailedSymbolsMap)[&JD] = {Name};
2639 return make_error<FailedToMaterialize>(
2640 getSymbolStringPool(), std::move(FailedSymbolsMap));
2641 }
2642
2643 // Otherwise this is a match.
2644
2645 // If this symbol is already in the required state then notify the
2646 // query, remove the symbol and continue.
2647 if (SymI->second.getState() >= Q->getRequiredState()) {
2649 << "matched, symbol already in required state\n");
2650 Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol());
2651 return true;
2652 }
2653
2654 // Otherwise this symbol does not yet meet the required state. Check
2655 // whether it has a materializer attached, and if so prepare to run
2656 // it.
2657 if (SymI->second.hasMaterializerAttached()) {
2658 assert(SymI->second.getAddress() == ExecutorAddr() &&
2659 "Symbol not resolved but already has address?");
2660 auto UMII = JD.UnmaterializedInfos.find(Name);
2661 assert(UMII != JD.UnmaterializedInfos.end() &&
2662 "Lazy symbol should have UnmaterializedInfo");
2663
2664 auto UMI = UMII->second;
2665 assert(UMI->MU && "Materializer should not be null");
2666 assert(UMI->RT && "Tracker should not be null");
2667 LLVM_DEBUG({
2668 dbgs() << "matched, preparing to dispatch MU@" << UMI->MU.get()
2669 << " (" << UMI->MU->getName() << ")\n";
2670 });
2671
2672 // Move all symbols associated with this MaterializationUnit into
2673 // materializing state.
2674 for (auto &KV : UMI->MU->getSymbols()) {
2675 auto SymK = JD.Symbols.find(KV.first);
2676 assert(SymK != JD.Symbols.end() &&
2677 "No entry for symbol covered by MaterializationUnit");
2678 SymK->second.setMaterializerAttached(false);
2679 SymK->second.setState(SymbolState::Materializing);
2680 JD.UnmaterializedInfos.erase(KV.first);
2681 }
2682
2683 // Add MU to the list of MaterializationUnits to be materialized.
2684 CollectedUMIs[&JD].push_back(std::move(UMI));
2685 } else
2686 LLVM_DEBUG(dbgs() << "matched, registering query");
2687
2688 // Add the query to the PendingQueries list and continue, deleting
2689 // the element from the lookup set.
2690 assert(SymI->second.getState() != SymbolState::NeverSearched &&
2691 SymI->second.getState() != SymbolState::Ready &&
2692 "By this line the symbol should be materializing");
2693 auto &MI = JD.MaterializingInfos[Name];
2694 MI.addQuery(Q);
2695 Q->addQueryDependence(JD, Name);
2696
2697 return true;
2698 });
2699
2700 JD.shrinkMaterializationInfoMemory();
2701
2702 // Handle failure.
2703 if (Err) {
2704
2705 LLVM_DEBUG({
2706 dbgs() << "Lookup failed. Detaching query and replacing MUs.\n";
2707 });
2708
2709 // Detach the query.
2710 Q->detach();
2711
2712 // Replace the MUs.
2713 for (auto &KV : CollectedUMIs) {
2714 auto &JD = *KV.first;
2715 for (auto &UMI : KV.second)
2716 for (auto &KV2 : UMI->MU->getSymbols()) {
2717 assert(!JD.UnmaterializedInfos.count(KV2.first) &&
2718 "Unexpected materializer in map");
2719 auto SymI = JD.Symbols.find(KV2.first);
2720 assert(SymI != JD.Symbols.end() && "Missing symbol entry");
2721 assert(SymI->second.getState() == SymbolState::Materializing &&
2722 "Can not replace symbol that is not materializing");
2723 assert(!SymI->second.hasMaterializerAttached() &&
2724 "MaterializerAttached flag should not be set");
2725 SymI->second.setMaterializerAttached(true);
2726 JD.UnmaterializedInfos[KV2.first] = UMI;
2727 }
2728 }
2729
2730 return Err;
2731 }
2732 }
2733
2734 LLVM_DEBUG(dbgs() << "Stripping unmatched weakly-referenced symbols\n");
2735 IPLS->LookupSet.forEachWithRemoval(
2736 [&](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2737 if (SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol) {
2738 Q->dropSymbol(Name);
2739 return true;
2740 } else
2741 return false;
2742 });
2743
2744 if (!IPLS->LookupSet.empty()) {
2745 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2746 return make_error<SymbolsNotFound>(getSymbolStringPool(),
2747 IPLS->LookupSet.getSymbolNames());
2748 }
2749
2750 // Record whether the query completed.
2751 QueryComplete = Q->isComplete();
2752
2753 LLVM_DEBUG({
2754 dbgs() << "Query successfully "
2755 << (QueryComplete ? "completed" : "lodged") << "\n";
2756 });
2757
2758 // Move the collected MUs to the OutstandingMUs list.
2759 if (!CollectedUMIs.empty()) {
2760 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2761
2762 LLVM_DEBUG(dbgs() << "Adding MUs to dispatch:\n");
2763 for (auto &KV : CollectedUMIs) {
2764 LLVM_DEBUG({
2765 auto &JD = *KV.first;
2766 dbgs() << " For " << JD.getName() << ": Adding " << KV.second.size()
2767 << " MUs.\n";
2768 });
2769 for (auto &UMI : KV.second) {
2770 auto MR = createMaterializationResponsibility(
2771 *UMI->RT, std::move(UMI->MU->SymbolFlags),
2772 std::move(UMI->MU->InitSymbol));
2773 OutstandingMUs.push_back(
2774 std::make_pair(std::move(UMI->MU), std::move(MR)));
2775 }
2776 }
2777 } else
2778 LLVM_DEBUG(dbgs() << "No MUs to dispatch.\n");
2779
2780 if (RegisterDependencies && !Q->QueryRegistrations.empty()) {
2781 LLVM_DEBUG(dbgs() << "Registering dependencies\n");
2782 RegisterDependencies(Q->QueryRegistrations);
2783 } else
2784 LLVM_DEBUG(dbgs() << "No dependencies to register\n");
2785
2786 return Error::success();
2787 });
2788
2789 if (LodgingErr) {
2790 LLVM_DEBUG(dbgs() << "Failing query\n");
2791 Q->detach();
2792 Q->handleFailed(std::move(LodgingErr));
2793 return;
2794 }
2795
2796 if (QueryComplete) {
2797 LLVM_DEBUG(dbgs() << "Completing query\n");
2798 Q->handleComplete(*this);
2799 }
2800
2801 dispatchOutstandingMUs();
2802}
2803
2804void ExecutionSession::OL_completeLookupFlags(
2805 std::unique_ptr<InProgressLookupState> IPLS,
2806 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
2807
2808 auto Result = runSessionLocked([&]() -> Expected<SymbolFlagsMap> {
2809 LLVM_DEBUG({
2810 dbgs() << "Entering OL_completeLookupFlags:\n"
2811 << " Lookup kind: " << IPLS->K << "\n"
2812 << " Search order: " << IPLS->SearchOrder
2813 << ", Current index = " << IPLS->CurSearchOrderIndex
2814 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2815 << " Lookup set: " << IPLS->LookupSet << "\n"
2816 << " Definition generator candidates: "
2817 << IPLS->DefGeneratorCandidates << "\n"
2818 << " Definition generator non-candidates: "
2819 << IPLS->DefGeneratorNonCandidates << "\n";
2820 });
2821
2823
2824 // Attempt to find flags for each symbol.
2825 for (auto &KV : IPLS->SearchOrder) {
2826 auto &JD = *KV.first;
2827 auto JDLookupFlags = KV.second;
2828 LLVM_DEBUG({
2829 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2830 << ") with lookup set " << IPLS->LookupSet << ":\n";
2831 });
2832
2833 IPLS->LookupSet.forEachWithRemoval([&](const SymbolStringPtr &Name,
2834 SymbolLookupFlags SymLookupFlags) {
2835 LLVM_DEBUG({
2836 dbgs() << " Attempting to match \"" << Name << "\" ("
2837 << SymLookupFlags << ")... ";
2838 });
2839
2840 // Search for the symbol. If not found then continue without removing
2841 // from the lookup set.
2842 auto SymI = JD.Symbols.find(Name);
2843 if (SymI == JD.Symbols.end()) {
2844 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2845 return false;
2846 }
2847
2848 // If this is a non-exported symbol then it doesn't match. Skip it.
2849 if (!SymI->second.getFlags().isExported() &&
2851 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2852 return false;
2853 }
2854
2855 LLVM_DEBUG({
2856 dbgs() << "matched, \"" << Name << "\" -> " << SymI->second.getFlags()
2857 << "\n";
2858 });
2859 Result[Name] = SymI->second.getFlags();
2860 return true;
2861 });
2862 }
2863
2864 // Remove any weakly referenced symbols that haven't been resolved.
2865 IPLS->LookupSet.remove_if(
2866 [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2867 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2868 });
2869
2870 if (!IPLS->LookupSet.empty()) {
2871 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2872 return make_error<SymbolsNotFound>(getSymbolStringPool(),
2873 IPLS->LookupSet.getSymbolNames());
2874 }
2875
2876 LLVM_DEBUG(dbgs() << "Succeded, result = " << Result << "\n");
2877 return Result;
2878 });
2879
2880 // Run the callback on the result.
2881 LLVM_DEBUG(dbgs() << "Sending result to handler.\n");
2882 OnComplete(std::move(Result));
2883}
2884
2885void ExecutionSession::OL_destroyMaterializationResponsibility(
2886 MaterializationResponsibility &MR) {
2887
2888 assert(MR.SymbolFlags.empty() &&
2889 "All symbols should have been explicitly materialized or failed");
2890 MR.JD.unlinkMaterializationResponsibility(MR);
2891}
2892
2893SymbolNameSet ExecutionSession::OL_getRequestedSymbols(
2894 const MaterializationResponsibility &MR) {
2895 return MR.JD.getRequestedSymbols(MR.SymbolFlags);
2896}
2897
2898Error ExecutionSession::OL_notifyResolved(MaterializationResponsibility &MR,
2899 const SymbolMap &Symbols) {
2900 LLVM_DEBUG({
2901 dbgs() << "In " << MR.JD.getName() << " resolving " << Symbols << "\n";
2902 });
2903#ifndef NDEBUG
2904 for (auto &KV : Symbols) {
2905 auto I = MR.SymbolFlags.find(KV.first);
2906 assert(I != MR.SymbolFlags.end() &&
2907 "Resolving symbol outside this responsibility set");
2908 assert(!I->second.hasMaterializationSideEffectsOnly() &&
2909 "Can't resolve materialization-side-effects-only symbol");
2910 if (I->second & JITSymbolFlags::Common) {
2911 auto WeakOrCommon = JITSymbolFlags::Weak | JITSymbolFlags::Common;
2912 assert((KV.second.getFlags() & WeakOrCommon) &&
2913 "Common symbols must be resolved as common or weak");
2914 assert((KV.second.getFlags() & ~WeakOrCommon) ==
2915 (I->second & ~JITSymbolFlags::Common) &&
2916 "Resolving symbol with incorrect flags");
2917 } else
2918 assert(KV.second.getFlags() == I->second &&
2919 "Resolving symbol with incorrect flags");
2920 }
2921#endif
2922
2923 return MR.JD.resolve(MR, Symbols);
2924}
2925
2926template <typename HandleNewDepFn>
2927void ExecutionSession::propagateExtraEmitDeps(
2928 std::deque<JITDylib::EmissionDepUnit *> Worklist, EDUInfosMap &EDUInfos,
2929 HandleNewDepFn HandleNewDep) {
2930
2931 // Iterate to a fixed-point to propagate extra-emit dependencies through the
2932 // EDU graph.
2933 while (!Worklist.empty()) {
2934 auto &EDU = *Worklist.front();
2935 Worklist.pop_front();
2936
2937 assert(EDUInfos.count(&EDU) && "No info entry for EDU");
2938 auto &EDUInfo = EDUInfos[&EDU];
2939
2940 // Propagate new dependencies to users.
2941 for (auto *UserEDU : EDUInfo.IntraEmitUsers) {
2942
2943 // UserEDUInfo only present if UserEDU has its own users.
2944 JITDylib::EmissionDepUnitInfo *UserEDUInfo = nullptr;
2945 {
2946 auto UserEDUInfoItr = EDUInfos.find(UserEDU);
2947 if (UserEDUInfoItr != EDUInfos.end())
2948 UserEDUInfo = &UserEDUInfoItr->second;
2949 }
2950
2951 for (auto &[DepJD, Deps] : EDUInfo.NewDeps) {
2952 auto &UserEDUDepsForJD = UserEDU->Dependencies[DepJD];
2953 DenseSet<NonOwningSymbolStringPtr> *UserEDUNewDepsForJD = nullptr;
2954 for (auto Dep : Deps) {
2955 if (UserEDUDepsForJD.insert(Dep).second) {
2956 HandleNewDep(*UserEDU, *DepJD, Dep);
2957 if (UserEDUInfo) {
2958 if (!UserEDUNewDepsForJD) {
2959 // If UserEDU has no new deps then it's not in the worklist
2960 // yet, so add it.
2961 if (UserEDUInfo->NewDeps.empty())
2962 Worklist.push_back(UserEDU);
2963 UserEDUNewDepsForJD = &UserEDUInfo->NewDeps[DepJD];
2964 }
2965 // Add (DepJD, Dep) to NewDeps.
2966 UserEDUNewDepsForJD->insert(Dep);
2967 }
2968 }
2969 }
2970 }
2971 }
2972
2973 EDUInfo.NewDeps.clear();
2974 }
2975}
2976
2977// Note: This method modifies the emitted set.
2978ExecutionSession::EDUInfosMap ExecutionSession::simplifyDepGroups(
2979 MaterializationResponsibility &MR,
2980 ArrayRef<SymbolDependenceGroup> EmittedDeps) {
2981
2982 auto &TargetJD = MR.getTargetJITDylib();
2983
2984 // 1. Build initial EmissionDepUnit -> EmissionDepUnitInfo and
2985 // Symbol -> EmissionDepUnit mappings.
2986 DenseMap<JITDylib::EmissionDepUnit *, JITDylib::EmissionDepUnitInfo> EDUInfos;
2987 EDUInfos.reserve(EmittedDeps.size());
2988 DenseMap<NonOwningSymbolStringPtr, JITDylib::EmissionDepUnit *> EDUForSymbol;
2989 for (auto &DG : EmittedDeps) {
2990 assert(!DG.Symbols.empty() && "DepGroup does not cover any symbols");
2991
2992 // Skip empty EDUs.
2993 if (DG.Dependencies.empty())
2994 continue;
2995
2996 auto TmpEDU = std::make_shared<JITDylib::EmissionDepUnit>(TargetJD);
2997 auto &EDUInfo = EDUInfos[TmpEDU.get()];
2998 EDUInfo.EDU = std::move(TmpEDU);
2999 for (const auto &Symbol : DG.Symbols) {
3000 NonOwningSymbolStringPtr NonOwningSymbol(Symbol);
3001 assert(!EDUForSymbol.count(NonOwningSymbol) &&
3002 "Symbol should not appear in more than one SymbolDependenceGroup");
3003 assert(MR.getSymbols().count(Symbol) &&
3004 "Symbol in DepGroups not in the emitted set");
3005 auto NewlyEmittedItr = MR.getSymbols().find(Symbol);
3006 EDUInfo.EDU->Symbols[NonOwningSymbol] = NewlyEmittedItr->second;
3007 EDUForSymbol[NonOwningSymbol] = EDUInfo.EDU.get();
3008 }
3009 }
3010
3011 // 2. Build a "residual" EDU to cover all symbols that have no dependencies.
3012 {
3013 DenseMap<NonOwningSymbolStringPtr, JITSymbolFlags> ResidualSymbolFlags;
3014 for (auto &[Sym, Flags] : MR.getSymbols()) {
3015 if (!EDUForSymbol.count(NonOwningSymbolStringPtr(Sym)))
3016 ResidualSymbolFlags[NonOwningSymbolStringPtr(Sym)] = Flags;
3017 }
3018 if (!ResidualSymbolFlags.empty()) {
3019 auto ResidualEDU = std::make_shared<JITDylib::EmissionDepUnit>(TargetJD);
3020 ResidualEDU->Symbols = std::move(ResidualSymbolFlags);
3021 auto &ResidualEDUInfo = EDUInfos[ResidualEDU.get()];
3022 ResidualEDUInfo.EDU = std::move(ResidualEDU);
3023
3024 // If the residual EDU is the only one then bail out early.
3025 if (EDUInfos.size() == 1)
3026 return EDUInfos;
3027
3028 // Otherwise add the residual EDU to the EDUForSymbol map.
3029 for (auto &[Sym, Flags] : ResidualEDUInfo.EDU->Symbols)
3030 EDUForSymbol[Sym] = ResidualEDUInfo.EDU.get();
3031 }
3032 }
3033
3034#ifndef NDEBUG
3035 assert(EDUForSymbol.size() == MR.getSymbols().size() &&
3036 "MR symbols not fully covered by EDUs?");
3037 for (auto &[Sym, Flags] : MR.getSymbols()) {
3038 assert(EDUForSymbol.count(NonOwningSymbolStringPtr(Sym)) &&
3039 "Sym in MR not covered by EDU");
3040 }
3041#endif // NDEBUG
3042
3043 // 3. Use the DepGroups array to build a graph of dependencies between
3044 // EmissionDepUnits in this finalization. We want to remove these
3045 // intra-finalization uses, propagating dependencies on symbols outside
3046 // this finalization. Add EDUs to the worklist.
3047 for (auto &DG : EmittedDeps) {
3048
3049 // Skip SymbolDependenceGroups with no dependencies.
3050 if (DG.Dependencies.empty())
3051 continue;
3052
3053 assert(EDUForSymbol.count(NonOwningSymbolStringPtr(*DG.Symbols.begin())) &&
3054 "No EDU for DG");
3055 auto &EDU =
3056 *EDUForSymbol.find(NonOwningSymbolStringPtr(*DG.Symbols.begin()))
3057 ->second;
3058
3059 for (auto &[DepJD, Deps] : DG.Dependencies) {
3060 DenseSet<NonOwningSymbolStringPtr> NewDepsForJD;
3061
3062 assert(!Deps.empty() && "Dependence set for DepJD is empty");
3063
3064 if (DepJD != &TargetJD) {
3065 // DepJD is some other JITDylib.There can't be any intra-finalization
3066 // edges here, so just skip.
3067 for (auto &Dep : Deps)
3068 NewDepsForJD.insert(NonOwningSymbolStringPtr(Dep));
3069 } else {
3070 // DepJD is the Target JITDylib. Check for intra-finaliztaion edges,
3071 // skipping any and recording the intra-finalization use instead.
3072 for (auto &Dep : Deps) {
3073 NonOwningSymbolStringPtr NonOwningDep(Dep);
3074 auto I = EDUForSymbol.find(NonOwningDep);
3075 if (I == EDUForSymbol.end()) {
3076 if (!MR.getSymbols().count(Dep))
3077 NewDepsForJD.insert(NonOwningDep);
3078 continue;
3079 }
3080
3081 if (I->second != &EDU)
3082 EDUInfos[I->second].IntraEmitUsers.insert(&EDU);
3083 }
3084 }
3085
3086 if (!NewDepsForJD.empty())
3087 EDU.Dependencies[DepJD] = std::move(NewDepsForJD);
3088 }
3089 }
3090
3091 // 4. Build the worklist.
3092 std::deque<JITDylib::EmissionDepUnit *> Worklist;
3093 for (auto &[EDU, EDUInfo] : EDUInfos) {
3094 // If this EDU has extra-finalization dependencies and intra-finalization
3095 // users then add it to the worklist.
3096 if (!EDU->Dependencies.empty()) {
3097 auto I = EDUInfos.find(EDU);
3098 if (I != EDUInfos.end()) {
3099 auto &EDUInfo = I->second;
3100 if (!EDUInfo.IntraEmitUsers.empty()) {
3101 EDUInfo.NewDeps = EDU->Dependencies;
3102 Worklist.push_back(EDU);
3103 }
3104 }
3105 }
3106 }
3107
3108 // 4. Propagate dependencies through the EDU graph.
3109 propagateExtraEmitDeps(
3110 Worklist, EDUInfos,
3111 [](JITDylib::EmissionDepUnit &, JITDylib &, NonOwningSymbolStringPtr) {});
3112
3113 return EDUInfos;
3114}
3115
3116void ExecutionSession::IL_makeEDUReady(
3117 std::shared_ptr<JITDylib::EmissionDepUnit> EDU,
3118 JITDylib::AsynchronousSymbolQuerySet &Queries) {
3119
3120 // The symbols for this EDU are ready.
3121 auto &JD = *EDU->JD;
3122
3123 for (auto &[Sym, Flags] : EDU->Symbols) {
3124 assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
3125 "JD does not have an entry for Sym");
3126 auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
3127
3128 assert(((Entry.getFlags().hasMaterializationSideEffectsOnly() &&
3129 Entry.getState() == SymbolState::Materializing) ||
3130 Entry.getState() == SymbolState::Resolved ||
3131 Entry.getState() == SymbolState::Emitted) &&
3132 "Emitting from state other than Resolved");
3133
3134 Entry.setState(SymbolState::Ready);
3135
3136 auto MII = JD.MaterializingInfos.find(SymbolStringPtr(Sym));
3137
3138 // Check for pending queries.
3139 if (MII == JD.MaterializingInfos.end())
3140 continue;
3141 auto &MI = MII->second;
3142
3143 for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) {
3144 Q->notifySymbolMetRequiredState(SymbolStringPtr(Sym), Entry.getSymbol());
3145 if (Q->isComplete())
3146 Queries.insert(Q);
3147 Q->removeQueryDependence(JD, SymbolStringPtr(Sym));
3148 }
3149
3150 JD.MaterializingInfos.erase(MII);
3151 }
3152
3153 JD.shrinkMaterializationInfoMemory();
3154}
3155
3156void ExecutionSession::IL_makeEDUEmitted(
3157 std::shared_ptr<JITDylib::EmissionDepUnit> EDU,
3158 JITDylib::AsynchronousSymbolQuerySet &Queries) {
3159
3160 // The symbols for this EDU are emitted, but not ready.
3161 auto &JD = *EDU->JD;
3162
3163 for (auto &[Sym, Flags] : EDU->Symbols) {
3164 assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
3165 "JD does not have an entry for Sym");
3166 auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
3167
3168 assert(((Entry.getFlags().hasMaterializationSideEffectsOnly() &&
3169 Entry.getState() == SymbolState::Materializing) ||
3170 Entry.getState() == SymbolState::Resolved ||
3171 Entry.getState() == SymbolState::Emitted) &&
3172 "Emitting from state other than Resolved");
3173
3174 if (Entry.getState() == SymbolState::Emitted) {
3175 // This was already emitted, so we can skip the rest of this loop.
3176#ifndef NDEBUG
3177 for (auto &[Sym, Flags] : EDU->Symbols) {
3178 assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
3179 "JD does not have an entry for Sym");
3180 auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
3181 assert(Entry.getState() == SymbolState::Emitted &&
3182 "Symbols for EDU in inconsistent state");
3183 assert(JD.MaterializingInfos.count(SymbolStringPtr(Sym)) &&
3184 "Emitted symbol has no MI");
3185 auto MI = JD.MaterializingInfos[SymbolStringPtr(Sym)];
3186 assert(MI.takeQueriesMeeting(SymbolState::Emitted).empty() &&
3187 "Already-emitted symbol has waiting-on-emitted queries");
3188 }
3189#endif // NDEBUG
3190 break;
3191 }
3192
3193 Entry.setState(SymbolState::Emitted);
3194 auto &MI = JD.MaterializingInfos[SymbolStringPtr(Sym)];
3195 MI.DefiningEDU = EDU;
3196
3197 for (auto &Q : MI.takeQueriesMeeting(SymbolState::Emitted)) {
3198 Q->notifySymbolMetRequiredState(SymbolStringPtr(Sym), Entry.getSymbol());
3199 if (Q->isComplete())
3200 Queries.insert(Q);
3201 Q->removeQueryDependence(JD, SymbolStringPtr(Sym));
3202 }
3203 }
3204
3205 for (auto &[DepJD, Deps] : EDU->Dependencies) {
3206 for (auto &Dep : Deps)
3207 DepJD->MaterializingInfos[SymbolStringPtr(Dep)].DependantEDUs.insert(
3208 EDU.get());
3209 }
3210}
3211
3212/// Removes the given dependence from EDU. If EDU's dependence set becomes
3213/// empty then this function adds an entry for it to the EDUInfos map.
3214/// Returns true if a new EDUInfosMap entry is added.
3215bool ExecutionSession::IL_removeEDUDependence(JITDylib::EmissionDepUnit &EDU,
3216 JITDylib &DepJD,
3217 NonOwningSymbolStringPtr DepSym,
3218 EDUInfosMap &EDUInfos) {
3219 assert(EDU.Dependencies.count(&DepJD) &&
3220 "JD does not appear in Dependencies of DependantEDU");
3221 assert(EDU.Dependencies[&DepJD].count(DepSym) &&
3222 "Symbol does not appear in Dependencies of DependantEDU");
3223 auto &JDDeps = EDU.Dependencies[&DepJD];
3224 JDDeps.erase(DepSym);
3225 if (JDDeps.empty()) {
3226 EDU.Dependencies.erase(&DepJD);
3227 if (EDU.Dependencies.empty()) {
3228 // If the dependencies set has become empty then EDU _may_ be ready
3229 // (we won't know for sure until we've propagated the extra-emit deps).
3230 // Create an EDUInfo for it (if it doesn't have one already) so that
3231 // it'll be visited after propagation.
3232 auto &DepEDUInfo = EDUInfos[&EDU];
3233 if (!DepEDUInfo.EDU) {
3234 assert(EDU.JD->Symbols.count(
3235 SymbolStringPtr(EDU.Symbols.begin()->first)) &&
3236 "Missing symbol entry for first symbol in EDU");
3237 auto DepEDUFirstMI = EDU.JD->MaterializingInfos.find(
3238 SymbolStringPtr(EDU.Symbols.begin()->first));
3239 assert(DepEDUFirstMI != EDU.JD->MaterializingInfos.end() &&
3240 "Missing MI for first symbol in DependantEDU");
3241 DepEDUInfo.EDU = DepEDUFirstMI->second.DefiningEDU;
3242 return true;
3243 }
3244 }
3245 }
3246 return false;
3247}
3248
3249Error ExecutionSession::makeJDClosedError(JITDylib::EmissionDepUnit &EDU,
3250 JITDylib &ClosedJD) {
3251 SymbolNameSet FailedSymbols;
3252 for (auto &[Sym, Flags] : EDU.Symbols)
3253 FailedSymbols.insert(SymbolStringPtr(Sym));
3254 SymbolDependenceMap BadDeps;
3255 for (auto &Dep : EDU.Dependencies[&ClosedJD])
3256 BadDeps[&ClosedJD].insert(SymbolStringPtr(Dep));
3257 return make_error<UnsatisfiedSymbolDependencies>(
3258 ClosedJD.getExecutionSession().getSymbolStringPool(), EDU.JD,
3259 std::move(FailedSymbols), std::move(BadDeps),
3260 ClosedJD.getName() + " is closed");
3261}
3262
3263Error ExecutionSession::makeUnsatisfiedDepsError(JITDylib::EmissionDepUnit &EDU,
3264 JITDylib &BadJD,
3265 SymbolNameSet BadDeps) {
3266 SymbolNameSet FailedSymbols;
3267 for (auto &[Sym, Flags] : EDU.Symbols)
3268 FailedSymbols.insert(SymbolStringPtr(Sym));
3269 SymbolDependenceMap BadDepsMap;
3270 BadDepsMap[&BadJD] = std::move(BadDeps);
3271 return make_error<UnsatisfiedSymbolDependencies>(
3272 BadJD.getExecutionSession().getSymbolStringPool(), &BadJD,
3273 std::move(FailedSymbols), std::move(BadDepsMap),
3274 "dependencies removed or in error state");
3275}
3276
3277Expected<JITDylib::AsynchronousSymbolQuerySet>
3278ExecutionSession::IL_emit(MaterializationResponsibility &MR,
3279 EDUInfosMap EDUInfos) {
3280
3281 if (MR.RT->isDefunct())
3282 return make_error<ResourceTrackerDefunct>(MR.RT);
3283
3284 auto &TargetJD = MR.getTargetJITDylib();
3285 if (TargetJD.State != JITDylib::Open)
3286 return make_error<StringError>("JITDylib " + TargetJD.getName() +
3287 " is defunct",
3289#ifdef EXPENSIVE_CHECKS
3290 verifySessionState("entering ExecutionSession::IL_emit");
3291#endif
3292
3293 // Walk all EDUs:
3294 // 1. Verifying that dependencies are available (not removed or in the error
3295 // state.
3296 // 2. Removing any dependencies that are already Ready.
3297 // 3. Lifting any EDUs for Emitted symbols into the EDUInfos map.
3298 // 4. Finding any dependant EDUs and lifting them into the EDUInfos map.
3299 std::deque<JITDylib::EmissionDepUnit *> Worklist;
3300 for (auto &[EDU, _] : EDUInfos)
3301 Worklist.push_back(EDU);
3302
3303 for (auto *EDU : Worklist) {
3304 auto *EDUInfo = &EDUInfos[EDU];
3305
3306 SmallVector<JITDylib *> DepJDsToRemove;
3307 for (auto &[DepJD, Deps] : EDU->Dependencies) {
3308 if (DepJD->State != JITDylib::Open)
3309 return makeJDClosedError(*EDU, *DepJD);
3310
3311 SymbolNameSet BadDeps;
3312 SmallVector<NonOwningSymbolStringPtr> DepsToRemove;
3313 for (auto &Dep : Deps) {
3314 auto DepEntryItr = DepJD->Symbols.find(SymbolStringPtr(Dep));
3315
3316 // If this dep has been removed or moved to the error state then add it
3317 // to the bad deps set. We aggregate these bad deps for more
3318 // comprehensive error messages.
3319 if (DepEntryItr == DepJD->Symbols.end() ||
3320 DepEntryItr->second.getFlags().hasError()) {
3321 BadDeps.insert(SymbolStringPtr(Dep));
3322 continue;
3323 }
3324
3325 // If this dep isn't emitted yet then just add it to the NewDeps set to
3326 // be propagated.
3327 auto &DepEntry = DepEntryItr->second;
3328 if (DepEntry.getState() < SymbolState::Emitted) {
3329 EDUInfo->NewDeps[DepJD].insert(Dep);
3330 continue;
3331 }
3332
3333 // This dep has been emitted, so add it to the list to be removed from
3334 // EDU.
3335 DepsToRemove.push_back(Dep);
3336
3337 // If Dep is Ready then there's nothing further to do.
3338 if (DepEntry.getState() == SymbolState::Ready) {
3339 assert(!DepJD->MaterializingInfos.count(SymbolStringPtr(Dep)) &&
3340 "Unexpected MaterializationInfo attached to ready symbol");
3341 continue;
3342 }
3343
3344 // If we get here thene Dep is Emitted. We need to look up its defining
3345 // EDU and add this EDU to the defining EDU's list of users (this means
3346 // creating an EDUInfos entry if the defining EDU doesn't have one
3347 // already).
3348 assert(DepJD->MaterializingInfos.count(SymbolStringPtr(Dep)) &&
3349 "Expected MaterializationInfo for emitted dependency");
3350 auto &DepMI = DepJD->MaterializingInfos[SymbolStringPtr(Dep)];
3351 assert(DepMI.DefiningEDU &&
3352 "Emitted symbol does not have a defining EDU");
3353 assert(!DepMI.DefiningEDU->Dependencies.empty() &&
3354 "Emitted symbol has empty dependencies (should be ready)");
3355 assert(DepMI.DependantEDUs.empty() &&
3356 "Already-emitted symbol has dependant EDUs?");
3357 auto &DepEDUInfo = EDUInfos[DepMI.DefiningEDU.get()];
3358 if (!DepEDUInfo.EDU) {
3359 // No EDUInfo yet -- build initial entry, and reset the EDUInfo
3360 // pointer, which we will have invalidated.
3361 EDUInfo = &EDUInfos[EDU];
3362 DepEDUInfo.EDU = DepMI.DefiningEDU;
3363 for (auto &[DepDepJD, DepDeps] : DepEDUInfo.EDU->Dependencies) {
3364 if (DepDepJD == &TargetJD) {
3365 for (auto &DepDep : DepDeps)
3366 if (!MR.getSymbols().count(SymbolStringPtr(DepDep)))
3367 DepEDUInfo.NewDeps[DepDepJD].insert(DepDep);
3368 } else
3369 DepEDUInfo.NewDeps[DepDepJD] = DepDeps;
3370 }
3371 }
3372 DepEDUInfo.IntraEmitUsers.insert(EDU);
3373 }
3374
3375 // Some dependencies were removed or in an error state -- error out.
3376 if (!BadDeps.empty())
3377 return makeUnsatisfiedDepsError(*EDU, *DepJD, std::move(BadDeps));
3378
3379 // Remove the emitted / ready deps from DepJD.
3380 for (auto &Dep : DepsToRemove)
3381 Deps.erase(Dep);
3382
3383 // If there are no further deps in DepJD then flag it for removal too.
3384 if (Deps.empty())
3385 DepJDsToRemove.push_back(DepJD);
3386 }
3387
3388 // Remove any JDs whose dependence sets have become empty.
3389 for (auto &DepJD : DepJDsToRemove) {
3390 assert(EDU->Dependencies.count(DepJD) &&
3391 "Trying to remove non-existent dep entries");
3392 EDU->Dependencies.erase(DepJD);
3393 }
3394
3395 // Now look for users of this EDU.
3396 for (auto &[Sym, Flags] : EDU->Symbols) {
3397 assert(TargetJD.Symbols.count(SymbolStringPtr(Sym)) &&
3398 "Sym not present in symbol table");
3399 assert((TargetJD.Symbols[SymbolStringPtr(Sym)].getState() ==
3401 TargetJD.Symbols[SymbolStringPtr(Sym)]
3402 .getFlags()
3403 .hasMaterializationSideEffectsOnly()) &&
3404 "Emitting symbol not in the resolved state");
3405 assert(!TargetJD.Symbols[SymbolStringPtr(Sym)].getFlags().hasError() &&
3406 "Symbol is already in an error state");
3407
3408 auto MII = TargetJD.MaterializingInfos.find(SymbolStringPtr(Sym));
3409 if (MII == TargetJD.MaterializingInfos.end() ||
3410 MII->second.DependantEDUs.empty())
3411 continue;
3412
3413 for (auto &DependantEDU : MII->second.DependantEDUs) {
3414 if (IL_removeEDUDependence(*DependantEDU, TargetJD, Sym, EDUInfos))
3415 EDUInfo = &EDUInfos[EDU];
3416 EDUInfo->IntraEmitUsers.insert(DependantEDU);
3417 }
3418 MII->second.DependantEDUs.clear();
3419 }
3420 }
3421
3422 Worklist.clear();
3423 for (auto &[EDU, EDUInfo] : EDUInfos) {
3424 if (!EDUInfo.IntraEmitUsers.empty() && !EDU->Dependencies.empty()) {
3425 if (EDUInfo.NewDeps.empty())
3426 EDUInfo.NewDeps = EDU->Dependencies;
3427 Worklist.push_back(EDU);
3428 }
3429 }
3430
3431 propagateExtraEmitDeps(
3432 Worklist, EDUInfos,
3433 [](JITDylib::EmissionDepUnit &EDU, JITDylib &JD,
3434 NonOwningSymbolStringPtr Sym) {
3435 JD.MaterializingInfos[SymbolStringPtr(Sym)].DependantEDUs.insert(&EDU);
3436 });
3437
3438 JITDylib::AsynchronousSymbolQuerySet CompletedQueries;
3439
3440 // Extract completed queries and lodge not-yet-ready EDUs in the
3441 // session.
3442 for (auto &[EDU, EDUInfo] : EDUInfos) {
3443 if (EDU->Dependencies.empty())
3444 IL_makeEDUReady(std::move(EDUInfo.EDU), CompletedQueries);
3445 else
3446 IL_makeEDUEmitted(std::move(EDUInfo.EDU), CompletedQueries);
3447 }
3448
3449#ifdef EXPENSIVE_CHECKS
3450 verifySessionState("exiting ExecutionSession::IL_emit");
3451#endif
3452
3453 return std::move(CompletedQueries);
3454}
3455
3456Error ExecutionSession::OL_notifyEmitted(
3457 MaterializationResponsibility &MR,
3458 ArrayRef<SymbolDependenceGroup> DepGroups) {
3459 LLVM_DEBUG({
3460 dbgs() << "In " << MR.JD.getName() << " emitting " << MR.SymbolFlags
3461 << "\n";
3462 if (!DepGroups.empty()) {
3463 dbgs() << " Initial dependencies:\n";
3464 for (auto &SDG : DepGroups) {
3465 dbgs() << " Symbols: " << SDG.Symbols
3466 << ", Dependencies: " << SDG.Dependencies << "\n";
3467 }
3468 }
3469 });
3470
3471#ifndef NDEBUG
3472 SymbolNameSet Visited;
3473 for (auto &DG : DepGroups) {
3474 for (auto &Sym : DG.Symbols) {
3475 assert(MR.SymbolFlags.count(Sym) &&
3476 "DG contains dependence for symbol outside this MR");
3477 assert(Visited.insert(Sym).second &&
3478 "DG contains duplicate entries for Name");
3479 }
3480 }
3481#endif // NDEBUG
3482
3483 auto EDUInfos = simplifyDepGroups(MR, DepGroups);
3484
3485 LLVM_DEBUG({
3486 dbgs() << " Simplified dependencies:\n";
3487 for (auto &[EDU, EDUInfo] : EDUInfos) {
3488 dbgs() << " Symbols: { ";
3489 for (auto &[Sym, Flags] : EDU->Symbols)
3490 dbgs() << Sym << " ";
3491 dbgs() << "}, Dependencies: { ";
3492 for (auto &[DepJD, Deps] : EDU->Dependencies) {
3493 dbgs() << "(" << DepJD->getName() << ", { ";
3494 for (auto &Dep : Deps)
3495 dbgs() << Dep << " ";
3496 dbgs() << "}) ";
3497 }
3498 dbgs() << "}\n";
3499 }
3500 });
3501
3502 auto CompletedQueries =
3503 runSessionLocked([&]() { return IL_emit(MR, EDUInfos); });
3504
3505 // On error bail out.
3506 if (!CompletedQueries)
3507 return CompletedQueries.takeError();
3508
3509 MR.SymbolFlags.clear();
3510
3511 // Otherwise notify all the completed queries.
3512 for (auto &Q : *CompletedQueries) {
3513 assert(Q->isComplete() && "Q is not complete");
3514 Q->handleComplete(*this);
3515 }
3516
3517 return Error::success();
3518}
3519
3520Error ExecutionSession::OL_defineMaterializing(
3521 MaterializationResponsibility &MR, SymbolFlagsMap NewSymbolFlags) {
3522
3523 LLVM_DEBUG({
3524 dbgs() << "In " << MR.JD.getName() << " defining materializing symbols "
3525 << NewSymbolFlags << "\n";
3526 });
3527 if (auto AcceptedDefs =
3528 MR.JD.defineMaterializing(MR, std::move(NewSymbolFlags))) {
3529 // Add all newly accepted symbols to this responsibility object.
3530 for (auto &KV : *AcceptedDefs)
3531 MR.SymbolFlags.insert(KV);
3532 return Error::success();
3533 } else
3534 return AcceptedDefs.takeError();
3535}
3536
3537std::pair<JITDylib::AsynchronousSymbolQuerySet,
3538 std::shared_ptr<SymbolDependenceMap>>
3539ExecutionSession::IL_failSymbols(JITDylib &JD,
3540 const SymbolNameVector &SymbolsToFail) {
3541
3542#ifdef EXPENSIVE_CHECKS
3543 verifySessionState("entering ExecutionSession::IL_failSymbols");
3544#endif
3545
3546 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3547 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
3548 auto ExtractFailedQueries = [&](JITDylib::MaterializingInfo &MI) {
3549 JITDylib::AsynchronousSymbolQueryList ToDetach;
3550 for (auto &Q : MI.pendingQueries()) {
3551 // Add the query to the list to be failed and detach it.
3552 FailedQueries.insert(Q);
3553 ToDetach.push_back(Q);
3554 }
3555 for (auto &Q : ToDetach)
3556 Q->detach();
3557 assert(!MI.hasQueriesPending() && "Queries still pending after detach");
3558 };
3559
3560 for (auto &Name : SymbolsToFail) {
3561 (*FailedSymbolsMap)[&JD].insert(Name);
3562
3563 // Look up the symbol to fail.
3564 auto SymI = JD.Symbols.find(Name);
3565
3566 // FIXME: Revisit this. We should be able to assert sequencing between
3567 // ResourceTracker removal and symbol failure.
3568 //
3569 // It's possible that this symbol has already been removed, e.g. if a
3570 // materialization failure happens concurrently with a ResourceTracker or
3571 // JITDylib removal. In that case we can safely skip this symbol and
3572 // continue.
3573 if (SymI == JD.Symbols.end())
3574 continue;
3575 auto &Sym = SymI->second;
3576
3577 // If the symbol is already in the error state then we must have visited
3578 // it earlier.
3579 if (Sym.getFlags().hasError()) {
3580 assert(!JD.MaterializingInfos.count(Name) &&
3581 "Symbol in error state still has MaterializingInfo");
3582 continue;
3583 }
3584
3585 // Move the symbol into the error state.
3586 Sym.setFlags(Sym.getFlags() | JITSymbolFlags::HasError);
3587
3588 // FIXME: Come up with a sane mapping of state to
3589 // presence-of-MaterializingInfo so that we can assert presence / absence
3590 // here, rather than testing it.
3591 auto MII = JD.MaterializingInfos.find(Name);
3592 if (MII == JD.MaterializingInfos.end())
3593 continue;
3594
3595 auto &MI = MII->second;
3596
3597 // Collect queries to be failed for this MII.
3598 ExtractFailedQueries(MI);
3599
3600 if (MI.DefiningEDU) {
3601 // If there is a DefiningEDU for this symbol then remove this
3602 // symbol from it.
3603 assert(MI.DependantEDUs.empty() &&
3604 "Symbol with DefiningEDU should not have DependantEDUs");
3605 assert(Sym.getState() >= SymbolState::Emitted &&
3606 "Symbol has EDU, should have been emitted");
3607 assert(MI.DefiningEDU->Symbols.count(NonOwningSymbolStringPtr(Name)) &&
3608 "Symbol does not appear in its DefiningEDU");
3609 MI.DefiningEDU->Symbols.erase(NonOwningSymbolStringPtr(Name));
3610 MI.DefiningEDU = nullptr;
3611 } else {
3612 // Otherwise if there are any EDUs waiting on this symbol then move
3613 // those symbols to the error state too, and deregister them from the
3614 // symbols that they depend on.
3615 // Note: We use a copy of DependantEDUs here since we'll be removing
3616 // from the original set as we go.
3617 for (auto &DependantEDU : MI.DependantEDUs) {
3618
3619 // Remove DependantEDU from all of its users DependantEDUs lists.
3620 for (auto &[DepJD, DepSyms] : DependantEDU->Dependencies) {
3621 for (auto DepSym : DepSyms) {
3622 // Skip self-reference to avoid invalidating the MI.DependantEDUs
3623 // map. We'll clear this later.
3624 if (DepJD == &JD && DepSym == Name)
3625 continue;
3626 assert(DepJD->Symbols.count(SymbolStringPtr(DepSym)) &&
3627 "DepSym not in DepJD?");
3628 assert(DepJD->MaterializingInfos.count(SymbolStringPtr(DepSym)) &&
3629 "DependantEDU not registered with symbol it depends on");
3630 auto &SymMI = DepJD->MaterializingInfos[SymbolStringPtr(DepSym)];
3631 assert(SymMI.DependantEDUs.count(DependantEDU) &&
3632 "DependantEDU missing from DependantEDUs list");
3633 SymMI.DependantEDUs.erase(DependantEDU);
3634 }
3635 }
3636
3637 // Move any symbols defined by DependantEDU into the error state and
3638 // fail any queries waiting on them.
3639 auto &DepJD = *DependantEDU->JD;
3640 auto DepEDUSymbols = std::move(DependantEDU->Symbols);
3641 for (auto &[DepName, Flags] : DepEDUSymbols) {
3642 auto DepSymItr = DepJD.Symbols.find(SymbolStringPtr(DepName));
3643 assert(DepSymItr != DepJD.Symbols.end() &&
3644 "Symbol not present in table");
3645 auto &DepSym = DepSymItr->second;
3646
3647 assert(DepSym.getState() >= SymbolState::Emitted &&
3648 "Symbol has EDU, should have been emitted");
3649 assert(!DepSym.getFlags().hasError() &&
3650 "Symbol is already in the error state?");
3651 DepSym.setFlags(DepSym.getFlags() | JITSymbolFlags::HasError);
3652 (*FailedSymbolsMap)[&DepJD].insert(SymbolStringPtr(DepName));
3653
3654 // This symbol has a defining EDU so its MaterializingInfo object must
3655 // exist.
3656 auto DepMIItr =
3657 DepJD.MaterializingInfos.find(SymbolStringPtr(DepName));
3658 assert(DepMIItr != DepJD.MaterializingInfos.end() &&
3659 "Symbol has defining EDU but not MaterializingInfo");
3660 auto &DepMI = DepMIItr->second;
3661 assert(DepMI.DefiningEDU.get() == DependantEDU &&
3662 "Bad EDU dependence edge");
3663 assert(DepMI.DependantEDUs.empty() &&
3664 "Symbol was emitted, should not have any DependantEDUs");
3665 ExtractFailedQueries(DepMI);
3666 DepJD.MaterializingInfos.erase(SymbolStringPtr(DepName));
3667 }
3668
3669 DepJD.shrinkMaterializationInfoMemory();
3670 }
3671
3672 MI.DependantEDUs.clear();
3673 }
3674
3675 assert(!MI.DefiningEDU && "DefiningEDU should have been reset");
3676 assert(MI.DependantEDUs.empty() &&
3677 "DependantEDUs should have been removed above");
3678 assert(!MI.hasQueriesPending() &&
3679 "Can not delete MaterializingInfo with queries pending");
3680 JD.MaterializingInfos.erase(Name);
3681 }
3682
3683 JD.shrinkMaterializationInfoMemory();
3684
3685#ifdef EXPENSIVE_CHECKS
3686 verifySessionState("exiting ExecutionSession::IL_failSymbols");
3687#endif
3688
3689 return std::make_pair(std::move(FailedQueries), std::move(FailedSymbolsMap));
3690}
3691
3692void ExecutionSession::OL_notifyFailed(MaterializationResponsibility &MR) {
3693
3694 LLVM_DEBUG({
3695 dbgs() << "In " << MR.JD.getName() << " failing materialization for "
3696 << MR.SymbolFlags << "\n";
3697 });
3698
3699 if (MR.SymbolFlags.empty())
3700 return;
3701
3702 SymbolNameVector SymbolsToFail;
3703 for (auto &[Name, Flags] : MR.SymbolFlags)
3704 SymbolsToFail.push_back(Name);
3705 MR.SymbolFlags.clear();
3706
3707 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3708 std::shared_ptr<SymbolDependenceMap> FailedSymbols;
3709
3710 std::tie(FailedQueries, FailedSymbols) = runSessionLocked([&]() {
3711 // If the tracker is defunct then there's nothing to do here.
3712 if (MR.RT->isDefunct())
3713 return std::pair<JITDylib::AsynchronousSymbolQuerySet,
3714 std::shared_ptr<SymbolDependenceMap>>();
3715 return IL_failSymbols(MR.getTargetJITDylib(), SymbolsToFail);
3716 });
3717
3718 for (auto &Q : FailedQueries)
3719 Q->handleFailed(
3720 make_error<FailedToMaterialize>(getSymbolStringPool(), FailedSymbols));
3721}
3722
3723Error ExecutionSession::OL_replace(MaterializationResponsibility &MR,
3724 std::unique_ptr<MaterializationUnit> MU) {
3725 for (auto &KV : MU->getSymbols()) {
3726 assert(MR.SymbolFlags.count(KV.first) &&
3727 "Replacing definition outside this responsibility set");
3728 MR.SymbolFlags.erase(KV.first);
3729 }
3730
3731 if (MU->getInitializerSymbol() == MR.InitSymbol)
3732 MR.InitSymbol = nullptr;
3733
3734 LLVM_DEBUG(MR.JD.getExecutionSession().runSessionLocked([&]() {
3735 dbgs() << "In " << MR.JD.getName() << " replacing symbols with " << *MU
3736 << "\n";
3737 }););
3738
3739 return MR.JD.replace(MR, std::move(MU));
3740}
3741
3742Expected<std::unique_ptr<MaterializationResponsibility>>
3743ExecutionSession::OL_delegate(MaterializationResponsibility &MR,
3744 const SymbolNameSet &Symbols) {
3745
3746 SymbolStringPtr DelegatedInitSymbol;
3747 SymbolFlagsMap DelegatedFlags;
3748
3749 for (auto &Name : Symbols) {
3750 auto I = MR.SymbolFlags.find(Name);
3751 assert(I != MR.SymbolFlags.end() &&
3752 "Symbol is not tracked by this MaterializationResponsibility "
3753 "instance");
3754
3755 DelegatedFlags[Name] = std::move(I->second);
3756 if (Name == MR.InitSymbol)
3757 std::swap(MR.InitSymbol, DelegatedInitSymbol);
3758
3759 MR.SymbolFlags.erase(I);
3760 }
3761
3762 return MR.JD.delegate(MR, std::move(DelegatedFlags),
3763 std::move(DelegatedInitSymbol));
3764}
3765
3766#ifndef NDEBUG
3767void ExecutionSession::dumpDispatchInfo(Task &T) {
3768 runSessionLocked([&]() {
3769 dbgs() << "Dispatching: ";
3770 T.printDescription(dbgs());
3771 dbgs() << "\n";
3772 });
3773}
3774#endif // NDEBUG
3775
3776} // End namespace orc.
3777} // End namespace llvm.
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Addr
std::string Name
Symbol * Sym
Definition: ELF_riscv.cpp:479
#define _
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
#define H(x, y, z)
Definition: MD5.cpp:57
while(!ToSimplify.empty())
if(PassOpts->AAPipeline)
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
raw_pwrite_stream & OS
static uint32_t getFlags(const Symbol *Sym)
Definition: TapiFile.cpp:27
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
const T & front() const
front - Get the first element.
Definition: ArrayRef.h:168
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
const T * data() const
Definition: ArrayRef.h:162
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
bool erase(const KeyT &Val)
Definition: DenseMap.h:336
unsigned size() const
Definition: DenseMap.h:99
bool empty() const
Definition: DenseMap.h:98
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
iterator end()
Definition: DenseMap.h:84
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:211
Helper for Errors used as out-parameters.
Definition: Error.h:1130
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:337
Tagged union holding either a T or a Error.
Definition: Error.h:481
void push_back(const T &Elt)
Definition: SmallVector.h:427
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1210
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Symbol info for RuntimeDyld.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
iterator find(const_arg_type_t< ValueT > V)
Definition: DenseSet.h:179
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:97
StringRef getName() const override
Return the name of this materialization unit.
Definition: Core.cpp:283
AbsoluteSymbolsMaterializationUnit(SymbolMap Symbols)
Definition: Core.cpp:279
AsynchronousSymbolQuery(const SymbolLookupSet &Symbols, SymbolState RequiredState, SymbolsResolvedCallback NotifyComplete)
Create a query for the given symbols.
Definition: Core.cpp:180
void notifySymbolMetRequiredState(const SymbolStringPtr &Name, ExecutorSymbolDef Sym)
Notify the query that a requested symbol has reached the required state.
Definition: Core.cpp:194
Definition generators can be attached to JITDylibs to generate new definitions for otherwise unresolv...
Definition: Core.h:946
An ExecutionSession represents a running JIT program.
Definition: Core.h:1431
Error endSession()
End the session.
Definition: Core.cpp:1624
void reportError(Error Err)
Report a error for this execution session.
Definition: Core.h:1566
friend class JITDylib
Definition: Core.h:1434
void lookupFlags(LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet Symbols, unique_function< void(Expected< SymbolFlagsMap >)> OnComplete)
Search the given JITDylibs to find the flags associated with each of the given symbols.
Definition: Core.cpp:1784
SymbolStringPtr intern(StringRef SymName)
Add a symbol name to the SymbolStringPool and return a pointer to it.
Definition: Core.h:1485
JITDylib * getJITDylibByName(StringRef Name)
Return a pointer to the "name" JITDylib.
Definition: Core.cpp:1663
friend class LookupState
Definition: Core.h:1435
JITDylib & createBareJITDylib(std::string Name)
Add a new bare JITDylib to this ExecutionSession.
Definition: Core.cpp:1672
std::shared_ptr< SymbolStringPool > getSymbolStringPool()
Get the SymbolStringPool for this instance.
Definition: Core.h:1480
void lookup(LookupKind K, const JITDylibSearchOrder &SearchOrder, SymbolLookupSet Symbols, SymbolState RequiredState, SymbolsResolvedCallback NotifyComplete, RegisterDependenciesFunction RegisterDependencies)
Search the given JITDylibs for the given symbols.
Definition: Core.cpp:1810
Error registerJITDispatchHandlers(JITDylib &JD, JITDispatchHandlerAssociationMap WFs)
For each tag symbol name, associate the corresponding AsyncHandlerWrapperFunction with the address of...
Definition: Core.cpp:1919
void registerResourceManager(ResourceManager &RM)
Register the given ResourceManager with this ExecutionSession.
Definition: Core.cpp:1646
~ExecutionSession()
Destroy an ExecutionSession.
Definition: Core.cpp:1618
void runJITDispatchHandler(SendResultFunction SendResult, ExecutorAddr HandlerFnTagAddr, ArrayRef< char > ArgBuffer)
Run a registered jit-side wrapper function.
Definition: Core.cpp:1950
void deregisterResourceManager(ResourceManager &RM)
Deregister the given ResourceManager with this ExecutionSession.
Definition: Core.cpp:1650
ExecutionSession(std::unique_ptr< ExecutorProcessControl > EPC)
Construct an ExecutionSession with the given ExecutorProcessControl object.
Definition: Core.cpp:1612
decltype(auto) runSessionLocked(Func &&F)
Run the given lambda with the session mutex locked.
Definition: Core.h:1495
void dump(raw_ostream &OS)
Dump the state of all the JITDylibs in this session.
Definition: Core.cpp:1971
Error removeJITDylibs(std::vector< JITDylibSP > JDsToRemove)
Removes the given JITDylibs from the ExecutionSession.
Definition: Core.cpp:1689
Expected< JITDylib & > createJITDylib(std::string Name)
Add a new JITDylib to this ExecutionSession.
Definition: Core.cpp:1681
void dispatchTask(std::unique_ptr< Task > T)
Materialize the given unit.
Definition: Core.h:1639
Represents an address in the executor process.
Represents a defining location for a JIT symbol.
FailedToMaterialize(std::shared_ptr< SymbolStringPool > SSP, std::shared_ptr< SymbolDependenceMap > Symbols)
Definition: Core.cpp:81
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:99
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:103
InProgressFullLookupState(LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet, SymbolState RequiredState, std::shared_ptr< AsynchronousSymbolQuery > Q, RegisterDependenciesFunction RegisterDependencies)
Definition: Core.cpp:585
void complete(std::unique_ptr< InProgressLookupState > IPLS) override
Definition: Core.cpp:595
void fail(Error Err) override
Definition: Core.cpp:601
void complete(std::unique_ptr< InProgressLookupState > IPLS) override
Definition: Core.cpp:572
void fail(Error Err) override
Definition: Core.cpp:577
InProgressLookupFlagsState(LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet, unique_function< void(Expected< SymbolFlagsMap >)> OnComplete)
Definition: Core.cpp:565
virtual ~InProgressLookupState()=default
SymbolLookupSet DefGeneratorCandidates
Definition: Core.cpp:552
JITDylibSearchOrder SearchOrder
Definition: Core.cpp:546
std::vector< std::weak_ptr< DefinitionGenerator > > CurDefGeneratorStack
Definition: Core.cpp:560
enum llvm::orc::InProgressLookupState::@480 GenState
SymbolLookupSet LookupSet
Definition: Core.cpp:547
virtual void complete(std::unique_ptr< InProgressLookupState > IPLS)=0
InProgressLookupState(LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet, SymbolState RequiredState)
Definition: Core.cpp:535
SymbolLookupSet DefGeneratorNonCandidates
Definition: Core.cpp:553
virtual void fail(Error Err)=0
Represents a JIT'd dynamic library.
Definition: Core.h:989
Error clear()
Calls remove on all trackers currently associated with this JITDylib.
Definition: Core.cpp:676
Error define(std::unique_ptr< MaterializationUnitType > &&MU, ResourceTrackerSP RT=nullptr)
Define all symbols provided by the materialization unit to be part of this JITDylib.
Definition: Core.h:1910
ExecutionSession & getExecutionSession() const
Get a reference to the ExecutionSession for this JITDylib.
Definition: Core.h:1008
ResourceTrackerSP createResourceTracker()
Create a resource tracker for this JITDylib.
Definition: Core.cpp:700
Expected< std::vector< JITDylibSP > > getReverseDFSLinkOrder()
Rteurn this JITDylib and its transitive dependencies in reverse DFS order based on linkage relationsh...
Definition: Core.cpp:1780
ResourceTrackerSP getDefaultResourceTracker()
Get the default resource tracker for this JITDylib.
Definition: Core.cpp:691
void removeGenerator(DefinitionGenerator &G)
Remove a definition generator from this JITDylib.
Definition: Core.cpp:708
Expected< std::vector< JITDylibSP > > getDFSLinkOrder()
Return this JITDylib and its transitive dependencies in DFS order based on linkage relationships.
Definition: Core.cpp:1776
Wraps state for a lookup-in-progress.
Definition: Core.h:921
void continueLookup(Error Err)
Continue the lookup.
Definition: Core.cpp:652
LookupState & operator=(LookupState &&)
void run() override
Definition: Core.cpp:1610
static char ID
Definition: Core.h:1420
void printDescription(raw_ostream &OS) override
Definition: Core.cpp:1608
Tracks responsibility for materialization, and mediates interactions between MaterializationUnits and...
Definition: Core.h:555
void printDescription(raw_ostream &OS) override
Definition: Core.cpp:1601
A MaterializationUnit represents a set of symbol definitions that can be materialized as a group,...
Definition: Core.h:693
SymbolFlagsMap SymbolFlags
Definition: Core.h:749
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:162
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:166
Non-owning SymbolStringPool entry pointer.
static void lookupInitSymbolsAsync(unique_function< void(Error)> OnComplete, ExecutionSession &ES, const DenseMap< JITDylib *, SymbolLookupSet > &InitSyms)
Performs an async lookup for the given symbols in each of the given JITDylibs, calling the given hand...
Definition: Core.cpp:1558
static Expected< DenseMap< JITDylib *, SymbolMap > > lookupInitSymbols(ExecutionSession &ES, const DenseMap< JITDylib *, SymbolLookupSet > &InitSyms)
A utility function for looking up initializer symbols.
Definition: Core.cpp:1509
StringRef getName() const override
Return the name of this materialization unit.
Definition: Core.cpp:326
ReExportsMaterializationUnit(JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags, SymbolAliasMap Aliases)
SourceJD is allowed to be nullptr, in which case the source JITDylib is taken to be whatever JITDylib...
Definition: Core.cpp:320
std::function< bool(SymbolStringPtr)> SymbolPredicate
Definition: Core.h:1999
Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD, JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &LookupSet) override
DefinitionGenerators should override this method to insert new definitions into the parent JITDylib.
Definition: Core.cpp:617
ReexportsGenerator(JITDylib &SourceJD, JITDylibLookupFlags SourceJDLookupFlags, SymbolPredicate Allow=SymbolPredicate())
Create a reexports generator.
Definition: Core.cpp:611
Listens for ResourceTracker operations.
Definition: Core.h:104
ResourceTrackerDefunct(ResourceTrackerSP RT)
Definition: Core.cpp:70
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:77
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:73
API to remove / transfer ownership of JIT resources.
Definition: Core.h:56
JITDylib & getJITDylib() const
Return the JITDylib targeted by this tracker.
Definition: Core.h:71
void transferTo(ResourceTracker &DstRT)
Transfer all resources associated with this key to the given tracker, which must target the same JITD...
Definition: Core.cpp:58
ResourceTracker(const ResourceTracker &)=delete
Error remove()
Remove all resources associated with this key.
Definition: Core.cpp:54
A set of symbols to look up, each associated with a SymbolLookupFlags value.
Definition: Core.h:183
static SymbolLookupSet fromMapKeys(const DenseMap< SymbolStringPtr, ValT > &M, SymbolLookupFlags Flags=SymbolLookupFlags::RequiredSymbol)
Construct a SymbolLookupSet from DenseMap keys.
Definition: Core.h:232
Pointer to a pooled string representing a symbol name.
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:154
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:158
SymbolsCouldNotBeRemoved(std::shared_ptr< SymbolStringPool > SSP, SymbolNameSet Symbols)
Definition: Core.cpp:148
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:144
SymbolsNotFound(std::shared_ptr< SymbolStringPool > SSP, SymbolNameSet Symbols)
Definition: Core.cpp:126
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:140
Represents an abstract task for ORC to run.
Definition: TaskDispatch.h:35
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:171
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:175
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:119
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:115
UnsatisfiedSymbolDependencies(std::shared_ptr< SymbolStringPool > SSP, JITDylibSP JD, SymbolNameSet FailedSymbols, SymbolDependenceMap BadDeps, std::string Explanation)
Definition: Core.cpp:107
static WrapperFunctionResult createOutOfBandError(const char *Msg)
Create an out-of-band error by copying the given string.
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:460
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
@ Entry
Definition: COFF.h:826
IntrusiveRefCntPtr< JITDylib > JITDylibSP
Definition: Core.h:51
JITDylibSearchOrder makeJITDylibSearchOrder(ArrayRef< JITDylib * > JDs, JITDylibLookupFlags Flags=JITDylibLookupFlags::MatchExportedSymbolsOnly)
Convenience function for creating a search order from an ArrayRef of JITDylib*, all with the same fla...
Definition: Core.h:166
std::vector< std::pair< JITDylib *, JITDylibLookupFlags > > JITDylibSearchOrder
A list of (JITDylib*, JITDylibLookupFlags) pairs to be used as a search order during symbol lookup.
Definition: Core.h:162
std::function< void(const SymbolDependenceMap &)> RegisterDependenciesFunction
Callback to register the dependencies for a given query.
Definition: Core.h:403
std::unique_ptr< ReExportsMaterializationUnit > symbolAliases(SymbolAliasMap Aliases)
Create a ReExportsMaterializationUnit with the given aliases.
Definition: Core.h:837
SymbolLookupFlags
Lookup flags that apply to each symbol in a lookup.
Definition: Core.h:145
std::unique_ptr< ReExportsMaterializationUnit > reexports(JITDylib &SourceJD, SymbolAliasMap Aliases, JITDylibLookupFlags SourceJDLookupFlags=JITDylibLookupFlags::MatchExportedSymbolsOnly)
Create a materialization unit for re-exporting symbols from another JITDylib with alternative names/f...
Definition: Core.h:846
DenseMap< SymbolStringPtr, SymbolAliasMapEntry > SymbolAliasMap
A map of Symbols to (Symbol, Flags) pairs.
Definition: Core.h:396
JITDylibLookupFlags
Lookup flags that apply to each dylib in the search order for a lookup.
Definition: Core.h:135
DenseMap< SymbolStringPtr, ExecutorSymbolDef > SymbolMap
A map from symbol names (as SymbolStringPtrs) to JITSymbols (address/flags pairs).
Definition: Core.h:121
DenseMap< SymbolStringPtr, JITSymbolFlags > SymbolFlagsMap
A map from symbol names (as SymbolStringPtrs) to JITSymbolFlags.
Definition: Core.h:124
unique_function< void(Expected< SymbolMap >)> SymbolsResolvedCallback
Callback to notify client that symbols have been resolved.
Definition: Core.h:399
DenseSet< SymbolStringPtr > SymbolNameSet
A set of symbol names (represented by SymbolStringPtrs for.
Definition: Core.h:114
LookupKind
Describes the kind of lookup being performed.
Definition: Core.h:157
RegisterDependenciesFunction NoDependenciesToRegister
This can be used as the value for a RegisterDependenciesFunction if there are no dependants to regist...
Definition: Core.cpp:37
std::vector< SymbolStringPtr > SymbolNameVector
A vector of symbol names.
Definition: Core.h:117
SymbolState
Represents the state that a symbol has reached during materialization.
Definition: Core.h:859
@ Materializing
Added to the symbol table, never queried.
@ NeverSearched
No symbol should be in this state.
@ Ready
Emitted to memory, but waiting on transitive dependencies.
@ Emitted
Assigned address, still materializing.
@ Resolved
Queried, materialization begun.
std::error_code orcError(OrcErrorCode ErrCode)
Definition: OrcError.cpp:84
DenseMap< JITDylib *, SymbolNameSet > SymbolDependenceMap
A map from JITDylibs to sets of symbols.
Definition: Core.h:127
Expected< SymbolAliasMap > buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols)
Build a SymbolAliasMap for the common case where you want to re-export symbols from another JITDylib ...
NodeAddr< DefNode * > Def
Definition: RDFGraph.h:384
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1742
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:98
auto formatv(const char *Fmt, Ts &&...Vals) -> formatv_object< decltype(std::make_tuple(support::detail::build_format_adapter(std::forward< Ts >(Vals))...))>
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2073
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:419
Error joinErrors(Error E1, Error E2)
Concatenate errors.
Definition: Error.h:438
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
Definition: Format.h:125
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1954
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1849
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1749
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition: STLExtras.h:1879
Implement std::hash so that hash_code can be used in STL containers.
Definition: BitVector.h:858
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define NDEBUG
Definition: regutils.h:48