LLVM 19.0.0git
Core.cpp
Go to the documentation of this file.
1//===--- Core.cpp - Core ORC APIs (MaterializationUnit, JITDylib, etc.) ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
10
11#include "llvm/ADT/STLExtras.h"
12#include "llvm/Config/llvm-config.h"
17
18#include <condition_variable>
19#include <future>
20#include <optional>
21
22#define DEBUG_TYPE "orc"
23
24namespace llvm {
25namespace orc {
26
29char SymbolsNotFound::ID = 0;
35char LookupTask::ID = 0;
36
39
40void MaterializationUnit::anchor() {}
41
43 assert((reinterpret_cast<uintptr_t>(JD.get()) & 0x1) == 0 &&
44 "JITDylib must be two byte aligned");
45 JD->Retain();
46 JDAndFlag.store(reinterpret_cast<uintptr_t>(JD.get()));
47}
48
50 getJITDylib().getExecutionSession().destroyResourceTracker(*this);
52}
53
55 return getJITDylib().getExecutionSession().removeResourceTracker(*this);
56}
57
59 getJITDylib().getExecutionSession().transferResourceTracker(DstRT, *this);
60}
61
62void ResourceTracker::makeDefunct() {
63 uintptr_t Val = JDAndFlag.load();
64 Val |= 0x1U;
65 JDAndFlag.store(Val);
66}
67
69
71 : RT(std::move(RT)) {}
72
75}
76
78 OS << "Resource tracker " << (void *)RT.get() << " became defunct";
79}
80
82 std::shared_ptr<SymbolStringPool> SSP,
83 std::shared_ptr<SymbolDependenceMap> Symbols)
84 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
85 assert(this->SSP && "String pool cannot be null");
86 assert(!this->Symbols->empty() && "Can not fail to resolve an empty set");
87
88 // FIXME: Use a new dep-map type for FailedToMaterialize errors so that we
89 // don't have to manually retain/release.
90 for (auto &[JD, Syms] : *this->Symbols)
91 JD->Retain();
92}
93
95 for (auto &[JD, Syms] : *Symbols)
96 JD->Release();
97}
98
101}
102
104 OS << "Failed to materialize symbols: " << *Symbols;
105}
106
108 std::shared_ptr<SymbolStringPool> SSP, JITDylibSP JD,
109 SymbolNameSet FailedSymbols, SymbolDependenceMap BadDeps,
110 std::string Explanation)
111 : SSP(std::move(SSP)), JD(std::move(JD)),
112 FailedSymbols(std::move(FailedSymbols)), BadDeps(std::move(BadDeps)),
113 Explanation(std::move(Explanation)) {}
114
117}
118
120 OS << "In " << JD->getName() << ", failed to materialize " << FailedSymbols
121 << ", due to unsatisfied dependencies " << BadDeps;
122 if (!Explanation.empty())
123 OS << " (" << Explanation << ")";
124}
125
126SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
127 SymbolNameSet Symbols)
128 : SSP(std::move(SSP)) {
129 for (auto &Sym : Symbols)
130 this->Symbols.push_back(Sym);
131 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
132}
133
134SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP,
135 SymbolNameVector Symbols)
136 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
137 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
138}
139
140std::error_code SymbolsNotFound::convertToErrorCode() const {
142}
143
145 OS << "Symbols not found: " << Symbols;
146}
147
149 std::shared_ptr<SymbolStringPool> SSP, SymbolNameSet Symbols)
150 : SSP(std::move(SSP)), Symbols(std::move(Symbols)) {
151 assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
152}
153
156}
157
159 OS << "Symbols could not be removed: " << Symbols;
160}
161
164}
165
167 OS << "Missing definitions in module " << ModuleName
168 << ": " << Symbols;
169}
170
173}
174
176 OS << "Unexpected definitions in module " << ModuleName
177 << ": " << Symbols;
178}
179
181 const SymbolLookupSet &Symbols, SymbolState RequiredState,
182 SymbolsResolvedCallback NotifyComplete)
183 : NotifyComplete(std::move(NotifyComplete)), RequiredState(RequiredState) {
184 assert(RequiredState >= SymbolState::Resolved &&
185 "Cannot query for a symbols that have not reached the resolve state "
186 "yet");
187
188 OutstandingSymbolsCount = Symbols.size();
189
190 for (auto &KV : Symbols)
191 ResolvedSymbols[KV.first] = ExecutorSymbolDef();
192}
193
196 auto I = ResolvedSymbols.find(Name);
197 assert(I != ResolvedSymbols.end() &&
198 "Resolving symbol outside the requested set");
199 assert(I->second == ExecutorSymbolDef() &&
200 "Redundantly resolving symbol Name");
201
202 // If this is a materialization-side-effects-only symbol then drop it,
203 // otherwise update its map entry with its resolved address.
204 if (Sym.getFlags().hasMaterializationSideEffectsOnly())
205 ResolvedSymbols.erase(I);
206 else
207 I->second = std::move(Sym);
208 --OutstandingSymbolsCount;
209}
210
211void AsynchronousSymbolQuery::handleComplete(ExecutionSession &ES) {
212 assert(OutstandingSymbolsCount == 0 &&
213 "Symbols remain, handleComplete called prematurely");
214
215 class RunQueryCompleteTask : public Task {
216 public:
217 RunQueryCompleteTask(SymbolMap ResolvedSymbols,
218 SymbolsResolvedCallback NotifyComplete)
219 : ResolvedSymbols(std::move(ResolvedSymbols)),
220 NotifyComplete(std::move(NotifyComplete)) {}
221 void printDescription(raw_ostream &OS) override {
222 OS << "Execute query complete callback for " << ResolvedSymbols;
223 }
224 void run() override { NotifyComplete(std::move(ResolvedSymbols)); }
225
226 private:
227 SymbolMap ResolvedSymbols;
228 SymbolsResolvedCallback NotifyComplete;
229 };
230
231 auto T = std::make_unique<RunQueryCompleteTask>(std::move(ResolvedSymbols),
232 std::move(NotifyComplete));
233 NotifyComplete = SymbolsResolvedCallback();
234 ES.dispatchTask(std::move(T));
235}
236
237void AsynchronousSymbolQuery::handleFailed(Error Err) {
238 assert(QueryRegistrations.empty() && ResolvedSymbols.empty() &&
239 OutstandingSymbolsCount == 0 &&
240 "Query should already have been abandoned");
241 NotifyComplete(std::move(Err));
242 NotifyComplete = SymbolsResolvedCallback();
243}
244
245void AsynchronousSymbolQuery::addQueryDependence(JITDylib &JD,
246 SymbolStringPtr Name) {
247 bool Added = QueryRegistrations[&JD].insert(std::move(Name)).second;
248 (void)Added;
249 assert(Added && "Duplicate dependence notification?");
250}
251
252void AsynchronousSymbolQuery::removeQueryDependence(
253 JITDylib &JD, const SymbolStringPtr &Name) {
254 auto QRI = QueryRegistrations.find(&JD);
255 assert(QRI != QueryRegistrations.end() &&
256 "No dependencies registered for JD");
257 assert(QRI->second.count(Name) && "No dependency on Name in JD");
258 QRI->second.erase(Name);
259 if (QRI->second.empty())
260 QueryRegistrations.erase(QRI);
261}
262
263void AsynchronousSymbolQuery::dropSymbol(const SymbolStringPtr &Name) {
264 auto I = ResolvedSymbols.find(Name);
265 assert(I != ResolvedSymbols.end() &&
266 "Redundant removal of weakly-referenced symbol");
267 ResolvedSymbols.erase(I);
268 --OutstandingSymbolsCount;
269}
270
271void AsynchronousSymbolQuery::detach() {
272 ResolvedSymbols.clear();
273 OutstandingSymbolsCount = 0;
274 for (auto &KV : QueryRegistrations)
275 KV.first->detachQueryHelper(*this, KV.second);
276 QueryRegistrations.clear();
277}
278
280 SymbolMap Symbols)
281 : MaterializationUnit(extractFlags(Symbols)), Symbols(std::move(Symbols)) {}
282
284 return "<Absolute Symbols>";
285}
286
287void AbsoluteSymbolsMaterializationUnit::materialize(
288 std::unique_ptr<MaterializationResponsibility> R) {
289 // Even though these are just absolute symbols we need to check for failure
290 // to resolve/emit: the tracker for these symbols may have been removed while
291 // the materialization was in flight (e.g. due to a failure in some action
292 // triggered by the queries attached to the resolution/emission of these
293 // symbols).
294 if (auto Err = R->notifyResolved(Symbols)) {
295 R->getExecutionSession().reportError(std::move(Err));
296 R->failMaterialization();
297 return;
298 }
299 if (auto Err = R->notifyEmitted({})) {
300 R->getExecutionSession().reportError(std::move(Err));
301 R->failMaterialization();
302 return;
303 }
304}
305
306void AbsoluteSymbolsMaterializationUnit::discard(const JITDylib &JD,
307 const SymbolStringPtr &Name) {
308 assert(Symbols.count(Name) && "Symbol is not part of this MU");
309 Symbols.erase(Name);
310}
311
312MaterializationUnit::Interface
313AbsoluteSymbolsMaterializationUnit::extractFlags(const SymbolMap &Symbols) {
315 for (const auto &KV : Symbols)
316 Flags[KV.first] = KV.second.getFlags();
317 return MaterializationUnit::Interface(std::move(Flags), nullptr);
318}
319
321 JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags,
322 SymbolAliasMap Aliases)
323 : MaterializationUnit(extractFlags(Aliases)), SourceJD(SourceJD),
324 SourceJDLookupFlags(SourceJDLookupFlags), Aliases(std::move(Aliases)) {}
325
327 return "<Reexports>";
328}
329
330void ReExportsMaterializationUnit::materialize(
331 std::unique_ptr<MaterializationResponsibility> R) {
332
333 auto &ES = R->getTargetJITDylib().getExecutionSession();
334 JITDylib &TgtJD = R->getTargetJITDylib();
335 JITDylib &SrcJD = SourceJD ? *SourceJD : TgtJD;
336
337 // Find the set of requested aliases and aliasees. Return any unrequested
338 // aliases back to the JITDylib so as to not prematurely materialize any
339 // aliasees.
340 auto RequestedSymbols = R->getRequestedSymbols();
341 SymbolAliasMap RequestedAliases;
342
343 for (auto &Name : RequestedSymbols) {
344 auto I = Aliases.find(Name);
345 assert(I != Aliases.end() && "Symbol not found in aliases map?");
346 RequestedAliases[Name] = std::move(I->second);
347 Aliases.erase(I);
348 }
349
350 LLVM_DEBUG({
351 ES.runSessionLocked([&]() {
352 dbgs() << "materializing reexports: target = " << TgtJD.getName()
353 << ", source = " << SrcJD.getName() << " " << RequestedAliases
354 << "\n";
355 });
356 });
357
358 if (!Aliases.empty()) {
359 auto Err = SourceJD ? R->replace(reexports(*SourceJD, std::move(Aliases),
360 SourceJDLookupFlags))
361 : R->replace(symbolAliases(std::move(Aliases)));
362
363 if (Err) {
364 // FIXME: Should this be reported / treated as failure to materialize?
365 // Or should this be treated as a sanctioned bailing-out?
366 ES.reportError(std::move(Err));
367 R->failMaterialization();
368 return;
369 }
370 }
371
372 // The OnResolveInfo struct will hold the aliases and responsibility for each
373 // query in the list.
374 struct OnResolveInfo {
375 OnResolveInfo(std::unique_ptr<MaterializationResponsibility> R,
376 SymbolAliasMap Aliases)
377 : R(std::move(R)), Aliases(std::move(Aliases)) {}
378
379 std::unique_ptr<MaterializationResponsibility> R;
380 SymbolAliasMap Aliases;
381 std::vector<SymbolDependenceGroup> SDGs;
382 };
383
384 // Build a list of queries to issue. In each round we build a query for the
385 // largest set of aliases that we can resolve without encountering a chain of
386 // aliases (e.g. Foo -> Bar, Bar -> Baz). Such a chain would deadlock as the
387 // query would be waiting on a symbol that it itself had to resolve. Creating
388 // a new query for each link in such a chain eliminates the possibility of
389 // deadlock. In practice chains are likely to be rare, and this algorithm will
390 // usually result in a single query to issue.
391
392 std::vector<std::pair<SymbolLookupSet, std::shared_ptr<OnResolveInfo>>>
393 QueryInfos;
394 while (!RequestedAliases.empty()) {
395 SymbolNameSet ResponsibilitySymbols;
396 SymbolLookupSet QuerySymbols;
397 SymbolAliasMap QueryAliases;
398
399 // Collect as many aliases as we can without including a chain.
400 for (auto &KV : RequestedAliases) {
401 // Chain detected. Skip this symbol for this round.
402 if (&SrcJD == &TgtJD && (QueryAliases.count(KV.second.Aliasee) ||
403 RequestedAliases.count(KV.second.Aliasee)))
404 continue;
405
406 ResponsibilitySymbols.insert(KV.first);
407 QuerySymbols.add(KV.second.Aliasee,
408 KV.second.AliasFlags.hasMaterializationSideEffectsOnly()
411 QueryAliases[KV.first] = std::move(KV.second);
412 }
413
414 // Remove the aliases collected this round from the RequestedAliases map.
415 for (auto &KV : QueryAliases)
416 RequestedAliases.erase(KV.first);
417
418 assert(!QuerySymbols.empty() && "Alias cycle detected!");
419
420 auto NewR = R->delegate(ResponsibilitySymbols);
421 if (!NewR) {
422 ES.reportError(NewR.takeError());
423 R->failMaterialization();
424 return;
425 }
426
427 auto QueryInfo = std::make_shared<OnResolveInfo>(std::move(*NewR),
428 std::move(QueryAliases));
429 QueryInfos.push_back(
430 make_pair(std::move(QuerySymbols), std::move(QueryInfo)));
431 }
432
433 // Issue the queries.
434 while (!QueryInfos.empty()) {
435 auto QuerySymbols = std::move(QueryInfos.back().first);
436 auto QueryInfo = std::move(QueryInfos.back().second);
437
438 QueryInfos.pop_back();
439
440 auto RegisterDependencies = [QueryInfo,
441 &SrcJD](const SymbolDependenceMap &Deps) {
442 // If there were no materializing symbols, just bail out.
443 if (Deps.empty())
444 return;
445
446 // Otherwise the only deps should be on SrcJD.
447 assert(Deps.size() == 1 && Deps.count(&SrcJD) &&
448 "Unexpected dependencies for reexports");
449
450 auto &SrcJDDeps = Deps.find(&SrcJD)->second;
451
452 for (auto &[Alias, AliasInfo] : QueryInfo->Aliases)
453 if (SrcJDDeps.count(AliasInfo.Aliasee))
454 QueryInfo->SDGs.push_back({{Alias}, {{&SrcJD, {AliasInfo.Aliasee}}}});
455 };
456
457 auto OnComplete = [QueryInfo](Expected<SymbolMap> Result) {
458 auto &ES = QueryInfo->R->getTargetJITDylib().getExecutionSession();
459 if (Result) {
460 SymbolMap ResolutionMap;
461 for (auto &KV : QueryInfo->Aliases) {
462 assert((KV.second.AliasFlags.hasMaterializationSideEffectsOnly() ||
463 Result->count(KV.second.Aliasee)) &&
464 "Result map missing entry?");
465 // Don't try to resolve materialization-side-effects-only symbols.
466 if (KV.second.AliasFlags.hasMaterializationSideEffectsOnly())
467 continue;
468
469 ResolutionMap[KV.first] = {(*Result)[KV.second.Aliasee].getAddress(),
470 KV.second.AliasFlags};
471 }
472 if (auto Err = QueryInfo->R->notifyResolved(ResolutionMap)) {
473 ES.reportError(std::move(Err));
474 QueryInfo->R->failMaterialization();
475 return;
476 }
477 if (auto Err = QueryInfo->R->notifyEmitted(QueryInfo->SDGs)) {
478 ES.reportError(std::move(Err));
479 QueryInfo->R->failMaterialization();
480 return;
481 }
482 } else {
483 ES.reportError(Result.takeError());
484 QueryInfo->R->failMaterialization();
485 }
486 };
487
489 JITDylibSearchOrder({{&SrcJD, SourceJDLookupFlags}}),
490 QuerySymbols, SymbolState::Resolved, std::move(OnComplete),
491 std::move(RegisterDependencies));
492 }
493}
494
495void ReExportsMaterializationUnit::discard(const JITDylib &JD,
496 const SymbolStringPtr &Name) {
497 assert(Aliases.count(Name) &&
498 "Symbol not covered by this MaterializationUnit");
499 Aliases.erase(Name);
500}
501
502MaterializationUnit::Interface
503ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) {
505 for (auto &KV : Aliases)
506 SymbolFlags[KV.first] = KV.second.AliasFlags;
507
508 return MaterializationUnit::Interface(std::move(SymbolFlags), nullptr);
509}
510
512 SymbolNameSet Symbols) {
513 SymbolLookupSet LookupSet(Symbols);
514 auto Flags = SourceJD.getExecutionSession().lookupFlags(
516 SymbolLookupSet(std::move(Symbols)));
517
518 if (!Flags)
519 return Flags.takeError();
520
522 for (auto &Name : Symbols) {
523 assert(Flags->count(Name) && "Missing entry in flags map");
524 Result[Name] = SymbolAliasMapEntry(Name, (*Flags)[Name]);
525 }
526
527 return Result;
528}
529
531public:
532 // FIXME: Reduce the number of SymbolStringPtrs here. See
533 // https://github.com/llvm/llvm-project/issues/55576.
534
540 }
541 virtual ~InProgressLookupState() = default;
542 virtual void complete(std::unique_ptr<InProgressLookupState> IPLS) = 0;
543 virtual void fail(Error Err) = 0;
544
549
551 bool NewJITDylib = true;
554
555 enum {
556 NotInGenerator, // Not currently using a generator.
557 ResumedForGenerator, // Resumed after being auto-suspended before generator.
558 InGenerator // Currently using generator.
560 std::vector<std::weak_ptr<DefinitionGenerator>> CurDefGeneratorStack;
561};
562
564public:
570 OnComplete(std::move(OnComplete)) {}
571
572 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
573 auto &ES = SearchOrder.front().first->getExecutionSession();
574 ES.OL_completeLookupFlags(std::move(IPLS), std::move(OnComplete));
575 }
576
577 void fail(Error Err) override { OnComplete(std::move(Err)); }
578
579private:
581};
582
584public:
588 std::shared_ptr<AsynchronousSymbolQuery> Q,
589 RegisterDependenciesFunction RegisterDependencies)
592 Q(std::move(Q)), RegisterDependencies(std::move(RegisterDependencies)) {
593 }
594
595 void complete(std::unique_ptr<InProgressLookupState> IPLS) override {
596 auto &ES = SearchOrder.front().first->getExecutionSession();
597 ES.OL_completeLookup(std::move(IPLS), std::move(Q),
598 std::move(RegisterDependencies));
599 }
600
601 void fail(Error Err) override {
602 Q->detach();
603 Q->handleFailed(std::move(Err));
604 }
605
606private:
607 std::shared_ptr<AsynchronousSymbolQuery> Q;
608 RegisterDependenciesFunction RegisterDependencies;
609};
610
612 JITDylibLookupFlags SourceJDLookupFlags,
613 SymbolPredicate Allow)
614 : SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags),
615 Allow(std::move(Allow)) {}
616
618 JITDylib &JD,
619 JITDylibLookupFlags JDLookupFlags,
620 const SymbolLookupSet &LookupSet) {
621 assert(&JD != &SourceJD && "Cannot re-export from the same dylib");
622
623 // Use lookupFlags to find the subset of symbols that match our lookup.
624 auto Flags = JD.getExecutionSession().lookupFlags(
625 K, {{&SourceJD, JDLookupFlags}}, LookupSet);
626 if (!Flags)
627 return Flags.takeError();
628
629 // Create an alias map.
630 orc::SymbolAliasMap AliasMap;
631 for (auto &KV : *Flags)
632 if (!Allow || Allow(KV.first))
633 AliasMap[KV.first] = SymbolAliasMapEntry(KV.first, KV.second);
634
635 if (AliasMap.empty())
636 return Error::success();
637
638 // Define the re-exports.
639 return JD.define(reexports(SourceJD, AliasMap, SourceJDLookupFlags));
640}
641
642LookupState::LookupState(std::unique_ptr<InProgressLookupState> IPLS)
643 : IPLS(std::move(IPLS)) {}
644
645void LookupState::reset(InProgressLookupState *IPLS) { this->IPLS.reset(IPLS); }
646
647LookupState::LookupState() = default;
648LookupState::LookupState(LookupState &&) = default;
649LookupState &LookupState::operator=(LookupState &&) = default;
650LookupState::~LookupState() = default;
651
653 assert(IPLS && "Cannot call continueLookup on empty LookupState");
654 auto &ES = IPLS->SearchOrder.begin()->first->getExecutionSession();
655 ES.OL_applyQueryPhase1(std::move(IPLS), std::move(Err));
656}
657
659 std::deque<LookupState> LookupsToFail;
660 {
661 std::lock_guard<std::mutex> Lock(M);
662 std::swap(PendingLookups, LookupsToFail);
663 InUse = false;
664 }
665
666 for (auto &LS : LookupsToFail)
667 LS.continueLookup(make_error<StringError>(
668 "Query waiting on DefinitionGenerator that was destroyed",
670}
671
673 LLVM_DEBUG(dbgs() << "Destroying JITDylib " << getName() << "\n");
674}
675
677 std::vector<ResourceTrackerSP> TrackersToRemove;
678 ES.runSessionLocked([&]() {
679 assert(State != Closed && "JD is defunct");
680 for (auto &KV : TrackerSymbols)
681 TrackersToRemove.push_back(KV.first);
682 TrackersToRemove.push_back(getDefaultResourceTracker());
683 });
684
685 Error Err = Error::success();
686 for (auto &RT : TrackersToRemove)
687 Err = joinErrors(std::move(Err), RT->remove());
688 return Err;
689}
690
692 return ES.runSessionLocked([this] {
693 assert(State != Closed && "JD is defunct");
694 if (!DefaultTracker)
695 DefaultTracker = new ResourceTracker(this);
696 return DefaultTracker;
697 });
698}
699
701 return ES.runSessionLocked([this] {
702 assert(State == Open && "JD is defunct");
703 ResourceTrackerSP RT = new ResourceTracker(this);
704 return RT;
705 });
706}
707
709 // DefGenerator moved into TmpDG to ensure that it's destroyed outside the
710 // session lock (since it may have to send errors to pending queries).
711 std::shared_ptr<DefinitionGenerator> TmpDG;
712
713 ES.runSessionLocked([&] {
714 assert(State == Open && "JD is defunct");
715 auto I = llvm::find_if(DefGenerators,
716 [&](const std::shared_ptr<DefinitionGenerator> &H) {
717 return H.get() == &G;
718 });
719 assert(I != DefGenerators.end() && "Generator not found");
720 TmpDG = std::move(*I);
721 DefGenerators.erase(I);
722 });
723}
724
726JITDylib::defineMaterializing(MaterializationResponsibility &FromMR,
727 SymbolFlagsMap SymbolFlags) {
728
729 return ES.runSessionLocked([&]() -> Expected<SymbolFlagsMap> {
730 if (FromMR.RT->isDefunct())
731 return make_error<ResourceTrackerDefunct>(FromMR.RT);
732
733 std::vector<NonOwningSymbolStringPtr> AddedSyms;
734 std::vector<NonOwningSymbolStringPtr> RejectedWeakDefs;
735
736 for (auto SFItr = SymbolFlags.begin(), SFEnd = SymbolFlags.end();
737 SFItr != SFEnd; ++SFItr) {
738
739 auto &Name = SFItr->first;
740 auto &Flags = SFItr->second;
741
742 auto EntryItr = Symbols.find(Name);
743
744 // If the entry already exists...
745 if (EntryItr != Symbols.end()) {
746
747 // If this is a strong definition then error out.
748 if (!Flags.isWeak()) {
749 // Remove any symbols already added.
750 for (auto &S : AddedSyms)
751 Symbols.erase(Symbols.find_as(S));
752
753 // FIXME: Return all duplicates.
754 return make_error<DuplicateDefinition>(std::string(*Name));
755 }
756
757 // Otherwise just make a note to discard this symbol after the loop.
758 RejectedWeakDefs.push_back(NonOwningSymbolStringPtr(Name));
759 continue;
760 } else
761 EntryItr =
762 Symbols.insert(std::make_pair(Name, SymbolTableEntry(Flags))).first;
763
764 AddedSyms.push_back(NonOwningSymbolStringPtr(Name));
765 EntryItr->second.setState(SymbolState::Materializing);
766 }
767
768 // Remove any rejected weak definitions from the SymbolFlags map.
769 while (!RejectedWeakDefs.empty()) {
770 SymbolFlags.erase(SymbolFlags.find_as(RejectedWeakDefs.back()));
771 RejectedWeakDefs.pop_back();
772 }
773
774 return SymbolFlags;
775 });
776}
777
778Error JITDylib::replace(MaterializationResponsibility &FromMR,
779 std::unique_ptr<MaterializationUnit> MU) {
780 assert(MU != nullptr && "Can not replace with a null MaterializationUnit");
781 std::unique_ptr<MaterializationUnit> MustRunMU;
782 std::unique_ptr<MaterializationResponsibility> MustRunMR;
783
784 auto Err =
785 ES.runSessionLocked([&, this]() -> Error {
786 if (FromMR.RT->isDefunct())
787 return make_error<ResourceTrackerDefunct>(std::move(FromMR.RT));
788
789#ifndef NDEBUG
790 for (auto &KV : MU->getSymbols()) {
791 auto SymI = Symbols.find(KV.first);
792 assert(SymI != Symbols.end() && "Replacing unknown symbol");
793 assert(SymI->second.getState() == SymbolState::Materializing &&
794 "Can not replace a symbol that ha is not materializing");
795 assert(!SymI->second.hasMaterializerAttached() &&
796 "Symbol should not have materializer attached already");
797 assert(UnmaterializedInfos.count(KV.first) == 0 &&
798 "Symbol being replaced should have no UnmaterializedInfo");
799 }
800#endif // NDEBUG
801
802 // If the tracker is defunct we need to bail out immediately.
803
804 // If any symbol has pending queries against it then we need to
805 // materialize MU immediately.
806 for (auto &KV : MU->getSymbols()) {
807 auto MII = MaterializingInfos.find(KV.first);
808 if (MII != MaterializingInfos.end()) {
809 if (MII->second.hasQueriesPending()) {
810 MustRunMR = ES.createMaterializationResponsibility(
811 *FromMR.RT, std::move(MU->SymbolFlags),
812 std::move(MU->InitSymbol));
813 MustRunMU = std::move(MU);
814 return Error::success();
815 }
816 }
817 }
818
819 // Otherwise, make MU responsible for all the symbols.
820 auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU),
821 FromMR.RT.get());
822 for (auto &KV : UMI->MU->getSymbols()) {
823 auto SymI = Symbols.find(KV.first);
824 assert(SymI->second.getState() == SymbolState::Materializing &&
825 "Can not replace a symbol that is not materializing");
826 assert(!SymI->second.hasMaterializerAttached() &&
827 "Can not replace a symbol that has a materializer attached");
828 assert(UnmaterializedInfos.count(KV.first) == 0 &&
829 "Unexpected materializer entry in map");
830 SymI->second.setAddress(SymI->second.getAddress());
831 SymI->second.setMaterializerAttached(true);
832
833 auto &UMIEntry = UnmaterializedInfos[KV.first];
834 assert((!UMIEntry || !UMIEntry->MU) &&
835 "Replacing symbol with materializer still attached");
836 UMIEntry = UMI;
837 }
838
839 return Error::success();
840 });
841
842 if (Err)
843 return Err;
844
845 if (MustRunMU) {
846 assert(MustRunMR && "MustRunMU set implies MustRunMR set");
847 ES.dispatchTask(std::make_unique<MaterializationTask>(
848 std::move(MustRunMU), std::move(MustRunMR)));
849 } else {
850 assert(!MustRunMR && "MustRunMU unset implies MustRunMR unset");
851 }
852
853 return Error::success();
854}
855
856Expected<std::unique_ptr<MaterializationResponsibility>>
857JITDylib::delegate(MaterializationResponsibility &FromMR,
858 SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol) {
859
860 return ES.runSessionLocked(
861 [&]() -> Expected<std::unique_ptr<MaterializationResponsibility>> {
862 if (FromMR.RT->isDefunct())
863 return make_error<ResourceTrackerDefunct>(std::move(FromMR.RT));
864
865 return ES.createMaterializationResponsibility(
866 *FromMR.RT, std::move(SymbolFlags), std::move(InitSymbol));
867 });
868}
869
871JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const {
872 return ES.runSessionLocked([&]() {
873 SymbolNameSet RequestedSymbols;
874
875 for (auto &KV : SymbolFlags) {
876 assert(Symbols.count(KV.first) && "JITDylib does not cover this symbol?");
877 assert(Symbols.find(KV.first)->second.getState() !=
878 SymbolState::NeverSearched &&
879 Symbols.find(KV.first)->second.getState() != SymbolState::Ready &&
880 "getRequestedSymbols can only be called for symbols that have "
881 "started materializing");
882 auto I = MaterializingInfos.find(KV.first);
883 if (I == MaterializingInfos.end())
884 continue;
885
886 if (I->second.hasQueriesPending())
887 RequestedSymbols.insert(KV.first);
888 }
889
890 return RequestedSymbols;
891 });
892}
893
894Error JITDylib::resolve(MaterializationResponsibility &MR,
895 const SymbolMap &Resolved) {
896 AsynchronousSymbolQuerySet CompletedQueries;
897
898 if (auto Err = ES.runSessionLocked([&, this]() -> Error {
899 if (MR.RT->isDefunct())
900 return make_error<ResourceTrackerDefunct>(MR.RT);
901
902 if (State != Open)
903 return make_error<StringError>("JITDylib " + getName() +
904 " is defunct",
905 inconvertibleErrorCode());
906
907 struct WorklistEntry {
908 SymbolTable::iterator SymI;
909 ExecutorSymbolDef ResolvedSym;
910 };
911
912 SymbolNameSet SymbolsInErrorState;
913 std::vector<WorklistEntry> Worklist;
914 Worklist.reserve(Resolved.size());
915
916 // Build worklist and check for any symbols in the error state.
917 for (const auto &KV : Resolved) {
918
919 assert(!KV.second.getFlags().hasError() &&
920 "Resolution result can not have error flag set");
921
922 auto SymI = Symbols.find(KV.first);
923
924 assert(SymI != Symbols.end() && "Symbol not found");
925 assert(!SymI->second.hasMaterializerAttached() &&
926 "Resolving symbol with materializer attached?");
927 assert(SymI->second.getState() == SymbolState::Materializing &&
928 "Symbol should be materializing");
929 assert(SymI->second.getAddress() == ExecutorAddr() &&
930 "Symbol has already been resolved");
931
932 if (SymI->second.getFlags().hasError())
933 SymbolsInErrorState.insert(KV.first);
934 else {
935 auto Flags = KV.second.getFlags();
936 Flags &= ~JITSymbolFlags::Common;
937 assert(Flags ==
938 (SymI->second.getFlags() & ~JITSymbolFlags::Common) &&
939 "Resolved flags should match the declared flags");
940
941 Worklist.push_back({SymI, {KV.second.getAddress(), Flags}});
942 }
943 }
944
945 // If any symbols were in the error state then bail out.
946 if (!SymbolsInErrorState.empty()) {
947 auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>();
948 (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState);
949 return make_error<FailedToMaterialize>(
950 getExecutionSession().getSymbolStringPool(),
951 std::move(FailedSymbolsDepMap));
952 }
953
954 while (!Worklist.empty()) {
955 auto SymI = Worklist.back().SymI;
956 auto ResolvedSym = Worklist.back().ResolvedSym;
957 Worklist.pop_back();
958
959 auto &Name = SymI->first;
960
961 // Resolved symbols can not be weak: discard the weak flag.
962 JITSymbolFlags ResolvedFlags = ResolvedSym.getFlags();
963 SymI->second.setAddress(ResolvedSym.getAddress());
964 SymI->second.setFlags(ResolvedFlags);
965 SymI->second.setState(SymbolState::Resolved);
966
967 auto MII = MaterializingInfos.find(Name);
968 if (MII == MaterializingInfos.end())
969 continue;
970
971 auto &MI = MII->second;
972 for (auto &Q : MI.takeQueriesMeeting(SymbolState::Resolved)) {
973 Q->notifySymbolMetRequiredState(Name, ResolvedSym);
974 Q->removeQueryDependence(*this, Name);
975 if (Q->isComplete())
976 CompletedQueries.insert(std::move(Q));
977 }
978 }
979
980 return Error::success();
981 }))
982 return Err;
983
984 // Otherwise notify all the completed queries.
985 for (auto &Q : CompletedQueries) {
986 assert(Q->isComplete() && "Q not completed");
987 Q->handleComplete(ES);
988 }
989
990 return Error::success();
991}
992
993void JITDylib::unlinkMaterializationResponsibility(
994 MaterializationResponsibility &MR) {
995 ES.runSessionLocked([&]() {
996 auto I = TrackerMRs.find(MR.RT.get());
997 assert(I != TrackerMRs.end() && "No MRs in TrackerMRs list for RT");
998 assert(I->second.count(&MR) && "MR not in TrackerMRs list for RT");
999 I->second.erase(&MR);
1000 if (I->second.empty())
1001 TrackerMRs.erase(MR.RT.get());
1002 });
1003}
1004
1005void JITDylib::shrinkMaterializationInfoMemory() {
1006 // DenseMap::erase never shrinks its storage; use clear to heuristically free
1007 // memory since we may have long-lived JDs after linking is done.
1008
1009 if (UnmaterializedInfos.empty())
1010 UnmaterializedInfos.clear();
1011
1012 if (MaterializingInfos.empty())
1013 MaterializingInfos.clear();
1014}
1015
1016void JITDylib::setLinkOrder(JITDylibSearchOrder NewLinkOrder,
1017 bool LinkAgainstThisJITDylibFirst) {
1018 ES.runSessionLocked([&]() {
1019 assert(State == Open && "JD is defunct");
1020 if (LinkAgainstThisJITDylibFirst) {
1021 LinkOrder.clear();
1022 if (NewLinkOrder.empty() || NewLinkOrder.front().first != this)
1023 LinkOrder.push_back(
1024 std::make_pair(this, JITDylibLookupFlags::MatchAllSymbols));
1025 llvm::append_range(LinkOrder, NewLinkOrder);
1026 } else
1027 LinkOrder = std::move(NewLinkOrder);
1028 });
1029}
1030
1031void JITDylib::addToLinkOrder(const JITDylibSearchOrder &NewLinks) {
1032 ES.runSessionLocked([&]() {
1033 for (auto &KV : NewLinks) {
1034 // Skip elements of NewLinks that are already in the link order.
1035 if (llvm::is_contained(LinkOrder, KV))
1036 continue;
1037
1038 LinkOrder.push_back(std::move(KV));
1039 }
1040 });
1041}
1042
1043void JITDylib::addToLinkOrder(JITDylib &JD, JITDylibLookupFlags JDLookupFlags) {
1044 ES.runSessionLocked([&]() { LinkOrder.push_back({&JD, JDLookupFlags}); });
1045}
1046
1047void JITDylib::replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD,
1048 JITDylibLookupFlags JDLookupFlags) {
1049 ES.runSessionLocked([&]() {
1050 assert(State == Open && "JD is defunct");
1051 for (auto &KV : LinkOrder)
1052 if (KV.first == &OldJD) {
1053 KV = {&NewJD, JDLookupFlags};
1054 break;
1055 }
1056 });
1057}
1058
1059void JITDylib::removeFromLinkOrder(JITDylib &JD) {
1060 ES.runSessionLocked([&]() {
1061 assert(State == Open && "JD is defunct");
1062 auto I = llvm::find_if(LinkOrder,
1063 [&](const JITDylibSearchOrder::value_type &KV) {
1064 return KV.first == &JD;
1065 });
1066 if (I != LinkOrder.end())
1067 LinkOrder.erase(I);
1068 });
1069}
1070
1071Error JITDylib::remove(const SymbolNameSet &Names) {
1072 return ES.runSessionLocked([&]() -> Error {
1073 assert(State == Open && "JD is defunct");
1074 using SymbolMaterializerItrPair =
1075 std::pair<SymbolTable::iterator, UnmaterializedInfosMap::iterator>;
1076 std::vector<SymbolMaterializerItrPair> SymbolsToRemove;
1077 SymbolNameSet Missing;
1079
1080 for (auto &Name : Names) {
1081 auto I = Symbols.find(Name);
1082
1083 // Note symbol missing.
1084 if (I == Symbols.end()) {
1085 Missing.insert(Name);
1086 continue;
1087 }
1088
1089 // Note symbol materializing.
1090 if (I->second.getState() != SymbolState::NeverSearched &&
1091 I->second.getState() != SymbolState::Ready) {
1092 Materializing.insert(Name);
1093 continue;
1094 }
1095
1096 auto UMII = I->second.hasMaterializerAttached()
1097 ? UnmaterializedInfos.find(Name)
1098 : UnmaterializedInfos.end();
1099 SymbolsToRemove.push_back(std::make_pair(I, UMII));
1100 }
1101
1102 // If any of the symbols are not defined, return an error.
1103 if (!Missing.empty())
1104 return make_error<SymbolsNotFound>(ES.getSymbolStringPool(),
1105 std::move(Missing));
1106
1107 // If any of the symbols are currently materializing, return an error.
1108 if (!Materializing.empty())
1109 return make_error<SymbolsCouldNotBeRemoved>(ES.getSymbolStringPool(),
1110 std::move(Materializing));
1111
1112 // Remove the symbols.
1113 for (auto &SymbolMaterializerItrPair : SymbolsToRemove) {
1114 auto UMII = SymbolMaterializerItrPair.second;
1115
1116 // If there is a materializer attached, call discard.
1117 if (UMII != UnmaterializedInfos.end()) {
1118 UMII->second->MU->doDiscard(*this, UMII->first);
1119 UnmaterializedInfos.erase(UMII);
1120 }
1121
1122 auto SymI = SymbolMaterializerItrPair.first;
1123 Symbols.erase(SymI);
1124 }
1125
1126 shrinkMaterializationInfoMemory();
1127
1128 return Error::success();
1129 });
1130}
1131
1132void JITDylib::dump(raw_ostream &OS) {
1133 ES.runSessionLocked([&, this]() {
1134 OS << "JITDylib \"" << getName() << "\" (ES: "
1135 << format("0x%016" PRIx64, reinterpret_cast<uintptr_t>(&ES))
1136 << ", State = ";
1137 switch (State) {
1138 case Open:
1139 OS << "Open";
1140 break;
1141 case Closing:
1142 OS << "Closing";
1143 break;
1144 case Closed:
1145 OS << "Closed";
1146 break;
1147 }
1148 OS << ")\n";
1149 if (State == Closed)
1150 return;
1151 OS << "Link order: " << LinkOrder << "\n"
1152 << "Symbol table:\n";
1153
1154 // Sort symbols so we get a deterministic order and can check them in tests.
1155 std::vector<std::pair<SymbolStringPtr, SymbolTableEntry *>> SymbolsSorted;
1156 for (auto &KV : Symbols)
1157 SymbolsSorted.emplace_back(KV.first, &KV.second);
1158 std::sort(SymbolsSorted.begin(), SymbolsSorted.end(),
1159 [](const auto &L, const auto &R) { return *L.first < *R.first; });
1160
1161 for (auto &KV : SymbolsSorted) {
1162 OS << " \"" << *KV.first << "\": ";
1163 if (auto Addr = KV.second->getAddress())
1164 OS << Addr;
1165 else
1166 OS << "<not resolved> ";
1167
1168 OS << " " << KV.second->getFlags() << " " << KV.second->getState();
1169
1170 if (KV.second->hasMaterializerAttached()) {
1171 OS << " (Materializer ";
1172 auto I = UnmaterializedInfos.find(KV.first);
1173 assert(I != UnmaterializedInfos.end() &&
1174 "Lazy symbol should have UnmaterializedInfo");
1175 OS << I->second->MU.get() << ", " << I->second->MU->getName() << ")\n";
1176 } else
1177 OS << "\n";
1178 }
1179
1180 if (!MaterializingInfos.empty())
1181 OS << " MaterializingInfos entries:\n";
1182 for (auto &KV : MaterializingInfos) {
1183 OS << " \"" << *KV.first << "\":\n"
1184 << " " << KV.second.pendingQueries().size()
1185 << " pending queries: { ";
1186 for (const auto &Q : KV.second.pendingQueries())
1187 OS << Q.get() << " (" << Q->getRequiredState() << ") ";
1188 OS << "}\n Defining EDU: ";
1189 if (KV.second.DefiningEDU) {
1190 OS << KV.second.DefiningEDU.get() << " { ";
1191 for (auto &[Name, Flags] : KV.second.DefiningEDU->Symbols)
1192 OS << Name << " ";
1193 OS << "}\n";
1194 OS << " Dependencies:\n";
1195 if (!KV.second.DefiningEDU->Dependencies.empty()) {
1196 for (auto &[DepJD, Deps] : KV.second.DefiningEDU->Dependencies) {
1197 OS << " " << DepJD->getName() << ": [ ";
1198 for (auto &Dep : Deps)
1199 OS << Dep << " ";
1200 OS << "]\n";
1201 }
1202 } else
1203 OS << " none\n";
1204 } else
1205 OS << "none\n";
1206 OS << " Dependant EDUs:\n";
1207 if (!KV.second.DependantEDUs.empty()) {
1208 for (auto &DependantEDU : KV.second.DependantEDUs) {
1209 OS << " " << DependantEDU << ": "
1210 << DependantEDU->JD->getName() << " { ";
1211 for (auto &[Name, Flags] : DependantEDU->Symbols)
1212 OS << Name << " ";
1213 OS << "}\n";
1214 }
1215 } else
1216 OS << " none\n";
1217 assert((Symbols[KV.first].getState() != SymbolState::Ready ||
1218 (KV.second.pendingQueries().empty() && !KV.second.DefiningEDU &&
1219 !KV.second.DependantEDUs.empty())) &&
1220 "Stale materializing info entry");
1221 }
1222 });
1223}
1224
1225void JITDylib::MaterializingInfo::addQuery(
1226 std::shared_ptr<AsynchronousSymbolQuery> Q) {
1227
1228 auto I = llvm::lower_bound(
1229 llvm::reverse(PendingQueries), Q->getRequiredState(),
1230 [](const std::shared_ptr<AsynchronousSymbolQuery> &V, SymbolState S) {
1231 return V->getRequiredState() <= S;
1232 });
1233 PendingQueries.insert(I.base(), std::move(Q));
1234}
1235
1236void JITDylib::MaterializingInfo::removeQuery(
1237 const AsynchronousSymbolQuery &Q) {
1238 // FIXME: Implement 'find_as' for shared_ptr<T>/T*.
1239 auto I = llvm::find_if(
1240 PendingQueries, [&Q](const std::shared_ptr<AsynchronousSymbolQuery> &V) {
1241 return V.get() == &Q;
1242 });
1243 assert(I != PendingQueries.end() &&
1244 "Query is not attached to this MaterializingInfo");
1245 PendingQueries.erase(I);
1246}
1247
1248JITDylib::AsynchronousSymbolQueryList
1249JITDylib::MaterializingInfo::takeQueriesMeeting(SymbolState RequiredState) {
1250 AsynchronousSymbolQueryList Result;
1251 while (!PendingQueries.empty()) {
1252 if (PendingQueries.back()->getRequiredState() > RequiredState)
1253 break;
1254
1255 Result.push_back(std::move(PendingQueries.back()));
1256 PendingQueries.pop_back();
1257 }
1258
1259 return Result;
1260}
1261
1262JITDylib::JITDylib(ExecutionSession &ES, std::string Name)
1263 : JITLinkDylib(std::move(Name)), ES(ES) {
1264 LinkOrder.push_back({this, JITDylibLookupFlags::MatchAllSymbols});
1265}
1266
1267std::pair<JITDylib::AsynchronousSymbolQuerySet,
1268 std::shared_ptr<SymbolDependenceMap>>
1269JITDylib::removeTracker(ResourceTracker &RT) {
1270 // Note: Should be called under the session lock.
1271 assert(State != Closed && "JD is defunct");
1272
1273 SymbolNameVector SymbolsToRemove;
1274 SymbolNameVector SymbolsToFail;
1275
1276 if (&RT == DefaultTracker.get()) {
1277 SymbolNameSet TrackedSymbols;
1278 for (auto &KV : TrackerSymbols)
1279 for (auto &Sym : KV.second)
1280 TrackedSymbols.insert(Sym);
1281
1282 for (auto &KV : Symbols) {
1283 auto &Sym = KV.first;
1284 if (!TrackedSymbols.count(Sym))
1285 SymbolsToRemove.push_back(Sym);
1286 }
1287
1288 DefaultTracker.reset();
1289 } else {
1290 /// Check for a non-default tracker.
1291 auto I = TrackerSymbols.find(&RT);
1292 if (I != TrackerSymbols.end()) {
1293 SymbolsToRemove = std::move(I->second);
1294 TrackerSymbols.erase(I);
1295 }
1296 // ... if not found this tracker was already defunct. Nothing to do.
1297 }
1298
1299 for (auto &Sym : SymbolsToRemove) {
1300 assert(Symbols.count(Sym) && "Symbol not in symbol table");
1301
1302 // If there is a MaterializingInfo then collect any queries to fail.
1303 auto MII = MaterializingInfos.find(Sym);
1304 if (MII != MaterializingInfos.end())
1305 SymbolsToFail.push_back(Sym);
1306 }
1307
1308 AsynchronousSymbolQuerySet QueriesToFail;
1309 auto Result = ES.runSessionLocked(
1310 [&]() { return ES.IL_failSymbols(*this, std::move(SymbolsToFail)); });
1311
1312 // Removed symbols should be taken out of the table altogether.
1313 for (auto &Sym : SymbolsToRemove) {
1314 auto I = Symbols.find(Sym);
1315 assert(I != Symbols.end() && "Symbol not present in table");
1316
1317 // Remove Materializer if present.
1318 if (I->second.hasMaterializerAttached()) {
1319 // FIXME: Should this discard the symbols?
1320 UnmaterializedInfos.erase(Sym);
1321 } else {
1322 assert(!UnmaterializedInfos.count(Sym) &&
1323 "Symbol has materializer attached");
1324 }
1325
1326 Symbols.erase(I);
1327 }
1328
1329 shrinkMaterializationInfoMemory();
1330
1331 return Result;
1332}
1333
1334void JITDylib::transferTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT) {
1335 assert(State != Closed && "JD is defunct");
1336 assert(&DstRT != &SrcRT && "No-op transfers shouldn't call transferTracker");
1337 assert(&DstRT.getJITDylib() == this && "DstRT is not for this JITDylib");
1338 assert(&SrcRT.getJITDylib() == this && "SrcRT is not for this JITDylib");
1339
1340 // Update trackers for any not-yet materialized units.
1341 for (auto &KV : UnmaterializedInfos) {
1342 if (KV.second->RT == &SrcRT)
1343 KV.second->RT = &DstRT;
1344 }
1345
1346 // Update trackers for any active materialization responsibilities.
1347 {
1348 auto I = TrackerMRs.find(&SrcRT);
1349 if (I != TrackerMRs.end()) {
1350 auto &SrcMRs = I->second;
1351 auto &DstMRs = TrackerMRs[&DstRT];
1352 for (auto *MR : SrcMRs)
1353 MR->RT = &DstRT;
1354 if (DstMRs.empty())
1355 DstMRs = std::move(SrcMRs);
1356 else
1357 for (auto *MR : SrcMRs)
1358 DstMRs.insert(MR);
1359 // Erase SrcRT entry in TrackerMRs. Use &SrcRT key rather than iterator I
1360 // for this, since I may have been invalidated by 'TrackerMRs[&DstRT]'.
1361 TrackerMRs.erase(&SrcRT);
1362 }
1363 }
1364
1365 // If we're transfering to the default tracker we just need to delete the
1366 // tracked symbols for the source tracker.
1367 if (&DstRT == DefaultTracker.get()) {
1368 TrackerSymbols.erase(&SrcRT);
1369 return;
1370 }
1371
1372 // If we're transferring from the default tracker we need to find all
1373 // currently untracked symbols.
1374 if (&SrcRT == DefaultTracker.get()) {
1375 assert(!TrackerSymbols.count(&SrcRT) &&
1376 "Default tracker should not appear in TrackerSymbols");
1377
1378 SymbolNameVector SymbolsToTrack;
1379
1380 SymbolNameSet CurrentlyTrackedSymbols;
1381 for (auto &KV : TrackerSymbols)
1382 for (auto &Sym : KV.second)
1383 CurrentlyTrackedSymbols.insert(Sym);
1384
1385 for (auto &KV : Symbols) {
1386 auto &Sym = KV.first;
1387 if (!CurrentlyTrackedSymbols.count(Sym))
1388 SymbolsToTrack.push_back(Sym);
1389 }
1390
1391 TrackerSymbols[&DstRT] = std::move(SymbolsToTrack);
1392 return;
1393 }
1394
1395 auto &DstTrackedSymbols = TrackerSymbols[&DstRT];
1396
1397 // Finally if neither SrtRT or DstRT are the default tracker then
1398 // just append DstRT's tracked symbols to SrtRT's.
1399 auto SI = TrackerSymbols.find(&SrcRT);
1400 if (SI == TrackerSymbols.end())
1401 return;
1402
1403 DstTrackedSymbols.reserve(DstTrackedSymbols.size() + SI->second.size());
1404 for (auto &Sym : SI->second)
1405 DstTrackedSymbols.push_back(std::move(Sym));
1406 TrackerSymbols.erase(SI);
1407}
1408
1409Error JITDylib::defineImpl(MaterializationUnit &MU) {
1410 LLVM_DEBUG({ dbgs() << " " << MU.getSymbols() << "\n"; });
1411
1412 SymbolNameSet Duplicates;
1413 std::vector<SymbolStringPtr> ExistingDefsOverridden;
1414 std::vector<SymbolStringPtr> MUDefsOverridden;
1415
1416 for (const auto &KV : MU.getSymbols()) {
1417 auto I = Symbols.find(KV.first);
1418
1419 if (I != Symbols.end()) {
1420 if (KV.second.isStrong()) {
1421 if (I->second.getFlags().isStrong() ||
1422 I->second.getState() > SymbolState::NeverSearched)
1423 Duplicates.insert(KV.first);
1424 else {
1425 assert(I->second.getState() == SymbolState::NeverSearched &&
1426 "Overridden existing def should be in the never-searched "
1427 "state");
1428 ExistingDefsOverridden.push_back(KV.first);
1429 }
1430 } else
1431 MUDefsOverridden.push_back(KV.first);
1432 }
1433 }
1434
1435 // If there were any duplicate definitions then bail out.
1436 if (!Duplicates.empty()) {
1437 LLVM_DEBUG(
1438 { dbgs() << " Error: Duplicate symbols " << Duplicates << "\n"; });
1439 return make_error<DuplicateDefinition>(std::string(**Duplicates.begin()));
1440 }
1441
1442 // Discard any overridden defs in this MU.
1443 LLVM_DEBUG({
1444 if (!MUDefsOverridden.empty())
1445 dbgs() << " Defs in this MU overridden: " << MUDefsOverridden << "\n";
1446 });
1447 for (auto &S : MUDefsOverridden)
1448 MU.doDiscard(*this, S);
1449
1450 // Discard existing overridden defs.
1451 LLVM_DEBUG({
1452 if (!ExistingDefsOverridden.empty())
1453 dbgs() << " Existing defs overridden by this MU: " << MUDefsOverridden
1454 << "\n";
1455 });
1456 for (auto &S : ExistingDefsOverridden) {
1457
1458 auto UMII = UnmaterializedInfos.find(S);
1459 assert(UMII != UnmaterializedInfos.end() &&
1460 "Overridden existing def should have an UnmaterializedInfo");
1461 UMII->second->MU->doDiscard(*this, S);
1462 }
1463
1464 // Finally, add the defs from this MU.
1465 for (auto &KV : MU.getSymbols()) {
1466 auto &SymEntry = Symbols[KV.first];
1467 SymEntry.setFlags(KV.second);
1468 SymEntry.setState(SymbolState::NeverSearched);
1469 SymEntry.setMaterializerAttached(true);
1470 }
1471
1472 return Error::success();
1473}
1474
1475void JITDylib::installMaterializationUnit(
1476 std::unique_ptr<MaterializationUnit> MU, ResourceTracker &RT) {
1477
1478 /// defineImpl succeeded.
1479 if (&RT != DefaultTracker.get()) {
1480 auto &TS = TrackerSymbols[&RT];
1481 TS.reserve(TS.size() + MU->getSymbols().size());
1482 for (auto &KV : MU->getSymbols())
1483 TS.push_back(KV.first);
1484 }
1485
1486 auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU), &RT);
1487 for (auto &KV : UMI->MU->getSymbols())
1488 UnmaterializedInfos[KV.first] = UMI;
1489}
1490
1491void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q,
1492 const SymbolNameSet &QuerySymbols) {
1493 for (auto &QuerySymbol : QuerySymbols) {
1494 assert(MaterializingInfos.count(QuerySymbol) &&
1495 "QuerySymbol does not have MaterializingInfo");
1496 auto &MI = MaterializingInfos[QuerySymbol];
1497 MI.removeQuery(Q);
1498 }
1499}
1500
1501Platform::~Platform() = default;
1502
1504 ExecutionSession &ES,
1505 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1506
1507 DenseMap<JITDylib *, SymbolMap> CompoundResult;
1508 Error CompoundErr = Error::success();
1509 std::mutex LookupMutex;
1510 std::condition_variable CV;
1511 uint64_t Count = InitSyms.size();
1512
1513 LLVM_DEBUG({
1514 dbgs() << "Issuing init-symbol lookup:\n";
1515 for (auto &KV : InitSyms)
1516 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1517 });
1518
1519 for (auto &KV : InitSyms) {
1520 auto *JD = KV.first;
1521 auto Names = std::move(KV.second);
1522 ES.lookup(
1525 std::move(Names), SymbolState::Ready,
1526 [&, JD](Expected<SymbolMap> Result) {
1527 {
1528 std::lock_guard<std::mutex> Lock(LookupMutex);
1529 --Count;
1530 if (Result) {
1531 assert(!CompoundResult.count(JD) &&
1532 "Duplicate JITDylib in lookup?");
1533 CompoundResult[JD] = std::move(*Result);
1534 } else
1535 CompoundErr =
1536 joinErrors(std::move(CompoundErr), Result.takeError());
1537 }
1538 CV.notify_one();
1539 },
1541 }
1542
1543 std::unique_lock<std::mutex> Lock(LookupMutex);
1544 CV.wait(Lock, [&] { return Count == 0 || CompoundErr; });
1545
1546 if (CompoundErr)
1547 return std::move(CompoundErr);
1548
1549 return std::move(CompoundResult);
1550}
1551
1553 unique_function<void(Error)> OnComplete, ExecutionSession &ES,
1554 const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) {
1555
1556 class TriggerOnComplete {
1557 public:
1558 using OnCompleteFn = unique_function<void(Error)>;
1559 TriggerOnComplete(OnCompleteFn OnComplete)
1560 : OnComplete(std::move(OnComplete)) {}
1561 ~TriggerOnComplete() { OnComplete(std::move(LookupResult)); }
1562 void reportResult(Error Err) {
1563 std::lock_guard<std::mutex> Lock(ResultMutex);
1564 LookupResult = joinErrors(std::move(LookupResult), std::move(Err));
1565 }
1566
1567 private:
1568 std::mutex ResultMutex;
1569 Error LookupResult{Error::success()};
1570 OnCompleteFn OnComplete;
1571 };
1572
1573 LLVM_DEBUG({
1574 dbgs() << "Issuing init-symbol lookup:\n";
1575 for (auto &KV : InitSyms)
1576 dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n";
1577 });
1578
1579 auto TOC = std::make_shared<TriggerOnComplete>(std::move(OnComplete));
1580
1581 for (auto &KV : InitSyms) {
1582 auto *JD = KV.first;
1583 auto Names = std::move(KV.second);
1584 ES.lookup(
1587 std::move(Names), SymbolState::Ready,
1589 TOC->reportResult(Result.takeError());
1590 },
1592 }
1593}
1594
1596 OS << "Materialization task: " << MU->getName() << " in "
1597 << MR->getTargetJITDylib().getName();
1598}
1599
1600void MaterializationTask::run() { MU->materialize(std::move(MR)); }
1601
1602void LookupTask::printDescription(raw_ostream &OS) { OS << "Lookup task"; }
1603
1605
1606ExecutionSession::ExecutionSession(std::unique_ptr<ExecutorProcessControl> EPC)
1607 : EPC(std::move(EPC)) {
1608 // Associated EPC and this.
1609 this->EPC->ES = this;
1610}
1611
1613 // You must call endSession prior to destroying the session.
1614 assert(!SessionOpen &&
1615 "Session still open. Did you forget to call endSession?");
1616}
1617
1619 LLVM_DEBUG(dbgs() << "Ending ExecutionSession " << this << "\n");
1620
1621 auto JDsToRemove = runSessionLocked([&] {
1622
1623#ifdef EXPENSIVE_CHECKS
1624 verifySessionState("Entering ExecutionSession::endSession");
1625#endif
1626
1627 SessionOpen = false;
1628 return JDs;
1629 });
1630
1631 std::reverse(JDsToRemove.begin(), JDsToRemove.end());
1632
1633 auto Err = removeJITDylibs(std::move(JDsToRemove));
1634
1635 Err = joinErrors(std::move(Err), EPC->disconnect());
1636
1637 return Err;
1638}
1639
1641 runSessionLocked([&] { ResourceManagers.push_back(&RM); });
1642}
1643
1645 runSessionLocked([&] {
1646 assert(!ResourceManagers.empty() && "No managers registered");
1647 if (ResourceManagers.back() == &RM)
1648 ResourceManagers.pop_back();
1649 else {
1650 auto I = llvm::find(ResourceManagers, &RM);
1651 assert(I != ResourceManagers.end() && "RM not registered");
1652 ResourceManagers.erase(I);
1653 }
1654 });
1655}
1656
1658 return runSessionLocked([&, this]() -> JITDylib * {
1659 for (auto &JD : JDs)
1660 if (JD->getName() == Name)
1661 return JD.get();
1662 return nullptr;
1663 });
1664}
1665
1667 assert(!getJITDylibByName(Name) && "JITDylib with that name already exists");
1668 return runSessionLocked([&, this]() -> JITDylib & {
1669 assert(SessionOpen && "Cannot create JITDylib after session is closed");
1670 JDs.push_back(new JITDylib(*this, std::move(Name)));
1671 return *JDs.back();
1672 });
1673}
1674
1676 auto &JD = createBareJITDylib(Name);
1677 if (P)
1678 if (auto Err = P->setupJITDylib(JD))
1679 return std::move(Err);
1680 return JD;
1681}
1682
1683Error ExecutionSession::removeJITDylibs(std::vector<JITDylibSP> JDsToRemove) {
1684 // Set JD to 'Closing' state and remove JD from the ExecutionSession.
1685 runSessionLocked([&] {
1686 for (auto &JD : JDsToRemove) {
1687 assert(JD->State == JITDylib::Open && "JD already closed");
1688 JD->State = JITDylib::Closing;
1689 auto I = llvm::find(JDs, JD);
1690 assert(I != JDs.end() && "JD does not appear in session JDs");
1691 JDs.erase(I);
1692 }
1693 });
1694
1695 // Clear JITDylibs and notify the platform.
1696 Error Err = Error::success();
1697 for (auto JD : JDsToRemove) {
1698 Err = joinErrors(std::move(Err), JD->clear());
1699 if (P)
1700 Err = joinErrors(std::move(Err), P->teardownJITDylib(*JD));
1701 }
1702
1703 // Set JD to closed state. Clear remaining data structures.
1704 runSessionLocked([&] {
1705 for (auto &JD : JDsToRemove) {
1706 assert(JD->State == JITDylib::Closing && "JD should be closing");
1707 JD->State = JITDylib::Closed;
1708 assert(JD->Symbols.empty() && "JD.Symbols is not empty after clear");
1709 assert(JD->UnmaterializedInfos.empty() &&
1710 "JD.UnmaterializedInfos is not empty after clear");
1711 assert(JD->MaterializingInfos.empty() &&
1712 "JD.MaterializingInfos is not empty after clear");
1713 assert(JD->TrackerSymbols.empty() &&
1714 "TrackerSymbols is not empty after clear");
1715 JD->DefGenerators.clear();
1716 JD->LinkOrder.clear();
1717 }
1718 });
1719
1720 return Err;
1721}
1722
1725 if (JDs.empty())
1726 return std::vector<JITDylibSP>();
1727
1728 auto &ES = JDs.front()->getExecutionSession();
1729 return ES.runSessionLocked([&]() -> Expected<std::vector<JITDylibSP>> {
1730 DenseSet<JITDylib *> Visited;
1731 std::vector<JITDylibSP> Result;
1732
1733 for (auto &JD : JDs) {
1734
1735 if (JD->State != Open)
1736 return make_error<StringError>(
1737 "Error building link order: " + JD->getName() + " is defunct",
1739 if (Visited.count(JD.get()))
1740 continue;
1741
1743 WorkStack.push_back(JD);
1744 Visited.insert(JD.get());
1745
1746 while (!WorkStack.empty()) {
1747 Result.push_back(std::move(WorkStack.back()));
1748 WorkStack.pop_back();
1749
1750 for (auto &KV : llvm::reverse(Result.back()->LinkOrder)) {
1751 auto &JD = *KV.first;
1752 if (!Visited.insert(&JD).second)
1753 continue;
1754 WorkStack.push_back(&JD);
1755 }
1756 }
1757 }
1758 return Result;
1759 });
1760}
1761
1764 auto Result = getDFSLinkOrder(JDs);
1765 if (Result)
1766 std::reverse(Result->begin(), Result->end());
1767 return Result;
1768}
1769
1771 return getDFSLinkOrder({this});
1772}
1773
1775 return getReverseDFSLinkOrder({this});
1776}
1777
1779 LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet,
1780 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
1781
1782 OL_applyQueryPhase1(std::make_unique<InProgressLookupFlagsState>(
1783 K, std::move(SearchOrder), std::move(LookupSet),
1784 std::move(OnComplete)),
1785 Error::success());
1786}
1787
1790 SymbolLookupSet LookupSet) {
1791
1792 std::promise<MSVCPExpected<SymbolFlagsMap>> ResultP;
1793 OL_applyQueryPhase1(std::make_unique<InProgressLookupFlagsState>(
1794 K, std::move(SearchOrder), std::move(LookupSet),
1795 [&ResultP](Expected<SymbolFlagsMap> Result) {
1796 ResultP.set_value(std::move(Result));
1797 }),
1798 Error::success());
1799
1800 auto ResultF = ResultP.get_future();
1801 return ResultF.get();
1802}
1803
1805 LookupKind K, const JITDylibSearchOrder &SearchOrder,
1806 SymbolLookupSet Symbols, SymbolState RequiredState,
1807 SymbolsResolvedCallback NotifyComplete,
1808 RegisterDependenciesFunction RegisterDependencies) {
1809
1810 LLVM_DEBUG({
1811 runSessionLocked([&]() {
1812 dbgs() << "Looking up " << Symbols << " in " << SearchOrder
1813 << " (required state: " << RequiredState << ")\n";
1814 });
1815 });
1816
1817 // lookup can be re-entered recursively if running on a single thread. Run any
1818 // outstanding MUs in case this query depends on them, otherwise this lookup
1819 // will starve waiting for a result from an MU that is stuck in the queue.
1820 dispatchOutstandingMUs();
1821
1822 auto Unresolved = std::move(Symbols);
1823 auto Q = std::make_shared<AsynchronousSymbolQuery>(Unresolved, RequiredState,
1824 std::move(NotifyComplete));
1825
1826 auto IPLS = std::make_unique<InProgressFullLookupState>(
1827 K, SearchOrder, std::move(Unresolved), RequiredState, std::move(Q),
1828 std::move(RegisterDependencies));
1829
1830 OL_applyQueryPhase1(std::move(IPLS), Error::success());
1831}
1832
1835 SymbolLookupSet Symbols, LookupKind K,
1836 SymbolState RequiredState,
1837 RegisterDependenciesFunction RegisterDependencies) {
1838#if LLVM_ENABLE_THREADS
1839 // In the threaded case we use promises to return the results.
1840 std::promise<SymbolMap> PromisedResult;
1841 Error ResolutionError = Error::success();
1842
1843 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1844 if (R)
1845 PromisedResult.set_value(std::move(*R));
1846 else {
1847 ErrorAsOutParameter _(&ResolutionError);
1848 ResolutionError = R.takeError();
1849 PromisedResult.set_value(SymbolMap());
1850 }
1851 };
1852
1853#else
1855 Error ResolutionError = Error::success();
1856
1857 auto NotifyComplete = [&](Expected<SymbolMap> R) {
1858 ErrorAsOutParameter _(&ResolutionError);
1859 if (R)
1860 Result = std::move(*R);
1861 else
1862 ResolutionError = R.takeError();
1863 };
1864#endif
1865
1866 // Perform the asynchronous lookup.
1867 lookup(K, SearchOrder, std::move(Symbols), RequiredState, NotifyComplete,
1868 RegisterDependencies);
1869
1870#if LLVM_ENABLE_THREADS
1871 auto ResultFuture = PromisedResult.get_future();
1872 auto Result = ResultFuture.get();
1873
1874 if (ResolutionError)
1875 return std::move(ResolutionError);
1876
1877 return std::move(Result);
1878
1879#else
1880 if (ResolutionError)
1881 return std::move(ResolutionError);
1882
1883 return Result;
1884#endif
1885}
1886
1889 SymbolStringPtr Name, SymbolState RequiredState) {
1890 SymbolLookupSet Names({Name});
1891
1892 if (auto ResultMap = lookup(SearchOrder, std::move(Names), LookupKind::Static,
1893 RequiredState, NoDependenciesToRegister)) {
1894 assert(ResultMap->size() == 1 && "Unexpected number of results");
1895 assert(ResultMap->count(Name) && "Missing result for symbol");
1896 return std::move(ResultMap->begin()->second);
1897 } else
1898 return ResultMap.takeError();
1899}
1900
1903 SymbolState RequiredState) {
1904 return lookup(makeJITDylibSearchOrder(SearchOrder), Name, RequiredState);
1905}
1906
1909 SymbolState RequiredState) {
1910 return lookup(SearchOrder, intern(Name), RequiredState);
1911}
1912
1915
1916 auto TagAddrs = lookup({{&JD, JITDylibLookupFlags::MatchAllSymbols}},
1919 if (!TagAddrs)
1920 return TagAddrs.takeError();
1921
1922 // Associate tag addresses with implementations.
1923 std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex);
1924 for (auto &KV : *TagAddrs) {
1925 auto TagAddr = KV.second.getAddress();
1926 if (JITDispatchHandlers.count(TagAddr))
1927 return make_error<StringError>("Tag " + formatv("{0:x16}", TagAddr) +
1928 " (for " + *KV.first +
1929 ") already registered",
1931 auto I = WFs.find(KV.first);
1932 assert(I != WFs.end() && I->second &&
1933 "JITDispatchHandler implementation missing");
1934 JITDispatchHandlers[KV.second.getAddress()] =
1935 std::make_shared<JITDispatchHandlerFunction>(std::move(I->second));
1936 LLVM_DEBUG({
1937 dbgs() << "Associated function tag \"" << *KV.first << "\" ("
1938 << formatv("{0:x}", KV.second.getAddress()) << ") with handler\n";
1939 });
1940 }
1941 return Error::success();
1942}
1943
1945 ExecutorAddr HandlerFnTagAddr,
1946 ArrayRef<char> ArgBuffer) {
1947
1948 std::shared_ptr<JITDispatchHandlerFunction> F;
1949 {
1950 std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex);
1951 auto I = JITDispatchHandlers.find(HandlerFnTagAddr);
1952 if (I != JITDispatchHandlers.end())
1953 F = I->second;
1954 }
1955
1956 if (F)
1957 (*F)(std::move(SendResult), ArgBuffer.data(), ArgBuffer.size());
1958 else
1960 ("No function registered for tag " +
1961 formatv("{0:x16}", HandlerFnTagAddr))
1962 .str()));
1963}
1964
1966 runSessionLocked([this, &OS]() {
1967 for (auto &JD : JDs)
1968 JD->dump(OS);
1969 });
1970}
1971
1972#ifdef EXPENSIVE_CHECKS
1973bool ExecutionSession::verifySessionState(Twine Phase) {
1974 return runSessionLocked([&]() {
1975 bool AllOk = true;
1976
1977 // We'll collect these and verify them later to avoid redundant checks.
1979
1980 for (auto &JD : JDs) {
1981
1982 auto LogFailure = [&]() -> raw_fd_ostream & {
1983 auto &Stream = errs();
1984 if (AllOk)
1985 Stream << "ERROR: Bad ExecutionSession state detected " << Phase
1986 << "\n";
1987 Stream << " In JITDylib " << JD->getName() << ", ";
1988 AllOk = false;
1989 return Stream;
1990 };
1991
1992 if (JD->State != JITDylib::Open) {
1993 LogFailure()
1994 << "state is not Open, but JD is in ExecutionSession list.";
1995 }
1996
1997 // Check symbol table.
1998 // 1. If the entry state isn't resolved then check that no address has
1999 // been set.
2000 // 2. Check that if the hasMaterializerAttached flag is set then there is
2001 // an UnmaterializedInfo entry, and vice-versa.
2002 for (auto &[Sym, Entry] : JD->Symbols) {
2003 // Check that unresolved symbols have null addresses.
2004 if (Entry.getState() < SymbolState::Resolved) {
2005 if (Entry.getAddress()) {
2006 LogFailure() << "symbol " << Sym << " has state "
2007 << Entry.getState()
2008 << " (not-yet-resolved) but non-null address "
2009 << Entry.getAddress() << ".\n";
2010 }
2011 }
2012
2013 // Check that the hasMaterializerAttached flag is correct.
2014 auto UMIItr = JD->UnmaterializedInfos.find(Sym);
2015 if (Entry.hasMaterializerAttached()) {
2016 if (UMIItr == JD->UnmaterializedInfos.end()) {
2017 LogFailure() << "symbol " << Sym
2018 << " entry claims materializer attached, but "
2019 "UnmaterializedInfos has no corresponding entry.\n";
2020 }
2021 } else if (UMIItr != JD->UnmaterializedInfos.end()) {
2022 LogFailure()
2023 << "symbol " << Sym
2024 << " entry claims no materializer attached, but "
2025 "UnmaterializedInfos has an unexpected entry for it.\n";
2026 }
2027 }
2028
2029 // Check that every UnmaterializedInfo entry has a corresponding entry
2030 // in the Symbols table.
2031 for (auto &[Sym, UMI] : JD->UnmaterializedInfos) {
2032 auto SymItr = JD->Symbols.find(Sym);
2033 if (SymItr == JD->Symbols.end()) {
2034 LogFailure()
2035 << "symbol " << Sym
2036 << " has UnmaterializedInfos entry, but no Symbols entry.\n";
2037 }
2038 }
2039
2040 // Check consistency of the MaterializingInfos table.
2041 for (auto &[Sym, MII] : JD->MaterializingInfos) {
2042
2043 auto SymItr = JD->Symbols.find(Sym);
2044 if (SymItr == JD->Symbols.end()) {
2045 // If there's no Symbols entry for this MaterializingInfos entry then
2046 // report that.
2047 LogFailure()
2048 << "symbol " << Sym
2049 << " has MaterializingInfos entry, but no Symbols entry.\n";
2050 } else {
2051 // Otherwise check consistency between Symbols and MaterializingInfos.
2052
2053 // Ready symbols should not have MaterializingInfos.
2054 if (SymItr->second.getState() == SymbolState::Ready) {
2055 LogFailure()
2056 << "symbol " << Sym
2057 << " is in Ready state, should not have MaterializingInfo.\n";
2058 }
2059
2060 // Pending queries should be for subsequent states.
2061 auto CurState = static_cast<SymbolState>(
2062 static_cast<std::underlying_type_t<SymbolState>>(
2063 SymItr->second.getState()) + 1);
2064 for (auto &Q : MII.PendingQueries) {
2065 if (Q->getRequiredState() != CurState) {
2066 if (Q->getRequiredState() > CurState)
2067 CurState = Q->getRequiredState();
2068 else
2069 LogFailure() << "symbol " << Sym
2070 << " has stale or misordered queries.\n";
2071 }
2072 }
2073
2074 // If there's a DefiningEDU then check that...
2075 // 1. The JD matches.
2076 // 2. The symbol is in the EDU's Symbols map.
2077 // 3. The symbol table entry is in the Emitted state.
2078 if (MII.DefiningEDU) {
2079
2080 EDUsToCheck.insert(MII.DefiningEDU.get());
2081
2082 if (MII.DefiningEDU->JD != JD.get()) {
2083 LogFailure() << "symbol " << Sym
2084 << " has DefiningEDU with incorrect JD"
2085 << (llvm::is_contained(JDs, MII.DefiningEDU->JD)
2086 ? " (JD not currently in ExecutionSession"
2087 : "")
2088 << "\n";
2089 }
2090
2091 if (SymItr->second.getState() != SymbolState::Emitted) {
2092 LogFailure()
2093 << "symbol " << Sym
2094 << " has DefiningEDU, but is not in Emitted state.\n";
2095 }
2096 }
2097
2098 // Check that JDs for any DependantEDUs are also in the session --
2099 // that guarantees that we'll also visit them during this loop.
2100 for (auto &DepEDU : MII.DependantEDUs) {
2101 if (!llvm::is_contained(JDs, DepEDU->JD)) {
2102 LogFailure() << "symbol " << Sym << " has DependantEDU "
2103 << (void *)DepEDU << " with JD (" << DepEDU->JD
2104 << ") that isn't in ExecutionSession.\n";
2105 }
2106 }
2107 }
2108 }
2109 }
2110
2111 // Check EDUs.
2112 for (auto *EDU : EDUsToCheck) {
2113 assert(EDU->JD->State == JITDylib::Open && "EDU->JD is not Open");
2114
2115 auto LogFailure = [&]() -> raw_fd_ostream & {
2116 AllOk = false;
2117 auto &Stream = errs();
2118 Stream << "In EDU defining " << EDU->JD->getName() << ": { ";
2119 for (auto &[Sym, Flags] : EDU->Symbols)
2120 Stream << Sym << " ";
2121 Stream << "}, ";
2122 return Stream;
2123 };
2124
2125 if (EDU->Symbols.empty())
2126 LogFailure() << "no symbols defined.\n";
2127 else {
2128 for (auto &[Sym, Flags] : EDU->Symbols) {
2129 if (!Sym)
2130 LogFailure() << "null symbol defined.\n";
2131 else {
2132 if (!EDU->JD->Symbols.count(SymbolStringPtr(Sym))) {
2133 LogFailure() << "symbol " << Sym
2134 << " isn't present in JD's symbol table.\n";
2135 }
2136 }
2137 }
2138 }
2139
2140 for (auto &[DepJD, Symbols] : EDU->Dependencies) {
2141 if (!llvm::is_contained(JDs, DepJD)) {
2142 LogFailure() << "dependant symbols listed for JD that isn't in "
2143 "ExecutionSession.\n";
2144 } else {
2145 for (auto &DepSym : Symbols) {
2146 if (!DepJD->Symbols.count(SymbolStringPtr(DepSym))) {
2147 LogFailure()
2148 << "dependant symbol " << DepSym
2149 << " does not appear in symbol table for dependant JD "
2150 << DepJD->getName() << ".\n";
2151 }
2152 }
2153 }
2154 }
2155 }
2156
2157 return AllOk;
2158 });
2159}
2160#endif // EXPENSIVE_CHECKS
2161
2162void ExecutionSession::dispatchOutstandingMUs() {
2163 LLVM_DEBUG(dbgs() << "Dispatching MaterializationUnits...\n");
2164 while (true) {
2165 std::optional<std::pair<std::unique_ptr<MaterializationUnit>,
2166 std::unique_ptr<MaterializationResponsibility>>>
2167 JMU;
2168
2169 {
2170 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2171 if (!OutstandingMUs.empty()) {
2172 JMU.emplace(std::move(OutstandingMUs.back()));
2173 OutstandingMUs.pop_back();
2174 }
2175 }
2176
2177 if (!JMU)
2178 break;
2179
2180 assert(JMU->first && "No MU?");
2181 LLVM_DEBUG(dbgs() << " Dispatching \"" << JMU->first->getName() << "\"\n");
2182 dispatchTask(std::make_unique<MaterializationTask>(std::move(JMU->first),
2183 std::move(JMU->second)));
2184 }
2185 LLVM_DEBUG(dbgs() << "Done dispatching MaterializationUnits.\n");
2186}
2187
2188Error ExecutionSession::removeResourceTracker(ResourceTracker &RT) {
2189 LLVM_DEBUG({
2190 dbgs() << "In " << RT.getJITDylib().getName() << " removing tracker "
2191 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2192 });
2193 std::vector<ResourceManager *> CurrentResourceManagers;
2194
2195 JITDylib::AsynchronousSymbolQuerySet QueriesToFail;
2196 std::shared_ptr<SymbolDependenceMap> FailedSymbols;
2197
2198 runSessionLocked([&] {
2199 CurrentResourceManagers = ResourceManagers;
2200 RT.makeDefunct();
2201 std::tie(QueriesToFail, FailedSymbols) = RT.getJITDylib().removeTracker(RT);
2202 });
2203
2204 Error Err = Error::success();
2205
2206 auto &JD = RT.getJITDylib();
2207 for (auto *L : reverse(CurrentResourceManagers))
2208 Err = joinErrors(std::move(Err),
2209 L->handleRemoveResources(JD, RT.getKeyUnsafe()));
2210
2211 for (auto &Q : QueriesToFail)
2212 Q->handleFailed(
2213 make_error<FailedToMaterialize>(getSymbolStringPool(), FailedSymbols));
2214
2215 return Err;
2216}
2217
2218void ExecutionSession::transferResourceTracker(ResourceTracker &DstRT,
2219 ResourceTracker &SrcRT) {
2220 LLVM_DEBUG({
2221 dbgs() << "In " << SrcRT.getJITDylib().getName()
2222 << " transfering resources from tracker "
2223 << formatv("{0:x}", SrcRT.getKeyUnsafe()) << " to tracker "
2224 << formatv("{0:x}", DstRT.getKeyUnsafe()) << "\n";
2225 });
2226
2227 // No-op transfers are allowed and do not invalidate the source.
2228 if (&DstRT == &SrcRT)
2229 return;
2230
2231 assert(&DstRT.getJITDylib() == &SrcRT.getJITDylib() &&
2232 "Can't transfer resources between JITDylibs");
2233 runSessionLocked([&]() {
2234 SrcRT.makeDefunct();
2235 auto &JD = DstRT.getJITDylib();
2236 JD.transferTracker(DstRT, SrcRT);
2237 for (auto *L : reverse(ResourceManagers))
2238 L->handleTransferResources(JD, DstRT.getKeyUnsafe(),
2239 SrcRT.getKeyUnsafe());
2240 });
2241}
2242
2243void ExecutionSession::destroyResourceTracker(ResourceTracker &RT) {
2244 runSessionLocked([&]() {
2245 LLVM_DEBUG({
2246 dbgs() << "In " << RT.getJITDylib().getName() << " destroying tracker "
2247 << formatv("{0:x}", RT.getKeyUnsafe()) << "\n";
2248 });
2249 if (!RT.isDefunct())
2250 transferResourceTracker(*RT.getJITDylib().getDefaultResourceTracker(),
2251 RT);
2252 });
2253}
2254
2255Error ExecutionSession::IL_updateCandidatesFor(
2256 JITDylib &JD, JITDylibLookupFlags JDLookupFlags,
2257 SymbolLookupSet &Candidates, SymbolLookupSet *NonCandidates) {
2258 return Candidates.forEachWithRemoval(
2259 [&](const SymbolStringPtr &Name,
2260 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2261 /// Search for the symbol. If not found then continue without
2262 /// removal.
2263 auto SymI = JD.Symbols.find(Name);
2264 if (SymI == JD.Symbols.end())
2265 return false;
2266
2267 // If this is a non-exported symbol and we're matching exported
2268 // symbols only then remove this symbol from the candidates list.
2269 //
2270 // If we're tracking non-candidates then add this to the non-candidate
2271 // list.
2272 if (!SymI->second.getFlags().isExported() &&
2274 if (NonCandidates)
2275 NonCandidates->add(Name, SymLookupFlags);
2276 return true;
2277 }
2278
2279 // If we match against a materialization-side-effects only symbol
2280 // then make sure it is weakly-referenced. Otherwise bail out with
2281 // an error.
2282 // FIXME: Use a "materialization-side-effects-only symbols must be
2283 // weakly referenced" specific error here to reduce confusion.
2284 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2286 return make_error<SymbolsNotFound>(getSymbolStringPool(),
2288
2289 // If we matched against this symbol but it is in the error state
2290 // then bail out and treat it as a failure to materialize.
2291 if (SymI->second.getFlags().hasError()) {
2292 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2293 (*FailedSymbolsMap)[&JD] = {Name};
2294 return make_error<FailedToMaterialize>(getSymbolStringPool(),
2295 std::move(FailedSymbolsMap));
2296 }
2297
2298 // Otherwise this is a match. Remove it from the candidate set.
2299 return true;
2300 });
2301}
2302
2303void ExecutionSession::OL_resumeLookupAfterGeneration(
2304 InProgressLookupState &IPLS) {
2305
2307 "Should not be called for not-in-generator lookups");
2309
2311
2312 if (auto DG = IPLS.CurDefGeneratorStack.back().lock()) {
2313 IPLS.CurDefGeneratorStack.pop_back();
2314 std::lock_guard<std::mutex> Lock(DG->M);
2315
2316 // If there are no pending lookups then mark the generator as free and
2317 // return.
2318 if (DG->PendingLookups.empty()) {
2319 DG->InUse = false;
2320 return;
2321 }
2322
2323 // Otherwise resume the next lookup.
2324 LS = std::move(DG->PendingLookups.front());
2325 DG->PendingLookups.pop_front();
2326 }
2327
2328 if (LS.IPLS) {
2330 dispatchTask(std::make_unique<LookupTask>(std::move(LS)));
2331 }
2332}
2333
2334void ExecutionSession::OL_applyQueryPhase1(
2335 std::unique_ptr<InProgressLookupState> IPLS, Error Err) {
2336
2337 LLVM_DEBUG({
2338 dbgs() << "Entering OL_applyQueryPhase1:\n"
2339 << " Lookup kind: " << IPLS->K << "\n"
2340 << " Search order: " << IPLS->SearchOrder
2341 << ", Current index = " << IPLS->CurSearchOrderIndex
2342 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2343 << " Lookup set: " << IPLS->LookupSet << "\n"
2344 << " Definition generator candidates: "
2345 << IPLS->DefGeneratorCandidates << "\n"
2346 << " Definition generator non-candidates: "
2347 << IPLS->DefGeneratorNonCandidates << "\n";
2348 });
2349
2350 if (IPLS->GenState == InProgressLookupState::InGenerator)
2351 OL_resumeLookupAfterGeneration(*IPLS);
2352
2353 assert(IPLS->GenState != InProgressLookupState::InGenerator &&
2354 "Lookup should not be in InGenerator state here");
2355
2356 // FIXME: We should attach the query as we go: This provides a result in a
2357 // single pass in the common case where all symbols have already reached the
2358 // required state. The query could be detached again in the 'fail' method on
2359 // IPLS. Phase 2 would be reduced to collecting and dispatching the MUs.
2360
2361 while (IPLS->CurSearchOrderIndex != IPLS->SearchOrder.size()) {
2362
2363 // If we've been handed an error or received one back from a generator then
2364 // fail the query. We don't need to unlink: At this stage the query hasn't
2365 // actually been lodged.
2366 if (Err)
2367 return IPLS->fail(std::move(Err));
2368
2369 // Get the next JITDylib and lookup flags.
2370 auto &KV = IPLS->SearchOrder[IPLS->CurSearchOrderIndex];
2371 auto &JD = *KV.first;
2372 auto JDLookupFlags = KV.second;
2373
2374 LLVM_DEBUG({
2375 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2376 << ") with lookup set " << IPLS->LookupSet << ":\n";
2377 });
2378
2379 // If we've just reached a new JITDylib then perform some setup.
2380 if (IPLS->NewJITDylib) {
2381 // Add any non-candidates from the last JITDylib (if any) back on to the
2382 // list of definition candidates for this JITDylib, reset definition
2383 // non-candidates to the empty set.
2384 SymbolLookupSet Tmp;
2385 std::swap(IPLS->DefGeneratorNonCandidates, Tmp);
2386 IPLS->DefGeneratorCandidates.append(std::move(Tmp));
2387
2388 LLVM_DEBUG({
2389 dbgs() << " First time visiting " << JD.getName()
2390 << ", resetting candidate sets and building generator stack\n";
2391 });
2392
2393 // Build the definition generator stack for this JITDylib.
2394 runSessionLocked([&] {
2395 IPLS->CurDefGeneratorStack.reserve(JD.DefGenerators.size());
2396 for (auto &DG : reverse(JD.DefGenerators))
2397 IPLS->CurDefGeneratorStack.push_back(DG);
2398 });
2399
2400 // Flag that we've done our initialization.
2401 IPLS->NewJITDylib = false;
2402 }
2403
2404 // Remove any generation candidates that are already defined (and match) in
2405 // this JITDylib.
2406 runSessionLocked([&] {
2407 // Update the list of candidates (and non-candidates) for definition
2408 // generation.
2409 LLVM_DEBUG(dbgs() << " Updating candidate set...\n");
2410 Err = IL_updateCandidatesFor(
2411 JD, JDLookupFlags, IPLS->DefGeneratorCandidates,
2412 JD.DefGenerators.empty() ? nullptr
2413 : &IPLS->DefGeneratorNonCandidates);
2414 LLVM_DEBUG({
2415 dbgs() << " Remaining candidates = " << IPLS->DefGeneratorCandidates
2416 << "\n";
2417 });
2418
2419 // If this lookup was resumed after auto-suspension but all candidates
2420 // have already been generated (by some previous call to the generator)
2421 // treat the lookup as if it had completed generation.
2422 if (IPLS->GenState == InProgressLookupState::ResumedForGenerator &&
2423 IPLS->DefGeneratorCandidates.empty())
2424 OL_resumeLookupAfterGeneration(*IPLS);
2425 });
2426
2427 // If we encountered an error while filtering generation candidates then
2428 // bail out.
2429 if (Err)
2430 return IPLS->fail(std::move(Err));
2431
2432 /// Apply any definition generators on the stack.
2433 LLVM_DEBUG({
2434 if (IPLS->CurDefGeneratorStack.empty())
2435 LLVM_DEBUG(dbgs() << " No generators to run for this JITDylib.\n");
2436 else if (IPLS->DefGeneratorCandidates.empty())
2437 LLVM_DEBUG(dbgs() << " No candidates to generate.\n");
2438 else
2439 dbgs() << " Running " << IPLS->CurDefGeneratorStack.size()
2440 << " remaining generators for "
2441 << IPLS->DefGeneratorCandidates.size() << " candidates\n";
2442 });
2443 while (!IPLS->CurDefGeneratorStack.empty() &&
2444 !IPLS->DefGeneratorCandidates.empty()) {
2445 auto DG = IPLS->CurDefGeneratorStack.back().lock();
2446
2447 if (!DG)
2448 return IPLS->fail(make_error<StringError>(
2449 "DefinitionGenerator removed while lookup in progress",
2451
2452 // At this point the lookup is in either the NotInGenerator state, or in
2453 // the ResumedForGenerator state.
2454 // If this lookup is in the NotInGenerator state then check whether the
2455 // generator is in use. If the generator is not in use then move the
2456 // lookup to the InGenerator state and continue. If the generator is
2457 // already in use then just add this lookup to the pending lookups list
2458 // and bail out.
2459 // If this lookup is in the ResumedForGenerator state then just move it
2460 // to InGenerator and continue.
2461 if (IPLS->GenState == InProgressLookupState::NotInGenerator) {
2462 std::lock_guard<std::mutex> Lock(DG->M);
2463 if (DG->InUse) {
2464 DG->PendingLookups.push_back(std::move(IPLS));
2465 return;
2466 }
2467 DG->InUse = true;
2468 }
2469
2470 IPLS->GenState = InProgressLookupState::InGenerator;
2471
2472 auto K = IPLS->K;
2473 auto &LookupSet = IPLS->DefGeneratorCandidates;
2474
2475 // Run the generator. If the generator takes ownership of QA then this
2476 // will break the loop.
2477 {
2478 LLVM_DEBUG(dbgs() << " Attempting to generate " << LookupSet << "\n");
2479 LookupState LS(std::move(IPLS));
2480 Err = DG->tryToGenerate(LS, K, JD, JDLookupFlags, LookupSet);
2481 IPLS = std::move(LS.IPLS);
2482 }
2483
2484 // If the lookup returned then pop the generator stack and unblock the
2485 // next lookup on this generator (if any).
2486 if (IPLS)
2487 OL_resumeLookupAfterGeneration(*IPLS);
2488
2489 // If there was an error then fail the query.
2490 if (Err) {
2491 LLVM_DEBUG({
2492 dbgs() << " Error attempting to generate " << LookupSet << "\n";
2493 });
2494 assert(IPLS && "LS cannot be retained if error is returned");
2495 return IPLS->fail(std::move(Err));
2496 }
2497
2498 // Otherwise if QA was captured then break the loop.
2499 if (!IPLS) {
2500 LLVM_DEBUG(
2501 { dbgs() << " LookupState captured. Exiting phase1 for now.\n"; });
2502 return;
2503 }
2504
2505 // Otherwise if we're continuing around the loop then update candidates
2506 // for the next round.
2507 runSessionLocked([&] {
2508 LLVM_DEBUG(dbgs() << " Updating candidate set post-generation\n");
2509 Err = IL_updateCandidatesFor(
2510 JD, JDLookupFlags, IPLS->DefGeneratorCandidates,
2511 JD.DefGenerators.empty() ? nullptr
2512 : &IPLS->DefGeneratorNonCandidates);
2513 });
2514
2515 // If updating candidates failed then fail the query.
2516 if (Err) {
2517 LLVM_DEBUG(dbgs() << " Error encountered while updating candidates\n");
2518 return IPLS->fail(std::move(Err));
2519 }
2520 }
2521
2522 if (IPLS->DefGeneratorCandidates.empty() &&
2523 IPLS->DefGeneratorNonCandidates.empty()) {
2524 // Early out if there are no remaining symbols.
2525 LLVM_DEBUG(dbgs() << "All symbols matched.\n");
2526 IPLS->CurSearchOrderIndex = IPLS->SearchOrder.size();
2527 break;
2528 } else {
2529 // If we get here then we've moved on to the next JITDylib with candidates
2530 // remaining.
2531 LLVM_DEBUG(dbgs() << "Phase 1 moving to next JITDylib.\n");
2532 ++IPLS->CurSearchOrderIndex;
2533 IPLS->NewJITDylib = true;
2534 }
2535 }
2536
2537 // Remove any weakly referenced candidates that could not be found/generated.
2538 IPLS->DefGeneratorCandidates.remove_if(
2539 [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2540 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2541 });
2542
2543 // If we get here then we've finished searching all JITDylibs.
2544 // If we matched all symbols then move to phase 2, otherwise fail the query
2545 // with a SymbolsNotFound error.
2546 if (IPLS->DefGeneratorCandidates.empty()) {
2547 LLVM_DEBUG(dbgs() << "Phase 1 succeeded.\n");
2548 IPLS->complete(std::move(IPLS));
2549 } else {
2550 LLVM_DEBUG(dbgs() << "Phase 1 failed with unresolved symbols.\n");
2551 IPLS->fail(make_error<SymbolsNotFound>(
2552 getSymbolStringPool(), IPLS->DefGeneratorCandidates.getSymbolNames()));
2553 }
2554}
2555
2556void ExecutionSession::OL_completeLookup(
2557 std::unique_ptr<InProgressLookupState> IPLS,
2558 std::shared_ptr<AsynchronousSymbolQuery> Q,
2559 RegisterDependenciesFunction RegisterDependencies) {
2560
2561 LLVM_DEBUG({
2562 dbgs() << "Entering OL_completeLookup:\n"
2563 << " Lookup kind: " << IPLS->K << "\n"
2564 << " Search order: " << IPLS->SearchOrder
2565 << ", Current index = " << IPLS->CurSearchOrderIndex
2566 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2567 << " Lookup set: " << IPLS->LookupSet << "\n"
2568 << " Definition generator candidates: "
2569 << IPLS->DefGeneratorCandidates << "\n"
2570 << " Definition generator non-candidates: "
2571 << IPLS->DefGeneratorNonCandidates << "\n";
2572 });
2573
2574 bool QueryComplete = false;
2575 DenseMap<JITDylib *, JITDylib::UnmaterializedInfosList> CollectedUMIs;
2576
2577 auto LodgingErr = runSessionLocked([&]() -> Error {
2578 for (auto &KV : IPLS->SearchOrder) {
2579 auto &JD = *KV.first;
2580 auto JDLookupFlags = KV.second;
2581 LLVM_DEBUG({
2582 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2583 << ") with lookup set " << IPLS->LookupSet << ":\n";
2584 });
2585
2586 auto Err = IPLS->LookupSet.forEachWithRemoval(
2587 [&](const SymbolStringPtr &Name,
2588 SymbolLookupFlags SymLookupFlags) -> Expected<bool> {
2589 LLVM_DEBUG({
2590 dbgs() << " Attempting to match \"" << Name << "\" ("
2591 << SymLookupFlags << ")... ";
2592 });
2593
2594 /// Search for the symbol. If not found then continue without
2595 /// removal.
2596 auto SymI = JD.Symbols.find(Name);
2597 if (SymI == JD.Symbols.end()) {
2598 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2599 return false;
2600 }
2601
2602 // If this is a non-exported symbol and we're matching exported
2603 // symbols only then skip this symbol without removal.
2604 if (!SymI->second.getFlags().isExported() &&
2605 JDLookupFlags ==
2607 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2608 return false;
2609 }
2610
2611 // If we match against a materialization-side-effects only symbol
2612 // then make sure it is weakly-referenced. Otherwise bail out with
2613 // an error.
2614 // FIXME: Use a "materialization-side-effects-only symbols must be
2615 // weakly referenced" specific error here to reduce confusion.
2616 if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() &&
2618 LLVM_DEBUG({
2619 dbgs() << "error: "
2620 "required, but symbol is has-side-effects-only\n";
2621 });
2622 return make_error<SymbolsNotFound>(getSymbolStringPool(),
2624 }
2625
2626 // If we matched against this symbol but it is in the error state
2627 // then bail out and treat it as a failure to materialize.
2628 if (SymI->second.getFlags().hasError()) {
2629 LLVM_DEBUG(dbgs() << "error: symbol is in error state\n");
2630 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
2631 (*FailedSymbolsMap)[&JD] = {Name};
2632 return make_error<FailedToMaterialize>(
2633 getSymbolStringPool(), std::move(FailedSymbolsMap));
2634 }
2635
2636 // Otherwise this is a match.
2637
2638 // If this symbol is already in the required state then notify the
2639 // query, remove the symbol and continue.
2640 if (SymI->second.getState() >= Q->getRequiredState()) {
2642 << "matched, symbol already in required state\n");
2643 Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol());
2644 return true;
2645 }
2646
2647 // Otherwise this symbol does not yet meet the required state. Check
2648 // whether it has a materializer attached, and if so prepare to run
2649 // it.
2650 if (SymI->second.hasMaterializerAttached()) {
2651 assert(SymI->second.getAddress() == ExecutorAddr() &&
2652 "Symbol not resolved but already has address?");
2653 auto UMII = JD.UnmaterializedInfos.find(Name);
2654 assert(UMII != JD.UnmaterializedInfos.end() &&
2655 "Lazy symbol should have UnmaterializedInfo");
2656
2657 auto UMI = UMII->second;
2658 assert(UMI->MU && "Materializer should not be null");
2659 assert(UMI->RT && "Tracker should not be null");
2660 LLVM_DEBUG({
2661 dbgs() << "matched, preparing to dispatch MU@" << UMI->MU.get()
2662 << " (" << UMI->MU->getName() << ")\n";
2663 });
2664
2665 // Move all symbols associated with this MaterializationUnit into
2666 // materializing state.
2667 for (auto &KV : UMI->MU->getSymbols()) {
2668 auto SymK = JD.Symbols.find(KV.first);
2669 assert(SymK != JD.Symbols.end() &&
2670 "No entry for symbol covered by MaterializationUnit");
2671 SymK->second.setMaterializerAttached(false);
2672 SymK->second.setState(SymbolState::Materializing);
2673 JD.UnmaterializedInfos.erase(KV.first);
2674 }
2675
2676 // Add MU to the list of MaterializationUnits to be materialized.
2677 CollectedUMIs[&JD].push_back(std::move(UMI));
2678 } else
2679 LLVM_DEBUG(dbgs() << "matched, registering query");
2680
2681 // Add the query to the PendingQueries list and continue, deleting
2682 // the element from the lookup set.
2683 assert(SymI->second.getState() != SymbolState::NeverSearched &&
2684 SymI->second.getState() != SymbolState::Ready &&
2685 "By this line the symbol should be materializing");
2686 auto &MI = JD.MaterializingInfos[Name];
2687 MI.addQuery(Q);
2688 Q->addQueryDependence(JD, Name);
2689
2690 return true;
2691 });
2692
2693 JD.shrinkMaterializationInfoMemory();
2694
2695 // Handle failure.
2696 if (Err) {
2697
2698 LLVM_DEBUG({
2699 dbgs() << "Lookup failed. Detaching query and replacing MUs.\n";
2700 });
2701
2702 // Detach the query.
2703 Q->detach();
2704
2705 // Replace the MUs.
2706 for (auto &KV : CollectedUMIs) {
2707 auto &JD = *KV.first;
2708 for (auto &UMI : KV.second)
2709 for (auto &KV2 : UMI->MU->getSymbols()) {
2710 assert(!JD.UnmaterializedInfos.count(KV2.first) &&
2711 "Unexpected materializer in map");
2712 auto SymI = JD.Symbols.find(KV2.first);
2713 assert(SymI != JD.Symbols.end() && "Missing symbol entry");
2714 assert(SymI->second.getState() == SymbolState::Materializing &&
2715 "Can not replace symbol that is not materializing");
2716 assert(!SymI->second.hasMaterializerAttached() &&
2717 "MaterializerAttached flag should not be set");
2718 SymI->second.setMaterializerAttached(true);
2719 JD.UnmaterializedInfos[KV2.first] = UMI;
2720 }
2721 }
2722
2723 return Err;
2724 }
2725 }
2726
2727 LLVM_DEBUG(dbgs() << "Stripping unmatched weakly-referenced symbols\n");
2728 IPLS->LookupSet.forEachWithRemoval(
2729 [&](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2730 if (SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol) {
2731 Q->dropSymbol(Name);
2732 return true;
2733 } else
2734 return false;
2735 });
2736
2737 if (!IPLS->LookupSet.empty()) {
2738 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2739 return make_error<SymbolsNotFound>(getSymbolStringPool(),
2740 IPLS->LookupSet.getSymbolNames());
2741 }
2742
2743 // Record whether the query completed.
2744 QueryComplete = Q->isComplete();
2745
2746 LLVM_DEBUG({
2747 dbgs() << "Query successfully "
2748 << (QueryComplete ? "completed" : "lodged") << "\n";
2749 });
2750
2751 // Move the collected MUs to the OutstandingMUs list.
2752 if (!CollectedUMIs.empty()) {
2753 std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex);
2754
2755 LLVM_DEBUG(dbgs() << "Adding MUs to dispatch:\n");
2756 for (auto &KV : CollectedUMIs) {
2757 LLVM_DEBUG({
2758 auto &JD = *KV.first;
2759 dbgs() << " For " << JD.getName() << ": Adding " << KV.second.size()
2760 << " MUs.\n";
2761 });
2762 for (auto &UMI : KV.second) {
2763 auto MR = createMaterializationResponsibility(
2764 *UMI->RT, std::move(UMI->MU->SymbolFlags),
2765 std::move(UMI->MU->InitSymbol));
2766 OutstandingMUs.push_back(
2767 std::make_pair(std::move(UMI->MU), std::move(MR)));
2768 }
2769 }
2770 } else
2771 LLVM_DEBUG(dbgs() << "No MUs to dispatch.\n");
2772
2773 if (RegisterDependencies && !Q->QueryRegistrations.empty()) {
2774 LLVM_DEBUG(dbgs() << "Registering dependencies\n");
2775 RegisterDependencies(Q->QueryRegistrations);
2776 } else
2777 LLVM_DEBUG(dbgs() << "No dependencies to register\n");
2778
2779 return Error::success();
2780 });
2781
2782 if (LodgingErr) {
2783 LLVM_DEBUG(dbgs() << "Failing query\n");
2784 Q->detach();
2785 Q->handleFailed(std::move(LodgingErr));
2786 return;
2787 }
2788
2789 if (QueryComplete) {
2790 LLVM_DEBUG(dbgs() << "Completing query\n");
2791 Q->handleComplete(*this);
2792 }
2793
2794 dispatchOutstandingMUs();
2795}
2796
2797void ExecutionSession::OL_completeLookupFlags(
2798 std::unique_ptr<InProgressLookupState> IPLS,
2799 unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) {
2800
2801 auto Result = runSessionLocked([&]() -> Expected<SymbolFlagsMap> {
2802 LLVM_DEBUG({
2803 dbgs() << "Entering OL_completeLookupFlags:\n"
2804 << " Lookup kind: " << IPLS->K << "\n"
2805 << " Search order: " << IPLS->SearchOrder
2806 << ", Current index = " << IPLS->CurSearchOrderIndex
2807 << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n"
2808 << " Lookup set: " << IPLS->LookupSet << "\n"
2809 << " Definition generator candidates: "
2810 << IPLS->DefGeneratorCandidates << "\n"
2811 << " Definition generator non-candidates: "
2812 << IPLS->DefGeneratorNonCandidates << "\n";
2813 });
2814
2816
2817 // Attempt to find flags for each symbol.
2818 for (auto &KV : IPLS->SearchOrder) {
2819 auto &JD = *KV.first;
2820 auto JDLookupFlags = KV.second;
2821 LLVM_DEBUG({
2822 dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags
2823 << ") with lookup set " << IPLS->LookupSet << ":\n";
2824 });
2825
2826 IPLS->LookupSet.forEachWithRemoval([&](const SymbolStringPtr &Name,
2827 SymbolLookupFlags SymLookupFlags) {
2828 LLVM_DEBUG({
2829 dbgs() << " Attempting to match \"" << Name << "\" ("
2830 << SymLookupFlags << ")... ";
2831 });
2832
2833 // Search for the symbol. If not found then continue without removing
2834 // from the lookup set.
2835 auto SymI = JD.Symbols.find(Name);
2836 if (SymI == JD.Symbols.end()) {
2837 LLVM_DEBUG(dbgs() << "skipping: not present\n");
2838 return false;
2839 }
2840
2841 // If this is a non-exported symbol then it doesn't match. Skip it.
2842 if (!SymI->second.getFlags().isExported() &&
2844 LLVM_DEBUG(dbgs() << "skipping: not exported\n");
2845 return false;
2846 }
2847
2848 LLVM_DEBUG({
2849 dbgs() << "matched, \"" << Name << "\" -> " << SymI->second.getFlags()
2850 << "\n";
2851 });
2852 Result[Name] = SymI->second.getFlags();
2853 return true;
2854 });
2855 }
2856
2857 // Remove any weakly referenced symbols that haven't been resolved.
2858 IPLS->LookupSet.remove_if(
2859 [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) {
2860 return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol;
2861 });
2862
2863 if (!IPLS->LookupSet.empty()) {
2864 LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n");
2865 return make_error<SymbolsNotFound>(getSymbolStringPool(),
2866 IPLS->LookupSet.getSymbolNames());
2867 }
2868
2869 LLVM_DEBUG(dbgs() << "Succeded, result = " << Result << "\n");
2870 return Result;
2871 });
2872
2873 // Run the callback on the result.
2874 LLVM_DEBUG(dbgs() << "Sending result to handler.\n");
2875 OnComplete(std::move(Result));
2876}
2877
2878void ExecutionSession::OL_destroyMaterializationResponsibility(
2879 MaterializationResponsibility &MR) {
2880
2881 assert(MR.SymbolFlags.empty() &&
2882 "All symbols should have been explicitly materialized or failed");
2883 MR.JD.unlinkMaterializationResponsibility(MR);
2884}
2885
2886SymbolNameSet ExecutionSession::OL_getRequestedSymbols(
2887 const MaterializationResponsibility &MR) {
2888 return MR.JD.getRequestedSymbols(MR.SymbolFlags);
2889}
2890
2891Error ExecutionSession::OL_notifyResolved(MaterializationResponsibility &MR,
2892 const SymbolMap &Symbols) {
2893 LLVM_DEBUG({
2894 dbgs() << "In " << MR.JD.getName() << " resolving " << Symbols << "\n";
2895 });
2896#ifndef NDEBUG
2897 for (auto &KV : Symbols) {
2898 auto I = MR.SymbolFlags.find(KV.first);
2899 assert(I != MR.SymbolFlags.end() &&
2900 "Resolving symbol outside this responsibility set");
2901 assert(!I->second.hasMaterializationSideEffectsOnly() &&
2902 "Can't resolve materialization-side-effects-only symbol");
2903 assert((KV.second.getFlags() & ~JITSymbolFlags::Common) ==
2904 (I->second & ~JITSymbolFlags::Common) &&
2905 "Resolving symbol with incorrect flags");
2906 }
2907#endif
2908
2909 return MR.JD.resolve(MR, Symbols);
2910}
2911
2912template <typename HandleNewDepFn>
2913void ExecutionSession::propagateExtraEmitDeps(
2914 std::deque<JITDylib::EmissionDepUnit *> Worklist, EDUInfosMap &EDUInfos,
2915 HandleNewDepFn HandleNewDep) {
2916
2917 // Iterate to a fixed-point to propagate extra-emit dependencies through the
2918 // EDU graph.
2919 while (!Worklist.empty()) {
2920 auto &EDU = *Worklist.front();
2921 Worklist.pop_front();
2922
2923 assert(EDUInfos.count(&EDU) && "No info entry for EDU");
2924 auto &EDUInfo = EDUInfos[&EDU];
2925
2926 // Propagate new dependencies to users.
2927 for (auto *UserEDU : EDUInfo.IntraEmitUsers) {
2928
2929 // UserEDUInfo only present if UserEDU has its own users.
2930 JITDylib::EmissionDepUnitInfo *UserEDUInfo = nullptr;
2931 {
2932 auto UserEDUInfoItr = EDUInfos.find(UserEDU);
2933 if (UserEDUInfoItr != EDUInfos.end())
2934 UserEDUInfo = &UserEDUInfoItr->second;
2935 }
2936
2937 for (auto &[DepJD, Deps] : EDUInfo.NewDeps) {
2938 auto &UserEDUDepsForJD = UserEDU->Dependencies[DepJD];
2939 DenseSet<NonOwningSymbolStringPtr> *UserEDUNewDepsForJD = nullptr;
2940 for (auto Dep : Deps) {
2941 if (UserEDUDepsForJD.insert(Dep).second) {
2942 HandleNewDep(*UserEDU, *DepJD, Dep);
2943 if (UserEDUInfo) {
2944 if (!UserEDUNewDepsForJD) {
2945 // If UserEDU has no new deps then it's not in the worklist
2946 // yet, so add it.
2947 if (UserEDUInfo->NewDeps.empty())
2948 Worklist.push_back(UserEDU);
2949 UserEDUNewDepsForJD = &UserEDUInfo->NewDeps[DepJD];
2950 }
2951 // Add (DepJD, Dep) to NewDeps.
2952 UserEDUNewDepsForJD->insert(Dep);
2953 }
2954 }
2955 }
2956 }
2957 }
2958
2959 EDUInfo.NewDeps.clear();
2960 }
2961}
2962
2963// Note: This method modifies the emitted set.
2964ExecutionSession::EDUInfosMap ExecutionSession::simplifyDepGroups(
2965 MaterializationResponsibility &MR,
2966 ArrayRef<SymbolDependenceGroup> EmittedDeps) {
2967
2968 auto &TargetJD = MR.getTargetJITDylib();
2969
2970 // 1. Build initial EmissionDepUnit -> EmissionDepUnitInfo and
2971 // Symbol -> EmissionDepUnit mappings.
2972 DenseMap<JITDylib::EmissionDepUnit *, JITDylib::EmissionDepUnitInfo> EDUInfos;
2973 EDUInfos.reserve(EmittedDeps.size());
2974 DenseMap<NonOwningSymbolStringPtr, JITDylib::EmissionDepUnit *> EDUForSymbol;
2975 for (auto &DG : EmittedDeps) {
2976 assert(!DG.Symbols.empty() && "DepGroup does not cover any symbols");
2977
2978 // Skip empty EDUs.
2979 if (DG.Dependencies.empty())
2980 continue;
2981
2982 auto TmpEDU = std::make_shared<JITDylib::EmissionDepUnit>(TargetJD);
2983 auto &EDUInfo = EDUInfos[TmpEDU.get()];
2984 EDUInfo.EDU = std::move(TmpEDU);
2985 for (const auto &Symbol : DG.Symbols) {
2986 NonOwningSymbolStringPtr NonOwningSymbol(Symbol);
2987 assert(!EDUForSymbol.count(NonOwningSymbol) &&
2988 "Symbol should not appear in more than one SymbolDependenceGroup");
2989 assert(MR.getSymbols().count(Symbol) &&
2990 "Symbol in DepGroups not in the emitted set");
2991 auto NewlyEmittedItr = MR.getSymbols().find(Symbol);
2992 EDUInfo.EDU->Symbols[NonOwningSymbol] = NewlyEmittedItr->second;
2993 EDUForSymbol[NonOwningSymbol] = EDUInfo.EDU.get();
2994 }
2995 }
2996
2997 // 2. Build a "residual" EDU to cover all symbols that have no dependencies.
2998 {
2999 DenseMap<NonOwningSymbolStringPtr, JITSymbolFlags> ResidualSymbolFlags;
3000 for (auto &[Sym, Flags] : MR.getSymbols()) {
3001 if (!EDUForSymbol.count(NonOwningSymbolStringPtr(Sym)))
3002 ResidualSymbolFlags[NonOwningSymbolStringPtr(Sym)] = Flags;
3003 }
3004 if (!ResidualSymbolFlags.empty()) {
3005 auto ResidualEDU = std::make_shared<JITDylib::EmissionDepUnit>(TargetJD);
3006 ResidualEDU->Symbols = std::move(ResidualSymbolFlags);
3007 auto &ResidualEDUInfo = EDUInfos[ResidualEDU.get()];
3008 ResidualEDUInfo.EDU = std::move(ResidualEDU);
3009
3010 // If the residual EDU is the only one then bail out early.
3011 if (EDUInfos.size() == 1)
3012 return EDUInfos;
3013
3014 // Otherwise add the residual EDU to the EDUForSymbol map.
3015 for (auto &[Sym, Flags] : ResidualEDUInfo.EDU->Symbols)
3016 EDUForSymbol[Sym] = ResidualEDUInfo.EDU.get();
3017 }
3018 }
3019
3020#ifndef NDEBUG
3021 assert(EDUForSymbol.size() == MR.getSymbols().size() &&
3022 "MR symbols not fully covered by EDUs?");
3023 for (auto &[Sym, Flags] : MR.getSymbols()) {
3024 assert(EDUForSymbol.count(NonOwningSymbolStringPtr(Sym)) &&
3025 "Sym in MR not covered by EDU");
3026 }
3027#endif // NDEBUG
3028
3029 // 3. Use the DepGroups array to build a graph of dependencies between
3030 // EmissionDepUnits in this finalization. We want to remove these
3031 // intra-finalization uses, propagating dependencies on symbols outside
3032 // this finalization. Add EDUs to the worklist.
3033 for (auto &DG : EmittedDeps) {
3034
3035 // Skip SymbolDependenceGroups with no dependencies.
3036 if (DG.Dependencies.empty())
3037 continue;
3038
3039 assert(EDUForSymbol.count(NonOwningSymbolStringPtr(*DG.Symbols.begin())) &&
3040 "No EDU for DG");
3041 auto &EDU =
3042 *EDUForSymbol.find(NonOwningSymbolStringPtr(*DG.Symbols.begin()))
3043 ->second;
3044
3045 for (auto &[DepJD, Deps] : DG.Dependencies) {
3046 DenseSet<NonOwningSymbolStringPtr> NewDepsForJD;
3047
3048 assert(!Deps.empty() && "Dependence set for DepJD is empty");
3049
3050 if (DepJD != &TargetJD) {
3051 // DepJD is some other JITDylib.There can't be any intra-finalization
3052 // edges here, so just skip.
3053 for (auto &Dep : Deps)
3054 NewDepsForJD.insert(NonOwningSymbolStringPtr(Dep));
3055 } else {
3056 // DepJD is the Target JITDylib. Check for intra-finaliztaion edges,
3057 // skipping any and recording the intra-finalization use instead.
3058 for (auto &Dep : Deps) {
3059 NonOwningSymbolStringPtr NonOwningDep(Dep);
3060 auto I = EDUForSymbol.find(NonOwningDep);
3061 if (I == EDUForSymbol.end()) {
3062 if (!MR.getSymbols().count(Dep))
3063 NewDepsForJD.insert(NonOwningDep);
3064 continue;
3065 }
3066
3067 if (I->second != &EDU)
3068 EDUInfos[I->second].IntraEmitUsers.insert(&EDU);
3069 }
3070 }
3071
3072 if (!NewDepsForJD.empty())
3073 EDU.Dependencies[DepJD] = std::move(NewDepsForJD);
3074 }
3075 }
3076
3077 // 4. Build the worklist.
3078 std::deque<JITDylib::EmissionDepUnit *> Worklist;
3079 for (auto &[EDU, EDUInfo] : EDUInfos) {
3080 // If this EDU has extra-finalization dependencies and intra-finalization
3081 // users then add it to the worklist.
3082 if (!EDU->Dependencies.empty()) {
3083 auto I = EDUInfos.find(EDU);
3084 if (I != EDUInfos.end()) {
3085 auto &EDUInfo = I->second;
3086 if (!EDUInfo.IntraEmitUsers.empty()) {
3087 EDUInfo.NewDeps = EDU->Dependencies;
3088 Worklist.push_back(EDU);
3089 }
3090 }
3091 }
3092 }
3093
3094 // 4. Propagate dependencies through the EDU graph.
3095 propagateExtraEmitDeps(
3096 Worklist, EDUInfos,
3097 [](JITDylib::EmissionDepUnit &, JITDylib &, NonOwningSymbolStringPtr) {});
3098
3099 return EDUInfos;
3100}
3101
3102void ExecutionSession::IL_makeEDUReady(
3103 std::shared_ptr<JITDylib::EmissionDepUnit> EDU,
3104 JITDylib::AsynchronousSymbolQuerySet &Queries) {
3105
3106 // The symbols for this EDU are ready.
3107 auto &JD = *EDU->JD;
3108
3109 for (auto &[Sym, Flags] : EDU->Symbols) {
3110 assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
3111 "JD does not have an entry for Sym");
3112 auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
3113
3114 assert(((Entry.getFlags().hasMaterializationSideEffectsOnly() &&
3115 Entry.getState() == SymbolState::Materializing) ||
3116 Entry.getState() == SymbolState::Resolved ||
3117 Entry.getState() == SymbolState::Emitted) &&
3118 "Emitting from state other than Resolved");
3119
3120 Entry.setState(SymbolState::Ready);
3121
3122 auto MII = JD.MaterializingInfos.find(SymbolStringPtr(Sym));
3123
3124 // Check for pending queries.
3125 if (MII == JD.MaterializingInfos.end())
3126 continue;
3127 auto &MI = MII->second;
3128
3129 for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) {
3130 Q->notifySymbolMetRequiredState(SymbolStringPtr(Sym), Entry.getSymbol());
3131 if (Q->isComplete())
3132 Queries.insert(Q);
3133 Q->removeQueryDependence(JD, SymbolStringPtr(Sym));
3134 }
3135
3136 JD.MaterializingInfos.erase(MII);
3137 }
3138
3139 JD.shrinkMaterializationInfoMemory();
3140}
3141
3142void ExecutionSession::IL_makeEDUEmitted(
3143 std::shared_ptr<JITDylib::EmissionDepUnit> EDU,
3144 JITDylib::AsynchronousSymbolQuerySet &Queries) {
3145
3146 // The symbols for this EDU are emitted, but not ready.
3147 auto &JD = *EDU->JD;
3148
3149 for (auto &[Sym, Flags] : EDU->Symbols) {
3150 assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
3151 "JD does not have an entry for Sym");
3152 auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
3153
3154 assert(((Entry.getFlags().hasMaterializationSideEffectsOnly() &&
3155 Entry.getState() == SymbolState::Materializing) ||
3156 Entry.getState() == SymbolState::Resolved ||
3157 Entry.getState() == SymbolState::Emitted) &&
3158 "Emitting from state other than Resolved");
3159
3160 if (Entry.getState() == SymbolState::Emitted) {
3161 // This was already emitted, so we can skip the rest of this loop.
3162#ifndef NDEBUG
3163 for (auto &[Sym, Flags] : EDU->Symbols) {
3164 assert(JD.Symbols.count(SymbolStringPtr(Sym)) &&
3165 "JD does not have an entry for Sym");
3166 auto &Entry = JD.Symbols[SymbolStringPtr(Sym)];
3167 assert(Entry.getState() == SymbolState::Emitted &&
3168 "Symbols for EDU in inconsistent state");
3169 assert(JD.MaterializingInfos.count(SymbolStringPtr(Sym)) &&
3170 "Emitted symbol has no MI");
3171 auto MI = JD.MaterializingInfos[SymbolStringPtr(Sym)];
3172 assert(MI.takeQueriesMeeting(SymbolState::Emitted).empty() &&
3173 "Already-emitted symbol has waiting-on-emitted queries");
3174 }
3175#endif // NDEBUG
3176 break;
3177 }
3178
3179 Entry.setState(SymbolState::Emitted);
3180 auto &MI = JD.MaterializingInfos[SymbolStringPtr(Sym)];
3181 MI.DefiningEDU = EDU;
3182
3183 for (auto &Q : MI.takeQueriesMeeting(SymbolState::Emitted)) {
3184 Q->notifySymbolMetRequiredState(SymbolStringPtr(Sym), Entry.getSymbol());
3185 if (Q->isComplete())
3186 Queries.insert(Q);
3187 Q->removeQueryDependence(JD, SymbolStringPtr(Sym));
3188 }
3189 }
3190
3191 for (auto &[DepJD, Deps] : EDU->Dependencies) {
3192 for (auto &Dep : Deps)
3193 DepJD->MaterializingInfos[SymbolStringPtr(Dep)].DependantEDUs.insert(
3194 EDU.get());
3195 }
3196}
3197
3198/// Removes the given dependence from EDU. If EDU's dependence set becomes
3199/// empty then this function adds an entry for it to the EDUInfos map.
3200/// Returns true if a new EDUInfosMap entry is added.
3201bool ExecutionSession::IL_removeEDUDependence(JITDylib::EmissionDepUnit &EDU,
3202 JITDylib &DepJD,
3203 NonOwningSymbolStringPtr DepSym,
3204 EDUInfosMap &EDUInfos) {
3205 assert(EDU.Dependencies.count(&DepJD) &&
3206 "JD does not appear in Dependencies of DependantEDU");
3207 assert(EDU.Dependencies[&DepJD].count(DepSym) &&
3208 "Symbol does not appear in Dependencies of DependantEDU");
3209 auto &JDDeps = EDU.Dependencies[&DepJD];
3210 JDDeps.erase(DepSym);
3211 if (JDDeps.empty()) {
3212 EDU.Dependencies.erase(&DepJD);
3213 if (EDU.Dependencies.empty()) {
3214 // If the dependencies set has become empty then EDU _may_ be ready
3215 // (we won't know for sure until we've propagated the extra-emit deps).
3216 // Create an EDUInfo for it (if it doesn't have one already) so that
3217 // it'll be visited after propagation.
3218 auto &DepEDUInfo = EDUInfos[&EDU];
3219 if (!DepEDUInfo.EDU) {
3220 assert(EDU.JD->Symbols.count(
3221 SymbolStringPtr(EDU.Symbols.begin()->first)) &&
3222 "Missing symbol entry for first symbol in EDU");
3223 auto DepEDUFirstMI = EDU.JD->MaterializingInfos.find(
3224 SymbolStringPtr(EDU.Symbols.begin()->first));
3225 assert(DepEDUFirstMI != EDU.JD->MaterializingInfos.end() &&
3226 "Missing MI for first symbol in DependantEDU");
3227 DepEDUInfo.EDU = DepEDUFirstMI->second.DefiningEDU;
3228 return true;
3229 }
3230 }
3231 }
3232 return false;
3233}
3234
3235Error ExecutionSession::makeJDClosedError(JITDylib::EmissionDepUnit &EDU,
3236 JITDylib &ClosedJD) {
3237 SymbolNameSet FailedSymbols;
3238 for (auto &[Sym, Flags] : EDU.Symbols)
3239 FailedSymbols.insert(SymbolStringPtr(Sym));
3240 SymbolDependenceMap BadDeps;
3241 for (auto &Dep : EDU.Dependencies[&ClosedJD])
3242 BadDeps[&ClosedJD].insert(SymbolStringPtr(Dep));
3243 return make_error<UnsatisfiedSymbolDependencies>(
3244 ClosedJD.getExecutionSession().getSymbolStringPool(), EDU.JD,
3245 std::move(FailedSymbols), std::move(BadDeps),
3246 ClosedJD.getName() + " is closed");
3247}
3248
3249Error ExecutionSession::makeUnsatisfiedDepsError(JITDylib::EmissionDepUnit &EDU,
3250 JITDylib &BadJD,
3251 SymbolNameSet BadDeps) {
3252 SymbolNameSet FailedSymbols;
3253 for (auto &[Sym, Flags] : EDU.Symbols)
3254 FailedSymbols.insert(SymbolStringPtr(Sym));
3255 SymbolDependenceMap BadDepsMap;
3256 BadDepsMap[&BadJD] = std::move(BadDeps);
3257 return make_error<UnsatisfiedSymbolDependencies>(
3258 BadJD.getExecutionSession().getSymbolStringPool(), &BadJD,
3259 std::move(FailedSymbols), std::move(BadDepsMap),
3260 "dependencies removed or in error state");
3261}
3262
3263Expected<JITDylib::AsynchronousSymbolQuerySet>
3264ExecutionSession::IL_emit(MaterializationResponsibility &MR,
3265 EDUInfosMap EDUInfos) {
3266
3267 if (MR.RT->isDefunct())
3268 return make_error<ResourceTrackerDefunct>(MR.RT);
3269
3270 auto &TargetJD = MR.getTargetJITDylib();
3271 if (TargetJD.State != JITDylib::Open)
3272 return make_error<StringError>("JITDylib " + TargetJD.getName() +
3273 " is defunct",
3275#ifdef EXPENSIVE_CHECKS
3276 verifySessionState("entering ExecutionSession::IL_emit");
3277#endif
3278
3279 // Walk all EDUs:
3280 // 1. Verifying that dependencies are available (not removed or in the error
3281 // state.
3282 // 2. Removing any dependencies that are already Ready.
3283 // 3. Lifting any EDUs for Emitted symbols into the EDUInfos map.
3284 // 4. Finding any dependant EDUs and lifting them into the EDUInfos map.
3285 std::deque<JITDylib::EmissionDepUnit *> Worklist;
3286 for (auto &[EDU, _] : EDUInfos)
3287 Worklist.push_back(EDU);
3288
3289 for (auto *EDU : Worklist) {
3290 auto *EDUInfo = &EDUInfos[EDU];
3291
3292 SmallVector<JITDylib *> DepJDsToRemove;
3293 for (auto &[DepJD, Deps] : EDU->Dependencies) {
3294 if (DepJD->State != JITDylib::Open)
3295 return makeJDClosedError(*EDU, *DepJD);
3296
3297 SymbolNameSet BadDeps;
3298 SmallVector<NonOwningSymbolStringPtr> DepsToRemove;
3299 for (auto &Dep : Deps) {
3300 auto DepEntryItr = DepJD->Symbols.find(SymbolStringPtr(Dep));
3301
3302 // If this dep has been removed or moved to the error state then add it
3303 // to the bad deps set. We aggregate these bad deps for more
3304 // comprehensive error messages.
3305 if (DepEntryItr == DepJD->Symbols.end() ||
3306 DepEntryItr->second.getFlags().hasError()) {
3307 BadDeps.insert(SymbolStringPtr(Dep));
3308 continue;
3309 }
3310
3311 // If this dep isn't emitted yet then just add it to the NewDeps set to
3312 // be propagated.
3313 auto &DepEntry = DepEntryItr->second;
3314 if (DepEntry.getState() < SymbolState::Emitted) {
3315 EDUInfo->NewDeps[DepJD].insert(Dep);
3316 continue;
3317 }
3318
3319 // This dep has been emitted, so add it to the list to be removed from
3320 // EDU.
3321 DepsToRemove.push_back(Dep);
3322
3323 // If Dep is Ready then there's nothing further to do.
3324 if (DepEntry.getState() == SymbolState::Ready) {
3325 assert(!DepJD->MaterializingInfos.count(SymbolStringPtr(Dep)) &&
3326 "Unexpected MaterializationInfo attached to ready symbol");
3327 continue;
3328 }
3329
3330 // If we get here thene Dep is Emitted. We need to look up its defining
3331 // EDU and add this EDU to the defining EDU's list of users (this means
3332 // creating an EDUInfos entry if the defining EDU doesn't have one
3333 // already).
3334 assert(DepJD->MaterializingInfos.count(SymbolStringPtr(Dep)) &&
3335 "Expected MaterializationInfo for emitted dependency");
3336 auto &DepMI = DepJD->MaterializingInfos[SymbolStringPtr(Dep)];
3337 assert(DepMI.DefiningEDU &&
3338 "Emitted symbol does not have a defining EDU");
3339 assert(!DepMI.DefiningEDU->Dependencies.empty() &&
3340 "Emitted symbol has empty dependencies (should be ready)");
3341 assert(DepMI.DependantEDUs.empty() &&
3342 "Already-emitted symbol has dependant EDUs?");
3343 auto &DepEDUInfo = EDUInfos[DepMI.DefiningEDU.get()];
3344 if (!DepEDUInfo.EDU) {
3345 // No EDUInfo yet -- build initial entry, and reset the EDUInfo
3346 // pointer, which we will have invalidated.
3347 EDUInfo = &EDUInfos[EDU];
3348 DepEDUInfo.EDU = DepMI.DefiningEDU;
3349 for (auto &[DepDepJD, DepDeps] : DepEDUInfo.EDU->Dependencies) {
3350 if (DepDepJD == &TargetJD) {
3351 for (auto &DepDep : DepDeps)
3352 if (!MR.getSymbols().count(SymbolStringPtr(DepDep)))
3353 DepEDUInfo.NewDeps[DepDepJD].insert(DepDep);
3354 } else
3355 DepEDUInfo.NewDeps[DepDepJD] = DepDeps;
3356 }
3357 }
3358 DepEDUInfo.IntraEmitUsers.insert(EDU);
3359 }
3360
3361 // Some dependencies were removed or in an error state -- error out.
3362 if (!BadDeps.empty())
3363 return makeUnsatisfiedDepsError(*EDU, *DepJD, std::move(BadDeps));
3364
3365 // Remove the emitted / ready deps from DepJD.
3366 for (auto &Dep : DepsToRemove)
3367 Deps.erase(Dep);
3368
3369 // If there are no further deps in DepJD then flag it for removal too.
3370 if (Deps.empty())
3371 DepJDsToRemove.push_back(DepJD);
3372 }
3373
3374 // Remove any JDs whose dependence sets have become empty.
3375 for (auto &DepJD : DepJDsToRemove) {
3376 assert(EDU->Dependencies.count(DepJD) &&
3377 "Trying to remove non-existent dep entries");
3378 EDU->Dependencies.erase(DepJD);
3379 }
3380
3381 // Now look for users of this EDU.
3382 for (auto &[Sym, Flags] : EDU->Symbols) {
3383 assert(TargetJD.Symbols.count(SymbolStringPtr(Sym)) &&
3384 "Sym not present in symbol table");
3385 assert((TargetJD.Symbols[SymbolStringPtr(Sym)].getState() ==
3387 TargetJD.Symbols[SymbolStringPtr(Sym)]
3388 .getFlags()
3389 .hasMaterializationSideEffectsOnly()) &&
3390 "Emitting symbol not in the resolved state");
3391 assert(!TargetJD.Symbols[SymbolStringPtr(Sym)].getFlags().hasError() &&
3392 "Symbol is already in an error state");
3393
3394 auto MII = TargetJD.MaterializingInfos.find(SymbolStringPtr(Sym));
3395 if (MII == TargetJD.MaterializingInfos.end() ||
3396 MII->second.DependantEDUs.empty())
3397 continue;
3398
3399 for (auto &DependantEDU : MII->second.DependantEDUs) {
3400 if (IL_removeEDUDependence(*DependantEDU, TargetJD, Sym, EDUInfos))
3401 EDUInfo = &EDUInfos[EDU];
3402 EDUInfo->IntraEmitUsers.insert(DependantEDU);
3403 }
3404 MII->second.DependantEDUs.clear();
3405 }
3406 }
3407
3408 Worklist.clear();
3409 for (auto &[EDU, EDUInfo] : EDUInfos) {
3410 if (!EDUInfo.IntraEmitUsers.empty() && !EDU->Dependencies.empty()) {
3411 if (EDUInfo.NewDeps.empty())
3412 EDUInfo.NewDeps = EDU->Dependencies;
3413 Worklist.push_back(EDU);
3414 }
3415 }
3416
3417 propagateExtraEmitDeps(
3418 Worklist, EDUInfos,
3419 [](JITDylib::EmissionDepUnit &EDU, JITDylib &JD,
3420 NonOwningSymbolStringPtr Sym) {
3421 JD.MaterializingInfos[SymbolStringPtr(Sym)].DependantEDUs.insert(&EDU);
3422 });
3423
3424 JITDylib::AsynchronousSymbolQuerySet CompletedQueries;
3425
3426 // Extract completed queries and lodge not-yet-ready EDUs in the
3427 // session.
3428 for (auto &[EDU, EDUInfo] : EDUInfos) {
3429 if (EDU->Dependencies.empty())
3430 IL_makeEDUReady(std::move(EDUInfo.EDU), CompletedQueries);
3431 else
3432 IL_makeEDUEmitted(std::move(EDUInfo.EDU), CompletedQueries);
3433 }
3434
3435#ifdef EXPENSIVE_CHECKS
3436 verifySessionState("exiting ExecutionSession::IL_emit");
3437#endif
3438
3439 return std::move(CompletedQueries);
3440}
3441
3442Error ExecutionSession::OL_notifyEmitted(
3443 MaterializationResponsibility &MR,
3444 ArrayRef<SymbolDependenceGroup> DepGroups) {
3445 LLVM_DEBUG({
3446 dbgs() << "In " << MR.JD.getName() << " emitting " << MR.SymbolFlags
3447 << "\n";
3448 if (!DepGroups.empty()) {
3449 dbgs() << " Initial dependencies:\n";
3450 for (auto &SDG : DepGroups) {
3451 dbgs() << " Symbols: " << SDG.Symbols
3452 << ", Dependencies: " << SDG.Dependencies << "\n";
3453 }
3454 }
3455 });
3456
3457#ifndef NDEBUG
3458 SymbolNameSet Visited;
3459 for (auto &DG : DepGroups) {
3460 for (auto &Sym : DG.Symbols) {
3461 assert(MR.SymbolFlags.count(Sym) &&
3462 "DG contains dependence for symbol outside this MR");
3463 assert(Visited.insert(Sym).second &&
3464 "DG contains duplicate entries for Name");
3465 }
3466 }
3467#endif // NDEBUG
3468
3469 auto EDUInfos = simplifyDepGroups(MR, DepGroups);
3470
3471 LLVM_DEBUG({
3472 dbgs() << " Simplified dependencies:\n";
3473 for (auto &[EDU, EDUInfo] : EDUInfos) {
3474 dbgs() << " Symbols: { ";
3475 for (auto &[Sym, Flags] : EDU->Symbols)
3476 dbgs() << Sym << " ";
3477 dbgs() << "}, Dependencies: { ";
3478 for (auto &[DepJD, Deps] : EDU->Dependencies) {
3479 dbgs() << "(" << DepJD->getName() << ", { ";
3480 for (auto &Dep : Deps)
3481 dbgs() << Dep << " ";
3482 dbgs() << "}) ";
3483 }
3484 dbgs() << "}\n";
3485 }
3486 });
3487
3488 auto CompletedQueries =
3489 runSessionLocked([&]() { return IL_emit(MR, EDUInfos); });
3490
3491 // On error bail out.
3492 if (!CompletedQueries)
3493 return CompletedQueries.takeError();
3494
3495 MR.SymbolFlags.clear();
3496
3497 // Otherwise notify all the completed queries.
3498 for (auto &Q : *CompletedQueries) {
3499 assert(Q->isComplete() && "Q is not complete");
3500 Q->handleComplete(*this);
3501 }
3502
3503 return Error::success();
3504}
3505
3506Error ExecutionSession::OL_defineMaterializing(
3507 MaterializationResponsibility &MR, SymbolFlagsMap NewSymbolFlags) {
3508
3509 LLVM_DEBUG({
3510 dbgs() << "In " << MR.JD.getName() << " defining materializing symbols "
3511 << NewSymbolFlags << "\n";
3512 });
3513 if (auto AcceptedDefs =
3514 MR.JD.defineMaterializing(MR, std::move(NewSymbolFlags))) {
3515 // Add all newly accepted symbols to this responsibility object.
3516 for (auto &KV : *AcceptedDefs)
3517 MR.SymbolFlags.insert(KV);
3518 return Error::success();
3519 } else
3520 return AcceptedDefs.takeError();
3521}
3522
3523std::pair<JITDylib::AsynchronousSymbolQuerySet,
3524 std::shared_ptr<SymbolDependenceMap>>
3525ExecutionSession::IL_failSymbols(JITDylib &JD,
3526 const SymbolNameVector &SymbolsToFail) {
3527
3528#ifdef EXPENSIVE_CHECKS
3529 verifySessionState("entering ExecutionSession::IL_failSymbols");
3530#endif
3531
3532 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3533 auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
3534 auto ExtractFailedQueries = [&](JITDylib::MaterializingInfo &MI) {
3535 JITDylib::AsynchronousSymbolQueryList ToDetach;
3536 for (auto &Q : MI.pendingQueries()) {
3537 // Add the query to the list to be failed and detach it.
3538 FailedQueries.insert(Q);
3539 ToDetach.push_back(Q);
3540 }
3541 for (auto &Q : ToDetach)
3542 Q->detach();
3543 assert(!MI.hasQueriesPending() && "Queries still pending after detach");
3544 };
3545
3546 for (auto &Name : SymbolsToFail) {
3547 (*FailedSymbolsMap)[&JD].insert(Name);
3548
3549 // Look up the symbol to fail.
3550 auto SymI = JD.Symbols.find(Name);
3551
3552 // FIXME: Revisit this. We should be able to assert sequencing between
3553 // ResourceTracker removal and symbol failure.
3554 //
3555 // It's possible that this symbol has already been removed, e.g. if a
3556 // materialization failure happens concurrently with a ResourceTracker or
3557 // JITDylib removal. In that case we can safely skip this symbol and
3558 // continue.
3559 if (SymI == JD.Symbols.end())
3560 continue;
3561 auto &Sym = SymI->second;
3562
3563 // If the symbol is already in the error state then we must have visited
3564 // it earlier.
3565 if (Sym.getFlags().hasError()) {
3566 assert(!JD.MaterializingInfos.count(Name) &&
3567 "Symbol in error state still has MaterializingInfo");
3568 continue;
3569 }
3570
3571 // Move the symbol into the error state.
3572 Sym.setFlags(Sym.getFlags() | JITSymbolFlags::HasError);
3573
3574 // FIXME: Come up with a sane mapping of state to
3575 // presence-of-MaterializingInfo so that we can assert presence / absence
3576 // here, rather than testing it.
3577 auto MII = JD.MaterializingInfos.find(Name);
3578 if (MII == JD.MaterializingInfos.end())
3579 continue;
3580
3581 auto &MI = MII->second;
3582
3583 // Collect queries to be failed for this MII.
3584 ExtractFailedQueries(MI);
3585
3586 if (MI.DefiningEDU) {
3587 // If there is a DefiningEDU for this symbol then remove this
3588 // symbol from it.
3589 assert(MI.DependantEDUs.empty() &&
3590 "Symbol with DefiningEDU should not have DependantEDUs");
3591 assert(Sym.getState() >= SymbolState::Emitted &&
3592 "Symbol has EDU, should have been emitted");
3593 assert(MI.DefiningEDU->Symbols.count(NonOwningSymbolStringPtr(Name)) &&
3594 "Symbol does not appear in its DefiningEDU");
3595 MI.DefiningEDU->Symbols.erase(NonOwningSymbolStringPtr(Name));
3596 MI.DefiningEDU = nullptr;
3597 } else {
3598 // Otherwise if there are any EDUs waiting on this symbol then move
3599 // those symbols to the error state too, and deregister them from the
3600 // symbols that they depend on.
3601 // Note: We use a copy of DependantEDUs here since we'll be removing
3602 // from the original set as we go.
3603 for (auto &DependantEDU : MI.DependantEDUs) {
3604
3605 // Remove DependantEDU from all of its users DependantEDUs lists.
3606 for (auto &[DepJD, DepSyms] : DependantEDU->Dependencies) {
3607 for (auto DepSym : DepSyms) {
3608 // Skip self-reference to avoid invalidating the MI.DependantEDUs
3609 // map. We'll clear this later.
3610 if (DepJD == &JD && DepSym == Name)
3611 continue;
3612 assert(DepJD->Symbols.count(SymbolStringPtr(DepSym)) &&
3613 "DepSym not in DepJD?");
3614 assert(DepJD->MaterializingInfos.count(SymbolStringPtr(DepSym)) &&
3615 "DependantEDU not registered with symbol it depends on");
3616 auto &SymMI = DepJD->MaterializingInfos[SymbolStringPtr(DepSym)];
3617 assert(SymMI.DependantEDUs.count(DependantEDU) &&
3618 "DependantEDU missing from DependantEDUs list");
3619 SymMI.DependantEDUs.erase(DependantEDU);
3620 }
3621 }
3622
3623 // Move any symbols defined by DependantEDU into the error state and
3624 // fail any queries waiting on them.
3625 auto &DepJD = *DependantEDU->JD;
3626 auto DepEDUSymbols = std::move(DependantEDU->Symbols);
3627 for (auto &[DepName, Flags] : DepEDUSymbols) {
3628 auto DepSymItr = DepJD.Symbols.find(SymbolStringPtr(DepName));
3629 assert(DepSymItr != DepJD.Symbols.end() &&
3630 "Symbol not present in table");
3631 auto &DepSym = DepSymItr->second;
3632
3633 assert(DepSym.getState() >= SymbolState::Emitted &&
3634 "Symbol has EDU, should have been emitted");
3635 assert(!DepSym.getFlags().hasError() &&
3636 "Symbol is already in the error state?");
3637 DepSym.setFlags(DepSym.getFlags() | JITSymbolFlags::HasError);
3638 (*FailedSymbolsMap)[&DepJD].insert(SymbolStringPtr(DepName));
3639
3640 // This symbol has a defining EDU so its MaterializingInfo object must
3641 // exist.
3642 auto DepMIItr =
3643 DepJD.MaterializingInfos.find(SymbolStringPtr(DepName));
3644 assert(DepMIItr != DepJD.MaterializingInfos.end() &&
3645 "Symbol has defining EDU but not MaterializingInfo");
3646 auto &DepMI = DepMIItr->second;
3647 assert(DepMI.DefiningEDU.get() == DependantEDU &&
3648 "Bad EDU dependence edge");
3649 assert(DepMI.DependantEDUs.empty() &&
3650 "Symbol was emitted, should not have any DependantEDUs");
3651 ExtractFailedQueries(DepMI);
3652 DepJD.MaterializingInfos.erase(SymbolStringPtr(DepName));
3653 }
3654
3655 DepJD.shrinkMaterializationInfoMemory();
3656 }
3657
3658 MI.DependantEDUs.clear();
3659 }
3660
3661 assert(!MI.DefiningEDU && "DefiningEDU should have been reset");
3662 assert(MI.DependantEDUs.empty() &&
3663 "DependantEDUs should have been removed above");
3664 assert(!MI.hasQueriesPending() &&
3665 "Can not delete MaterializingInfo with queries pending");
3666 JD.MaterializingInfos.erase(Name);
3667 }
3668
3669 JD.shrinkMaterializationInfoMemory();
3670
3671#ifdef EXPENSIVE_CHECKS
3672 verifySessionState("exiting ExecutionSession::IL_failSymbols");
3673#endif
3674
3675 return std::make_pair(std::move(FailedQueries), std::move(FailedSymbolsMap));
3676}
3677
3678void ExecutionSession::OL_notifyFailed(MaterializationResponsibility &MR) {
3679
3680 LLVM_DEBUG({
3681 dbgs() << "In " << MR.JD.getName() << " failing materialization for "
3682 << MR.SymbolFlags << "\n";
3683 });
3684
3685 if (MR.SymbolFlags.empty())
3686 return;
3687
3688 SymbolNameVector SymbolsToFail;
3689 for (auto &[Name, Flags] : MR.SymbolFlags)
3690 SymbolsToFail.push_back(Name);
3691 MR.SymbolFlags.clear();
3692
3693 JITDylib::AsynchronousSymbolQuerySet FailedQueries;
3694 std::shared_ptr<SymbolDependenceMap> FailedSymbols;
3695
3696 std::tie(FailedQueries, FailedSymbols) = runSessionLocked([&]() {
3697 // If the tracker is defunct then there's nothing to do here.
3698 if (MR.RT->isDefunct())
3699 return std::pair<JITDylib::AsynchronousSymbolQuerySet,
3700 std::shared_ptr<SymbolDependenceMap>>();
3701 return IL_failSymbols(MR.getTargetJITDylib(), SymbolsToFail);
3702 });
3703
3704 for (auto &Q : FailedQueries)
3705 Q->handleFailed(
3706 make_error<FailedToMaterialize>(getSymbolStringPool(), FailedSymbols));
3707}
3708
3709Error ExecutionSession::OL_replace(MaterializationResponsibility &MR,
3710 std::unique_ptr<MaterializationUnit> MU) {
3711 for (auto &KV : MU->getSymbols()) {
3712 assert(MR.SymbolFlags.count(KV.first) &&
3713 "Replacing definition outside this responsibility set");
3714 MR.SymbolFlags.erase(KV.first);
3715 }
3716
3717 if (MU->getInitializerSymbol() == MR.InitSymbol)
3718 MR.InitSymbol = nullptr;
3719
3720 LLVM_DEBUG(MR.JD.getExecutionSession().runSessionLocked([&]() {
3721 dbgs() << "In " << MR.JD.getName() << " replacing symbols with " << *MU
3722 << "\n";
3723 }););
3724
3725 return MR.JD.replace(MR, std::move(MU));
3726}
3727
3728Expected<std::unique_ptr<MaterializationResponsibility>>
3729ExecutionSession::OL_delegate(MaterializationResponsibility &MR,
3730 const SymbolNameSet &Symbols) {
3731
3732 SymbolStringPtr DelegatedInitSymbol;
3733 SymbolFlagsMap DelegatedFlags;
3734
3735 for (auto &Name : Symbols) {
3736 auto I = MR.SymbolFlags.find(Name);
3737 assert(I != MR.SymbolFlags.end() &&
3738 "Symbol is not tracked by this MaterializationResponsibility "
3739 "instance");
3740
3741 DelegatedFlags[Name] = std::move(I->second);
3742 if (Name == MR.InitSymbol)
3743 std::swap(MR.InitSymbol, DelegatedInitSymbol);
3744
3745 MR.SymbolFlags.erase(I);
3746 }
3747
3748 return MR.JD.delegate(MR, std::move(DelegatedFlags),
3749 std::move(DelegatedInitSymbol));
3750}
3751
3752#ifndef NDEBUG
3753void ExecutionSession::dumpDispatchInfo(Task &T) {
3754 runSessionLocked([&]() {
3755 dbgs() << "Dispatching: ";
3756 T.printDescription(dbgs());
3757 dbgs() << "\n";
3758 });
3759}
3760#endif // NDEBUG
3761
3762} // End namespace orc.
3763} // End namespace llvm.
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Addr
std::string Name
Symbol * Sym
Definition: ELF_riscv.cpp:479
#define _
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
#define H(x, y, z)
Definition: MD5.cpp:57
while(!ToSimplify.empty())
if(VerifyEach)
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
raw_pwrite_stream & OS
static uint32_t getFlags(const Symbol *Sym)
Definition: TapiFile.cpp:27
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
const T & front() const
front - Get the first element.
Definition: ArrayRef.h:168
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
const T * data() const
Definition: ArrayRef.h:162
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
bool erase(const KeyT &Val)
Definition: DenseMap.h:329
unsigned size() const
Definition: DenseMap.h:99
bool empty() const
Definition: DenseMap.h:98
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
iterator end()
Definition: DenseMap.h:84
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:220
Helper for Errors used as out-parameters.
Definition: Error.h:1102
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:334
Tagged union holding either a T or a Error.
Definition: Error.h:474
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Symbol info for RuntimeDyld.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
iterator find(const_arg_type_t< ValueT > V)
Definition: DenseSet.h:179
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
Definition: DenseSet.h:97
StringRef getName() const override
Return the name of this materialization unit.
Definition: Core.cpp:283
AbsoluteSymbolsMaterializationUnit(SymbolMap Symbols)
Definition: Core.cpp:279
AsynchronousSymbolQuery(const SymbolLookupSet &Symbols, SymbolState RequiredState, SymbolsResolvedCallback NotifyComplete)
Create a query for the given symbols.
Definition: Core.cpp:180
void notifySymbolMetRequiredState(const SymbolStringPtr &Name, ExecutorSymbolDef Sym)
Notify the query that a requested symbol has reached the required state.
Definition: Core.cpp:194
Definition generators can be attached to JITDylibs to generate new definitions for otherwise unresolv...
Definition: Core.h:946
An ExecutionSession represents a running JIT program.
Definition: Core.h:1431
Error endSession()
End the session.
Definition: Core.cpp:1618
void reportError(Error Err)
Report a error for this execution session.
Definition: Core.h:1569
friend class JITDylib
Definition: Core.h:1434
void lookupFlags(LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet Symbols, unique_function< void(Expected< SymbolFlagsMap >)> OnComplete)
Search the given JITDylibs to find the flags associated with each of the given symbols.
Definition: Core.cpp:1778
SymbolStringPtr intern(StringRef SymName)
Add a symbol name to the SymbolStringPool and return a pointer to it.
Definition: Core.h:1488
JITDylib * getJITDylibByName(StringRef Name)
Return a pointer to the "name" JITDylib.
Definition: Core.cpp:1657
friend class LookupState
Definition: Core.h:1435
JITDylib & createBareJITDylib(std::string Name)
Add a new bare JITDylib to this ExecutionSession.
Definition: Core.cpp:1666
std::shared_ptr< SymbolStringPool > getSymbolStringPool()
Get the SymbolStringPool for this instance.
Definition: Core.h:1483
void lookup(LookupKind K, const JITDylibSearchOrder &SearchOrder, SymbolLookupSet Symbols, SymbolState RequiredState, SymbolsResolvedCallback NotifyComplete, RegisterDependenciesFunction RegisterDependencies)
Search the given JITDylibs for the given symbols.
Definition: Core.cpp:1804
Error registerJITDispatchHandlers(JITDylib &JD, JITDispatchHandlerAssociationMap WFs)
For each tag symbol name, associate the corresponding AsyncHandlerWrapperFunction with the address of...
Definition: Core.cpp:1913
void registerResourceManager(ResourceManager &RM)
Register the given ResourceManager with this ExecutionSession.
Definition: Core.cpp:1640
~ExecutionSession()
Destroy an ExecutionSession.
Definition: Core.cpp:1612
void runJITDispatchHandler(SendResultFunction SendResult, ExecutorAddr HandlerFnTagAddr, ArrayRef< char > ArgBuffer)
Run a registered jit-side wrapper function.
Definition: Core.cpp:1944
void deregisterResourceManager(ResourceManager &RM)
Deregister the given ResourceManager with this ExecutionSession.
Definition: Core.cpp:1644
ExecutionSession(std::unique_ptr< ExecutorProcessControl > EPC)
Construct an ExecutionSession with the given ExecutorProcessControl object.
Definition: Core.cpp:1606
decltype(auto) runSessionLocked(Func &&F)
Run the given lambda with the session mutex locked.
Definition: Core.h:1498
void dump(raw_ostream &OS)
Dump the state of all the JITDylibs in this session.
Definition: Core.cpp:1965
Error removeJITDylibs(std::vector< JITDylibSP > JDsToRemove)
Removes the given JITDylibs from the ExecutionSession.
Definition: Core.cpp:1683
Expected< JITDylib & > createJITDylib(std::string Name)
Add a new JITDylib to this ExecutionSession.
Definition: Core.cpp:1675
void dispatchTask(std::unique_ptr< Task > T)
Materialize the given unit.
Definition: Core.h:1648
Represents an address in the executor process.
Represents a defining location for a JIT symbol.
FailedToMaterialize(std::shared_ptr< SymbolStringPool > SSP, std::shared_ptr< SymbolDependenceMap > Symbols)
Definition: Core.cpp:81
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:99
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:103
InProgressFullLookupState(LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet, SymbolState RequiredState, std::shared_ptr< AsynchronousSymbolQuery > Q, RegisterDependenciesFunction RegisterDependencies)
Definition: Core.cpp:585
void complete(std::unique_ptr< InProgressLookupState > IPLS) override
Definition: Core.cpp:595
void fail(Error Err) override
Definition: Core.cpp:601
void complete(std::unique_ptr< InProgressLookupState > IPLS) override
Definition: Core.cpp:572
void fail(Error Err) override
Definition: Core.cpp:577
InProgressLookupFlagsState(LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet, unique_function< void(Expected< SymbolFlagsMap >)> OnComplete)
Definition: Core.cpp:565
virtual ~InProgressLookupState()=default
SymbolLookupSet DefGeneratorCandidates
Definition: Core.cpp:552
JITDylibSearchOrder SearchOrder
Definition: Core.cpp:546
std::vector< std::weak_ptr< DefinitionGenerator > > CurDefGeneratorStack
Definition: Core.cpp:560
enum llvm::orc::InProgressLookupState::@467 GenState
SymbolLookupSet LookupSet
Definition: Core.cpp:547
virtual void complete(std::unique_ptr< InProgressLookupState > IPLS)=0
InProgressLookupState(LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet, SymbolState RequiredState)
Definition: Core.cpp:535
SymbolLookupSet DefGeneratorNonCandidates
Definition: Core.cpp:553
virtual void fail(Error Err)=0
Represents a JIT'd dynamic library.
Definition: Core.h:989
Error clear()
Calls remove on all trackers currently associated with this JITDylib.
Definition: Core.cpp:676
Error define(std::unique_ptr< MaterializationUnitType > &&MU, ResourceTrackerSP RT=nullptr)
Define all symbols provided by the materialization unit to be part of this JITDylib.
Definition: Core.h:1922
ExecutionSession & getExecutionSession() const
Get a reference to the ExecutionSession for this JITDylib.
Definition: Core.h:1008
ResourceTrackerSP createResourceTracker()
Create a resource tracker for this JITDylib.
Definition: Core.cpp:700
Expected< std::vector< JITDylibSP > > getReverseDFSLinkOrder()
Rteurn this JITDylib and its transitive dependencies in reverse DFS order based on linkage relationsh...
Definition: Core.cpp:1774
ResourceTrackerSP getDefaultResourceTracker()
Get the default resource tracker for this JITDylib.
Definition: Core.cpp:691
void removeGenerator(DefinitionGenerator &G)
Remove a definition generator from this JITDylib.
Definition: Core.cpp:708
Expected< std::vector< JITDylibSP > > getDFSLinkOrder()
Return this JITDylib and its transitive dependencies in DFS order based on linkage relationships.
Definition: Core.cpp:1770
Wraps state for a lookup-in-progress.
Definition: Core.h:921
void continueLookup(Error Err)
Continue the lookup.
Definition: Core.cpp:652
LookupState & operator=(LookupState &&)
void run() override
Definition: Core.cpp:1604
static char ID
Definition: Core.h:1420
void printDescription(raw_ostream &OS) override
Definition: Core.cpp:1602
Tracks responsibility for materialization, and mediates interactions between MaterializationUnits and...
Definition: Core.h:555
void printDescription(raw_ostream &OS) override
Definition: Core.cpp:1595
A MaterializationUnit represents a set of symbol definitions that can be materialized as a group,...
Definition: Core.h:693
SymbolFlagsMap SymbolFlags
Definition: Core.h:749
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:162
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:166
Non-owning SymbolStringPool entry pointer.
static void lookupInitSymbolsAsync(unique_function< void(Error)> OnComplete, ExecutionSession &ES, const DenseMap< JITDylib *, SymbolLookupSet > &InitSyms)
Performs an async lookup for the given symbols in each of the given JITDylibs, calling the given hand...
Definition: Core.cpp:1552
static Expected< DenseMap< JITDylib *, SymbolMap > > lookupInitSymbols(ExecutionSession &ES, const DenseMap< JITDylib *, SymbolLookupSet > &InitSyms)
A utility function for looking up initializer symbols.
Definition: Core.cpp:1503
StringRef getName() const override
Return the name of this materialization unit.
Definition: Core.cpp:326
ReExportsMaterializationUnit(JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags, SymbolAliasMap Aliases)
SourceJD is allowed to be nullptr, in which case the source JITDylib is taken to be whatever JITDylib...
Definition: Core.cpp:320
std::function< bool(SymbolStringPtr)> SymbolPredicate
Definition: Core.h:2011
Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD, JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &LookupSet) override
DefinitionGenerators should override this method to insert new definitions into the parent JITDylib.
Definition: Core.cpp:617
ReexportsGenerator(JITDylib &SourceJD, JITDylibLookupFlags SourceJDLookupFlags, SymbolPredicate Allow=SymbolPredicate())
Create a reexports generator.
Definition: Core.cpp:611
Listens for ResourceTracker operations.
Definition: Core.h:104
ResourceTrackerDefunct(ResourceTrackerSP RT)
Definition: Core.cpp:70
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:77
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:73
API to remove / transfer ownership of JIT resources.
Definition: Core.h:56
JITDylib & getJITDylib() const
Return the JITDylib targeted by this tracker.
Definition: Core.h:71
void transferTo(ResourceTracker &DstRT)
Transfer all resources associated with this key to the given tracker, which must target the same JITD...
Definition: Core.cpp:58
ResourceTracker(const ResourceTracker &)=delete
Error remove()
Remove all resources associated with this key.
Definition: Core.cpp:54
A set of symbols to look up, each associated with a SymbolLookupFlags value.
Definition: Core.h:183
static SymbolLookupSet fromMapKeys(const DenseMap< SymbolStringPtr, ValT > &M, SymbolLookupFlags Flags=SymbolLookupFlags::RequiredSymbol)
Construct a SymbolLookupSet from DenseMap keys.
Definition: Core.h:232
Pointer to a pooled string representing a symbol name.
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:154
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:158
SymbolsCouldNotBeRemoved(std::shared_ptr< SymbolStringPool > SSP, SymbolNameSet Symbols)
Definition: Core.cpp:148
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:144
SymbolsNotFound(std::shared_ptr< SymbolStringPool > SSP, SymbolNameSet Symbols)
Definition: Core.cpp:126
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:140
Represents an abstract task for ORC to run.
Definition: TaskDispatch.h:34
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:171
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:175
void log(raw_ostream &OS) const override
Print an error message to an output stream.
Definition: Core.cpp:119
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Core.cpp:115
UnsatisfiedSymbolDependencies(std::shared_ptr< SymbolStringPool > SSP, JITDylibSP JD, SymbolNameSet FailedSymbols, SymbolDependenceMap BadDeps, std::string Explanation)
Definition: Core.cpp:107
static WrapperFunctionResult createOutOfBandError(const char *Msg)
Create an out-of-band error by copying the given string.
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:470
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
IntrusiveRefCntPtr< JITDylib > JITDylibSP
Definition: Core.h:51
JITDylibSearchOrder makeJITDylibSearchOrder(ArrayRef< JITDylib * > JDs, JITDylibLookupFlags Flags=JITDylibLookupFlags::MatchExportedSymbolsOnly)
Convenience function for creating a search order from an ArrayRef of JITDylib*, all with the same fla...
Definition: Core.h:166
std::vector< std::pair< JITDylib *, JITDylibLookupFlags > > JITDylibSearchOrder
A list of (JITDylib*, JITDylibLookupFlags) pairs to be used as a search order during symbol lookup.
Definition: Core.h:162
std::function< void(const SymbolDependenceMap &)> RegisterDependenciesFunction
Callback to register the dependencies for a given query.
Definition: Core.h:403
std::unique_ptr< ReExportsMaterializationUnit > symbolAliases(SymbolAliasMap Aliases)
Create a ReExportsMaterializationUnit with the given aliases.
Definition: Core.h:837
SymbolLookupFlags
Lookup flags that apply to each symbol in a lookup.
Definition: Core.h:145
std::unique_ptr< ReExportsMaterializationUnit > reexports(JITDylib &SourceJD, SymbolAliasMap Aliases, JITDylibLookupFlags SourceJDLookupFlags=JITDylibLookupFlags::MatchExportedSymbolsOnly)
Create a materialization unit for re-exporting symbols from another JITDylib with alternative names/f...
Definition: Core.h:846
DenseMap< SymbolStringPtr, SymbolAliasMapEntry > SymbolAliasMap
A map of Symbols to (Symbol, Flags) pairs.
Definition: Core.h:396
JITDylibLookupFlags
Lookup flags that apply to each dylib in the search order for a lookup.
Definition: Core.h:135
DenseMap< SymbolStringPtr, ExecutorSymbolDef > SymbolMap
A map from symbol names (as SymbolStringPtrs) to JITSymbols (address/flags pairs).
Definition: Core.h:121
DenseMap< SymbolStringPtr, JITSymbolFlags > SymbolFlagsMap
A map from symbol names (as SymbolStringPtrs) to JITSymbolFlags.
Definition: Core.h:124
unique_function< void(Expected< SymbolMap >)> SymbolsResolvedCallback
Callback to notify client that symbols have been resolved.
Definition: Core.h:399
DenseSet< SymbolStringPtr > SymbolNameSet
A set of symbol names (represented by SymbolStringPtrs for.
Definition: Core.h:114
LookupKind
Describes the kind of lookup being performed.
Definition: Core.h:157
RegisterDependenciesFunction NoDependenciesToRegister
This can be used as the value for a RegisterDependenciesFunction if there are no dependants to regist...
Definition: Core.cpp:37
std::vector< SymbolStringPtr > SymbolNameVector
A vector of symbol names.
Definition: Core.h:117
SymbolState
Represents the state that a symbol has reached during materialization.
Definition: Core.h:859
@ Materializing
Added to the symbol table, never queried.
@ NeverSearched
No symbol should be in this state.
@ Ready
Emitted to memory, but waiting on transitive dependencies.
@ Emitted
Assigned address, still materializing.
@ Resolved
Queried, materialization begun.
std::error_code orcError(OrcErrorCode ErrCode)
Definition: OrcError.cpp:84
DenseMap< JITDylib *, SymbolNameSet > SymbolDependenceMap
A map from JITDylibs to sets of symbols.
Definition: Core.h:127
Expected< SymbolAliasMap > buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols)
Build a SymbolAliasMap for the common case where you want to re-export symbols from another JITDylib ...
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1742
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:90
auto formatv(const char *Fmt, Ts &&...Vals) -> formatv_object< decltype(std::make_tuple(support::detail::build_format_adapter(std::forward< Ts >(Vals))...))>
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2073
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:419
Error joinErrors(Error E1, Error E2)
Concatenate errors.
Definition: Error.h:431
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
Definition: Format.h:125
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1954
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1849
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1749
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition: STLExtras.h:1879
Implement std::hash so that hash_code can be used in STL containers.
Definition: BitVector.h:858
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define NDEBUG
Definition: regutils.h:48