16#define DEBUG_TYPE "orc"
24 : ES(ES), ErrorHandlerAddr(ErrorHandlerAddr), TP(TP) {}
29 assert(TP &&
"TrampolinePool not set");
31 std::lock_guard<std::mutex> Lock(LCTMMutex);
35 return Trampoline.takeError();
37 Reexports[*Trampoline] =
ReexportsEntry{&SourceJD, std::move(SymbolName)};
38 Notifiers[*Trampoline] = std::move(NotifyResolved);
44 return ErrorHandlerAddr;
49 std::lock_guard<std::mutex> Lock(LCTMMutex);
50 auto I = Reexports.find(TrampolineAddr);
51 if (
I == Reexports.end())
53 "Missing reexport for trampoline address %p" +
54 formatv(
"{0:x}", TrampolineAddr));
62 std::lock_guard<std::mutex> Lock(LCTMMutex);
63 auto I = Notifiers.find(TrampolineAddr);
64 if (
I != Notifiers.end()) {
65 NotifyResolved = std::move(
I->second);
70 return NotifyResolved ? NotifyResolved(ResolvedAddr) :
Error::success();
84 auto Callback = [
this, TrampolineAddr, SymbolName = Entry->SymbolName,
85 NotifyLandingResolved = std::move(NotifyLandingResolved)](
89 assert(
Result->count(SymbolName) &&
"Unexpected result value");
90 ExecutorAddr LandingAddr = (*Result)[SymbolName].getAddress();
95 NotifyLandingResolved(LandingAddr);
111 switch (
T.getArch()) {
113 return make_error<StringError>(
114 std::string(
"No callback manager available for ") +
T.str(),
119 return LocalLazyCallThroughManager::Create<OrcAArch64>(ES,
123 return LocalLazyCallThroughManager::Create<OrcI386>(ES, ErrorHandlerAddr);
126 return LocalLazyCallThroughManager::Create<OrcLoongArch64>(
127 ES, ErrorHandlerAddr);
130 return LocalLazyCallThroughManager::Create<OrcMips32Be>(ES,
134 return LocalLazyCallThroughManager::Create<OrcMips32Le>(ES,
139 return LocalLazyCallThroughManager::Create<OrcMips64>(ES, ErrorHandlerAddr);
142 return LocalLazyCallThroughManager::Create<OrcRiscv64>(ES,
147 return LocalLazyCallThroughManager::Create<OrcX86_64_Win32>(
148 ES, ErrorHandlerAddr);
150 return LocalLazyCallThroughManager::Create<OrcX86_64_SysV>(
151 ES, ErrorHandlerAddr);
159 LCTManager(LCTManager), RSManager(RSManager), SourceJD(SourceJD),
160 CallableAliases(
std::
move(CallableAliases)), AliaseeTable(SrcJDLoc) {}
163 return "<Lazy Reexports>";
166void LazyReexportsMaterializationUnit::materialize(
167 std::unique_ptr<MaterializationResponsibility> R) {
168 auto RequestedSymbols = R->getRequestedSymbols();
171 for (
auto &RequestedSymbol : RequestedSymbols) {
172 auto I = CallableAliases.
find(RequestedSymbol);
173 assert(
I != CallableAliases.
end() &&
"Symbol not found in alias map?");
174 RequestedAliases[
I->first] = std::move(
I->second);
178 if (!CallableAliases.
empty())
179 if (
auto Err = R->replace(
lazyReexports(LCTManager, RSManager, SourceJD,
180 std::move(CallableAliases),
182 R->getExecutionSession().reportError(std::move(Err));
183 R->failMaterialization();
188 for (
auto &Alias : RequestedAliases) {
190 SourceJD, Alias.second.Aliasee,
191 [&TargetJD = R->getTargetJITDylib(), &RSManager = this->RSManager,
192 StubSym = Alias.first](ExecutorAddr ResolvedAddr) -> Error {
193 return RSManager.redirect(TargetJD, StubSym,
194 ExecutorSymbolDef(ResolvedAddr, {}));
197 if (!CallThroughTrampoline) {
198 R->getExecutionSession().reportError(CallThroughTrampoline.takeError());
199 R->failMaterialization();
203 Inits[Alias.first] = {*CallThroughTrampoline, Alias.second.AliasFlags};
206 if (AliaseeTable !=
nullptr && !RequestedAliases.empty())
207 AliaseeTable->trackImpls(RequestedAliases, &SourceJD);
209 if (
auto Err =
R->replace(std::make_unique<RedirectableMaterializationUnit>(
210 RSManager, std::move(Inits)))) {
211 R->getExecutionSession().reportError(std::move(Err));
212 return R->failMaterialization();
216void LazyReexportsMaterializationUnit::discard(
const JITDylib &JD,
217 const SymbolStringPtr &
Name) {
219 "Symbol not covered by this MaterializationUnit");
220 CallableAliases.erase(
Name);
223MaterializationUnit::Interface
224LazyReexportsMaterializationUnit::extractFlags(
const SymbolAliasMap &Aliases) {
226 for (
auto &KV : Aliases) {
227 assert(KV.second.AliasFlags.isCallable() &&
228 "Lazy re-exports must be callable symbols");
231 return MaterializationUnit::Interface(std::move(SymbolFlags),
nullptr);
238 Reexports(
std::
move(Reexports)) {}
243 for (
auto &[Alias, AI] : Reexports)
244 SF[Alias] = AI.AliasFlags;
245 return {std::move(SF),
nullptr};
250 void materialize(std::unique_ptr<MaterializationResponsibility> R)
override {
251 LRMgr.emitReentryTrampolines(std::move(R), std::move(Reexports));
254 void discard(
const JITDylib &JD,
const SymbolStringPtr &
Name)
override {
258 LazyReexportsManager &LRMgr;
269 return Error::success();
273 return Error::success();
283LazyReexportsManager::Listener::~Listener() =
default;
291 std::move(EmitTrampolines), RSMgr, PlatformJD, L, Err));
293 return std::move(Err);
294 return std::move(LRM);
299 auto I = KeyToReentryAddrs.find(K);
300 if (
I == KeyToReentryAddrs.end())
303 auto &ReentryAddrs =
I->second;
304 for (
auto &ReentryAddr : ReentryAddrs) {
305 assert(CallThroughs.count(ReentryAddr) &&
"CallTrhough missing");
306 CallThroughs.erase(ReentryAddr);
308 KeyToReentryAddrs.erase(
I);
313void LazyReexportsManager::handleTransferResources(
JITDylib &JD,
316 auto I = KeyToReentryAddrs.find(SrcK);
317 if (
I != KeyToReentryAddrs.end()) {
318 auto J = KeyToReentryAddrs.find(DstK);
319 if (J == KeyToReentryAddrs.end()) {
320 auto Tmp = std::move(
I->second);
321 KeyToReentryAddrs.erase(
I);
322 KeyToReentryAddrs[DstK] = std::move(Tmp);
324 auto &SrcAddrs =
I->second;
325 auto &DstAddrs = J->second;
326 DstAddrs.insert(DstAddrs.end(), SrcAddrs.begin(), SrcAddrs.end());
327 KeyToReentryAddrs.erase(
I);
330 L->onLazyReexportsTransfered(JD, DstK, SrcK);
334LazyReexportsManager::LazyReexportsManager(EmitTrampolinesFn EmitTrampolines,
338 : ES(PlatformJD.getExecutionSession()),
339 EmitTrampolines(
std::
move(EmitTrampolines)), RSMgr(RSMgr), L(L) {
341 using namespace shared;
347 WFs[ES.
intern(
"__orc_rt_resolve_tag")] =
349 this, &LazyReexportsManager::resolve);
354std::unique_ptr<MaterializationUnit>
355LazyReexportsManager::createLazyReexports(
SymbolAliasMap Reexports) {
356 return std::make_unique<MU>(*
this, std::move(Reexports));
359void LazyReexportsManager::emitReentryTrampolines(
360 std::unique_ptr<MaterializationResponsibility> MR,
362 size_t NumTrampolines = Reexports.size();
363 auto RT = MR->getResourceTracker();
365 std::move(RT), NumTrampolines,
366 [
this, MR = std::move(MR), Reexports = std::move(Reexports)](
367 Expected<std::vector<ExecutorSymbolDef>> ReentryPoints)
mutable {
368 emitRedirectableSymbols(std::move(MR), std::move(Reexports),
369 std::move(ReentryPoints));
373void LazyReexportsManager::emitRedirectableSymbols(
374 std::unique_ptr<MaterializationResponsibility> MR,
SymbolAliasMap Reexports,
375 Expected<std::vector<ExecutorSymbolDef>> ReentryPoints) {
377 if (!ReentryPoints) {
378 MR->getExecutionSession().reportError(ReentryPoints.takeError());
379 MR->failMaterialization();
383 assert(Reexports.size() == ReentryPoints->size() &&
384 "Number of reentry points doesn't match number of reexports");
389 for (
auto &[
Name, AI] : Reexports)
390 Redirs[
Name] = (*ReentryPoints)[
I++];
393 if (!Reexports.empty()) {
394 if (
auto Err = MR->withResourceKeyDo([&](
ResourceKey K) {
395 auto &JD = MR->getTargetJITDylib();
396 auto &ReentryAddrsForK = KeyToReentryAddrs[K];
397 for (auto &[Name, AI] : Reexports) {
398 const auto &ReentryPoint = (*ReentryPoints)[I++];
399 CallThroughs[ReentryPoint.getAddress()] = {&JD, Name, AI.Aliasee};
400 ReentryAddrsForK.push_back(ReentryPoint.getAddress());
403 L->onLazyReexportsCreated(JD, K, Reexports);
405 MR->getExecutionSession().reportError(std::move(Err));
406 MR->failMaterialization();
414void LazyReexportsManager::resolve(ResolveSendResultFn SendResult,
415 ExecutorAddr ReentryStubAddr) {
417 CallThroughInfo LandingInfo;
419 ES.runSessionLocked([&]() {
420 auto I = CallThroughs.find(ReentryStubAddr);
421 if (
I == CallThroughs.end())
422 return SendResult(make_error<StringError>(
423 "Reentry address " +
formatv(
"{0:x}", ReentryStubAddr) +
426 LandingInfo =
I->second;
430 L->onLazyReexportCalled(LandingInfo);
432 SymbolInstance LandingSym(LandingInfo.JD, std::move(LandingInfo.BodyName));
433 LandingSym.lookupAsync([
this, JD = std::move(LandingInfo.JD),
434 ReentryName = std::move(LandingInfo.Name),
435 SendResult = std::move(SendResult)](
436 Expected<ExecutorSymbolDef> Result)
mutable {
440 if (auto Err = RSMgr.redirect(*JD, ReentryName, *Result))
441 SendResult(std::move(Err));
443 SendResult(std::move(Result));
445 SendResult(std::move(Result));
455 OS <<
"Speculative Lookup Task";
460 S->doNextSpeculativeLookup();
464 std::weak_ptr<SimpleLazyReexportsSpeculator>
Speculator;
467SimpleLazyReexportsSpeculator::~SimpleLazyReexportsSpeculator() {
468 for (
auto &[JD,
_] : LazyReexports)
472void SimpleLazyReexportsSpeculator::onLazyReexportsCreated(
474 if (!LazyReexports.count(&JD))
476 auto &BodiesVec = LazyReexports[&JD][K];
477 for (
auto &[
Name, AI] : Reexports)
478 BodiesVec.push_back(AI.Aliasee);
479 if (!SpeculateTaskActive) {
480 SpeculateTaskActive =
true;
481 ES.dispatchTask(std::make_unique<SpeculateTask>(WeakThis));
485void SimpleLazyReexportsSpeculator::onLazyReexportsTransfered(
488 auto I = LazyReexports.find(&JD);
489 if (
I == LazyReexports.end())
492 auto &MapForJD =
I->second;
493 auto J = MapForJD.find(SrcK);
494 if (J == MapForJD.end())
498 auto K = MapForJD.find(DstK);
499 if (K == MapForJD.end()) {
500 auto Tmp = std::move(J->second);
502 MapForJD[DstK] = std::move(Tmp);
504 auto &SrcNames = J->second;
505 auto &DstNames = K->second;
506 DstNames.insert(DstNames.end(), SrcNames.begin(), SrcNames.end());
514 auto I = LazyReexports.find(&JD);
515 if (
I == LazyReexports.end())
518 auto &MapForJD =
I->second;
521 if (MapForJD.empty()) {
522 LazyReexports.erase(
I);
529void SimpleLazyReexportsSpeculator::onLazyReexportCalled(
535void SimpleLazyReexportsSpeculator::addSpeculationSuggestions(
536 std::vector<std::pair<std::string, SymbolStringPtr>> NewSuggestions) {
537 ES.runSessionLocked([&]() {
538 for (
auto &[JDName, SymbolName] : NewSuggestions)
539 SpeculateSuggestions.push_back(
540 {std::move(JDName), std::move(SymbolName)});
544bool SimpleLazyReexportsSpeculator::doNextSpeculativeLookup() {
550 auto SpeculateAgain = ES.runSessionLocked([&]() {
551 while (!SpeculateSuggestions.empty()) {
552 auto [JDName, SymbolName] = std::move(SpeculateSuggestions.front());
553 SpeculateSuggestions.pop_front();
555 if (auto *JD = ES.getJITDylibByName(JDName)) {
557 SpeculateFn = std::move(SymbolName);
563 assert(!LazyReexports.empty() &&
"LazyReexports map is empty");
565 std::next(LazyReexports.begin(), rand() % LazyReexports.size());
566 auto &[JD, KeyToFnBodies] = *LRItr;
568 assert(!KeyToFnBodies.empty() &&
"Key to function bodies map empty");
569 auto KeyToFnBodiesItr =
570 std::next(KeyToFnBodies.begin(), rand() % KeyToFnBodies.size());
571 auto &[Key, FnBodies] = *KeyToFnBodiesItr;
573 assert(!FnBodies.empty() &&
"Function bodies list empty");
574 auto FnBodyItr = std::next(FnBodies.begin(), rand() % FnBodies.size());
576 SpeculateJD = JITDylibSP(JD);
577 SpeculateFn = std::move(*FnBodyItr);
579 FnBodies.erase(FnBodyItr);
580 if (FnBodies.empty()) {
581 KeyToFnBodies.erase(KeyToFnBodiesItr);
582 if (KeyToFnBodies.empty()) {
583 LRItr->first->Release();
584 LazyReexports.erase(LRItr);
589 SpeculateTaskActive =
590 !SpeculateSuggestions.empty() || !LazyReexports.empty();
591 return SpeculateTaskActive;
595 dbgs() <<
"Issuing speculative lookup for ( " << SpeculateJD->getName()
596 <<
", " << SpeculateFn <<
" )...\n";
601 {{std::move(SpeculateFn), SymbolLookupFlags::WeaklyReferencedSymbol}},
607 ES.dispatchTask(std::make_unique<SpeculateTask>(WeakThis));
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
iterator find(const_arg_type_t< KeyT > Val)
bool erase(const KeyT &Val)
Helper for Errors used as out-parameters.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
StringRef - Represent a constant reference to a string, i.e.
Triple - Helper class for working with autoconf configuration names.
An ExecutionSession represents a running JIT program.
void reportError(Error Err)
Report a error for this execution session.
SymbolStringPtr intern(StringRef SymName)
Add a symbol name to the SymbolStringPool and return a pointer to it.
static JITDispatchHandlerFunction wrapAsyncWithSPS(HandlerT &&H)
Wrap a handler that takes concrete argument types (and a sender for a concrete return type) to produc...
void lookup(LookupKind K, const JITDylibSearchOrder &SearchOrder, SymbolLookupSet Symbols, SymbolState RequiredState, SymbolsResolvedCallback NotifyComplete, RegisterDependenciesFunction RegisterDependencies)
Search the given JITDylibs for the given symbols.
Error registerJITDispatchHandlers(JITDylib &JD, JITDispatchHandlerAssociationMap WFs)
For each tag symbol name, associate the corresponding AsyncHandlerWrapperFunction with the address of...
decltype(auto) runSessionLocked(Func &&F)
Run the given lambda with the session mutex locked.
Represents an address in the executor process.
IdleTask can be used as the basis for low-priority tasks, e.g.
Represents a JIT'd dynamic library.
ExecutionSession & getExecutionSession() const
Get a reference to the ExecutionSession for this JITDylib.
Manages a set of 'lazy call-through' trampolines.
ExecutorAddr reportCallThroughError(Error Err)
Expected< ReexportsEntry > findReexport(ExecutorAddr TrampolineAddr)
Error notifyResolved(ExecutorAddr TrampolineAddr, ExecutorAddr ResolvedAddr)
void resolveTrampolineLandingAddress(ExecutorAddr TrampolineAddr, TrampolinePool::NotifyLandingResolvedFunction NotifyLandingResolved)
LazyCallThroughManager(ExecutionSession &ES, ExecutorAddr ErrorHandlerAddr, TrampolinePool *TP)
Expected< ExecutorAddr > getCallThroughTrampoline(JITDylib &SourceJD, SymbolStringPtr SymbolName, NotifyResolvedFunction NotifyResolved)
MU(LazyReexportsManager &LRMgr, SymbolAliasMap Reexports)
void modifyPassConfig(MaterializationResponsibility &MR, jitlink::LinkGraph &G, jitlink::PassConfiguration &Config) override
void notifyTransferringResources(JITDylib &JD, ResourceKey DstKey, ResourceKey SrcKey) override
Error notifyFailed(MaterializationResponsibility &MR) override
Error notifyRemovingResources(JITDylib &JD, ResourceKey K) override
LazyReexportsMaterializationUnit(LazyCallThroughManager &LCTManager, RedirectableSymbolManager &RSManager, JITDylib &SourceJD, SymbolAliasMap CallableAliases, ImplSymbolMap *SrcJDLoc)
StringRef getName() const override
Return the name of this materialization unit.
Tracks responsibility for materialization, and mediates interactions between MaterializationUnits and...
A MaterializationUnit represents a set of symbol definitions that can be materialized as a group,...
Base class for managing redirectable symbols in which a call gets redirected to another symbol in run...
virtual void emitRedirectableSymbols(std::unique_ptr< MaterializationResponsibility > MR, SymbolMap InitialDests)=0
Emit redirectable symbol.
SpeculateTask(std::weak_ptr< SimpleLazyReexportsSpeculator > Speculator)
void printDescription(raw_ostream &OS) override
A set of symbols to look up, each associated with a SymbolLookupFlags value.
Pointer to a pooled string representing a symbol name.
Base class for pools of compiler re-entry trampolines.
Expected< ExecutorAddr > getTrampoline()
Get an available trampoline address.
This class implements an extremely fast bulk output stream that can only output to a stream.
unique_function is a type-erasing functor similar to std::function.
JITDylibSearchOrder makeJITDylibSearchOrder(ArrayRef< JITDylib * > JDs, JITDylibLookupFlags Flags=JITDylibLookupFlags::MatchExportedSymbolsOnly)
Convenience function for creating a search order from an ArrayRef of JITDylib*, all with the same fla...
DenseMap< SymbolStringPtr, SymbolAliasMapEntry > SymbolAliasMap
A map of Symbols to (Symbol, Flags) pairs.
DenseMap< SymbolStringPtr, ExecutorSymbolDef > SymbolMap
A map from symbol names (as SymbolStringPtrs) to JITSymbols (address/flags pairs).
DenseMap< SymbolStringPtr, JITSymbolFlags > SymbolFlagsMap
A map from symbol names (as SymbolStringPtrs) to JITSymbolFlags.
Expected< std::unique_ptr< LazyCallThroughManager > > createLocalLazyCallThroughManager(const Triple &T, ExecutionSession &ES, ExecutorAddr ErrorHandlerAddr)
Create a LocalLazyCallThroughManager from the given triple and execution session.
std::unique_ptr< LazyReexportsMaterializationUnit > lazyReexports(LazyCallThroughManager &LCTManager, RedirectableSymbolManager &RSManager, JITDylib &SourceJD, SymbolAliasMap CallableAliases, ImplSymbolMap *SrcJDLoc=nullptr)
Define lazy-reexports based on the given SymbolAliasMap.
RegisterDependenciesFunction NoDependenciesToRegister
This can be used as the value for a RegisterDependenciesFunction if there are no dependants to regist...
@ Ready
Emitted to memory, but waiting on transitive dependencies.
This is an optimization pass for GlobalISel generic memory operations.
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Error createStringError(std::error_code EC, char const *Fmt, const Ts &... Vals)
Create formatted StringError object.
auto formatv(bool Validate, const char *Fmt, Ts &&...Vals)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
OutputIt move(R &&Range, OutputIt Out)
Provide wrappers to std::move which take ranges instead of having to pass begin/end explicitly.
void consumeError(Error Err)
Consume a Error without doing anything.
Implement std::hash so that hash_code can be used in STL containers.
An LinkGraph pass configuration, consisting of a list of pre-prune, post-prune, and post-fixup passes...