LLVM 20.0.0git
ReOptimizeLayer.cpp
Go to the documentation of this file.
3
4using namespace llvm;
5using namespace orc;
6
7bool ReOptimizeLayer::ReOptMaterializationUnitState::tryStartReoptimize() {
8 std::unique_lock<std::mutex> Lock(Mutex);
9 if (Reoptimizing)
10 return false;
11
12 Reoptimizing = true;
13 return true;
14}
15
16void ReOptimizeLayer::ReOptMaterializationUnitState::reoptimizeSucceeded() {
17 std::unique_lock<std::mutex> Lock(Mutex);
18 assert(Reoptimizing && "Tried to mark unstarted reoptimization as done");
19 Reoptimizing = false;
20 CurVersion++;
21}
22
23void ReOptimizeLayer::ReOptMaterializationUnitState::reoptimizeFailed() {
24 std::unique_lock<std::mutex> Lock(Mutex);
25 assert(Reoptimizing && "Tried to mark unstarted reoptimization as done");
26 Reoptimizing = false;
27}
28
31 using ReoptimizeSPSSig = shared::SPSError(uint64_t, uint32_t);
32 WFs[Mangle("__orc_rt_reoptimize_tag")] =
33 ES.wrapAsyncWithSPS<ReoptimizeSPSSig>(this,
34 &ReOptimizeLayer::rt_reoptimize);
35 return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs));
36}
37
38void ReOptimizeLayer::emit(std::unique_ptr<MaterializationResponsibility> R,
39 ThreadSafeModule TSM) {
40 auto &JD = R->getTargetJITDylib();
41
42 bool HasNonCallable = false;
43 for (auto &KV : R->getSymbols()) {
44 auto &Flags = KV.second;
45 if (!Flags.isCallable())
46 HasNonCallable = true;
47 }
48
49 if (HasNonCallable) {
50 BaseLayer.emit(std::move(R), std::move(TSM));
51 return;
52 }
53
54 auto &MUState = createMaterializationUnitState(TSM);
55
56 if (auto Err = R->withResourceKeyDo([&](ResourceKey Key) {
57 registerMaterializationUnitResource(Key, MUState);
58 })) {
59 ES.reportError(std::move(Err));
60 R->failMaterialization();
61 return;
62 }
63
64 if (auto Err =
65 ProfilerFunc(*this, MUState.getID(), MUState.getCurVersion(), TSM)) {
66 ES.reportError(std::move(Err));
67 R->failMaterialization();
68 return;
69 }
70
71 auto InitialDests =
72 emitMUImplSymbols(MUState, MUState.getCurVersion(), JD, std::move(TSM));
73 if (!InitialDests) {
74 ES.reportError(InitialDests.takeError());
75 R->failMaterialization();
76 return;
77 }
78
79 RSManager.emitRedirectableSymbols(std::move(R), std::move(*InitialDests));
80}
81
84 unsigned CurVersion,
85 ThreadSafeModule &TSM) {
86 return TSM.withModuleDo([&](Module &M) -> Error {
87 Type *I64Ty = Type::getInt64Ty(M.getContext());
88 GlobalVariable *Counter = new GlobalVariable(
89 M, I64Ty, false, GlobalValue::InternalLinkage,
90 Constant::getNullValue(I64Ty), "__orc_reopt_counter");
91 auto ArgBufferConst = createReoptimizeArgBuffer(M, MUID, CurVersion);
92 if (auto Err = ArgBufferConst.takeError())
93 return Err;
94 GlobalVariable *ArgBuffer =
95 new GlobalVariable(M, (*ArgBufferConst)->getType(), true,
96 GlobalValue::InternalLinkage, (*ArgBufferConst));
97 for (auto &F : M) {
98 if (F.isDeclaration())
99 continue;
100 auto &BB = F.getEntryBlock();
101 auto *IP = &*BB.getFirstInsertionPt();
102 IRBuilder<> IRB(IP);
103 Value *Threshold = ConstantInt::get(I64Ty, CallCountThreshold, true);
104 Value *Cnt = IRB.CreateLoad(I64Ty, Counter);
105 // Use EQ to prevent further reoptimize calls.
106 Value *Cmp = IRB.CreateICmpEQ(Cnt, Threshold);
107 Value *Added = IRB.CreateAdd(Cnt, ConstantInt::get(I64Ty, 1));
108 (void)IRB.CreateStore(Added, Counter);
109 Instruction *SplitTerminator = SplitBlockAndInsertIfThen(Cmp, IP, false);
110 createReoptimizeCall(M, *SplitTerminator, ArgBuffer);
111 }
112 return Error::success();
113 });
114}
115
117ReOptimizeLayer::emitMUImplSymbols(ReOptMaterializationUnitState &MUState,
119 ThreadSafeModule TSM) {
121 cantFail(TSM.withModuleDo([&](Module &M) -> Error {
122 MangleAndInterner Mangle(ES, M.getDataLayout());
123 for (auto &F : M)
124 if (!F.isDeclaration()) {
125 std::string NewName =
126 (F.getName() + ".__def__." + Twine(Version)).str();
127 RenamedMap[Mangle(F.getName())] = Mangle(NewName);
128 F.setName(NewName);
129 }
130 return Error::success();
131 }));
132
133 auto RT = JD.createResourceTracker();
134 if (auto Err =
135 JD.define(std::make_unique<BasicIRLayerMaterializationUnit>(
136 BaseLayer, *getManglingOptions(), std::move(TSM)),
137 RT))
138 return Err;
139 MUState.setResourceTracker(RT);
140
141 SymbolLookupSet LookupSymbols;
142 for (auto [K, V] : RenamedMap)
143 LookupSymbols.add(V);
144
145 auto ImplSymbols =
146 ES.lookup({{&JD, JITDylibLookupFlags::MatchAllSymbols}}, LookupSymbols,
148 if (auto Err = ImplSymbols.takeError())
149 return Err;
150
152 for (auto [K, V] : RenamedMap)
153 Result[K] = (*ImplSymbols)[V];
154
155 return Result;
156}
157
158void ReOptimizeLayer::rt_reoptimize(SendErrorFn SendResult,
159 ReOptMaterializationUnitID MUID,
160 uint32_t CurVersion) {
161 auto &MUState = getMaterializationUnitState(MUID);
162 if (CurVersion < MUState.getCurVersion() || !MUState.tryStartReoptimize()) {
163 SendResult(Error::success());
164 return;
165 }
166
167 ThreadSafeModule TSM = cloneToNewContext(MUState.getThreadSafeModule());
168 auto OldRT = MUState.getResourceTracker();
169 auto &JD = OldRT->getJITDylib();
170
171 if (auto Err = ReOptFunc(*this, MUID, CurVersion + 1, OldRT, TSM)) {
172 ES.reportError(std::move(Err));
173 MUState.reoptimizeFailed();
174 SendResult(Error::success());
175 return;
176 }
177
178 auto SymbolDests =
179 emitMUImplSymbols(MUState, CurVersion + 1, JD, std::move(TSM));
180 if (!SymbolDests) {
181 ES.reportError(SymbolDests.takeError());
182 MUState.reoptimizeFailed();
183 SendResult(Error::success());
184 return;
185 }
186
187 if (auto Err = RSManager.redirect(JD, std::move(*SymbolDests))) {
188 ES.reportError(std::move(Err));
189 MUState.reoptimizeFailed();
190 SendResult(Error::success());
191 return;
192 }
193
194 MUState.reoptimizeSucceeded();
195 SendResult(Error::success());
196}
197
198Expected<Constant *> ReOptimizeLayer::createReoptimizeArgBuffer(
199 Module &M, ReOptMaterializationUnitID MUID, uint32_t CurVersion) {
200 size_t ArgBufferSize = SPSReoptimizeArgList::size(MUID, CurVersion);
201 std::vector<char> ArgBuffer(ArgBufferSize);
202 shared::SPSOutputBuffer OB(ArgBuffer.data(), ArgBuffer.size());
203 if (!SPSReoptimizeArgList::serialize(OB, MUID, CurVersion))
204 return make_error<StringError>("Could not serealize args list",
206 return ConstantDataArray::get(M.getContext(), ArrayRef(ArgBuffer));
207}
208
210 GlobalVariable *ArgBuffer) {
211 GlobalVariable *DispatchCtx =
212 M.getGlobalVariable("__orc_rt_jit_dispatch_ctx");
213 if (!DispatchCtx)
214 DispatchCtx = new GlobalVariable(M, PointerType::get(M.getContext(), 0),
216 nullptr, "__orc_rt_jit_dispatch_ctx");
217 GlobalVariable *ReoptimizeTag =
218 M.getGlobalVariable("__orc_rt_reoptimize_tag");
219 if (!ReoptimizeTag)
220 ReoptimizeTag = new GlobalVariable(M, PointerType::get(M.getContext(), 0),
222 nullptr, "__orc_rt_reoptimize_tag");
223 Function *DispatchFunc = M.getFunction("__orc_rt_jit_dispatch");
224 if (!DispatchFunc) {
225 std::vector<Type *> Args = {PointerType::get(M.getContext(), 0),
226 PointerType::get(M.getContext(), 0),
227 PointerType::get(M.getContext(), 0),
228 IntegerType::get(M.getContext(), 64)};
229 FunctionType *FuncTy =
230 FunctionType::get(Type::getVoidTy(M.getContext()), Args, false);
231 DispatchFunc = Function::Create(FuncTy, GlobalValue::ExternalLinkage,
232 "__orc_rt_jit_dispatch", &M);
233 }
234 size_t ArgBufferSizeConst =
235 SPSReoptimizeArgList::size(ReOptMaterializationUnitID{}, uint32_t{});
236 Constant *ArgBufferSize = ConstantInt::get(
237 IntegerType::get(M.getContext(), 64), ArgBufferSizeConst, false);
238 IRBuilder<> IRB(&IP);
239 (void)IRB.CreateCall(DispatchFunc,
240 {DispatchCtx, ReoptimizeTag, ArgBuffer, ArgBufferSize});
241}
242
243ReOptimizeLayer::ReOptMaterializationUnitState &
244ReOptimizeLayer::createMaterializationUnitState(const ThreadSafeModule &TSM) {
245 std::unique_lock<std::mutex> Lock(Mutex);
246 ReOptMaterializationUnitID MUID = NextID;
247 MUStates.emplace(MUID,
248 ReOptMaterializationUnitState(MUID, cloneToNewContext(TSM)));
249 ++NextID;
250 return MUStates.at(MUID);
251}
252
253ReOptimizeLayer::ReOptMaterializationUnitState &
254ReOptimizeLayer::getMaterializationUnitState(ReOptMaterializationUnitID MUID) {
255 std::unique_lock<std::mutex> Lock(Mutex);
256 return MUStates.at(MUID);
257}
258
259void ReOptimizeLayer::registerMaterializationUnitResource(
260 ResourceKey Key, ReOptMaterializationUnitState &State) {
261 std::unique_lock<std::mutex> Lock(Mutex);
262 MUResources[Key].insert(State.getID());
263}
264
266 std::unique_lock<std::mutex> Lock(Mutex);
267 for (auto MUID : MUResources[K])
268 MUStates.erase(MUID);
269
270 MUResources.erase(K);
271 return Error::success();
272}
273
275 ResourceKey SrcK) {
276 std::unique_lock<std::mutex> Lock(Mutex);
277 MUResources[DstK].insert(MUResources[SrcK].begin(), MUResources[SrcK].end());
278 MUResources.erase(SrcK);
279}
#define F(x, y, z)
Definition: MD5.cpp:55
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
Definition: Constants.h:709
This is an important base class in LLVM.
Definition: Constant.h:42
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:373
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:337
Tagged union holding either a T or a Error.
Definition: Error.h:481
Class to represent function types.
Definition: DerivedTypes.h:105
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:173
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:59
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:52
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2273
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Definition: IRBuilder.h:1813
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Definition: IRBuilder.h:1826
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1350
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2444
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2697
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:311
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
LLVM Value Representation.
Definition: Value.h:74
void reportError(Error Err)
Report a error for this execution session.
Definition: Core.h:1474
static JITDispatchHandlerFunction wrapAsyncWithSPS(HandlerT &&H)
Wrap a handler that takes concrete argument types (and a sender for a concrete return type) to produc...
Definition: Core.h:1607
Error registerJITDispatchHandlers(JITDylib &JD, JITDispatchHandlerAssociationMap WFs)
For each tag symbol name, associate the corresponding AsyncHandlerWrapperFunction with the address of...
Definition: Core.cpp:1897
virtual void emit(std::unique_ptr< MaterializationResponsibility > R, ThreadSafeModule TSM)=0
Emit should materialize the given IR.
Represents a JIT'd dynamic library.
Definition: Core.h:897
void emit(std::unique_ptr< MaterializationResponsibility > R, ThreadSafeModule TSM) override
Emits the given module.
Error reigsterRuntimeFunctions(JITDylib &PlatformJD)
Registers reoptimize runtime dispatch handlers to given PlatformJD.
static void createReoptimizeCall(Module &M, Instruction &IP, GlobalVariable *ArgBuffer)
void handleTransferResources(JITDylib &JD, ResourceKey DstK, ResourceKey SrcK) override
This function will be called inside the session lock.
static Error reoptimizeIfCallFrequent(ReOptimizeLayer &Parent, ReOptMaterializationUnitID MUID, unsigned CurVersion, ThreadSafeModule &TSM)
Basic AddProfilerFunc that reoptimizes the function when the call count exceeds CallCountThreshold.
Error handleRemoveResources(JITDylib &JD, ResourceKey K) override
This function will be called outside the session lock.
static const uint64_t CallCountThreshold
virtual void emitRedirectableSymbols(std::unique_ptr< MaterializationResponsibility > MR, SymbolMap InitialDests)=0
Emit redirectable symbol.
virtual Error redirect(JITDylib &JD, const SymbolMap &NewDests)=0
Change the redirection destination of given symbols to new destination symbols.
A set of symbols to look up, each associated with a SymbolLookupFlags value.
Definition: Core.h:194
SymbolLookupSet & add(SymbolStringPtr Name, SymbolLookupFlags Flags=SymbolLookupFlags::RequiredSymbol)
Add an element to the set.
Definition: Core.h:260
An LLVM Module together with a shared ThreadSafeContext.
decltype(auto) withModuleDo(Func &&F)
Locks the associated ThreadSafeContext and calls the given function on the contained Module.
Output char buffer with overflow check.
Key
PAL metadata keys.
@ OB
OB - OneByte - Set if this instruction has a one byte opcode.
Definition: X86BaseInfo.h:732
ThreadSafeModule cloneToNewContext(const ThreadSafeModule &TSMW, GVPredicate ShouldCloneDef=GVPredicate(), GVModifier UpdateClonedDefSource=GVModifier())
Clones the given module on to a new context.
@ Resolved
Queried, materialization begun.
uintptr_t ResourceKey
Definition: Core.h:74
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:98
void cantFail(Error Err, const char *Msg=nullptr)
Report a fatal error if Err is a failure value.
Definition: Error.h:756
Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...