File: | llvm/lib/Transforms/Coroutines/CoroSplit.cpp |
Warning: | line 461, column 19 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- CoroSplit.cpp - Converts a coroutine into a state machine ----------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // This pass builds the coroutine frame and outlines resume and destroy parts | |||
9 | // of the coroutine into separate functions. | |||
10 | // | |||
11 | // We present a coroutine to an LLVM as an ordinary function with suspension | |||
12 | // points marked up with intrinsics. We let the optimizer party on the coroutine | |||
13 | // as a single function for as long as possible. Shortly before the coroutine is | |||
14 | // eligible to be inlined into its callers, we split up the coroutine into parts | |||
15 | // corresponding to an initial, resume and destroy invocations of the coroutine, | |||
16 | // add them to the current SCC and restart the IPO pipeline to optimize the | |||
17 | // coroutine subfunctions we extracted before proceeding to the caller of the | |||
18 | // coroutine. | |||
19 | //===----------------------------------------------------------------------===// | |||
20 | ||||
21 | #include "llvm/Transforms/Coroutines/CoroSplit.h" | |||
22 | #include "CoroInstr.h" | |||
23 | #include "CoroInternal.h" | |||
24 | #include "llvm/ADT/DenseMap.h" | |||
25 | #include "llvm/ADT/SmallPtrSet.h" | |||
26 | #include "llvm/ADT/SmallVector.h" | |||
27 | #include "llvm/ADT/StringRef.h" | |||
28 | #include "llvm/ADT/Twine.h" | |||
29 | #include "llvm/Analysis/CallGraph.h" | |||
30 | #include "llvm/Analysis/CallGraphSCCPass.h" | |||
31 | #include "llvm/Analysis/LazyCallGraph.h" | |||
32 | #include "llvm/IR/Argument.h" | |||
33 | #include "llvm/IR/Attributes.h" | |||
34 | #include "llvm/IR/BasicBlock.h" | |||
35 | #include "llvm/IR/CFG.h" | |||
36 | #include "llvm/IR/CallingConv.h" | |||
37 | #include "llvm/IR/Constants.h" | |||
38 | #include "llvm/IR/DataLayout.h" | |||
39 | #include "llvm/IR/DerivedTypes.h" | |||
40 | #include "llvm/IR/Function.h" | |||
41 | #include "llvm/IR/GlobalValue.h" | |||
42 | #include "llvm/IR/GlobalVariable.h" | |||
43 | #include "llvm/IR/IRBuilder.h" | |||
44 | #include "llvm/IR/InstIterator.h" | |||
45 | #include "llvm/IR/InstrTypes.h" | |||
46 | #include "llvm/IR/Instruction.h" | |||
47 | #include "llvm/IR/Instructions.h" | |||
48 | #include "llvm/IR/IntrinsicInst.h" | |||
49 | #include "llvm/IR/LLVMContext.h" | |||
50 | #include "llvm/IR/LegacyPassManager.h" | |||
51 | #include "llvm/IR/Module.h" | |||
52 | #include "llvm/IR/Type.h" | |||
53 | #include "llvm/IR/Value.h" | |||
54 | #include "llvm/IR/Verifier.h" | |||
55 | #include "llvm/InitializePasses.h" | |||
56 | #include "llvm/Pass.h" | |||
57 | #include "llvm/Support/Casting.h" | |||
58 | #include "llvm/Support/Debug.h" | |||
59 | #include "llvm/Support/PrettyStackTrace.h" | |||
60 | #include "llvm/Support/raw_ostream.h" | |||
61 | #include "llvm/Transforms/Scalar.h" | |||
62 | #include "llvm/Transforms/Utils/BasicBlockUtils.h" | |||
63 | #include "llvm/Transforms/Utils/CallGraphUpdater.h" | |||
64 | #include "llvm/Transforms/Utils/Cloning.h" | |||
65 | #include "llvm/Transforms/Utils/Local.h" | |||
66 | #include "llvm/Transforms/Utils/ValueMapper.h" | |||
67 | #include <cassert> | |||
68 | #include <cstddef> | |||
69 | #include <cstdint> | |||
70 | #include <initializer_list> | |||
71 | #include <iterator> | |||
72 | ||||
73 | using namespace llvm; | |||
74 | ||||
75 | #define DEBUG_TYPE"coro-split" "coro-split" | |||
76 | ||||
77 | namespace { | |||
78 | ||||
79 | /// A little helper class for building | |||
80 | class CoroCloner { | |||
81 | public: | |||
82 | enum class Kind { | |||
83 | /// The shared resume function for a switch lowering. | |||
84 | SwitchResume, | |||
85 | ||||
86 | /// The shared unwind function for a switch lowering. | |||
87 | SwitchUnwind, | |||
88 | ||||
89 | /// The shared cleanup function for a switch lowering. | |||
90 | SwitchCleanup, | |||
91 | ||||
92 | /// An individual continuation function. | |||
93 | Continuation, | |||
94 | ||||
95 | /// An async resume function. | |||
96 | Async, | |||
97 | }; | |||
98 | ||||
99 | private: | |||
100 | Function &OrigF; | |||
101 | Function *NewF; | |||
102 | const Twine &Suffix; | |||
103 | coro::Shape &Shape; | |||
104 | Kind FKind; | |||
105 | ValueToValueMapTy VMap; | |||
106 | IRBuilder<> Builder; | |||
107 | Value *NewFramePtr = nullptr; | |||
108 | ||||
109 | /// The active suspend instruction; meaningful only for continuation and async | |||
110 | /// ABIs. | |||
111 | AnyCoroSuspendInst *ActiveSuspend = nullptr; | |||
112 | ||||
113 | public: | |||
114 | /// Create a cloner for a switch lowering. | |||
115 | CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, | |||
116 | Kind FKind) | |||
117 | : OrigF(OrigF), NewF(nullptr), Suffix(Suffix), Shape(Shape), | |||
118 | FKind(FKind), Builder(OrigF.getContext()) { | |||
119 | assert(Shape.ABI == coro::ABI::Switch)((Shape.ABI == coro::ABI::Switch) ? static_cast<void> ( 0) : __assert_fail ("Shape.ABI == coro::ABI::Switch", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 119, __PRETTY_FUNCTION__)); | |||
120 | } | |||
121 | ||||
122 | /// Create a cloner for a continuation lowering. | |||
123 | CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape, | |||
124 | Function *NewF, AnyCoroSuspendInst *ActiveSuspend) | |||
125 | : OrigF(OrigF), NewF(NewF), Suffix(Suffix), Shape(Shape), | |||
126 | FKind(Shape.ABI == coro::ABI::Async ? Kind::Async : Kind::Continuation), | |||
127 | Builder(OrigF.getContext()), ActiveSuspend(ActiveSuspend) { | |||
128 | assert(Shape.ABI == coro::ABI::Retcon ||((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async) ? static_cast<void> ( 0) : __assert_fail ("Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 129, __PRETTY_FUNCTION__)) | |||
129 | Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async)((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async) ? static_cast<void> ( 0) : __assert_fail ("Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 129, __PRETTY_FUNCTION__)); | |||
130 | assert(NewF && "need existing function for continuation")((NewF && "need existing function for continuation") ? static_cast<void> (0) : __assert_fail ("NewF && \"need existing function for continuation\"" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 130, __PRETTY_FUNCTION__)); | |||
131 | assert(ActiveSuspend && "need active suspend point for continuation")((ActiveSuspend && "need active suspend point for continuation" ) ? static_cast<void> (0) : __assert_fail ("ActiveSuspend && \"need active suspend point for continuation\"" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 131, __PRETTY_FUNCTION__)); | |||
132 | } | |||
133 | ||||
134 | Function *getFunction() const { | |||
135 | assert(NewF != nullptr && "declaration not yet set")((NewF != nullptr && "declaration not yet set") ? static_cast <void> (0) : __assert_fail ("NewF != nullptr && \"declaration not yet set\"" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 135, __PRETTY_FUNCTION__)); | |||
136 | return NewF; | |||
137 | } | |||
138 | ||||
139 | void create(); | |||
140 | ||||
141 | private: | |||
142 | bool isSwitchDestroyFunction() { | |||
143 | switch (FKind) { | |||
144 | case Kind::Async: | |||
145 | case Kind::Continuation: | |||
146 | case Kind::SwitchResume: | |||
147 | return false; | |||
148 | case Kind::SwitchUnwind: | |||
149 | case Kind::SwitchCleanup: | |||
150 | return true; | |||
151 | } | |||
152 | llvm_unreachable("Unknown CoroCloner::Kind enum")::llvm::llvm_unreachable_internal("Unknown CoroCloner::Kind enum" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 152); | |||
153 | } | |||
154 | ||||
155 | void replaceEntryBlock(); | |||
156 | Value *deriveNewFramePointer(); | |||
157 | void replaceRetconOrAsyncSuspendUses(); | |||
158 | void replaceCoroSuspends(); | |||
159 | void replaceCoroEnds(); | |||
160 | void replaceSwiftErrorOps(); | |||
161 | void salvageDebugInfo(); | |||
162 | void handleFinalSuspend(); | |||
163 | }; | |||
164 | ||||
165 | } // end anonymous namespace | |||
166 | ||||
167 | static void maybeFreeRetconStorage(IRBuilder<> &Builder, | |||
168 | const coro::Shape &Shape, Value *FramePtr, | |||
169 | CallGraph *CG) { | |||
170 | assert(Shape.ABI == coro::ABI::Retcon ||((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce ) ? static_cast<void> (0) : __assert_fail ("Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 171, __PRETTY_FUNCTION__)) | |||
171 | Shape.ABI == coro::ABI::RetconOnce)((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce ) ? static_cast<void> (0) : __assert_fail ("Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 171, __PRETTY_FUNCTION__)); | |||
172 | if (Shape.RetconLowering.IsFrameInlineInStorage) | |||
173 | return; | |||
174 | ||||
175 | Shape.emitDealloc(Builder, FramePtr, CG); | |||
176 | } | |||
177 | ||||
178 | /// Replace an llvm.coro.end.async. | |||
179 | /// Will inline the must tail call function call if there is one. | |||
180 | /// \returns true if cleanup of the coro.end block is needed, false otherwise. | |||
181 | static bool replaceCoroEndAsync(AnyCoroEndInst *End) { | |||
182 | IRBuilder<> Builder(End); | |||
183 | ||||
184 | auto *EndAsync = dyn_cast<CoroAsyncEndInst>(End); | |||
185 | if (!EndAsync) { | |||
186 | Builder.CreateRetVoid(); | |||
187 | return true /*needs cleanup of coro.end block*/; | |||
188 | } | |||
189 | ||||
190 | auto *MustTailCallFunc = EndAsync->getMustTailCallFunction(); | |||
191 | if (!MustTailCallFunc) { | |||
192 | Builder.CreateRetVoid(); | |||
193 | return true /*needs cleanup of coro.end block*/; | |||
194 | } | |||
195 | ||||
196 | // Move the must tail call from the predecessor block into the end block. | |||
197 | auto *CoroEndBlock = End->getParent(); | |||
198 | auto *MustTailCallFuncBlock = CoroEndBlock->getSinglePredecessor(); | |||
199 | assert(MustTailCallFuncBlock && "Must have a single predecessor block")((MustTailCallFuncBlock && "Must have a single predecessor block" ) ? static_cast<void> (0) : __assert_fail ("MustTailCallFuncBlock && \"Must have a single predecessor block\"" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 199, __PRETTY_FUNCTION__)); | |||
200 | auto It = MustTailCallFuncBlock->getTerminator()->getIterator(); | |||
201 | auto *MustTailCall = cast<CallInst>(&*std::prev(It)); | |||
202 | CoroEndBlock->getInstList().splice( | |||
203 | End->getIterator(), MustTailCallFuncBlock->getInstList(), MustTailCall); | |||
204 | ||||
205 | // Insert the return instruction. | |||
206 | Builder.SetInsertPoint(End); | |||
207 | Builder.CreateRetVoid(); | |||
208 | InlineFunctionInfo FnInfo; | |||
209 | ||||
210 | // Remove the rest of the block, by splitting it into an unreachable block. | |||
211 | auto *BB = End->getParent(); | |||
212 | BB->splitBasicBlock(End); | |||
213 | BB->getTerminator()->eraseFromParent(); | |||
214 | ||||
215 | auto InlineRes = InlineFunction(*MustTailCall, FnInfo); | |||
216 | assert(InlineRes.isSuccess() && "Expected inlining to succeed")((InlineRes.isSuccess() && "Expected inlining to succeed" ) ? static_cast<void> (0) : __assert_fail ("InlineRes.isSuccess() && \"Expected inlining to succeed\"" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 216, __PRETTY_FUNCTION__)); | |||
217 | (void)InlineRes; | |||
218 | ||||
219 | // We have cleaned up the coro.end block above. | |||
220 | return false; | |||
221 | } | |||
222 | ||||
223 | /// Replace a non-unwind call to llvm.coro.end. | |||
224 | static void replaceFallthroughCoroEnd(AnyCoroEndInst *End, | |||
225 | const coro::Shape &Shape, Value *FramePtr, | |||
226 | bool InResume, CallGraph *CG) { | |||
227 | // Start inserting right before the coro.end. | |||
228 | IRBuilder<> Builder(End); | |||
229 | ||||
230 | // Create the return instruction. | |||
231 | switch (Shape.ABI) { | |||
232 | // The cloned functions in switch-lowering always return void. | |||
233 | case coro::ABI::Switch: | |||
234 | // coro.end doesn't immediately end the coroutine in the main function | |||
235 | // in this lowering, because we need to deallocate the coroutine. | |||
236 | if (!InResume) | |||
237 | return; | |||
238 | Builder.CreateRetVoid(); | |||
239 | break; | |||
240 | ||||
241 | // In async lowering this returns. | |||
242 | case coro::ABI::Async: { | |||
243 | bool CoroEndBlockNeedsCleanup = replaceCoroEndAsync(End); | |||
244 | if (!CoroEndBlockNeedsCleanup) | |||
245 | return; | |||
246 | break; | |||
247 | } | |||
248 | ||||
249 | // In unique continuation lowering, the continuations always return void. | |||
250 | // But we may have implicitly allocated storage. | |||
251 | case coro::ABI::RetconOnce: | |||
252 | maybeFreeRetconStorage(Builder, Shape, FramePtr, CG); | |||
253 | Builder.CreateRetVoid(); | |||
254 | break; | |||
255 | ||||
256 | // In non-unique continuation lowering, we signal completion by returning | |||
257 | // a null continuation. | |||
258 | case coro::ABI::Retcon: { | |||
259 | maybeFreeRetconStorage(Builder, Shape, FramePtr, CG); | |||
260 | auto RetTy = Shape.getResumeFunctionType()->getReturnType(); | |||
261 | auto RetStructTy = dyn_cast<StructType>(RetTy); | |||
262 | PointerType *ContinuationTy = | |||
263 | cast<PointerType>(RetStructTy ? RetStructTy->getElementType(0) : RetTy); | |||
264 | ||||
265 | Value *ReturnValue = ConstantPointerNull::get(ContinuationTy); | |||
266 | if (RetStructTy) { | |||
267 | ReturnValue = Builder.CreateInsertValue(UndefValue::get(RetStructTy), | |||
268 | ReturnValue, 0); | |||
269 | } | |||
270 | Builder.CreateRet(ReturnValue); | |||
271 | break; | |||
272 | } | |||
273 | } | |||
274 | ||||
275 | // Remove the rest of the block, by splitting it into an unreachable block. | |||
276 | auto *BB = End->getParent(); | |||
277 | BB->splitBasicBlock(End); | |||
278 | BB->getTerminator()->eraseFromParent(); | |||
279 | } | |||
280 | ||||
281 | /// Replace an unwind call to llvm.coro.end. | |||
282 | static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, | |||
283 | Value *FramePtr, bool InResume, | |||
284 | CallGraph *CG) { | |||
285 | IRBuilder<> Builder(End); | |||
286 | ||||
287 | switch (Shape.ABI) { | |||
288 | // In switch-lowering, this does nothing in the main function. | |||
289 | case coro::ABI::Switch: | |||
290 | if (!InResume) | |||
291 | return; | |||
292 | break; | |||
293 | // In async lowering this does nothing. | |||
294 | case coro::ABI::Async: | |||
295 | break; | |||
296 | // In continuation-lowering, this frees the continuation storage. | |||
297 | case coro::ABI::Retcon: | |||
298 | case coro::ABI::RetconOnce: | |||
299 | maybeFreeRetconStorage(Builder, Shape, FramePtr, CG); | |||
300 | break; | |||
301 | } | |||
302 | ||||
303 | // If coro.end has an associated bundle, add cleanupret instruction. | |||
304 | if (auto Bundle = End->getOperandBundle(LLVMContext::OB_funclet)) { | |||
305 | auto *FromPad = cast<CleanupPadInst>(Bundle->Inputs[0]); | |||
306 | auto *CleanupRet = Builder.CreateCleanupRet(FromPad, nullptr); | |||
307 | End->getParent()->splitBasicBlock(End); | |||
308 | CleanupRet->getParent()->getTerminator()->eraseFromParent(); | |||
309 | } | |||
310 | } | |||
311 | ||||
312 | static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, | |||
313 | Value *FramePtr, bool InResume, CallGraph *CG) { | |||
314 | if (End->isUnwind()) | |||
315 | replaceUnwindCoroEnd(End, Shape, FramePtr, InResume, CG); | |||
316 | else | |||
317 | replaceFallthroughCoroEnd(End, Shape, FramePtr, InResume, CG); | |||
318 | ||||
319 | auto &Context = End->getContext(); | |||
320 | End->replaceAllUsesWith(InResume ? ConstantInt::getTrue(Context) | |||
321 | : ConstantInt::getFalse(Context)); | |||
322 | End->eraseFromParent(); | |||
323 | } | |||
324 | ||||
325 | // Create an entry block for a resume function with a switch that will jump to | |||
326 | // suspend points. | |||
327 | static void createResumeEntryBlock(Function &F, coro::Shape &Shape) { | |||
328 | assert(Shape.ABI == coro::ABI::Switch)((Shape.ABI == coro::ABI::Switch) ? static_cast<void> ( 0) : __assert_fail ("Shape.ABI == coro::ABI::Switch", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 328, __PRETTY_FUNCTION__)); | |||
329 | LLVMContext &C = F.getContext(); | |||
330 | ||||
331 | // resume.entry: | |||
332 | // %index.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0, | |||
333 | // i32 2 | |||
334 | // % index = load i32, i32* %index.addr | |||
335 | // switch i32 %index, label %unreachable [ | |||
336 | // i32 0, label %resume.0 | |||
337 | // i32 1, label %resume.1 | |||
338 | // ... | |||
339 | // ] | |||
340 | ||||
341 | auto *NewEntry = BasicBlock::Create(C, "resume.entry", &F); | |||
342 | auto *UnreachBB = BasicBlock::Create(C, "unreachable", &F); | |||
343 | ||||
344 | IRBuilder<> Builder(NewEntry); | |||
345 | auto *FramePtr = Shape.FramePtr; | |||
346 | auto *FrameTy = Shape.FrameTy; | |||
347 | auto *GepIndex = Builder.CreateStructGEP( | |||
348 | FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr"); | |||
349 | auto *Index = Builder.CreateLoad(Shape.getIndexType(), GepIndex, "index"); | |||
350 | auto *Switch = | |||
351 | Builder.CreateSwitch(Index, UnreachBB, Shape.CoroSuspends.size()); | |||
352 | Shape.SwitchLowering.ResumeSwitch = Switch; | |||
353 | ||||
354 | size_t SuspendIndex = 0; | |||
355 | for (auto *AnyS : Shape.CoroSuspends) { | |||
356 | auto *S = cast<CoroSuspendInst>(AnyS); | |||
357 | ConstantInt *IndexVal = Shape.getIndex(SuspendIndex); | |||
358 | ||||
359 | // Replace CoroSave with a store to Index: | |||
360 | // %index.addr = getelementptr %f.frame... (index field number) | |||
361 | // store i32 0, i32* %index.addr1 | |||
362 | auto *Save = S->getCoroSave(); | |||
363 | Builder.SetInsertPoint(Save); | |||
364 | if (S->isFinal()) { | |||
365 | // Final suspend point is represented by storing zero in ResumeFnAddr. | |||
366 | auto *GepIndex = Builder.CreateStructGEP(FrameTy, FramePtr, | |||
367 | coro::Shape::SwitchFieldIndex::Resume, | |||
368 | "ResumeFn.addr"); | |||
369 | auto *NullPtr = ConstantPointerNull::get(cast<PointerType>( | |||
370 | cast<PointerType>(GepIndex->getType())->getElementType())); | |||
371 | Builder.CreateStore(NullPtr, GepIndex); | |||
372 | } else { | |||
373 | auto *GepIndex = Builder.CreateStructGEP( | |||
374 | FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr"); | |||
375 | Builder.CreateStore(IndexVal, GepIndex); | |||
376 | } | |||
377 | Save->replaceAllUsesWith(ConstantTokenNone::get(C)); | |||
378 | Save->eraseFromParent(); | |||
379 | ||||
380 | // Split block before and after coro.suspend and add a jump from an entry | |||
381 | // switch: | |||
382 | // | |||
383 | // whateverBB: | |||
384 | // whatever | |||
385 | // %0 = call i8 @llvm.coro.suspend(token none, i1 false) | |||
386 | // switch i8 %0, label %suspend[i8 0, label %resume | |||
387 | // i8 1, label %cleanup] | |||
388 | // becomes: | |||
389 | // | |||
390 | // whateverBB: | |||
391 | // whatever | |||
392 | // br label %resume.0.landing | |||
393 | // | |||
394 | // resume.0: ; <--- jump from the switch in the resume.entry | |||
395 | // %0 = tail call i8 @llvm.coro.suspend(token none, i1 false) | |||
396 | // br label %resume.0.landing | |||
397 | // | |||
398 | // resume.0.landing: | |||
399 | // %1 = phi i8[-1, %whateverBB], [%0, %resume.0] | |||
400 | // switch i8 % 1, label %suspend [i8 0, label %resume | |||
401 | // i8 1, label %cleanup] | |||
402 | ||||
403 | auto *SuspendBB = S->getParent(); | |||
404 | auto *ResumeBB = | |||
405 | SuspendBB->splitBasicBlock(S, "resume." + Twine(SuspendIndex)); | |||
406 | auto *LandingBB = ResumeBB->splitBasicBlock( | |||
407 | S->getNextNode(), ResumeBB->getName() + Twine(".landing")); | |||
408 | Switch->addCase(IndexVal, ResumeBB); | |||
409 | ||||
410 | cast<BranchInst>(SuspendBB->getTerminator())->setSuccessor(0, LandingBB); | |||
411 | auto *PN = PHINode::Create(Builder.getInt8Ty(), 2, "", &LandingBB->front()); | |||
412 | S->replaceAllUsesWith(PN); | |||
413 | PN->addIncoming(Builder.getInt8(-1), SuspendBB); | |||
414 | PN->addIncoming(S, ResumeBB); | |||
415 | ||||
416 | ++SuspendIndex; | |||
417 | } | |||
418 | ||||
419 | Builder.SetInsertPoint(UnreachBB); | |||
420 | Builder.CreateUnreachable(); | |||
421 | ||||
422 | Shape.SwitchLowering.ResumeEntryBlock = NewEntry; | |||
423 | } | |||
424 | ||||
425 | ||||
426 | // Rewrite final suspend point handling. We do not use suspend index to | |||
427 | // represent the final suspend point. Instead we zero-out ResumeFnAddr in the | |||
428 | // coroutine frame, since it is undefined behavior to resume a coroutine | |||
429 | // suspended at the final suspend point. Thus, in the resume function, we can | |||
430 | // simply remove the last case (when coro::Shape is built, the final suspend | |||
431 | // point (if present) is always the last element of CoroSuspends array). | |||
432 | // In the destroy function, we add a code sequence to check if ResumeFnAddress | |||
433 | // is Null, and if so, jump to the appropriate label to handle cleanup from the | |||
434 | // final suspend point. | |||
435 | void CoroCloner::handleFinalSuspend() { | |||
436 | assert(Shape.ABI == coro::ABI::Switch &&((Shape.ABI == coro::ABI::Switch && Shape.SwitchLowering .HasFinalSuspend) ? static_cast<void> (0) : __assert_fail ("Shape.ABI == coro::ABI::Switch && Shape.SwitchLowering.HasFinalSuspend" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 437, __PRETTY_FUNCTION__)) | |||
437 | Shape.SwitchLowering.HasFinalSuspend)((Shape.ABI == coro::ABI::Switch && Shape.SwitchLowering .HasFinalSuspend) ? static_cast<void> (0) : __assert_fail ("Shape.ABI == coro::ABI::Switch && Shape.SwitchLowering.HasFinalSuspend" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 437, __PRETTY_FUNCTION__)); | |||
438 | auto *Switch = cast<SwitchInst>(VMap[Shape.SwitchLowering.ResumeSwitch]); | |||
439 | auto FinalCaseIt = std::prev(Switch->case_end()); | |||
440 | BasicBlock *ResumeBB = FinalCaseIt->getCaseSuccessor(); | |||
441 | Switch->removeCase(FinalCaseIt); | |||
442 | if (isSwitchDestroyFunction()) { | |||
443 | BasicBlock *OldSwitchBB = Switch->getParent(); | |||
444 | auto *NewSwitchBB = OldSwitchBB->splitBasicBlock(Switch, "Switch"); | |||
445 | Builder.SetInsertPoint(OldSwitchBB->getTerminator()); | |||
446 | auto *GepIndex = Builder.CreateStructGEP(Shape.FrameTy, NewFramePtr, | |||
447 | coro::Shape::SwitchFieldIndex::Resume, | |||
448 | "ResumeFn.addr"); | |||
449 | auto *Load = Builder.CreateLoad(Shape.getSwitchResumePointerType(), | |||
450 | GepIndex); | |||
451 | auto *Cond = Builder.CreateIsNull(Load); | |||
452 | Builder.CreateCondBr(Cond, ResumeBB, NewSwitchBB); | |||
453 | OldSwitchBB->getTerminator()->eraseFromParent(); | |||
454 | } | |||
455 | } | |||
456 | ||||
457 | static FunctionType * | |||
458 | getFunctionTypeFromAsyncSuspend(AnyCoroSuspendInst *Suspend) { | |||
459 | auto *AsyncSuspend = cast<CoroSuspendAsyncInst>(Suspend); | |||
460 | auto *StructTy = cast<StructType>(AsyncSuspend->getType()); | |||
461 | auto &Context = Suspend->getParent()->getParent()->getContext(); | |||
| ||||
462 | auto *VoidTy = Type::getVoidTy(Context); | |||
463 | return FunctionType::get(VoidTy, StructTy->elements(), false); | |||
464 | } | |||
465 | ||||
466 | static Function *createCloneDeclaration(Function &OrigF, coro::Shape &Shape, | |||
467 | const Twine &Suffix, | |||
468 | Module::iterator InsertBefore, | |||
469 | AnyCoroSuspendInst *ActiveSuspend) { | |||
470 | Module *M = OrigF.getParent(); | |||
471 | auto *FnTy = (Shape.ABI != coro::ABI::Async) | |||
472 | ? Shape.getResumeFunctionType() | |||
473 | : getFunctionTypeFromAsyncSuspend(ActiveSuspend); | |||
474 | ||||
475 | Function *NewF = | |||
476 | Function::Create(FnTy, GlobalValue::LinkageTypes::InternalLinkage, | |||
477 | OrigF.getName() + Suffix); | |||
478 | NewF->addParamAttr(0, Attribute::NonNull); | |||
479 | ||||
480 | // For the async lowering ABI we can't guarantee that the context argument is | |||
481 | // not access via a different pointer not based on the argument. | |||
482 | if (Shape.ABI != coro::ABI::Async) | |||
483 | NewF->addParamAttr(0, Attribute::NoAlias); | |||
484 | ||||
485 | M->getFunctionList().insert(InsertBefore, NewF); | |||
486 | ||||
487 | return NewF; | |||
488 | } | |||
489 | ||||
490 | /// Replace uses of the active llvm.coro.suspend.retcon/async call with the | |||
491 | /// arguments to the continuation function. | |||
492 | /// | |||
493 | /// This assumes that the builder has a meaningful insertion point. | |||
494 | void CoroCloner::replaceRetconOrAsyncSuspendUses() { | |||
495 | assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce ||((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async) ? static_cast<void> ( 0) : __assert_fail ("Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 496, __PRETTY_FUNCTION__)) | |||
496 | Shape.ABI == coro::ABI::Async)((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async) ? static_cast<void> ( 0) : __assert_fail ("Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 496, __PRETTY_FUNCTION__)); | |||
497 | ||||
498 | auto NewS = VMap[ActiveSuspend]; | |||
499 | if (NewS->use_empty()) return; | |||
500 | ||||
501 | // Copy out all the continuation arguments after the buffer pointer into | |||
502 | // an easily-indexed data structure for convenience. | |||
503 | SmallVector<Value*, 8> Args; | |||
504 | // The async ABI includes all arguments -- including the first argument. | |||
505 | bool IsAsyncABI = Shape.ABI == coro::ABI::Async; | |||
506 | for (auto I = IsAsyncABI ? NewF->arg_begin() : std::next(NewF->arg_begin()), | |||
507 | E = NewF->arg_end(); | |||
508 | I != E; ++I) | |||
509 | Args.push_back(&*I); | |||
510 | ||||
511 | // If the suspend returns a single scalar value, we can just do a simple | |||
512 | // replacement. | |||
513 | if (!isa<StructType>(NewS->getType())) { | |||
514 | assert(Args.size() == 1)((Args.size() == 1) ? static_cast<void> (0) : __assert_fail ("Args.size() == 1", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 514, __PRETTY_FUNCTION__)); | |||
515 | NewS->replaceAllUsesWith(Args.front()); | |||
516 | return; | |||
517 | } | |||
518 | ||||
519 | // Try to peephole extracts of an aggregate return. | |||
520 | for (auto UI = NewS->use_begin(), UE = NewS->use_end(); UI != UE; ) { | |||
521 | auto EVI = dyn_cast<ExtractValueInst>((UI++)->getUser()); | |||
522 | if (!EVI || EVI->getNumIndices() != 1) | |||
523 | continue; | |||
524 | ||||
525 | EVI->replaceAllUsesWith(Args[EVI->getIndices().front()]); | |||
526 | EVI->eraseFromParent(); | |||
527 | } | |||
528 | ||||
529 | // If we have no remaining uses, we're done. | |||
530 | if (NewS->use_empty()) return; | |||
531 | ||||
532 | // Otherwise, we need to create an aggregate. | |||
533 | Value *Agg = UndefValue::get(NewS->getType()); | |||
534 | for (size_t I = 0, E = Args.size(); I != E; ++I) | |||
535 | Agg = Builder.CreateInsertValue(Agg, Args[I], I); | |||
536 | ||||
537 | NewS->replaceAllUsesWith(Agg); | |||
538 | } | |||
539 | ||||
540 | void CoroCloner::replaceCoroSuspends() { | |||
541 | Value *SuspendResult; | |||
542 | ||||
543 | switch (Shape.ABI) { | |||
544 | // In switch lowering, replace coro.suspend with the appropriate value | |||
545 | // for the type of function we're extracting. | |||
546 | // Replacing coro.suspend with (0) will result in control flow proceeding to | |||
547 | // a resume label associated with a suspend point, replacing it with (1) will | |||
548 | // result in control flow proceeding to a cleanup label associated with this | |||
549 | // suspend point. | |||
550 | case coro::ABI::Switch: | |||
551 | SuspendResult = Builder.getInt8(isSwitchDestroyFunction() ? 1 : 0); | |||
552 | break; | |||
553 | ||||
554 | // In async lowering there are no uses of the result. | |||
555 | case coro::ABI::Async: | |||
556 | return; | |||
557 | ||||
558 | // In returned-continuation lowering, the arguments from earlier | |||
559 | // continuations are theoretically arbitrary, and they should have been | |||
560 | // spilled. | |||
561 | case coro::ABI::RetconOnce: | |||
562 | case coro::ABI::Retcon: | |||
563 | return; | |||
564 | } | |||
565 | ||||
566 | for (AnyCoroSuspendInst *CS : Shape.CoroSuspends) { | |||
567 | // The active suspend was handled earlier. | |||
568 | if (CS == ActiveSuspend) continue; | |||
569 | ||||
570 | auto *MappedCS = cast<AnyCoroSuspendInst>(VMap[CS]); | |||
571 | MappedCS->replaceAllUsesWith(SuspendResult); | |||
572 | MappedCS->eraseFromParent(); | |||
573 | } | |||
574 | } | |||
575 | ||||
576 | void CoroCloner::replaceCoroEnds() { | |||
577 | for (AnyCoroEndInst *CE : Shape.CoroEnds) { | |||
578 | // We use a null call graph because there's no call graph node for | |||
579 | // the cloned function yet. We'll just be rebuilding that later. | |||
580 | auto *NewCE = cast<AnyCoroEndInst>(VMap[CE]); | |||
581 | replaceCoroEnd(NewCE, Shape, NewFramePtr, /*in resume*/ true, nullptr); | |||
582 | } | |||
583 | } | |||
584 | ||||
585 | static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape, | |||
586 | ValueToValueMapTy *VMap) { | |||
587 | if (Shape.ABI == coro::ABI::Async && Shape.CoroSuspends.empty()) | |||
588 | return; | |||
589 | Value *CachedSlot = nullptr; | |||
590 | auto getSwiftErrorSlot = [&](Type *ValueTy) -> Value * { | |||
591 | if (CachedSlot) { | |||
592 | assert(CachedSlot->getType()->getPointerElementType() == ValueTy &&((CachedSlot->getType()->getPointerElementType() == ValueTy && "multiple swifterror slots in function with different types" ) ? static_cast<void> (0) : __assert_fail ("CachedSlot->getType()->getPointerElementType() == ValueTy && \"multiple swifterror slots in function with different types\"" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 593, __PRETTY_FUNCTION__)) | |||
593 | "multiple swifterror slots in function with different types")((CachedSlot->getType()->getPointerElementType() == ValueTy && "multiple swifterror slots in function with different types" ) ? static_cast<void> (0) : __assert_fail ("CachedSlot->getType()->getPointerElementType() == ValueTy && \"multiple swifterror slots in function with different types\"" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 593, __PRETTY_FUNCTION__)); | |||
594 | return CachedSlot; | |||
595 | } | |||
596 | ||||
597 | // Check if the function has a swifterror argument. | |||
598 | for (auto &Arg : F.args()) { | |||
599 | if (Arg.isSwiftError()) { | |||
600 | CachedSlot = &Arg; | |||
601 | assert(Arg.getType()->getPointerElementType() == ValueTy &&((Arg.getType()->getPointerElementType() == ValueTy && "swifterror argument does not have expected type") ? static_cast <void> (0) : __assert_fail ("Arg.getType()->getPointerElementType() == ValueTy && \"swifterror argument does not have expected type\"" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 602, __PRETTY_FUNCTION__)) | |||
602 | "swifterror argument does not have expected type")((Arg.getType()->getPointerElementType() == ValueTy && "swifterror argument does not have expected type") ? static_cast <void> (0) : __assert_fail ("Arg.getType()->getPointerElementType() == ValueTy && \"swifterror argument does not have expected type\"" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 602, __PRETTY_FUNCTION__)); | |||
603 | return &Arg; | |||
604 | } | |||
605 | } | |||
606 | ||||
607 | // Create a swifterror alloca. | |||
608 | IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHIOrDbg()); | |||
609 | auto Alloca = Builder.CreateAlloca(ValueTy); | |||
610 | Alloca->setSwiftError(true); | |||
611 | ||||
612 | CachedSlot = Alloca; | |||
613 | return Alloca; | |||
614 | }; | |||
615 | ||||
616 | for (CallInst *Op : Shape.SwiftErrorOps) { | |||
617 | auto MappedOp = VMap ? cast<CallInst>((*VMap)[Op]) : Op; | |||
618 | IRBuilder<> Builder(MappedOp); | |||
619 | ||||
620 | // If there are no arguments, this is a 'get' operation. | |||
621 | Value *MappedResult; | |||
622 | if (Op->getNumArgOperands() == 0) { | |||
623 | auto ValueTy = Op->getType(); | |||
624 | auto Slot = getSwiftErrorSlot(ValueTy); | |||
625 | MappedResult = Builder.CreateLoad(ValueTy, Slot); | |||
626 | } else { | |||
627 | assert(Op->getNumArgOperands() == 1)((Op->getNumArgOperands() == 1) ? static_cast<void> ( 0) : __assert_fail ("Op->getNumArgOperands() == 1", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 627, __PRETTY_FUNCTION__)); | |||
628 | auto Value = MappedOp->getArgOperand(0); | |||
629 | auto ValueTy = Value->getType(); | |||
630 | auto Slot = getSwiftErrorSlot(ValueTy); | |||
631 | Builder.CreateStore(Value, Slot); | |||
632 | MappedResult = Slot; | |||
633 | } | |||
634 | ||||
635 | MappedOp->replaceAllUsesWith(MappedResult); | |||
636 | MappedOp->eraseFromParent(); | |||
637 | } | |||
638 | ||||
639 | // If we're updating the original function, we've invalidated SwiftErrorOps. | |||
640 | if (VMap == nullptr) { | |||
641 | Shape.SwiftErrorOps.clear(); | |||
642 | } | |||
643 | } | |||
644 | ||||
645 | void CoroCloner::replaceSwiftErrorOps() { | |||
646 | ::replaceSwiftErrorOps(*NewF, Shape, &VMap); | |||
647 | } | |||
648 | ||||
649 | void CoroCloner::salvageDebugInfo() { | |||
650 | SmallVector<DbgDeclareInst *, 8> Worklist; | |||
651 | SmallDenseMap<llvm::Value *, llvm::AllocaInst *, 4> DbgPtrAllocaCache; | |||
652 | for (auto &BB : *NewF) | |||
653 | for (auto &I : BB) | |||
654 | if (auto *DDI = dyn_cast<DbgDeclareInst>(&I)) | |||
655 | Worklist.push_back(DDI); | |||
656 | for (DbgDeclareInst *DDI : Worklist) | |||
657 | coro::salvageDebugInfo(DbgPtrAllocaCache, DDI); | |||
658 | ||||
659 | // Remove all salvaged dbg.declare intrinsics that became | |||
660 | // either unreachable or stale due to the CoroSplit transformation. | |||
661 | auto IsUnreachableBlock = [&](BasicBlock *BB) { | |||
662 | return BB->hasNPredecessors(0) && BB != &NewF->getEntryBlock(); | |||
663 | }; | |||
664 | for (DbgDeclareInst *DDI : Worklist) { | |||
665 | if (IsUnreachableBlock(DDI->getParent())) | |||
666 | DDI->eraseFromParent(); | |||
667 | else if (dyn_cast_or_null<AllocaInst>(DDI->getAddress())) { | |||
668 | // Count all non-debuginfo uses in reachable blocks. | |||
669 | unsigned Uses = 0; | |||
670 | for (auto *User : DDI->getAddress()->users()) | |||
671 | if (auto *I = dyn_cast<Instruction>(User)) | |||
672 | if (!isa<AllocaInst>(I) && !IsUnreachableBlock(I->getParent())) | |||
673 | ++Uses; | |||
674 | if (!Uses) | |||
675 | DDI->eraseFromParent(); | |||
676 | } | |||
677 | } | |||
678 | } | |||
679 | ||||
680 | void CoroCloner::replaceEntryBlock() { | |||
681 | // In the original function, the AllocaSpillBlock is a block immediately | |||
682 | // following the allocation of the frame object which defines GEPs for | |||
683 | // all the allocas that have been moved into the frame, and it ends by | |||
684 | // branching to the original beginning of the coroutine. Make this | |||
685 | // the entry block of the cloned function. | |||
686 | auto *Entry = cast<BasicBlock>(VMap[Shape.AllocaSpillBlock]); | |||
687 | auto *OldEntry = &NewF->getEntryBlock(); | |||
688 | Entry->setName("entry" + Suffix); | |||
689 | Entry->moveBefore(OldEntry); | |||
690 | Entry->getTerminator()->eraseFromParent(); | |||
691 | ||||
692 | // Clear all predecessors of the new entry block. There should be | |||
693 | // exactly one predecessor, which we created when splitting out | |||
694 | // AllocaSpillBlock to begin with. | |||
695 | assert(Entry->hasOneUse())((Entry->hasOneUse()) ? static_cast<void> (0) : __assert_fail ("Entry->hasOneUse()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 695, __PRETTY_FUNCTION__)); | |||
696 | auto BranchToEntry = cast<BranchInst>(Entry->user_back()); | |||
697 | assert(BranchToEntry->isUnconditional())((BranchToEntry->isUnconditional()) ? static_cast<void> (0) : __assert_fail ("BranchToEntry->isUnconditional()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 697, __PRETTY_FUNCTION__)); | |||
698 | Builder.SetInsertPoint(BranchToEntry); | |||
699 | Builder.CreateUnreachable(); | |||
700 | BranchToEntry->eraseFromParent(); | |||
701 | ||||
702 | // Branch from the entry to the appropriate place. | |||
703 | Builder.SetInsertPoint(Entry); | |||
704 | switch (Shape.ABI) { | |||
705 | case coro::ABI::Switch: { | |||
706 | // In switch-lowering, we built a resume-entry block in the original | |||
707 | // function. Make the entry block branch to this. | |||
708 | auto *SwitchBB = | |||
709 | cast<BasicBlock>(VMap[Shape.SwitchLowering.ResumeEntryBlock]); | |||
710 | Builder.CreateBr(SwitchBB); | |||
711 | break; | |||
712 | } | |||
713 | case coro::ABI::Async: | |||
714 | case coro::ABI::Retcon: | |||
715 | case coro::ABI::RetconOnce: { | |||
716 | // In continuation ABIs, we want to branch to immediately after the | |||
717 | // active suspend point. Earlier phases will have put the suspend in its | |||
718 | // own basic block, so just thread our jump directly to its successor. | |||
719 | assert((Shape.ABI == coro::ABI::Async &&(((Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst >(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape .ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst >(ActiveSuspend))) ? static_cast<void> (0) : __assert_fail ("(Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst>(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst>(ActiveSuspend))" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 723, __PRETTY_FUNCTION__)) | |||
720 | isa<CoroSuspendAsyncInst>(ActiveSuspend)) ||(((Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst >(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape .ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst >(ActiveSuspend))) ? static_cast<void> (0) : __assert_fail ("(Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst>(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst>(ActiveSuspend))" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 723, __PRETTY_FUNCTION__)) | |||
721 | ((Shape.ABI == coro::ABI::Retcon ||(((Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst >(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape .ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst >(ActiveSuspend))) ? static_cast<void> (0) : __assert_fail ("(Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst>(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst>(ActiveSuspend))" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 723, __PRETTY_FUNCTION__)) | |||
722 | Shape.ABI == coro::ABI::RetconOnce) &&(((Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst >(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape .ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst >(ActiveSuspend))) ? static_cast<void> (0) : __assert_fail ("(Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst>(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst>(ActiveSuspend))" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 723, __PRETTY_FUNCTION__)) | |||
723 | isa<CoroSuspendRetconInst>(ActiveSuspend)))(((Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst >(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape .ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst >(ActiveSuspend))) ? static_cast<void> (0) : __assert_fail ("(Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst>(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst>(ActiveSuspend))" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 723, __PRETTY_FUNCTION__)); | |||
724 | auto *MappedCS = cast<AnyCoroSuspendInst>(VMap[ActiveSuspend]); | |||
725 | auto Branch = cast<BranchInst>(MappedCS->getNextNode()); | |||
726 | assert(Branch->isUnconditional())((Branch->isUnconditional()) ? static_cast<void> (0) : __assert_fail ("Branch->isUnconditional()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 726, __PRETTY_FUNCTION__)); | |||
727 | Builder.CreateBr(Branch->getSuccessor(0)); | |||
728 | break; | |||
729 | } | |||
730 | } | |||
731 | ||||
732 | // Any static alloca that's still being used but not reachable from the new | |||
733 | // entry needs to be moved to the new entry. | |||
734 | Function *F = OldEntry->getParent(); | |||
735 | DominatorTree DT{*F}; | |||
736 | for (auto IT = inst_begin(F), End = inst_end(F); IT != End;) { | |||
737 | Instruction &I = *IT++; | |||
738 | auto *Alloca = dyn_cast<AllocaInst>(&I); | |||
739 | if (!Alloca || I.use_empty()) | |||
740 | continue; | |||
741 | if (DT.isReachableFromEntry(I.getParent()) || | |||
742 | !isa<ConstantInt>(Alloca->getArraySize())) | |||
743 | continue; | |||
744 | I.moveBefore(*Entry, Entry->getFirstInsertionPt()); | |||
745 | } | |||
746 | } | |||
747 | ||||
748 | /// Derive the value of the new frame pointer. | |||
749 | Value *CoroCloner::deriveNewFramePointer() { | |||
750 | // Builder should be inserting to the front of the new entry block. | |||
751 | ||||
752 | switch (Shape.ABI) { | |||
753 | // In switch-lowering, the argument is the frame pointer. | |||
754 | case coro::ABI::Switch: | |||
755 | return &*NewF->arg_begin(); | |||
756 | // In async-lowering, one of the arguments is an async context as determined | |||
757 | // by the `llvm.coro.id.async` intrinsic. We can retrieve the async context of | |||
758 | // the resume function from the async context projection function associated | |||
759 | // with the active suspend. The frame is located as a tail to the async | |||
760 | // context header. | |||
761 | case coro::ABI::Async: { | |||
762 | auto *ActiveAsyncSuspend = cast<CoroSuspendAsyncInst>(ActiveSuspend); | |||
763 | auto *CalleeContext = | |||
764 | NewF->getArg(ActiveAsyncSuspend->getStorageArgumentIndex()); | |||
765 | auto *FramePtrTy = Shape.FrameTy->getPointerTo(); | |||
766 | auto *ProjectionFunc = | |||
767 | ActiveAsyncSuspend->getAsyncContextProjectionFunction(); | |||
768 | auto DbgLoc = | |||
769 | cast<CoroSuspendAsyncInst>(VMap[ActiveSuspend])->getDebugLoc(); | |||
770 | // Calling i8* (i8*) | |||
771 | auto *CallerContext = Builder.CreateCall( | |||
772 | cast<FunctionType>(ProjectionFunc->getType()->getPointerElementType()), | |||
773 | ProjectionFunc, CalleeContext); | |||
774 | CallerContext->setCallingConv(ProjectionFunc->getCallingConv()); | |||
775 | CallerContext->setDebugLoc(DbgLoc); | |||
776 | // The frame is located after the async_context header. | |||
777 | auto &Context = Builder.getContext(); | |||
778 | auto *FramePtrAddr = Builder.CreateConstInBoundsGEP1_32( | |||
779 | Type::getInt8Ty(Context), CallerContext, | |||
780 | Shape.AsyncLowering.FrameOffset, "async.ctx.frameptr"); | |||
781 | // Inline the projection function. | |||
782 | InlineFunctionInfo InlineInfo; | |||
783 | auto InlineRes = InlineFunction(*CallerContext, InlineInfo); | |||
784 | assert(InlineRes.isSuccess())((InlineRes.isSuccess()) ? static_cast<void> (0) : __assert_fail ("InlineRes.isSuccess()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 784, __PRETTY_FUNCTION__)); | |||
785 | (void)InlineRes; | |||
786 | return Builder.CreateBitCast(FramePtrAddr, FramePtrTy); | |||
787 | } | |||
788 | // In continuation-lowering, the argument is the opaque storage. | |||
789 | case coro::ABI::Retcon: | |||
790 | case coro::ABI::RetconOnce: { | |||
791 | Argument *NewStorage = &*NewF->arg_begin(); | |||
792 | auto FramePtrTy = Shape.FrameTy->getPointerTo(); | |||
793 | ||||
794 | // If the storage is inline, just bitcast to the storage to the frame type. | |||
795 | if (Shape.RetconLowering.IsFrameInlineInStorage) | |||
796 | return Builder.CreateBitCast(NewStorage, FramePtrTy); | |||
797 | ||||
798 | // Otherwise, load the real frame from the opaque storage. | |||
799 | auto FramePtrPtr = | |||
800 | Builder.CreateBitCast(NewStorage, FramePtrTy->getPointerTo()); | |||
801 | return Builder.CreateLoad(FramePtrTy, FramePtrPtr); | |||
802 | } | |||
803 | } | |||
804 | llvm_unreachable("bad ABI")::llvm::llvm_unreachable_internal("bad ABI", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 804); | |||
805 | } | |||
806 | ||||
807 | static void addFramePointerAttrs(AttributeList &Attrs, LLVMContext &Context, | |||
808 | unsigned ParamIndex, | |||
809 | uint64_t Size, Align Alignment) { | |||
810 | AttrBuilder ParamAttrs; | |||
811 | ParamAttrs.addAttribute(Attribute::NonNull); | |||
812 | ParamAttrs.addAttribute(Attribute::NoAlias); | |||
813 | ParamAttrs.addAlignmentAttr(Alignment); | |||
814 | ParamAttrs.addDereferenceableAttr(Size); | |||
815 | Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs); | |||
816 | } | |||
817 | ||||
818 | /// Clone the body of the original function into a resume function of | |||
819 | /// some sort. | |||
820 | void CoroCloner::create() { | |||
821 | // Create the new function if we don't already have one. | |||
822 | if (!NewF) { | |||
823 | NewF = createCloneDeclaration(OrigF, Shape, Suffix, | |||
824 | OrigF.getParent()->end(), ActiveSuspend); | |||
825 | } | |||
826 | ||||
827 | // Replace all args with undefs. The buildCoroutineFrame algorithm already | |||
828 | // rewritten access to the args that occurs after suspend points with loads | |||
829 | // and stores to/from the coroutine frame. | |||
830 | for (Argument &A : OrigF.args()) | |||
831 | VMap[&A] = UndefValue::get(A.getType()); | |||
832 | ||||
833 | SmallVector<ReturnInst *, 4> Returns; | |||
834 | ||||
835 | // Ignore attempts to change certain attributes of the function. | |||
836 | // TODO: maybe there should be a way to suppress this during cloning? | |||
837 | auto savedVisibility = NewF->getVisibility(); | |||
838 | auto savedUnnamedAddr = NewF->getUnnamedAddr(); | |||
839 | auto savedDLLStorageClass = NewF->getDLLStorageClass(); | |||
840 | ||||
841 | // NewF's linkage (which CloneFunctionInto does *not* change) might not | |||
842 | // be compatible with the visibility of OrigF (which it *does* change), | |||
843 | // so protect against that. | |||
844 | auto savedLinkage = NewF->getLinkage(); | |||
845 | NewF->setLinkage(llvm::GlobalValue::ExternalLinkage); | |||
846 | ||||
847 | CloneFunctionInto(NewF, &OrigF, VMap, | |||
848 | CloneFunctionChangeType::LocalChangesOnly, Returns); | |||
849 | // For async functions / continuations, adjust the scope line of the | |||
850 | // clone to the line number of the suspend point. The scope line is | |||
851 | // associated with all pre-prologue instructions. This avoids a jump | |||
852 | // in the linetable from the function declaration to the suspend point. | |||
853 | if (DISubprogram *SP = NewF->getSubprogram()) { | |||
854 | assert(SP != OrigF.getSubprogram() && SP->isDistinct())((SP != OrigF.getSubprogram() && SP->isDistinct()) ? static_cast<void> (0) : __assert_fail ("SP != OrigF.getSubprogram() && SP->isDistinct()" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 854, __PRETTY_FUNCTION__)); | |||
855 | if (ActiveSuspend) | |||
856 | if (auto DL = ActiveSuspend->getDebugLoc()) | |||
857 | SP->setScopeLine(DL->getLine()); | |||
858 | } | |||
859 | ||||
860 | NewF->setLinkage(savedLinkage); | |||
861 | NewF->setVisibility(savedVisibility); | |||
862 | NewF->setUnnamedAddr(savedUnnamedAddr); | |||
863 | NewF->setDLLStorageClass(savedDLLStorageClass); | |||
864 | ||||
865 | auto &Context = NewF->getContext(); | |||
866 | ||||
867 | // Replace the attributes of the new function: | |||
868 | auto OrigAttrs = NewF->getAttributes(); | |||
869 | auto NewAttrs = AttributeList(); | |||
870 | ||||
871 | switch (Shape.ABI) { | |||
872 | case coro::ABI::Switch: | |||
873 | // Bootstrap attributes by copying function attributes from the | |||
874 | // original function. This should include optimization settings and so on. | |||
875 | NewAttrs = NewAttrs.addAttributes(Context, AttributeList::FunctionIndex, | |||
876 | OrigAttrs.getFnAttributes()); | |||
877 | ||||
878 | addFramePointerAttrs(NewAttrs, Context, 0, | |||
879 | Shape.FrameSize, Shape.FrameAlign); | |||
880 | break; | |||
881 | case coro::ABI::Async: | |||
882 | break; | |||
883 | case coro::ABI::Retcon: | |||
884 | case coro::ABI::RetconOnce: | |||
885 | // If we have a continuation prototype, just use its attributes, | |||
886 | // full-stop. | |||
887 | NewAttrs = Shape.RetconLowering.ResumePrototype->getAttributes(); | |||
888 | ||||
889 | addFramePointerAttrs(NewAttrs, Context, 0, | |||
890 | Shape.getRetconCoroId()->getStorageSize(), | |||
891 | Shape.getRetconCoroId()->getStorageAlignment()); | |||
892 | break; | |||
893 | } | |||
894 | ||||
895 | switch (Shape.ABI) { | |||
896 | // In these ABIs, the cloned functions always return 'void', and the | |||
897 | // existing return sites are meaningless. Note that for unique | |||
898 | // continuations, this includes the returns associated with suspends; | |||
899 | // this is fine because we can't suspend twice. | |||
900 | case coro::ABI::Switch: | |||
901 | case coro::ABI::RetconOnce: | |||
902 | // Remove old returns. | |||
903 | for (ReturnInst *Return : Returns) | |||
904 | changeToUnreachable(Return, /*UseLLVMTrap=*/false); | |||
905 | break; | |||
906 | ||||
907 | // With multi-suspend continuations, we'll already have eliminated the | |||
908 | // original returns and inserted returns before all the suspend points, | |||
909 | // so we want to leave any returns in place. | |||
910 | case coro::ABI::Retcon: | |||
911 | break; | |||
912 | // Async lowering will insert musttail call functions at all suspend points | |||
913 | // followed by a return. | |||
914 | // Don't change returns to unreachable because that will trip up the verifier. | |||
915 | // These returns should be unreachable from the clone. | |||
916 | case coro::ABI::Async: | |||
917 | break; | |||
918 | } | |||
919 | ||||
920 | NewF->setAttributes(NewAttrs); | |||
921 | NewF->setCallingConv(Shape.getResumeFunctionCC()); | |||
922 | ||||
923 | // Set up the new entry block. | |||
924 | replaceEntryBlock(); | |||
925 | ||||
926 | Builder.SetInsertPoint(&NewF->getEntryBlock().front()); | |||
927 | NewFramePtr = deriveNewFramePointer(); | |||
928 | ||||
929 | // Remap frame pointer. | |||
930 | Value *OldFramePtr = VMap[Shape.FramePtr]; | |||
931 | NewFramePtr->takeName(OldFramePtr); | |||
932 | OldFramePtr->replaceAllUsesWith(NewFramePtr); | |||
933 | ||||
934 | // Remap vFrame pointer. | |||
935 | auto *NewVFrame = Builder.CreateBitCast( | |||
936 | NewFramePtr, Type::getInt8PtrTy(Builder.getContext()), "vFrame"); | |||
937 | Value *OldVFrame = cast<Value>(VMap[Shape.CoroBegin]); | |||
938 | OldVFrame->replaceAllUsesWith(NewVFrame); | |||
939 | ||||
940 | switch (Shape.ABI) { | |||
941 | case coro::ABI::Switch: | |||
942 | // Rewrite final suspend handling as it is not done via switch (allows to | |||
943 | // remove final case from the switch, since it is undefined behavior to | |||
944 | // resume the coroutine suspended at the final suspend point. | |||
945 | if (Shape.SwitchLowering.HasFinalSuspend) | |||
946 | handleFinalSuspend(); | |||
947 | break; | |||
948 | case coro::ABI::Async: | |||
949 | case coro::ABI::Retcon: | |||
950 | case coro::ABI::RetconOnce: | |||
951 | // Replace uses of the active suspend with the corresponding | |||
952 | // continuation-function arguments. | |||
953 | assert(ActiveSuspend != nullptr &&((ActiveSuspend != nullptr && "no active suspend when lowering a continuation-style coroutine" ) ? static_cast<void> (0) : __assert_fail ("ActiveSuspend != nullptr && \"no active suspend when lowering a continuation-style coroutine\"" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 954, __PRETTY_FUNCTION__)) | |||
954 | "no active suspend when lowering a continuation-style coroutine")((ActiveSuspend != nullptr && "no active suspend when lowering a continuation-style coroutine" ) ? static_cast<void> (0) : __assert_fail ("ActiveSuspend != nullptr && \"no active suspend when lowering a continuation-style coroutine\"" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 954, __PRETTY_FUNCTION__)); | |||
955 | replaceRetconOrAsyncSuspendUses(); | |||
956 | break; | |||
957 | } | |||
958 | ||||
959 | // Handle suspends. | |||
960 | replaceCoroSuspends(); | |||
961 | ||||
962 | // Handle swifterror. | |||
963 | replaceSwiftErrorOps(); | |||
964 | ||||
965 | // Remove coro.end intrinsics. | |||
966 | replaceCoroEnds(); | |||
967 | ||||
968 | // Salvage debug info that points into the coroutine frame. | |||
969 | salvageDebugInfo(); | |||
970 | ||||
971 | // Eliminate coro.free from the clones, replacing it with 'null' in cleanup, | |||
972 | // to suppress deallocation code. | |||
973 | if (Shape.ABI == coro::ABI::Switch) | |||
974 | coro::replaceCoroFree(cast<CoroIdInst>(VMap[Shape.CoroBegin->getId()]), | |||
975 | /*Elide=*/ FKind == CoroCloner::Kind::SwitchCleanup); | |||
976 | } | |||
977 | ||||
978 | // Create a resume clone by cloning the body of the original function, setting | |||
979 | // new entry block and replacing coro.suspend an appropriate value to force | |||
980 | // resume or cleanup pass for every suspend point. | |||
981 | static Function *createClone(Function &F, const Twine &Suffix, | |||
982 | coro::Shape &Shape, CoroCloner::Kind FKind) { | |||
983 | CoroCloner Cloner(F, Suffix, Shape, FKind); | |||
984 | Cloner.create(); | |||
985 | return Cloner.getFunction(); | |||
986 | } | |||
987 | ||||
988 | /// Remove calls to llvm.coro.end in the original function. | |||
989 | static void removeCoroEnds(const coro::Shape &Shape, CallGraph *CG) { | |||
990 | for (auto End : Shape.CoroEnds) { | |||
991 | replaceCoroEnd(End, Shape, Shape.FramePtr, /*in resume*/ false, CG); | |||
992 | } | |||
993 | } | |||
994 | ||||
995 | static void updateAsyncFuncPointerContextSize(coro::Shape &Shape) { | |||
996 | assert(Shape.ABI == coro::ABI::Async)((Shape.ABI == coro::ABI::Async) ? static_cast<void> (0 ) : __assert_fail ("Shape.ABI == coro::ABI::Async", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 996, __PRETTY_FUNCTION__)); | |||
997 | ||||
998 | auto *FuncPtrStruct = cast<ConstantStruct>( | |||
999 | Shape.AsyncLowering.AsyncFuncPointer->getInitializer()); | |||
1000 | auto *OrigRelativeFunOffset = FuncPtrStruct->getOperand(0); | |||
1001 | auto *OrigContextSize = FuncPtrStruct->getOperand(1); | |||
1002 | auto *NewContextSize = ConstantInt::get(OrigContextSize->getType(), | |||
1003 | Shape.AsyncLowering.ContextSize); | |||
1004 | auto *NewFuncPtrStruct = ConstantStruct::get( | |||
1005 | FuncPtrStruct->getType(), OrigRelativeFunOffset, NewContextSize); | |||
1006 | ||||
1007 | Shape.AsyncLowering.AsyncFuncPointer->setInitializer(NewFuncPtrStruct); | |||
1008 | } | |||
1009 | ||||
1010 | static void replaceFrameSize(coro::Shape &Shape) { | |||
1011 | if (Shape.ABI == coro::ABI::Async) | |||
1012 | updateAsyncFuncPointerContextSize(Shape); | |||
1013 | ||||
1014 | if (Shape.CoroSizes.empty()) | |||
1015 | return; | |||
1016 | ||||
1017 | // In the same function all coro.sizes should have the same result type. | |||
1018 | auto *SizeIntrin = Shape.CoroSizes.back(); | |||
1019 | Module *M = SizeIntrin->getModule(); | |||
1020 | const DataLayout &DL = M->getDataLayout(); | |||
1021 | auto Size = DL.getTypeAllocSize(Shape.FrameTy); | |||
1022 | auto *SizeConstant = ConstantInt::get(SizeIntrin->getType(), Size); | |||
1023 | ||||
1024 | for (CoroSizeInst *CS : Shape.CoroSizes) { | |||
1025 | CS->replaceAllUsesWith(SizeConstant); | |||
1026 | CS->eraseFromParent(); | |||
1027 | } | |||
1028 | } | |||
1029 | ||||
1030 | // Create a global constant array containing pointers to functions provided and | |||
1031 | // set Info parameter of CoroBegin to point at this constant. Example: | |||
1032 | // | |||
1033 | // @f.resumers = internal constant [2 x void(%f.frame*)*] | |||
1034 | // [void(%f.frame*)* @f.resume, void(%f.frame*)* @f.destroy] | |||
1035 | // define void @f() { | |||
1036 | // ... | |||
1037 | // call i8* @llvm.coro.begin(i8* null, i32 0, i8* null, | |||
1038 | // i8* bitcast([2 x void(%f.frame*)*] * @f.resumers to i8*)) | |||
1039 | // | |||
1040 | // Assumes that all the functions have the same signature. | |||
1041 | static void setCoroInfo(Function &F, coro::Shape &Shape, | |||
1042 | ArrayRef<Function *> Fns) { | |||
1043 | // This only works under the switch-lowering ABI because coro elision | |||
1044 | // only works on the switch-lowering ABI. | |||
1045 | assert(Shape.ABI == coro::ABI::Switch)((Shape.ABI == coro::ABI::Switch) ? static_cast<void> ( 0) : __assert_fail ("Shape.ABI == coro::ABI::Switch", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 1045, __PRETTY_FUNCTION__)); | |||
1046 | ||||
1047 | SmallVector<Constant *, 4> Args(Fns.begin(), Fns.end()); | |||
1048 | assert(!Args.empty())((!Args.empty()) ? static_cast<void> (0) : __assert_fail ("!Args.empty()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 1048, __PRETTY_FUNCTION__)); | |||
1049 | Function *Part = *Fns.begin(); | |||
1050 | Module *M = Part->getParent(); | |||
1051 | auto *ArrTy = ArrayType::get(Part->getType(), Args.size()); | |||
1052 | ||||
1053 | auto *ConstVal = ConstantArray::get(ArrTy, Args); | |||
1054 | auto *GV = new GlobalVariable(*M, ConstVal->getType(), /*isConstant=*/true, | |||
1055 | GlobalVariable::PrivateLinkage, ConstVal, | |||
1056 | F.getName() + Twine(".resumers")); | |||
1057 | ||||
1058 | // Update coro.begin instruction to refer to this constant. | |||
1059 | LLVMContext &C = F.getContext(); | |||
1060 | auto *BC = ConstantExpr::getPointerCast(GV, Type::getInt8PtrTy(C)); | |||
1061 | Shape.getSwitchCoroId()->setInfo(BC); | |||
1062 | } | |||
1063 | ||||
1064 | // Store addresses of Resume/Destroy/Cleanup functions in the coroutine frame. | |||
1065 | static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn, | |||
1066 | Function *DestroyFn, Function *CleanupFn) { | |||
1067 | assert(Shape.ABI == coro::ABI::Switch)((Shape.ABI == coro::ABI::Switch) ? static_cast<void> ( 0) : __assert_fail ("Shape.ABI == coro::ABI::Switch", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 1067, __PRETTY_FUNCTION__)); | |||
1068 | ||||
1069 | IRBuilder<> Builder(Shape.FramePtr->getNextNode()); | |||
1070 | auto *ResumeAddr = Builder.CreateStructGEP( | |||
1071 | Shape.FrameTy, Shape.FramePtr, coro::Shape::SwitchFieldIndex::Resume, | |||
1072 | "resume.addr"); | |||
1073 | Builder.CreateStore(ResumeFn, ResumeAddr); | |||
1074 | ||||
1075 | Value *DestroyOrCleanupFn = DestroyFn; | |||
1076 | ||||
1077 | CoroIdInst *CoroId = Shape.getSwitchCoroId(); | |||
1078 | if (CoroAllocInst *CA = CoroId->getCoroAlloc()) { | |||
1079 | // If there is a CoroAlloc and it returns false (meaning we elide the | |||
1080 | // allocation, use CleanupFn instead of DestroyFn). | |||
1081 | DestroyOrCleanupFn = Builder.CreateSelect(CA, DestroyFn, CleanupFn); | |||
1082 | } | |||
1083 | ||||
1084 | auto *DestroyAddr = Builder.CreateStructGEP( | |||
1085 | Shape.FrameTy, Shape.FramePtr, coro::Shape::SwitchFieldIndex::Destroy, | |||
1086 | "destroy.addr"); | |||
1087 | Builder.CreateStore(DestroyOrCleanupFn, DestroyAddr); | |||
1088 | } | |||
1089 | ||||
1090 | static void postSplitCleanup(Function &F) { | |||
1091 | removeUnreachableBlocks(F); | |||
1092 | ||||
1093 | // For now, we do a mandatory verification step because we don't | |||
1094 | // entirely trust this pass. Note that we don't want to add a verifier | |||
1095 | // pass to FPM below because it will also verify all the global data. | |||
1096 | if (verifyFunction(F, &errs())) | |||
1097 | report_fatal_error("Broken function"); | |||
1098 | ||||
1099 | legacy::FunctionPassManager FPM(F.getParent()); | |||
1100 | ||||
1101 | FPM.add(createSCCPPass()); | |||
1102 | FPM.add(createCFGSimplificationPass()); | |||
1103 | FPM.add(createEarlyCSEPass()); | |||
1104 | FPM.add(createCFGSimplificationPass()); | |||
1105 | ||||
1106 | FPM.doInitialization(); | |||
1107 | FPM.run(F); | |||
1108 | FPM.doFinalization(); | |||
1109 | } | |||
1110 | ||||
1111 | // Assuming we arrived at the block NewBlock from Prev instruction, store | |||
1112 | // PHI's incoming values in the ResolvedValues map. | |||
1113 | static void | |||
1114 | scanPHIsAndUpdateValueMap(Instruction *Prev, BasicBlock *NewBlock, | |||
1115 | DenseMap<Value *, Value *> &ResolvedValues) { | |||
1116 | auto *PrevBB = Prev->getParent(); | |||
1117 | for (PHINode &PN : NewBlock->phis()) { | |||
1118 | auto V = PN.getIncomingValueForBlock(PrevBB); | |||
1119 | // See if we already resolved it. | |||
1120 | auto VI = ResolvedValues.find(V); | |||
1121 | if (VI != ResolvedValues.end()) | |||
1122 | V = VI->second; | |||
1123 | // Remember the value. | |||
1124 | ResolvedValues[&PN] = V; | |||
1125 | } | |||
1126 | } | |||
1127 | ||||
1128 | // Replace a sequence of branches leading to a ret, with a clone of a ret | |||
1129 | // instruction. Suspend instruction represented by a switch, track the PHI | |||
1130 | // values and select the correct case successor when possible. | |||
1131 | static bool simplifyTerminatorLeadingToRet(Instruction *InitialInst) { | |||
1132 | DenseMap<Value *, Value *> ResolvedValues; | |||
1133 | BasicBlock *UnconditionalSucc = nullptr; | |||
1134 | ||||
1135 | Instruction *I = InitialInst; | |||
1136 | while (I->isTerminator() || | |||
1137 | (isa<CmpInst>(I) && I->getNextNode()->isTerminator())) { | |||
1138 | if (isa<ReturnInst>(I)) { | |||
1139 | if (I != InitialInst) { | |||
1140 | // If InitialInst is an unconditional branch, | |||
1141 | // remove PHI values that come from basic block of InitialInst | |||
1142 | if (UnconditionalSucc) | |||
1143 | UnconditionalSucc->removePredecessor(InitialInst->getParent(), true); | |||
1144 | ReplaceInstWithInst(InitialInst, I->clone()); | |||
1145 | } | |||
1146 | return true; | |||
1147 | } | |||
1148 | if (auto *BR = dyn_cast<BranchInst>(I)) { | |||
1149 | if (BR->isUnconditional()) { | |||
1150 | BasicBlock *BB = BR->getSuccessor(0); | |||
1151 | if (I == InitialInst) | |||
1152 | UnconditionalSucc = BB; | |||
1153 | scanPHIsAndUpdateValueMap(I, BB, ResolvedValues); | |||
1154 | I = BB->getFirstNonPHIOrDbgOrLifetime(); | |||
1155 | continue; | |||
1156 | } | |||
1157 | } else if (auto *CondCmp = dyn_cast<CmpInst>(I)) { | |||
1158 | auto *BR = dyn_cast<BranchInst>(I->getNextNode()); | |||
1159 | if (BR && BR->isConditional() && CondCmp == BR->getCondition()) { | |||
1160 | // If the case number of suspended switch instruction is reduced to | |||
1161 | // 1, then it is simplified to CmpInst in llvm::ConstantFoldTerminator. | |||
1162 | // And the comparsion looks like : %cond = icmp eq i8 %V, constant. | |||
1163 | ConstantInt *CondConst = dyn_cast<ConstantInt>(CondCmp->getOperand(1)); | |||
1164 | if (CondConst && CondCmp->getPredicate() == CmpInst::ICMP_EQ) { | |||
1165 | Value *V = CondCmp->getOperand(0); | |||
1166 | auto it = ResolvedValues.find(V); | |||
1167 | if (it != ResolvedValues.end()) | |||
1168 | V = it->second; | |||
1169 | ||||
1170 | if (ConstantInt *Cond0 = dyn_cast<ConstantInt>(V)) { | |||
1171 | BasicBlock *BB = Cond0->equalsInt(CondConst->getZExtValue()) | |||
1172 | ? BR->getSuccessor(0) | |||
1173 | : BR->getSuccessor(1); | |||
1174 | scanPHIsAndUpdateValueMap(I, BB, ResolvedValues); | |||
1175 | I = BB->getFirstNonPHIOrDbgOrLifetime(); | |||
1176 | continue; | |||
1177 | } | |||
1178 | } | |||
1179 | } | |||
1180 | } else if (auto *SI = dyn_cast<SwitchInst>(I)) { | |||
1181 | Value *V = SI->getCondition(); | |||
1182 | auto it = ResolvedValues.find(V); | |||
1183 | if (it != ResolvedValues.end()) | |||
1184 | V = it->second; | |||
1185 | if (ConstantInt *Cond = dyn_cast<ConstantInt>(V)) { | |||
1186 | BasicBlock *BB = SI->findCaseValue(Cond)->getCaseSuccessor(); | |||
1187 | scanPHIsAndUpdateValueMap(I, BB, ResolvedValues); | |||
1188 | I = BB->getFirstNonPHIOrDbgOrLifetime(); | |||
1189 | continue; | |||
1190 | } | |||
1191 | } | |||
1192 | return false; | |||
1193 | } | |||
1194 | return false; | |||
1195 | } | |||
1196 | ||||
1197 | // Check whether CI obeys the rules of musttail attribute. | |||
1198 | static bool shouldBeMustTail(const CallInst &CI, const Function &F) { | |||
1199 | if (CI.isInlineAsm()) | |||
1200 | return false; | |||
1201 | ||||
1202 | // Match prototypes and calling conventions of resume function. | |||
1203 | FunctionType *CalleeTy = CI.getFunctionType(); | |||
1204 | if (!CalleeTy->getReturnType()->isVoidTy() || (CalleeTy->getNumParams() != 1)) | |||
1205 | return false; | |||
1206 | ||||
1207 | Type *CalleeParmTy = CalleeTy->getParamType(0); | |||
1208 | if (!CalleeParmTy->isPointerTy() || | |||
1209 | (CalleeParmTy->getPointerAddressSpace() != 0)) | |||
1210 | return false; | |||
1211 | ||||
1212 | if (CI.getCallingConv() != F.getCallingConv()) | |||
1213 | return false; | |||
1214 | ||||
1215 | // CI should not has any ABI-impacting function attributes. | |||
1216 | static const Attribute::AttrKind ABIAttrs[] = { | |||
1217 | Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca, | |||
1218 | Attribute::Preallocated, Attribute::InReg, Attribute::Returned, | |||
1219 | Attribute::SwiftSelf, Attribute::SwiftError}; | |||
1220 | AttributeList Attrs = CI.getAttributes(); | |||
1221 | for (auto AK : ABIAttrs) | |||
1222 | if (Attrs.hasParamAttribute(0, AK)) | |||
1223 | return false; | |||
1224 | ||||
1225 | return true; | |||
1226 | } | |||
1227 | ||||
1228 | // Add musttail to any resume instructions that is immediately followed by a | |||
1229 | // suspend (i.e. ret). We do this even in -O0 to support guaranteed tail call | |||
1230 | // for symmetrical coroutine control transfer (C++ Coroutines TS extension). | |||
1231 | // This transformation is done only in the resume part of the coroutine that has | |||
1232 | // identical signature and calling convention as the coro.resume call. | |||
1233 | static void addMustTailToCoroResumes(Function &F) { | |||
1234 | bool changed = false; | |||
1235 | ||||
1236 | // Collect potential resume instructions. | |||
1237 | SmallVector<CallInst *, 4> Resumes; | |||
1238 | for (auto &I : instructions(F)) | |||
1239 | if (auto *Call = dyn_cast<CallInst>(&I)) | |||
1240 | if (shouldBeMustTail(*Call, F)) | |||
1241 | Resumes.push_back(Call); | |||
1242 | ||||
1243 | // Set musttail on those that are followed by a ret instruction. | |||
1244 | for (CallInst *Call : Resumes) | |||
1245 | if (simplifyTerminatorLeadingToRet(Call->getNextNode())) { | |||
1246 | Call->setTailCallKind(CallInst::TCK_MustTail); | |||
1247 | changed = true; | |||
1248 | } | |||
1249 | ||||
1250 | if (changed) | |||
1251 | removeUnreachableBlocks(F); | |||
1252 | } | |||
1253 | ||||
1254 | // Coroutine has no suspend points. Remove heap allocation for the coroutine | |||
1255 | // frame if possible. | |||
1256 | static void handleNoSuspendCoroutine(coro::Shape &Shape) { | |||
1257 | auto *CoroBegin = Shape.CoroBegin; | |||
1258 | auto *CoroId = CoroBegin->getId(); | |||
1259 | auto *AllocInst = CoroId->getCoroAlloc(); | |||
1260 | switch (Shape.ABI) { | |||
1261 | case coro::ABI::Switch: { | |||
1262 | auto SwitchId = cast<CoroIdInst>(CoroId); | |||
1263 | coro::replaceCoroFree(SwitchId, /*Elide=*/AllocInst != nullptr); | |||
1264 | if (AllocInst) { | |||
1265 | IRBuilder<> Builder(AllocInst); | |||
1266 | auto *Frame = Builder.CreateAlloca(Shape.FrameTy); | |||
1267 | Frame->setAlignment(Shape.FrameAlign); | |||
1268 | auto *VFrame = Builder.CreateBitCast(Frame, Builder.getInt8PtrTy()); | |||
1269 | AllocInst->replaceAllUsesWith(Builder.getFalse()); | |||
1270 | AllocInst->eraseFromParent(); | |||
1271 | CoroBegin->replaceAllUsesWith(VFrame); | |||
1272 | } else { | |||
1273 | CoroBegin->replaceAllUsesWith(CoroBegin->getMem()); | |||
1274 | } | |||
1275 | break; | |||
1276 | } | |||
1277 | case coro::ABI::Async: | |||
1278 | case coro::ABI::Retcon: | |||
1279 | case coro::ABI::RetconOnce: | |||
1280 | CoroBegin->replaceAllUsesWith(UndefValue::get(CoroBegin->getType())); | |||
1281 | break; | |||
1282 | } | |||
1283 | ||||
1284 | CoroBegin->eraseFromParent(); | |||
1285 | } | |||
1286 | ||||
1287 | // SimplifySuspendPoint needs to check that there is no calls between | |||
1288 | // coro_save and coro_suspend, since any of the calls may potentially resume | |||
1289 | // the coroutine and if that is the case we cannot eliminate the suspend point. | |||
1290 | static bool hasCallsInBlockBetween(Instruction *From, Instruction *To) { | |||
1291 | for (Instruction *I = From; I != To; I = I->getNextNode()) { | |||
1292 | // Assume that no intrinsic can resume the coroutine. | |||
1293 | if (isa<IntrinsicInst>(I)) | |||
1294 | continue; | |||
1295 | ||||
1296 | if (isa<CallBase>(I)) | |||
1297 | return true; | |||
1298 | } | |||
1299 | return false; | |||
1300 | } | |||
1301 | ||||
1302 | static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB) { | |||
1303 | SmallPtrSet<BasicBlock *, 8> Set; | |||
1304 | SmallVector<BasicBlock *, 8> Worklist; | |||
1305 | ||||
1306 | Set.insert(SaveBB); | |||
1307 | Worklist.push_back(ResDesBB); | |||
1308 | ||||
1309 | // Accumulate all blocks between SaveBB and ResDesBB. Because CoroSaveIntr | |||
1310 | // returns a token consumed by suspend instruction, all blocks in between | |||
1311 | // will have to eventually hit SaveBB when going backwards from ResDesBB. | |||
1312 | while (!Worklist.empty()) { | |||
1313 | auto *BB = Worklist.pop_back_val(); | |||
1314 | Set.insert(BB); | |||
1315 | for (auto *Pred : predecessors(BB)) | |||
1316 | if (Set.count(Pred) == 0) | |||
1317 | Worklist.push_back(Pred); | |||
1318 | } | |||
1319 | ||||
1320 | // SaveBB and ResDesBB are checked separately in hasCallsBetween. | |||
1321 | Set.erase(SaveBB); | |||
1322 | Set.erase(ResDesBB); | |||
1323 | ||||
1324 | for (auto *BB : Set) | |||
1325 | if (hasCallsInBlockBetween(BB->getFirstNonPHI(), nullptr)) | |||
1326 | return true; | |||
1327 | ||||
1328 | return false; | |||
1329 | } | |||
1330 | ||||
1331 | static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy) { | |||
1332 | auto *SaveBB = Save->getParent(); | |||
1333 | auto *ResumeOrDestroyBB = ResumeOrDestroy->getParent(); | |||
1334 | ||||
1335 | if (SaveBB == ResumeOrDestroyBB) | |||
1336 | return hasCallsInBlockBetween(Save->getNextNode(), ResumeOrDestroy); | |||
1337 | ||||
1338 | // Any calls from Save to the end of the block? | |||
1339 | if (hasCallsInBlockBetween(Save->getNextNode(), nullptr)) | |||
1340 | return true; | |||
1341 | ||||
1342 | // Any calls from begging of the block up to ResumeOrDestroy? | |||
1343 | if (hasCallsInBlockBetween(ResumeOrDestroyBB->getFirstNonPHI(), | |||
1344 | ResumeOrDestroy)) | |||
1345 | return true; | |||
1346 | ||||
1347 | // Any calls in all of the blocks between SaveBB and ResumeOrDestroyBB? | |||
1348 | if (hasCallsInBlocksBetween(SaveBB, ResumeOrDestroyBB)) | |||
1349 | return true; | |||
1350 | ||||
1351 | return false; | |||
1352 | } | |||
1353 | ||||
1354 | // If a SuspendIntrin is preceded by Resume or Destroy, we can eliminate the | |||
1355 | // suspend point and replace it with nornal control flow. | |||
1356 | static bool simplifySuspendPoint(CoroSuspendInst *Suspend, | |||
1357 | CoroBeginInst *CoroBegin) { | |||
1358 | Instruction *Prev = Suspend->getPrevNode(); | |||
1359 | if (!Prev) { | |||
1360 | auto *Pred = Suspend->getParent()->getSinglePredecessor(); | |||
1361 | if (!Pred) | |||
1362 | return false; | |||
1363 | Prev = Pred->getTerminator(); | |||
1364 | } | |||
1365 | ||||
1366 | CallBase *CB = dyn_cast<CallBase>(Prev); | |||
1367 | if (!CB) | |||
1368 | return false; | |||
1369 | ||||
1370 | auto *Callee = CB->getCalledOperand()->stripPointerCasts(); | |||
1371 | ||||
1372 | // See if the callsite is for resumption or destruction of the coroutine. | |||
1373 | auto *SubFn = dyn_cast<CoroSubFnInst>(Callee); | |||
1374 | if (!SubFn) | |||
1375 | return false; | |||
1376 | ||||
1377 | // Does not refer to the current coroutine, we cannot do anything with it. | |||
1378 | if (SubFn->getFrame() != CoroBegin) | |||
1379 | return false; | |||
1380 | ||||
1381 | // See if the transformation is safe. Specifically, see if there are any | |||
1382 | // calls in between Save and CallInstr. They can potenitally resume the | |||
1383 | // coroutine rendering this optimization unsafe. | |||
1384 | auto *Save = Suspend->getCoroSave(); | |||
1385 | if (hasCallsBetween(Save, CB)) | |||
1386 | return false; | |||
1387 | ||||
1388 | // Replace llvm.coro.suspend with the value that results in resumption over | |||
1389 | // the resume or cleanup path. | |||
1390 | Suspend->replaceAllUsesWith(SubFn->getRawIndex()); | |||
1391 | Suspend->eraseFromParent(); | |||
1392 | Save->eraseFromParent(); | |||
1393 | ||||
1394 | // No longer need a call to coro.resume or coro.destroy. | |||
1395 | if (auto *Invoke = dyn_cast<InvokeInst>(CB)) { | |||
1396 | BranchInst::Create(Invoke->getNormalDest(), Invoke); | |||
1397 | } | |||
1398 | ||||
1399 | // Grab the CalledValue from CB before erasing the CallInstr. | |||
1400 | auto *CalledValue = CB->getCalledOperand(); | |||
1401 | CB->eraseFromParent(); | |||
1402 | ||||
1403 | // If no more users remove it. Usually it is a bitcast of SubFn. | |||
1404 | if (CalledValue != SubFn && CalledValue->user_empty()) | |||
1405 | if (auto *I = dyn_cast<Instruction>(CalledValue)) | |||
1406 | I->eraseFromParent(); | |||
1407 | ||||
1408 | // Now we are good to remove SubFn. | |||
1409 | if (SubFn->user_empty()) | |||
1410 | SubFn->eraseFromParent(); | |||
1411 | ||||
1412 | return true; | |||
1413 | } | |||
1414 | ||||
1415 | // Remove suspend points that are simplified. | |||
1416 | static void simplifySuspendPoints(coro::Shape &Shape) { | |||
1417 | // Currently, the only simplification we do is switch-lowering-specific. | |||
1418 | if (Shape.ABI != coro::ABI::Switch) | |||
1419 | return; | |||
1420 | ||||
1421 | auto &S = Shape.CoroSuspends; | |||
1422 | size_t I = 0, N = S.size(); | |||
1423 | if (N == 0) | |||
1424 | return; | |||
1425 | while (true) { | |||
1426 | auto SI = cast<CoroSuspendInst>(S[I]); | |||
1427 | // Leave final.suspend to handleFinalSuspend since it is undefined behavior | |||
1428 | // to resume a coroutine suspended at the final suspend point. | |||
1429 | if (!SI->isFinal() && simplifySuspendPoint(SI, Shape.CoroBegin)) { | |||
1430 | if (--N == I) | |||
1431 | break; | |||
1432 | std::swap(S[I], S[N]); | |||
1433 | continue; | |||
1434 | } | |||
1435 | if (++I == N) | |||
1436 | break; | |||
1437 | } | |||
1438 | S.resize(N); | |||
1439 | } | |||
1440 | ||||
1441 | static void splitSwitchCoroutine(Function &F, coro::Shape &Shape, | |||
1442 | SmallVectorImpl<Function *> &Clones) { | |||
1443 | assert(Shape.ABI == coro::ABI::Switch)((Shape.ABI == coro::ABI::Switch) ? static_cast<void> ( 0) : __assert_fail ("Shape.ABI == coro::ABI::Switch", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 1443, __PRETTY_FUNCTION__)); | |||
1444 | ||||
1445 | createResumeEntryBlock(F, Shape); | |||
1446 | auto ResumeClone = createClone(F, ".resume", Shape, | |||
1447 | CoroCloner::Kind::SwitchResume); | |||
1448 | auto DestroyClone = createClone(F, ".destroy", Shape, | |||
1449 | CoroCloner::Kind::SwitchUnwind); | |||
1450 | auto CleanupClone = createClone(F, ".cleanup", Shape, | |||
1451 | CoroCloner::Kind::SwitchCleanup); | |||
1452 | ||||
1453 | postSplitCleanup(*ResumeClone); | |||
1454 | postSplitCleanup(*DestroyClone); | |||
1455 | postSplitCleanup(*CleanupClone); | |||
1456 | ||||
1457 | addMustTailToCoroResumes(*ResumeClone); | |||
1458 | ||||
1459 | // Store addresses resume/destroy/cleanup functions in the coroutine frame. | |||
1460 | updateCoroFrame(Shape, ResumeClone, DestroyClone, CleanupClone); | |||
1461 | ||||
1462 | assert(Clones.empty())((Clones.empty()) ? static_cast<void> (0) : __assert_fail ("Clones.empty()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 1462, __PRETTY_FUNCTION__)); | |||
1463 | Clones.push_back(ResumeClone); | |||
1464 | Clones.push_back(DestroyClone); | |||
1465 | Clones.push_back(CleanupClone); | |||
1466 | ||||
1467 | // Create a constant array referring to resume/destroy/clone functions pointed | |||
1468 | // by the last argument of @llvm.coro.info, so that CoroElide pass can | |||
1469 | // determined correct function to call. | |||
1470 | setCoroInfo(F, Shape, Clones); | |||
1471 | } | |||
1472 | ||||
1473 | static void replaceAsyncResumeFunction(CoroSuspendAsyncInst *Suspend, | |||
1474 | Value *Continuation) { | |||
1475 | auto *ResumeIntrinsic = Suspend->getResumeFunction(); | |||
1476 | auto &Context = Suspend->getParent()->getParent()->getContext(); | |||
1477 | auto *Int8PtrTy = Type::getInt8PtrTy(Context); | |||
1478 | ||||
1479 | IRBuilder<> Builder(ResumeIntrinsic); | |||
1480 | auto *Val = Builder.CreateBitOrPointerCast(Continuation, Int8PtrTy); | |||
1481 | ResumeIntrinsic->replaceAllUsesWith(Val); | |||
1482 | ResumeIntrinsic->eraseFromParent(); | |||
1483 | Suspend->setOperand(CoroSuspendAsyncInst::ResumeFunctionArg, | |||
1484 | UndefValue::get(Int8PtrTy)); | |||
1485 | } | |||
1486 | ||||
1487 | /// Coerce the arguments in \p FnArgs according to \p FnTy in \p CallArgs. | |||
1488 | static void coerceArguments(IRBuilder<> &Builder, FunctionType *FnTy, | |||
1489 | ArrayRef<Value *> FnArgs, | |||
1490 | SmallVectorImpl<Value *> &CallArgs) { | |||
1491 | size_t ArgIdx = 0; | |||
1492 | for (auto paramTy : FnTy->params()) { | |||
1493 | assert(ArgIdx < FnArgs.size())((ArgIdx < FnArgs.size()) ? static_cast<void> (0) : __assert_fail ("ArgIdx < FnArgs.size()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 1493, __PRETTY_FUNCTION__)); | |||
1494 | if (paramTy != FnArgs[ArgIdx]->getType()) | |||
1495 | CallArgs.push_back( | |||
1496 | Builder.CreateBitOrPointerCast(FnArgs[ArgIdx], paramTy)); | |||
1497 | else | |||
1498 | CallArgs.push_back(FnArgs[ArgIdx]); | |||
1499 | ++ArgIdx; | |||
1500 | } | |||
1501 | } | |||
1502 | ||||
1503 | CallInst *coro::createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, | |||
1504 | ArrayRef<Value *> Arguments, | |||
1505 | IRBuilder<> &Builder) { | |||
1506 | auto *FnTy = | |||
1507 | cast<FunctionType>(MustTailCallFn->getType()->getPointerElementType()); | |||
1508 | // Coerce the arguments, llvm optimizations seem to ignore the types in | |||
1509 | // vaarg functions and throws away casts in optimized mode. | |||
1510 | SmallVector<Value *, 8> CallArgs; | |||
1511 | coerceArguments(Builder, FnTy, Arguments, CallArgs); | |||
1512 | ||||
1513 | auto *TailCall = Builder.CreateCall(FnTy, MustTailCallFn, CallArgs); | |||
1514 | TailCall->setTailCallKind(CallInst::TCK_MustTail); | |||
1515 | TailCall->setDebugLoc(Loc); | |||
1516 | TailCall->setCallingConv(MustTailCallFn->getCallingConv()); | |||
1517 | return TailCall; | |||
1518 | } | |||
1519 | ||||
1520 | static void splitAsyncCoroutine(Function &F, coro::Shape &Shape, | |||
1521 | SmallVectorImpl<Function *> &Clones) { | |||
1522 | assert(Shape.ABI == coro::ABI::Async)((Shape.ABI == coro::ABI::Async) ? static_cast<void> (0 ) : __assert_fail ("Shape.ABI == coro::ABI::Async", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 1522, __PRETTY_FUNCTION__)); | |||
1523 | assert(Clones.empty())((Clones.empty()) ? static_cast<void> (0) : __assert_fail ("Clones.empty()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 1523, __PRETTY_FUNCTION__)); | |||
1524 | // Reset various things that the optimizer might have decided it | |||
1525 | // "knows" about the coroutine function due to not seeing a return. | |||
1526 | F.removeFnAttr(Attribute::NoReturn); | |||
1527 | F.removeAttribute(AttributeList::ReturnIndex, Attribute::NoAlias); | |||
1528 | F.removeAttribute(AttributeList::ReturnIndex, Attribute::NonNull); | |||
1529 | ||||
1530 | auto &Context = F.getContext(); | |||
1531 | auto *Int8PtrTy = Type::getInt8PtrTy(Context); | |||
1532 | ||||
1533 | auto *Id = cast<CoroIdAsyncInst>(Shape.CoroBegin->getId()); | |||
1534 | IRBuilder<> Builder(Id); | |||
1535 | ||||
1536 | auto *FramePtr = Id->getStorage(); | |||
1537 | FramePtr = Builder.CreateBitOrPointerCast(FramePtr, Int8PtrTy); | |||
1538 | FramePtr = Builder.CreateConstInBoundsGEP1_32( | |||
1539 | Type::getInt8Ty(Context), FramePtr, Shape.AsyncLowering.FrameOffset, | |||
1540 | "async.ctx.frameptr"); | |||
1541 | ||||
1542 | // Map all uses of llvm.coro.begin to the allocated frame pointer. | |||
1543 | { | |||
1544 | // Make sure we don't invalidate Shape.FramePtr. | |||
1545 | TrackingVH<Instruction> Handle(Shape.FramePtr); | |||
1546 | Shape.CoroBegin->replaceAllUsesWith(FramePtr); | |||
1547 | Shape.FramePtr = Handle.getValPtr(); | |||
1548 | } | |||
1549 | ||||
1550 | // Create all the functions in order after the main function. | |||
1551 | auto NextF = std::next(F.getIterator()); | |||
1552 | ||||
1553 | // Create a continuation function for each of the suspend points. | |||
1554 | Clones.reserve(Shape.CoroSuspends.size()); | |||
1555 | for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) { | |||
1556 | auto *Suspend = cast<CoroSuspendAsyncInst>(Shape.CoroSuspends[Idx]); | |||
1557 | ||||
1558 | // Create the clone declaration. | |||
1559 | auto *Continuation = createCloneDeclaration( | |||
1560 | F, Shape, ".resume." + Twine(Idx), NextF, Suspend); | |||
1561 | Clones.push_back(Continuation); | |||
1562 | ||||
1563 | // Insert a branch to a new return block immediately before the suspend | |||
1564 | // point. | |||
1565 | auto *SuspendBB = Suspend->getParent(); | |||
1566 | auto *NewSuspendBB = SuspendBB->splitBasicBlock(Suspend); | |||
1567 | auto *Branch = cast<BranchInst>(SuspendBB->getTerminator()); | |||
1568 | ||||
1569 | // Place it before the first suspend. | |||
1570 | auto *ReturnBB = | |||
1571 | BasicBlock::Create(F.getContext(), "coro.return", &F, NewSuspendBB); | |||
1572 | Branch->setSuccessor(0, ReturnBB); | |||
1573 | ||||
1574 | IRBuilder<> Builder(ReturnBB); | |||
1575 | ||||
1576 | // Insert the call to the tail call function and inline it. | |||
1577 | auto *Fn = Suspend->getMustTailCallFunction(); | |||
1578 | SmallVector<Value *, 8> Args(Suspend->args()); | |||
1579 | auto FnArgs = ArrayRef<Value *>(Args).drop_front( | |||
1580 | CoroSuspendAsyncInst::MustTailCallFuncArg + 1); | |||
1581 | auto *TailCall = | |||
1582 | coro::createMustTailCall(Suspend->getDebugLoc(), Fn, FnArgs, Builder); | |||
1583 | Builder.CreateRetVoid(); | |||
1584 | InlineFunctionInfo FnInfo; | |||
1585 | auto InlineRes = InlineFunction(*TailCall, FnInfo); | |||
1586 | assert(InlineRes.isSuccess() && "Expected inlining to succeed")((InlineRes.isSuccess() && "Expected inlining to succeed" ) ? static_cast<void> (0) : __assert_fail ("InlineRes.isSuccess() && \"Expected inlining to succeed\"" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 1586, __PRETTY_FUNCTION__)); | |||
1587 | (void)InlineRes; | |||
1588 | ||||
1589 | // Replace the lvm.coro.async.resume intrisic call. | |||
1590 | replaceAsyncResumeFunction(Suspend, Continuation); | |||
1591 | } | |||
1592 | ||||
1593 | assert(Clones.size() == Shape.CoroSuspends.size())((Clones.size() == Shape.CoroSuspends.size()) ? static_cast< void> (0) : __assert_fail ("Clones.size() == Shape.CoroSuspends.size()" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 1593, __PRETTY_FUNCTION__)); | |||
1594 | for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) { | |||
1595 | auto *Suspend = Shape.CoroSuspends[Idx]; | |||
1596 | auto *Clone = Clones[Idx]; | |||
1597 | ||||
1598 | CoroCloner(F, "resume." + Twine(Idx), Shape, Clone, Suspend).create(); | |||
1599 | } | |||
1600 | } | |||
1601 | ||||
1602 | static void splitRetconCoroutine(Function &F, coro::Shape &Shape, | |||
1603 | SmallVectorImpl<Function *> &Clones) { | |||
1604 | assert(Shape.ABI == coro::ABI::Retcon ||((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce ) ? static_cast<void> (0) : __assert_fail ("Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 1605, __PRETTY_FUNCTION__)) | |||
1605 | Shape.ABI == coro::ABI::RetconOnce)((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce ) ? static_cast<void> (0) : __assert_fail ("Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 1605, __PRETTY_FUNCTION__)); | |||
1606 | assert(Clones.empty())((Clones.empty()) ? static_cast<void> (0) : __assert_fail ("Clones.empty()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 1606, __PRETTY_FUNCTION__)); | |||
1607 | ||||
1608 | // Reset various things that the optimizer might have decided it | |||
1609 | // "knows" about the coroutine function due to not seeing a return. | |||
1610 | F.removeFnAttr(Attribute::NoReturn); | |||
1611 | F.removeAttribute(AttributeList::ReturnIndex, Attribute::NoAlias); | |||
1612 | F.removeAttribute(AttributeList::ReturnIndex, Attribute::NonNull); | |||
1613 | ||||
1614 | // Allocate the frame. | |||
1615 | auto *Id = cast<AnyCoroIdRetconInst>(Shape.CoroBegin->getId()); | |||
1616 | Value *RawFramePtr; | |||
1617 | if (Shape.RetconLowering.IsFrameInlineInStorage) { | |||
1618 | RawFramePtr = Id->getStorage(); | |||
1619 | } else { | |||
1620 | IRBuilder<> Builder(Id); | |||
1621 | ||||
1622 | // Determine the size of the frame. | |||
1623 | const DataLayout &DL = F.getParent()->getDataLayout(); | |||
1624 | auto Size = DL.getTypeAllocSize(Shape.FrameTy); | |||
1625 | ||||
1626 | // Allocate. We don't need to update the call graph node because we're | |||
1627 | // going to recompute it from scratch after splitting. | |||
1628 | // FIXME: pass the required alignment | |||
1629 | RawFramePtr = Shape.emitAlloc(Builder, Builder.getInt64(Size), nullptr); | |||
1630 | RawFramePtr = | |||
1631 | Builder.CreateBitCast(RawFramePtr, Shape.CoroBegin->getType()); | |||
1632 | ||||
1633 | // Stash the allocated frame pointer in the continuation storage. | |||
1634 | auto Dest = Builder.CreateBitCast(Id->getStorage(), | |||
1635 | RawFramePtr->getType()->getPointerTo()); | |||
1636 | Builder.CreateStore(RawFramePtr, Dest); | |||
1637 | } | |||
1638 | ||||
1639 | // Map all uses of llvm.coro.begin to the allocated frame pointer. | |||
1640 | { | |||
1641 | // Make sure we don't invalidate Shape.FramePtr. | |||
1642 | TrackingVH<Instruction> Handle(Shape.FramePtr); | |||
1643 | Shape.CoroBegin->replaceAllUsesWith(RawFramePtr); | |||
1644 | Shape.FramePtr = Handle.getValPtr(); | |||
1645 | } | |||
1646 | ||||
1647 | // Create a unique return block. | |||
1648 | BasicBlock *ReturnBB = nullptr; | |||
1649 | SmallVector<PHINode *, 4> ReturnPHIs; | |||
1650 | ||||
1651 | // Create all the functions in order after the main function. | |||
1652 | auto NextF = std::next(F.getIterator()); | |||
1653 | ||||
1654 | // Create a continuation function for each of the suspend points. | |||
1655 | Clones.reserve(Shape.CoroSuspends.size()); | |||
1656 | for (size_t i = 0, e = Shape.CoroSuspends.size(); i != e; ++i) { | |||
1657 | auto Suspend = cast<CoroSuspendRetconInst>(Shape.CoroSuspends[i]); | |||
1658 | ||||
1659 | // Create the clone declaration. | |||
1660 | auto Continuation = | |||
1661 | createCloneDeclaration(F, Shape, ".resume." + Twine(i), NextF, nullptr); | |||
1662 | Clones.push_back(Continuation); | |||
1663 | ||||
1664 | // Insert a branch to the unified return block immediately before | |||
1665 | // the suspend point. | |||
1666 | auto SuspendBB = Suspend->getParent(); | |||
1667 | auto NewSuspendBB = SuspendBB->splitBasicBlock(Suspend); | |||
1668 | auto Branch = cast<BranchInst>(SuspendBB->getTerminator()); | |||
1669 | ||||
1670 | // Create the unified return block. | |||
1671 | if (!ReturnBB) { | |||
1672 | // Place it before the first suspend. | |||
1673 | ReturnBB = BasicBlock::Create(F.getContext(), "coro.return", &F, | |||
1674 | NewSuspendBB); | |||
1675 | Shape.RetconLowering.ReturnBlock = ReturnBB; | |||
1676 | ||||
1677 | IRBuilder<> Builder(ReturnBB); | |||
1678 | ||||
1679 | // Create PHIs for all the return values. | |||
1680 | assert(ReturnPHIs.empty())((ReturnPHIs.empty()) ? static_cast<void> (0) : __assert_fail ("ReturnPHIs.empty()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 1680, __PRETTY_FUNCTION__)); | |||
1681 | ||||
1682 | // First, the continuation. | |||
1683 | ReturnPHIs.push_back(Builder.CreatePHI(Continuation->getType(), | |||
1684 | Shape.CoroSuspends.size())); | |||
1685 | ||||
1686 | // Next, all the directly-yielded values. | |||
1687 | for (auto ResultTy : Shape.getRetconResultTypes()) | |||
1688 | ReturnPHIs.push_back(Builder.CreatePHI(ResultTy, | |||
1689 | Shape.CoroSuspends.size())); | |||
1690 | ||||
1691 | // Build the return value. | |||
1692 | auto RetTy = F.getReturnType(); | |||
1693 | ||||
1694 | // Cast the continuation value if necessary. | |||
1695 | // We can't rely on the types matching up because that type would | |||
1696 | // have to be infinite. | |||
1697 | auto CastedContinuationTy = | |||
1698 | (ReturnPHIs.size() == 1 ? RetTy : RetTy->getStructElementType(0)); | |||
1699 | auto *CastedContinuation = | |||
1700 | Builder.CreateBitCast(ReturnPHIs[0], CastedContinuationTy); | |||
1701 | ||||
1702 | Value *RetV; | |||
1703 | if (ReturnPHIs.size() == 1) { | |||
1704 | RetV = CastedContinuation; | |||
1705 | } else { | |||
1706 | RetV = UndefValue::get(RetTy); | |||
1707 | RetV = Builder.CreateInsertValue(RetV, CastedContinuation, 0); | |||
1708 | for (size_t I = 1, E = ReturnPHIs.size(); I != E; ++I) | |||
1709 | RetV = Builder.CreateInsertValue(RetV, ReturnPHIs[I], I); | |||
1710 | } | |||
1711 | ||||
1712 | Builder.CreateRet(RetV); | |||
1713 | } | |||
1714 | ||||
1715 | // Branch to the return block. | |||
1716 | Branch->setSuccessor(0, ReturnBB); | |||
1717 | ReturnPHIs[0]->addIncoming(Continuation, SuspendBB); | |||
1718 | size_t NextPHIIndex = 1; | |||
1719 | for (auto &VUse : Suspend->value_operands()) | |||
1720 | ReturnPHIs[NextPHIIndex++]->addIncoming(&*VUse, SuspendBB); | |||
1721 | assert(NextPHIIndex == ReturnPHIs.size())((NextPHIIndex == ReturnPHIs.size()) ? static_cast<void> (0) : __assert_fail ("NextPHIIndex == ReturnPHIs.size()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 1721, __PRETTY_FUNCTION__)); | |||
1722 | } | |||
1723 | ||||
1724 | assert(Clones.size() == Shape.CoroSuspends.size())((Clones.size() == Shape.CoroSuspends.size()) ? static_cast< void> (0) : __assert_fail ("Clones.size() == Shape.CoroSuspends.size()" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 1724, __PRETTY_FUNCTION__)); | |||
1725 | for (size_t i = 0, e = Shape.CoroSuspends.size(); i != e; ++i) { | |||
1726 | auto Suspend = Shape.CoroSuspends[i]; | |||
1727 | auto Clone = Clones[i]; | |||
1728 | ||||
1729 | CoroCloner(F, "resume." + Twine(i), Shape, Clone, Suspend).create(); | |||
1730 | } | |||
1731 | } | |||
1732 | ||||
1733 | namespace { | |||
1734 | class PrettyStackTraceFunction : public PrettyStackTraceEntry { | |||
1735 | Function &F; | |||
1736 | public: | |||
1737 | PrettyStackTraceFunction(Function &F) : F(F) {} | |||
1738 | void print(raw_ostream &OS) const override { | |||
1739 | OS << "While splitting coroutine "; | |||
1740 | F.printAsOperand(OS, /*print type*/ false, F.getParent()); | |||
1741 | OS << "\n"; | |||
1742 | } | |||
1743 | }; | |||
1744 | } | |||
1745 | ||||
1746 | static coro::Shape splitCoroutine(Function &F, | |||
1747 | SmallVectorImpl<Function *> &Clones, | |||
1748 | bool ReuseFrameSlot) { | |||
1749 | PrettyStackTraceFunction prettyStackTrace(F); | |||
1750 | ||||
1751 | // The suspend-crossing algorithm in buildCoroutineFrame get tripped | |||
1752 | // up by uses in unreachable blocks, so remove them as a first pass. | |||
1753 | removeUnreachableBlocks(F); | |||
1754 | ||||
1755 | coro::Shape Shape(F, ReuseFrameSlot); | |||
1756 | if (!Shape.CoroBegin) | |||
1757 | return Shape; | |||
1758 | ||||
1759 | simplifySuspendPoints(Shape); | |||
1760 | buildCoroutineFrame(F, Shape); | |||
1761 | replaceFrameSize(Shape); | |||
1762 | ||||
1763 | // If there are no suspend points, no split required, just remove | |||
1764 | // the allocation and deallocation blocks, they are not needed. | |||
1765 | if (Shape.CoroSuspends.empty()) { | |||
1766 | handleNoSuspendCoroutine(Shape); | |||
1767 | } else { | |||
1768 | switch (Shape.ABI) { | |||
1769 | case coro::ABI::Switch: | |||
1770 | splitSwitchCoroutine(F, Shape, Clones); | |||
1771 | break; | |||
1772 | case coro::ABI::Async: | |||
1773 | splitAsyncCoroutine(F, Shape, Clones); | |||
1774 | break; | |||
1775 | case coro::ABI::Retcon: | |||
1776 | case coro::ABI::RetconOnce: | |||
1777 | splitRetconCoroutine(F, Shape, Clones); | |||
1778 | break; | |||
1779 | } | |||
1780 | } | |||
1781 | ||||
1782 | // Replace all the swifterror operations in the original function. | |||
1783 | // This invalidates SwiftErrorOps in the Shape. | |||
1784 | replaceSwiftErrorOps(F, Shape, nullptr); | |||
1785 | ||||
1786 | return Shape; | |||
1787 | } | |||
1788 | ||||
1789 | static void | |||
1790 | updateCallGraphAfterCoroutineSplit(Function &F, const coro::Shape &Shape, | |||
1791 | const SmallVectorImpl<Function *> &Clones, | |||
1792 | CallGraph &CG, CallGraphSCC &SCC) { | |||
1793 | if (!Shape.CoroBegin) | |||
1794 | return; | |||
1795 | ||||
1796 | removeCoroEnds(Shape, &CG); | |||
1797 | postSplitCleanup(F); | |||
1798 | ||||
1799 | // Update call graph and add the functions we created to the SCC. | |||
1800 | coro::updateCallGraph(F, Clones, CG, SCC); | |||
1801 | } | |||
1802 | ||||
1803 | static void updateCallGraphAfterCoroutineSplit( | |||
1804 | LazyCallGraph::Node &N, const coro::Shape &Shape, | |||
1805 | const SmallVectorImpl<Function *> &Clones, LazyCallGraph::SCC &C, | |||
1806 | LazyCallGraph &CG, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, | |||
1807 | FunctionAnalysisManager &FAM) { | |||
1808 | if (!Shape.CoroBegin) | |||
1809 | return; | |||
1810 | ||||
1811 | for (llvm::AnyCoroEndInst *End : Shape.CoroEnds) { | |||
1812 | auto &Context = End->getContext(); | |||
1813 | End->replaceAllUsesWith(ConstantInt::getFalse(Context)); | |||
1814 | End->eraseFromParent(); | |||
1815 | } | |||
1816 | ||||
1817 | if (!Clones.empty()) { | |||
1818 | switch (Shape.ABI) { | |||
1819 | case coro::ABI::Switch: | |||
1820 | // Each clone in the Switch lowering is independent of the other clones. | |||
1821 | // Let the LazyCallGraph know about each one separately. | |||
1822 | for (Function *Clone : Clones) | |||
1823 | CG.addSplitFunction(N.getFunction(), *Clone); | |||
1824 | break; | |||
1825 | case coro::ABI::Async: | |||
1826 | case coro::ABI::Retcon: | |||
1827 | case coro::ABI::RetconOnce: | |||
1828 | // Each clone in the Async/Retcon lowering references of the other clones. | |||
1829 | // Let the LazyCallGraph know about all of them at once. | |||
1830 | if (!Clones.empty()) | |||
1831 | CG.addSplitRefRecursiveFunctions(N.getFunction(), Clones); | |||
1832 | break; | |||
1833 | } | |||
1834 | ||||
1835 | // Let the CGSCC infra handle the changes to the original function. | |||
1836 | updateCGAndAnalysisManagerForCGSCCPass(CG, C, N, AM, UR, FAM); | |||
1837 | } | |||
1838 | ||||
1839 | // Do some cleanup and let the CGSCC infra see if we've cleaned up any edges | |||
1840 | // to the split functions. | |||
1841 | postSplitCleanup(N.getFunction()); | |||
1842 | updateCGAndAnalysisManagerForFunctionPass(CG, C, N, AM, UR, FAM); | |||
1843 | } | |||
1844 | ||||
1845 | // When we see the coroutine the first time, we insert an indirect call to a | |||
1846 | // devirt trigger function and mark the coroutine that it is now ready for | |||
1847 | // split. | |||
1848 | // Async lowering uses this after it has split the function to restart the | |||
1849 | // pipeline. | |||
1850 | static void prepareForSplit(Function &F, CallGraph &CG, | |||
1851 | bool MarkForAsyncRestart = false) { | |||
1852 | Module &M = *F.getParent(); | |||
1853 | LLVMContext &Context = F.getContext(); | |||
1854 | #ifndef NDEBUG | |||
1855 | Function *DevirtFn = M.getFunction(CORO_DEVIRT_TRIGGER_FN"coro.devirt.trigger"); | |||
1856 | assert(DevirtFn && "coro.devirt.trigger function not found")((DevirtFn && "coro.devirt.trigger function not found" ) ? static_cast<void> (0) : __assert_fail ("DevirtFn && \"coro.devirt.trigger function not found\"" , "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp" , 1856, __PRETTY_FUNCTION__)); | |||
1857 | #endif | |||
1858 | ||||
1859 | F.addFnAttr(CORO_PRESPLIT_ATTR"coroutine.presplit", MarkForAsyncRestart | |||
1860 | ? ASYNC_RESTART_AFTER_SPLIT"2" | |||
1861 | : PREPARED_FOR_SPLIT"1"); | |||
1862 | ||||
1863 | // Insert an indirect call sequence that will be devirtualized by CoroElide | |||
1864 | // pass: | |||
1865 | // %0 = call i8* @llvm.coro.subfn.addr(i8* null, i8 -1) | |||
1866 | // %1 = bitcast i8* %0 to void(i8*)* | |||
1867 | // call void %1(i8* null) | |||
1868 | coro::LowererBase Lowerer(M); | |||
1869 | Instruction *InsertPt = | |||
1870 | MarkForAsyncRestart ? F.getEntryBlock().getFirstNonPHIOrDbgOrLifetime() | |||
1871 | : F.getEntryBlock().getTerminator(); | |||
1872 | auto *Null = ConstantPointerNull::get(Type::getInt8PtrTy(Context)); | |||
1873 | auto *DevirtFnAddr = | |||
1874 | Lowerer.makeSubFnCall(Null, CoroSubFnInst::RestartTrigger, InsertPt); | |||
1875 | FunctionType *FnTy = FunctionType::get(Type::getVoidTy(Context), | |||
1876 | {Type::getInt8PtrTy(Context)}, false); | |||
1877 | auto *IndirectCall = CallInst::Create(FnTy, DevirtFnAddr, Null, "", InsertPt); | |||
1878 | ||||
1879 | // Update CG graph with an indirect call we just added. | |||
1880 | CG[&F]->addCalledFunction(IndirectCall, CG.getCallsExternalNode()); | |||
1881 | } | |||
1882 | ||||
1883 | // Make sure that there is a devirtualization trigger function that the | |||
1884 | // coro-split pass uses to force a restart of the CGSCC pipeline. If the devirt | |||
1885 | // trigger function is not found, we will create one and add it to the current | |||
1886 | // SCC. | |||
1887 | static void createDevirtTriggerFunc(CallGraph &CG, CallGraphSCC &SCC) { | |||
1888 | Module &M = CG.getModule(); | |||
1889 | if (M.getFunction(CORO_DEVIRT_TRIGGER_FN"coro.devirt.trigger")) | |||
1890 | return; | |||
1891 | ||||
1892 | LLVMContext &C = M.getContext(); | |||
1893 | auto *FnTy = FunctionType::get(Type::getVoidTy(C), Type::getInt8PtrTy(C), | |||
1894 | /*isVarArg=*/false); | |||
1895 | Function *DevirtFn = | |||
1896 | Function::Create(FnTy, GlobalValue::LinkageTypes::PrivateLinkage, | |||
1897 | CORO_DEVIRT_TRIGGER_FN"coro.devirt.trigger", &M); | |||
1898 | DevirtFn->addFnAttr(Attribute::AlwaysInline); | |||
1899 | auto *Entry = BasicBlock::Create(C, "entry", DevirtFn); | |||
1900 | ReturnInst::Create(C, Entry); | |||
1901 | ||||
1902 | auto *Node = CG.getOrInsertFunction(DevirtFn); | |||
1903 | ||||
1904 | SmallVector<CallGraphNode *, 8> Nodes(SCC.begin(), SCC.end()); | |||
1905 | Nodes.push_back(Node); | |||
1906 | SCC.initialize(Nodes); | |||
1907 | } | |||
1908 | ||||
1909 | /// Replace a call to llvm.coro.prepare.retcon. | |||
1910 | static void replacePrepare(CallInst *Prepare, LazyCallGraph &CG, | |||
1911 | LazyCallGraph::SCC &C) { | |||
1912 | auto CastFn = Prepare->getArgOperand(0); // as an i8* | |||
1913 | auto Fn = CastFn->stripPointerCasts(); // as its original type | |||
1914 | ||||
1915 | // Attempt to peephole this pattern: | |||
1916 | // %0 = bitcast [[TYPE]] @some_function to i8* | |||
1917 | // %1 = call @llvm.coro.prepare.retcon(i8* %0) | |||
1918 | // %2 = bitcast %1 to [[TYPE]] | |||
1919 | // ==> | |||
1920 | // %2 = @some_function | |||
1921 | for (auto UI = Prepare->use_begin(), UE = Prepare->use_end(); UI != UE;) { | |||
1922 | // Look for bitcasts back to the original function type. | |||
1923 | auto *Cast = dyn_cast<BitCastInst>((UI++)->getUser()); | |||
1924 | if (!Cast || Cast->getType() != Fn->getType()) | |||
1925 | continue; | |||
1926 | ||||
1927 | // Replace and remove the cast. | |||
1928 | Cast->replaceAllUsesWith(Fn); | |||
1929 | Cast->eraseFromParent(); | |||
1930 | } | |||
1931 | ||||
1932 | // Replace any remaining uses with the function as an i8*. | |||
1933 | // This can never directly be a callee, so we don't need to update CG. | |||
1934 | Prepare->replaceAllUsesWith(CastFn); | |||
1935 | Prepare->eraseFromParent(); | |||
1936 | ||||
1937 | // Kill dead bitcasts. | |||
1938 | while (auto *Cast = dyn_cast<BitCastInst>(CastFn)) { | |||
1939 | if (!Cast->use_empty()) | |||
1940 | break; | |||
1941 | CastFn = Cast->getOperand(0); | |||
1942 | Cast->eraseFromParent(); | |||
1943 | } | |||
1944 | } | |||
1945 | /// Replace a call to llvm.coro.prepare.retcon. | |||
1946 | static void replacePrepare(CallInst *Prepare, CallGraph &CG) { | |||
1947 | auto CastFn = Prepare->getArgOperand(0); // as an i8* | |||
1948 | auto Fn = CastFn->stripPointerCasts(); // as its original type | |||
1949 | ||||
1950 | // Find call graph nodes for the preparation. | |||
1951 | CallGraphNode *PrepareUserNode = nullptr, *FnNode = nullptr; | |||
1952 | if (auto ConcreteFn = dyn_cast<Function>(Fn)) { | |||
1953 | PrepareUserNode = CG[Prepare->getFunction()]; | |||
1954 | FnNode = CG[ConcreteFn]; | |||
1955 | } | |||
1956 | ||||
1957 | // Attempt to peephole this pattern: | |||
1958 | // %0 = bitcast [[TYPE]] @some_function to i8* | |||
1959 | // %1 = call @llvm.coro.prepare.retcon(i8* %0) | |||
1960 | // %2 = bitcast %1 to [[TYPE]] | |||
1961 | // ==> | |||
1962 | // %2 = @some_function | |||
1963 | for (auto UI = Prepare->use_begin(), UE = Prepare->use_end(); | |||
1964 | UI != UE; ) { | |||
1965 | // Look for bitcasts back to the original function type. | |||
1966 | auto *Cast = dyn_cast<BitCastInst>((UI++)->getUser()); | |||
1967 | if (!Cast || Cast->getType() != Fn->getType()) continue; | |||
1968 | ||||
1969 | // Check whether the replacement will introduce new direct calls. | |||
1970 | // If so, we'll need to update the call graph. | |||
1971 | if (PrepareUserNode) { | |||
1972 | for (auto &Use : Cast->uses()) { | |||
1973 | if (auto *CB = dyn_cast<CallBase>(Use.getUser())) { | |||
1974 | if (!CB->isCallee(&Use)) | |||
1975 | continue; | |||
1976 | PrepareUserNode->removeCallEdgeFor(*CB); | |||
1977 | PrepareUserNode->addCalledFunction(CB, FnNode); | |||
1978 | } | |||
1979 | } | |||
1980 | } | |||
1981 | ||||
1982 | // Replace and remove the cast. | |||
1983 | Cast->replaceAllUsesWith(Fn); | |||
1984 | Cast->eraseFromParent(); | |||
1985 | } | |||
1986 | ||||
1987 | // Replace any remaining uses with the function as an i8*. | |||
1988 | // This can never directly be a callee, so we don't need to update CG. | |||
1989 | Prepare->replaceAllUsesWith(CastFn); | |||
1990 | Prepare->eraseFromParent(); | |||
1991 | ||||
1992 | // Kill dead bitcasts. | |||
1993 | while (auto *Cast = dyn_cast<BitCastInst>(CastFn)) { | |||
1994 | if (!Cast->use_empty()) break; | |||
1995 | CastFn = Cast->getOperand(0); | |||
1996 | Cast->eraseFromParent(); | |||
1997 | } | |||
1998 | } | |||
1999 | ||||
2000 | static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG, | |||
2001 | LazyCallGraph::SCC &C) { | |||
2002 | bool Changed = false; | |||
2003 | for (auto PI = PrepareFn->use_begin(), PE = PrepareFn->use_end(); PI != PE;) { | |||
2004 | // Intrinsics can only be used in calls. | |||
2005 | auto *Prepare = cast<CallInst>((PI++)->getUser()); | |||
2006 | replacePrepare(Prepare, CG, C); | |||
2007 | Changed = true; | |||
2008 | } | |||
2009 | ||||
2010 | return Changed; | |||
2011 | } | |||
2012 | ||||
2013 | /// Remove calls to llvm.coro.prepare.retcon, a barrier meant to prevent | |||
2014 | /// IPO from operating on calls to a retcon coroutine before it's been | |||
2015 | /// split. This is only safe to do after we've split all retcon | |||
2016 | /// coroutines in the module. We can do that this in this pass because | |||
2017 | /// this pass does promise to split all retcon coroutines (as opposed to | |||
2018 | /// switch coroutines, which are lowered in multiple stages). | |||
2019 | static bool replaceAllPrepares(Function *PrepareFn, CallGraph &CG) { | |||
2020 | bool Changed = false; | |||
2021 | for (auto PI = PrepareFn->use_begin(), PE = PrepareFn->use_end(); | |||
2022 | PI != PE; ) { | |||
2023 | // Intrinsics can only be used in calls. | |||
2024 | auto *Prepare = cast<CallInst>((PI++)->getUser()); | |||
2025 | replacePrepare(Prepare, CG); | |||
2026 | Changed = true; | |||
2027 | } | |||
2028 | ||||
2029 | return Changed; | |||
2030 | } | |||
2031 | ||||
2032 | static bool declaresCoroSplitIntrinsics(const Module &M) { | |||
2033 | return coro::declaresIntrinsics(M, {"llvm.coro.begin", | |||
2034 | "llvm.coro.prepare.retcon", | |||
2035 | "llvm.coro.prepare.async"}); | |||
2036 | } | |||
2037 | ||||
2038 | static void addPrepareFunction(const Module &M, | |||
2039 | SmallVectorImpl<Function *> &Fns, | |||
2040 | StringRef Name) { | |||
2041 | auto *PrepareFn = M.getFunction(Name); | |||
2042 | if (PrepareFn && !PrepareFn->use_empty()) | |||
2043 | Fns.push_back(PrepareFn); | |||
2044 | } | |||
2045 | ||||
2046 | PreservedAnalyses CoroSplitPass::run(LazyCallGraph::SCC &C, | |||
2047 | CGSCCAnalysisManager &AM, | |||
2048 | LazyCallGraph &CG, CGSCCUpdateResult &UR) { | |||
2049 | // NB: One invariant of a valid LazyCallGraph::SCC is that it must contain a | |||
2050 | // non-zero number of nodes, so we assume that here and grab the first | |||
2051 | // node's function's module. | |||
2052 | Module &M = *C.begin()->getFunction().getParent(); | |||
2053 | auto &FAM = | |||
2054 | AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager(); | |||
2055 | ||||
2056 | if (!declaresCoroSplitIntrinsics(M)) | |||
2057 | return PreservedAnalyses::all(); | |||
2058 | ||||
2059 | // Check for uses of llvm.coro.prepare.retcon/async. | |||
2060 | SmallVector<Function *, 2> PrepareFns; | |||
2061 | addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.retcon"); | |||
2062 | addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.async"); | |||
2063 | ||||
2064 | // Find coroutines for processing. | |||
2065 | SmallVector<LazyCallGraph::Node *, 4> Coroutines; | |||
2066 | for (LazyCallGraph::Node &N : C) | |||
2067 | if (N.getFunction().hasFnAttribute(CORO_PRESPLIT_ATTR"coroutine.presplit")) | |||
2068 | Coroutines.push_back(&N); | |||
2069 | ||||
2070 | if (Coroutines.empty() && PrepareFns.empty()) | |||
2071 | return PreservedAnalyses::all(); | |||
2072 | ||||
2073 | if (Coroutines.empty()) { | |||
2074 | for (auto *PrepareFn : PrepareFns) { | |||
2075 | replaceAllPrepares(PrepareFn, CG, C); | |||
2076 | } | |||
2077 | } | |||
2078 | ||||
2079 | // Split all the coroutines. | |||
2080 | for (LazyCallGraph::Node *N : Coroutines) { | |||
2081 | Function &F = N->getFunction(); | |||
2082 | Attribute Attr = F.getFnAttribute(CORO_PRESPLIT_ATTR"coroutine.presplit"); | |||
2083 | StringRef Value = Attr.getValueAsString(); | |||
2084 | LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F.getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("coro-split")) { dbgs() << "CoroSplit: Processing coroutine '" << F.getName() << "' state: " << Value << "\n"; } } while (false) | |||
2085 | << "' state: " << Value << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("coro-split")) { dbgs() << "CoroSplit: Processing coroutine '" << F.getName() << "' state: " << Value << "\n"; } } while (false); | |||
2086 | if (Value == UNPREPARED_FOR_SPLIT"0") { | |||
2087 | // Enqueue a second iteration of the CGSCC pipeline on this SCC. | |||
2088 | UR.CWorklist.insert(&C); | |||
2089 | F.addFnAttr(CORO_PRESPLIT_ATTR"coroutine.presplit", PREPARED_FOR_SPLIT"1"); | |||
2090 | continue; | |||
2091 | } | |||
2092 | F.removeFnAttr(CORO_PRESPLIT_ATTR"coroutine.presplit"); | |||
2093 | ||||
2094 | SmallVector<Function *, 4> Clones; | |||
2095 | const coro::Shape Shape = splitCoroutine(F, Clones, ReuseFrameSlot); | |||
2096 | updateCallGraphAfterCoroutineSplit(*N, Shape, Clones, C, CG, AM, UR, FAM); | |||
2097 | ||||
2098 | if ((Shape.ABI == coro::ABI::Async || Shape.ABI == coro::ABI::Retcon || | |||
2099 | Shape.ABI == coro::ABI::RetconOnce) && | |||
2100 | !Shape.CoroSuspends.empty()) { | |||
2101 | // Run the CGSCC pipeline on the newly split functions. | |||
2102 | // All clones will be in the same RefSCC, so choose a random clone. | |||
2103 | UR.RCWorklist.insert(CG.lookupRefSCC(CG.get(*Clones[0]))); | |||
2104 | } | |||
2105 | } | |||
2106 | ||||
2107 | if (!PrepareFns.empty()) { | |||
2108 | for (auto *PrepareFn : PrepareFns) { | |||
2109 | replaceAllPrepares(PrepareFn, CG, C); | |||
2110 | } | |||
2111 | } | |||
2112 | ||||
2113 | return PreservedAnalyses::none(); | |||
2114 | } | |||
2115 | ||||
2116 | namespace { | |||
2117 | ||||
2118 | // We present a coroutine to LLVM as an ordinary function with suspension | |||
2119 | // points marked up with intrinsics. We let the optimizer party on the coroutine | |||
2120 | // as a single function for as long as possible. Shortly before the coroutine is | |||
2121 | // eligible to be inlined into its callers, we split up the coroutine into parts | |||
2122 | // corresponding to initial, resume and destroy invocations of the coroutine, | |||
2123 | // add them to the current SCC and restart the IPO pipeline to optimize the | |||
2124 | // coroutine subfunctions we extracted before proceeding to the caller of the | |||
2125 | // coroutine. | |||
2126 | struct CoroSplitLegacy : public CallGraphSCCPass { | |||
2127 | static char ID; // Pass identification, replacement for typeid | |||
2128 | ||||
2129 | CoroSplitLegacy(bool ReuseFrameSlot = false) | |||
2130 | : CallGraphSCCPass(ID), ReuseFrameSlot(ReuseFrameSlot) { | |||
2131 | initializeCoroSplitLegacyPass(*PassRegistry::getPassRegistry()); | |||
2132 | } | |||
2133 | ||||
2134 | bool Run = false; | |||
2135 | bool ReuseFrameSlot; | |||
2136 | ||||
2137 | // A coroutine is identified by the presence of coro.begin intrinsic, if | |||
2138 | // we don't have any, this pass has nothing to do. | |||
2139 | bool doInitialization(CallGraph &CG) override { | |||
2140 | Run = declaresCoroSplitIntrinsics(CG.getModule()); | |||
2141 | return CallGraphSCCPass::doInitialization(CG); | |||
2142 | } | |||
2143 | ||||
2144 | bool runOnSCC(CallGraphSCC &SCC) override { | |||
2145 | if (!Run) | |||
| ||||
2146 | return false; | |||
2147 | ||||
2148 | // Check for uses of llvm.coro.prepare.retcon. | |||
2149 | SmallVector<Function *, 2> PrepareFns; | |||
2150 | auto &M = SCC.getCallGraph().getModule(); | |||
2151 | addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.retcon"); | |||
2152 | addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.async"); | |||
2153 | ||||
2154 | // Find coroutines for processing. | |||
2155 | SmallVector<Function *, 4> Coroutines; | |||
2156 | for (CallGraphNode *CGN : SCC) | |||
2157 | if (auto *F = CGN->getFunction()) | |||
2158 | if (F->hasFnAttribute(CORO_PRESPLIT_ATTR"coroutine.presplit")) | |||
2159 | Coroutines.push_back(F); | |||
2160 | ||||
2161 | if (Coroutines.empty() && PrepareFns.empty()) | |||
2162 | return false; | |||
2163 | ||||
2164 | CallGraph &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph(); | |||
2165 | ||||
2166 | if (Coroutines.empty()) { | |||
2167 | bool Changed = false; | |||
2168 | for (auto *PrepareFn : PrepareFns) | |||
2169 | Changed |= replaceAllPrepares(PrepareFn, CG); | |||
2170 | return Changed; | |||
2171 | } | |||
2172 | ||||
2173 | createDevirtTriggerFunc(CG, SCC); | |||
2174 | ||||
2175 | // Split all the coroutines. | |||
2176 | for (Function *F : Coroutines) { | |||
2177 | Attribute Attr = F->getFnAttribute(CORO_PRESPLIT_ATTR"coroutine.presplit"); | |||
2178 | StringRef Value = Attr.getValueAsString(); | |||
2179 | LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("coro-split")) { dbgs() << "CoroSplit: Processing coroutine '" << F->getName() << "' state: " << Value << "\n"; } } while (false) | |||
2180 | << "' state: " << Value << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("coro-split")) { dbgs() << "CoroSplit: Processing coroutine '" << F->getName() << "' state: " << Value << "\n"; } } while (false); | |||
2181 | // Async lowering marks coroutines to trigger a restart of the pipeline | |||
2182 | // after it has split them. | |||
2183 | if (Value == ASYNC_RESTART_AFTER_SPLIT"2") { | |||
2184 | F->removeFnAttr(CORO_PRESPLIT_ATTR"coroutine.presplit"); | |||
2185 | continue; | |||
2186 | } | |||
2187 | if (Value == UNPREPARED_FOR_SPLIT"0") { | |||
2188 | prepareForSplit(*F, CG); | |||
2189 | continue; | |||
2190 | } | |||
2191 | F->removeFnAttr(CORO_PRESPLIT_ATTR"coroutine.presplit"); | |||
2192 | ||||
2193 | SmallVector<Function *, 4> Clones; | |||
2194 | const coro::Shape Shape = splitCoroutine(*F, Clones, ReuseFrameSlot); | |||
2195 | updateCallGraphAfterCoroutineSplit(*F, Shape, Clones, CG, SCC); | |||
2196 | if (Shape.ABI == coro::ABI::Async) { | |||
2197 | // Restart SCC passes. | |||
2198 | // Mark function for CoroElide pass. It will devirtualize causing a | |||
2199 | // restart of the SCC pipeline. | |||
2200 | prepareForSplit(*F, CG, true /*MarkForAsyncRestart*/); | |||
2201 | } | |||
2202 | } | |||
2203 | ||||
2204 | for (auto *PrepareFn : PrepareFns) | |||
2205 | replaceAllPrepares(PrepareFn, CG); | |||
2206 | ||||
2207 | return true; | |||
2208 | } | |||
2209 | ||||
2210 | void getAnalysisUsage(AnalysisUsage &AU) const override { | |||
2211 | CallGraphSCCPass::getAnalysisUsage(AU); | |||
2212 | } | |||
2213 | ||||
2214 | StringRef getPassName() const override { return "Coroutine Splitting"; } | |||
2215 | }; | |||
2216 | ||||
2217 | } // end anonymous namespace | |||
2218 | ||||
2219 | char CoroSplitLegacy::ID = 0; | |||
2220 | ||||
2221 | INITIALIZE_PASS_BEGIN(static void *initializeCoroSplitLegacyPassOnce(PassRegistry & Registry) { | |||
2222 | CoroSplitLegacy, "coro-split",static void *initializeCoroSplitLegacyPassOnce(PassRegistry & Registry) { | |||
2223 | "Split coroutine into a set of functions driving its state machine", false,static void *initializeCoroSplitLegacyPassOnce(PassRegistry & Registry) { | |||
2224 | false)static void *initializeCoroSplitLegacyPassOnce(PassRegistry & Registry) { | |||
2225 | INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass)initializeCallGraphWrapperPassPass(Registry); | |||
2226 | INITIALIZE_PASS_END(PassInfo *PI = new PassInfo( "Split coroutine into a set of functions driving its state machine" , "coro-split", &CoroSplitLegacy::ID, PassInfo::NormalCtor_t (callDefaultCtor<CoroSplitLegacy>), false, false); Registry .registerPass(*PI, true); return PI; } static llvm::once_flag InitializeCoroSplitLegacyPassFlag; void llvm::initializeCoroSplitLegacyPass (PassRegistry &Registry) { llvm::call_once(InitializeCoroSplitLegacyPassFlag , initializeCoroSplitLegacyPassOnce, std::ref(Registry)); } | |||
2227 | CoroSplitLegacy, "coro-split",PassInfo *PI = new PassInfo( "Split coroutine into a set of functions driving its state machine" , "coro-split", &CoroSplitLegacy::ID, PassInfo::NormalCtor_t (callDefaultCtor<CoroSplitLegacy>), false, false); Registry .registerPass(*PI, true); return PI; } static llvm::once_flag InitializeCoroSplitLegacyPassFlag; void llvm::initializeCoroSplitLegacyPass (PassRegistry &Registry) { llvm::call_once(InitializeCoroSplitLegacyPassFlag , initializeCoroSplitLegacyPassOnce, std::ref(Registry)); } | |||
2228 | "Split coroutine into a set of functions driving its state machine", false,PassInfo *PI = new PassInfo( "Split coroutine into a set of functions driving its state machine" , "coro-split", &CoroSplitLegacy::ID, PassInfo::NormalCtor_t (callDefaultCtor<CoroSplitLegacy>), false, false); Registry .registerPass(*PI, true); return PI; } static llvm::once_flag InitializeCoroSplitLegacyPassFlag; void llvm::initializeCoroSplitLegacyPass (PassRegistry &Registry) { llvm::call_once(InitializeCoroSplitLegacyPassFlag , initializeCoroSplitLegacyPassOnce, std::ref(Registry)); } | |||
2229 | false)PassInfo *PI = new PassInfo( "Split coroutine into a set of functions driving its state machine" , "coro-split", &CoroSplitLegacy::ID, PassInfo::NormalCtor_t (callDefaultCtor<CoroSplitLegacy>), false, false); Registry .registerPass(*PI, true); return PI; } static llvm::once_flag InitializeCoroSplitLegacyPassFlag; void llvm::initializeCoroSplitLegacyPass (PassRegistry &Registry) { llvm::call_once(InitializeCoroSplitLegacyPassFlag , initializeCoroSplitLegacyPassOnce, std::ref(Registry)); } | |||
2230 | ||||
2231 | Pass *llvm::createCoroSplitLegacyPass(bool ReuseFrameSlot) { | |||
2232 | return new CoroSplitLegacy(ReuseFrameSlot); | |||
2233 | } |