Bug Summary

File:llvm/lib/Transforms/Coroutines/CoroSplit.cpp
Warning:line 461, column 19
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name CoroSplit.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/build-llvm/lib/Transforms/Coroutines -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/build-llvm/lib/Transforms/Coroutines -I /build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines -I /build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/build-llvm/lib/Transforms/Coroutines -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2021-03-08-182450-10039-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
1//===- CoroSplit.cpp - Converts a coroutine into a state machine ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// This pass builds the coroutine frame and outlines resume and destroy parts
9// of the coroutine into separate functions.
10//
11// We present a coroutine to an LLVM as an ordinary function with suspension
12// points marked up with intrinsics. We let the optimizer party on the coroutine
13// as a single function for as long as possible. Shortly before the coroutine is
14// eligible to be inlined into its callers, we split up the coroutine into parts
15// corresponding to an initial, resume and destroy invocations of the coroutine,
16// add them to the current SCC and restart the IPO pipeline to optimize the
17// coroutine subfunctions we extracted before proceeding to the caller of the
18// coroutine.
19//===----------------------------------------------------------------------===//
20
21#include "llvm/Transforms/Coroutines/CoroSplit.h"
22#include "CoroInstr.h"
23#include "CoroInternal.h"
24#include "llvm/ADT/DenseMap.h"
25#include "llvm/ADT/SmallPtrSet.h"
26#include "llvm/ADT/SmallVector.h"
27#include "llvm/ADT/StringRef.h"
28#include "llvm/ADT/Twine.h"
29#include "llvm/Analysis/CallGraph.h"
30#include "llvm/Analysis/CallGraphSCCPass.h"
31#include "llvm/Analysis/LazyCallGraph.h"
32#include "llvm/IR/Argument.h"
33#include "llvm/IR/Attributes.h"
34#include "llvm/IR/BasicBlock.h"
35#include "llvm/IR/CFG.h"
36#include "llvm/IR/CallingConv.h"
37#include "llvm/IR/Constants.h"
38#include "llvm/IR/DataLayout.h"
39#include "llvm/IR/DerivedTypes.h"
40#include "llvm/IR/Function.h"
41#include "llvm/IR/GlobalValue.h"
42#include "llvm/IR/GlobalVariable.h"
43#include "llvm/IR/IRBuilder.h"
44#include "llvm/IR/InstIterator.h"
45#include "llvm/IR/InstrTypes.h"
46#include "llvm/IR/Instruction.h"
47#include "llvm/IR/Instructions.h"
48#include "llvm/IR/IntrinsicInst.h"
49#include "llvm/IR/LLVMContext.h"
50#include "llvm/IR/LegacyPassManager.h"
51#include "llvm/IR/Module.h"
52#include "llvm/IR/Type.h"
53#include "llvm/IR/Value.h"
54#include "llvm/IR/Verifier.h"
55#include "llvm/InitializePasses.h"
56#include "llvm/Pass.h"
57#include "llvm/Support/Casting.h"
58#include "llvm/Support/Debug.h"
59#include "llvm/Support/PrettyStackTrace.h"
60#include "llvm/Support/raw_ostream.h"
61#include "llvm/Transforms/Scalar.h"
62#include "llvm/Transforms/Utils/BasicBlockUtils.h"
63#include "llvm/Transforms/Utils/CallGraphUpdater.h"
64#include "llvm/Transforms/Utils/Cloning.h"
65#include "llvm/Transforms/Utils/Local.h"
66#include "llvm/Transforms/Utils/ValueMapper.h"
67#include <cassert>
68#include <cstddef>
69#include <cstdint>
70#include <initializer_list>
71#include <iterator>
72
73using namespace llvm;
74
75#define DEBUG_TYPE"coro-split" "coro-split"
76
77namespace {
78
79/// A little helper class for building
80class CoroCloner {
81public:
82 enum class Kind {
83 /// The shared resume function for a switch lowering.
84 SwitchResume,
85
86 /// The shared unwind function for a switch lowering.
87 SwitchUnwind,
88
89 /// The shared cleanup function for a switch lowering.
90 SwitchCleanup,
91
92 /// An individual continuation function.
93 Continuation,
94
95 /// An async resume function.
96 Async,
97 };
98
99private:
100 Function &OrigF;
101 Function *NewF;
102 const Twine &Suffix;
103 coro::Shape &Shape;
104 Kind FKind;
105 ValueToValueMapTy VMap;
106 IRBuilder<> Builder;
107 Value *NewFramePtr = nullptr;
108
109 /// The active suspend instruction; meaningful only for continuation and async
110 /// ABIs.
111 AnyCoroSuspendInst *ActiveSuspend = nullptr;
20
Null pointer value stored to 'Cloner.ActiveSuspend'
112
113public:
114 /// Create a cloner for a switch lowering.
115 CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
116 Kind FKind)
117 : OrigF(OrigF), NewF(nullptr), Suffix(Suffix), Shape(Shape),
118 FKind(FKind), Builder(OrigF.getContext()) {
119 assert(Shape.ABI == coro::ABI::Switch)((Shape.ABI == coro::ABI::Switch) ? static_cast<void> (
0) : __assert_fail ("Shape.ABI == coro::ABI::Switch", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 119, __PRETTY_FUNCTION__))
;
21
Assuming field 'ABI' is equal to Switch
22
'?' condition is true
120 }
121
122 /// Create a cloner for a continuation lowering.
123 CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
124 Function *NewF, AnyCoroSuspendInst *ActiveSuspend)
125 : OrigF(OrigF), NewF(NewF), Suffix(Suffix), Shape(Shape),
126 FKind(Shape.ABI == coro::ABI::Async ? Kind::Async : Kind::Continuation),
127 Builder(OrigF.getContext()), ActiveSuspend(ActiveSuspend) {
128 assert(Shape.ABI == coro::ABI::Retcon ||((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce
|| Shape.ABI == coro::ABI::Async) ? static_cast<void> (
0) : __assert_fail ("Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 129, __PRETTY_FUNCTION__))
129 Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async)((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce
|| Shape.ABI == coro::ABI::Async) ? static_cast<void> (
0) : __assert_fail ("Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 129, __PRETTY_FUNCTION__))
;
130 assert(NewF && "need existing function for continuation")((NewF && "need existing function for continuation") ?
static_cast<void> (0) : __assert_fail ("NewF && \"need existing function for continuation\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 130, __PRETTY_FUNCTION__))
;
131 assert(ActiveSuspend && "need active suspend point for continuation")((ActiveSuspend && "need active suspend point for continuation"
) ? static_cast<void> (0) : __assert_fail ("ActiveSuspend && \"need active suspend point for continuation\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 131, __PRETTY_FUNCTION__))
;
132 }
133
134 Function *getFunction() const {
135 assert(NewF != nullptr && "declaration not yet set")((NewF != nullptr && "declaration not yet set") ? static_cast
<void> (0) : __assert_fail ("NewF != nullptr && \"declaration not yet set\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 135, __PRETTY_FUNCTION__))
;
136 return NewF;
137 }
138
139 void create();
140
141private:
142 bool isSwitchDestroyFunction() {
143 switch (FKind) {
144 case Kind::Async:
145 case Kind::Continuation:
146 case Kind::SwitchResume:
147 return false;
148 case Kind::SwitchUnwind:
149 case Kind::SwitchCleanup:
150 return true;
151 }
152 llvm_unreachable("Unknown CoroCloner::Kind enum")::llvm::llvm_unreachable_internal("Unknown CoroCloner::Kind enum"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 152)
;
153 }
154
155 void replaceEntryBlock();
156 Value *deriveNewFramePointer();
157 void replaceRetconOrAsyncSuspendUses();
158 void replaceCoroSuspends();
159 void replaceCoroEnds();
160 void replaceSwiftErrorOps();
161 void salvageDebugInfo();
162 void handleFinalSuspend();
163};
164
165} // end anonymous namespace
166
167static void maybeFreeRetconStorage(IRBuilder<> &Builder,
168 const coro::Shape &Shape, Value *FramePtr,
169 CallGraph *CG) {
170 assert(Shape.ABI == coro::ABI::Retcon ||((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce
) ? static_cast<void> (0) : __assert_fail ("Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 171, __PRETTY_FUNCTION__))
171 Shape.ABI == coro::ABI::RetconOnce)((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce
) ? static_cast<void> (0) : __assert_fail ("Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 171, __PRETTY_FUNCTION__))
;
172 if (Shape.RetconLowering.IsFrameInlineInStorage)
173 return;
174
175 Shape.emitDealloc(Builder, FramePtr, CG);
176}
177
178/// Replace an llvm.coro.end.async.
179/// Will inline the must tail call function call if there is one.
180/// \returns true if cleanup of the coro.end block is needed, false otherwise.
181static bool replaceCoroEndAsync(AnyCoroEndInst *End) {
182 IRBuilder<> Builder(End);
183
184 auto *EndAsync = dyn_cast<CoroAsyncEndInst>(End);
185 if (!EndAsync) {
186 Builder.CreateRetVoid();
187 return true /*needs cleanup of coro.end block*/;
188 }
189
190 auto *MustTailCallFunc = EndAsync->getMustTailCallFunction();
191 if (!MustTailCallFunc) {
192 Builder.CreateRetVoid();
193 return true /*needs cleanup of coro.end block*/;
194 }
195
196 // Move the must tail call from the predecessor block into the end block.
197 auto *CoroEndBlock = End->getParent();
198 auto *MustTailCallFuncBlock = CoroEndBlock->getSinglePredecessor();
199 assert(MustTailCallFuncBlock && "Must have a single predecessor block")((MustTailCallFuncBlock && "Must have a single predecessor block"
) ? static_cast<void> (0) : __assert_fail ("MustTailCallFuncBlock && \"Must have a single predecessor block\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 199, __PRETTY_FUNCTION__))
;
200 auto It = MustTailCallFuncBlock->getTerminator()->getIterator();
201 auto *MustTailCall = cast<CallInst>(&*std::prev(It));
202 CoroEndBlock->getInstList().splice(
203 End->getIterator(), MustTailCallFuncBlock->getInstList(), MustTailCall);
204
205 // Insert the return instruction.
206 Builder.SetInsertPoint(End);
207 Builder.CreateRetVoid();
208 InlineFunctionInfo FnInfo;
209
210 // Remove the rest of the block, by splitting it into an unreachable block.
211 auto *BB = End->getParent();
212 BB->splitBasicBlock(End);
213 BB->getTerminator()->eraseFromParent();
214
215 auto InlineRes = InlineFunction(*MustTailCall, FnInfo);
216 assert(InlineRes.isSuccess() && "Expected inlining to succeed")((InlineRes.isSuccess() && "Expected inlining to succeed"
) ? static_cast<void> (0) : __assert_fail ("InlineRes.isSuccess() && \"Expected inlining to succeed\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 216, __PRETTY_FUNCTION__))
;
217 (void)InlineRes;
218
219 // We have cleaned up the coro.end block above.
220 return false;
221}
222
223/// Replace a non-unwind call to llvm.coro.end.
224static void replaceFallthroughCoroEnd(AnyCoroEndInst *End,
225 const coro::Shape &Shape, Value *FramePtr,
226 bool InResume, CallGraph *CG) {
227 // Start inserting right before the coro.end.
228 IRBuilder<> Builder(End);
229
230 // Create the return instruction.
231 switch (Shape.ABI) {
232 // The cloned functions in switch-lowering always return void.
233 case coro::ABI::Switch:
234 // coro.end doesn't immediately end the coroutine in the main function
235 // in this lowering, because we need to deallocate the coroutine.
236 if (!InResume)
237 return;
238 Builder.CreateRetVoid();
239 break;
240
241 // In async lowering this returns.
242 case coro::ABI::Async: {
243 bool CoroEndBlockNeedsCleanup = replaceCoroEndAsync(End);
244 if (!CoroEndBlockNeedsCleanup)
245 return;
246 break;
247 }
248
249 // In unique continuation lowering, the continuations always return void.
250 // But we may have implicitly allocated storage.
251 case coro::ABI::RetconOnce:
252 maybeFreeRetconStorage(Builder, Shape, FramePtr, CG);
253 Builder.CreateRetVoid();
254 break;
255
256 // In non-unique continuation lowering, we signal completion by returning
257 // a null continuation.
258 case coro::ABI::Retcon: {
259 maybeFreeRetconStorage(Builder, Shape, FramePtr, CG);
260 auto RetTy = Shape.getResumeFunctionType()->getReturnType();
261 auto RetStructTy = dyn_cast<StructType>(RetTy);
262 PointerType *ContinuationTy =
263 cast<PointerType>(RetStructTy ? RetStructTy->getElementType(0) : RetTy);
264
265 Value *ReturnValue = ConstantPointerNull::get(ContinuationTy);
266 if (RetStructTy) {
267 ReturnValue = Builder.CreateInsertValue(UndefValue::get(RetStructTy),
268 ReturnValue, 0);
269 }
270 Builder.CreateRet(ReturnValue);
271 break;
272 }
273 }
274
275 // Remove the rest of the block, by splitting it into an unreachable block.
276 auto *BB = End->getParent();
277 BB->splitBasicBlock(End);
278 BB->getTerminator()->eraseFromParent();
279}
280
281/// Replace an unwind call to llvm.coro.end.
282static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape,
283 Value *FramePtr, bool InResume,
284 CallGraph *CG) {
285 IRBuilder<> Builder(End);
286
287 switch (Shape.ABI) {
288 // In switch-lowering, this does nothing in the main function.
289 case coro::ABI::Switch:
290 if (!InResume)
291 return;
292 break;
293 // In async lowering this does nothing.
294 case coro::ABI::Async:
295 break;
296 // In continuation-lowering, this frees the continuation storage.
297 case coro::ABI::Retcon:
298 case coro::ABI::RetconOnce:
299 maybeFreeRetconStorage(Builder, Shape, FramePtr, CG);
300 break;
301 }
302
303 // If coro.end has an associated bundle, add cleanupret instruction.
304 if (auto Bundle = End->getOperandBundle(LLVMContext::OB_funclet)) {
305 auto *FromPad = cast<CleanupPadInst>(Bundle->Inputs[0]);
306 auto *CleanupRet = Builder.CreateCleanupRet(FromPad, nullptr);
307 End->getParent()->splitBasicBlock(End);
308 CleanupRet->getParent()->getTerminator()->eraseFromParent();
309 }
310}
311
312static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape,
313 Value *FramePtr, bool InResume, CallGraph *CG) {
314 if (End->isUnwind())
315 replaceUnwindCoroEnd(End, Shape, FramePtr, InResume, CG);
316 else
317 replaceFallthroughCoroEnd(End, Shape, FramePtr, InResume, CG);
318
319 auto &Context = End->getContext();
320 End->replaceAllUsesWith(InResume ? ConstantInt::getTrue(Context)
321 : ConstantInt::getFalse(Context));
322 End->eraseFromParent();
323}
324
325// Create an entry block for a resume function with a switch that will jump to
326// suspend points.
327static void createResumeEntryBlock(Function &F, coro::Shape &Shape) {
328 assert(Shape.ABI == coro::ABI::Switch)((Shape.ABI == coro::ABI::Switch) ? static_cast<void> (
0) : __assert_fail ("Shape.ABI == coro::ABI::Switch", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 328, __PRETTY_FUNCTION__))
;
329 LLVMContext &C = F.getContext();
330
331 // resume.entry:
332 // %index.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0,
333 // i32 2
334 // % index = load i32, i32* %index.addr
335 // switch i32 %index, label %unreachable [
336 // i32 0, label %resume.0
337 // i32 1, label %resume.1
338 // ...
339 // ]
340
341 auto *NewEntry = BasicBlock::Create(C, "resume.entry", &F);
342 auto *UnreachBB = BasicBlock::Create(C, "unreachable", &F);
343
344 IRBuilder<> Builder(NewEntry);
345 auto *FramePtr = Shape.FramePtr;
346 auto *FrameTy = Shape.FrameTy;
347 auto *GepIndex = Builder.CreateStructGEP(
348 FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
349 auto *Index = Builder.CreateLoad(Shape.getIndexType(), GepIndex, "index");
350 auto *Switch =
351 Builder.CreateSwitch(Index, UnreachBB, Shape.CoroSuspends.size());
352 Shape.SwitchLowering.ResumeSwitch = Switch;
353
354 size_t SuspendIndex = 0;
355 for (auto *AnyS : Shape.CoroSuspends) {
356 auto *S = cast<CoroSuspendInst>(AnyS);
357 ConstantInt *IndexVal = Shape.getIndex(SuspendIndex);
358
359 // Replace CoroSave with a store to Index:
360 // %index.addr = getelementptr %f.frame... (index field number)
361 // store i32 0, i32* %index.addr1
362 auto *Save = S->getCoroSave();
363 Builder.SetInsertPoint(Save);
364 if (S->isFinal()) {
365 // Final suspend point is represented by storing zero in ResumeFnAddr.
366 auto *GepIndex = Builder.CreateStructGEP(FrameTy, FramePtr,
367 coro::Shape::SwitchFieldIndex::Resume,
368 "ResumeFn.addr");
369 auto *NullPtr = ConstantPointerNull::get(cast<PointerType>(
370 cast<PointerType>(GepIndex->getType())->getElementType()));
371 Builder.CreateStore(NullPtr, GepIndex);
372 } else {
373 auto *GepIndex = Builder.CreateStructGEP(
374 FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
375 Builder.CreateStore(IndexVal, GepIndex);
376 }
377 Save->replaceAllUsesWith(ConstantTokenNone::get(C));
378 Save->eraseFromParent();
379
380 // Split block before and after coro.suspend and add a jump from an entry
381 // switch:
382 //
383 // whateverBB:
384 // whatever
385 // %0 = call i8 @llvm.coro.suspend(token none, i1 false)
386 // switch i8 %0, label %suspend[i8 0, label %resume
387 // i8 1, label %cleanup]
388 // becomes:
389 //
390 // whateverBB:
391 // whatever
392 // br label %resume.0.landing
393 //
394 // resume.0: ; <--- jump from the switch in the resume.entry
395 // %0 = tail call i8 @llvm.coro.suspend(token none, i1 false)
396 // br label %resume.0.landing
397 //
398 // resume.0.landing:
399 // %1 = phi i8[-1, %whateverBB], [%0, %resume.0]
400 // switch i8 % 1, label %suspend [i8 0, label %resume
401 // i8 1, label %cleanup]
402
403 auto *SuspendBB = S->getParent();
404 auto *ResumeBB =
405 SuspendBB->splitBasicBlock(S, "resume." + Twine(SuspendIndex));
406 auto *LandingBB = ResumeBB->splitBasicBlock(
407 S->getNextNode(), ResumeBB->getName() + Twine(".landing"));
408 Switch->addCase(IndexVal, ResumeBB);
409
410 cast<BranchInst>(SuspendBB->getTerminator())->setSuccessor(0, LandingBB);
411 auto *PN = PHINode::Create(Builder.getInt8Ty(), 2, "", &LandingBB->front());
412 S->replaceAllUsesWith(PN);
413 PN->addIncoming(Builder.getInt8(-1), SuspendBB);
414 PN->addIncoming(S, ResumeBB);
415
416 ++SuspendIndex;
417 }
418
419 Builder.SetInsertPoint(UnreachBB);
420 Builder.CreateUnreachable();
421
422 Shape.SwitchLowering.ResumeEntryBlock = NewEntry;
423}
424
425
426// Rewrite final suspend point handling. We do not use suspend index to
427// represent the final suspend point. Instead we zero-out ResumeFnAddr in the
428// coroutine frame, since it is undefined behavior to resume a coroutine
429// suspended at the final suspend point. Thus, in the resume function, we can
430// simply remove the last case (when coro::Shape is built, the final suspend
431// point (if present) is always the last element of CoroSuspends array).
432// In the destroy function, we add a code sequence to check if ResumeFnAddress
433// is Null, and if so, jump to the appropriate label to handle cleanup from the
434// final suspend point.
435void CoroCloner::handleFinalSuspend() {
436 assert(Shape.ABI == coro::ABI::Switch &&((Shape.ABI == coro::ABI::Switch && Shape.SwitchLowering
.HasFinalSuspend) ? static_cast<void> (0) : __assert_fail
("Shape.ABI == coro::ABI::Switch && Shape.SwitchLowering.HasFinalSuspend"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 437, __PRETTY_FUNCTION__))
437 Shape.SwitchLowering.HasFinalSuspend)((Shape.ABI == coro::ABI::Switch && Shape.SwitchLowering
.HasFinalSuspend) ? static_cast<void> (0) : __assert_fail
("Shape.ABI == coro::ABI::Switch && Shape.SwitchLowering.HasFinalSuspend"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 437, __PRETTY_FUNCTION__))
;
438 auto *Switch = cast<SwitchInst>(VMap[Shape.SwitchLowering.ResumeSwitch]);
439 auto FinalCaseIt = std::prev(Switch->case_end());
440 BasicBlock *ResumeBB = FinalCaseIt->getCaseSuccessor();
441 Switch->removeCase(FinalCaseIt);
442 if (isSwitchDestroyFunction()) {
443 BasicBlock *OldSwitchBB = Switch->getParent();
444 auto *NewSwitchBB = OldSwitchBB->splitBasicBlock(Switch, "Switch");
445 Builder.SetInsertPoint(OldSwitchBB->getTerminator());
446 auto *GepIndex = Builder.CreateStructGEP(Shape.FrameTy, NewFramePtr,
447 coro::Shape::SwitchFieldIndex::Resume,
448 "ResumeFn.addr");
449 auto *Load = Builder.CreateLoad(Shape.getSwitchResumePointerType(),
450 GepIndex);
451 auto *Cond = Builder.CreateIsNull(Load);
452 Builder.CreateCondBr(Cond, ResumeBB, NewSwitchBB);
453 OldSwitchBB->getTerminator()->eraseFromParent();
454 }
455}
456
457static FunctionType *
458getFunctionTypeFromAsyncSuspend(AnyCoroSuspendInst *Suspend) {
459 auto *AsyncSuspend = cast<CoroSuspendAsyncInst>(Suspend);
460 auto *StructTy = cast<StructType>(AsyncSuspend->getType());
33
The object is a 'StructType'
461 auto &Context = Suspend->getParent()->getParent()->getContext();
34
Called C++ object pointer is null
462 auto *VoidTy = Type::getVoidTy(Context);
463 return FunctionType::get(VoidTy, StructTy->elements(), false);
464}
465
466static Function *createCloneDeclaration(Function &OrigF, coro::Shape &Shape,
467 const Twine &Suffix,
468 Module::iterator InsertBefore,
469 AnyCoroSuspendInst *ActiveSuspend) {
470 Module *M = OrigF.getParent();
471 auto *FnTy = (Shape.ABI != coro::ABI::Async)
29
Assuming field 'ABI' is equal to Async
30
'?' condition is false
472 ? Shape.getResumeFunctionType()
473 : getFunctionTypeFromAsyncSuspend(ActiveSuspend);
31
Passing null pointer value via 1st parameter 'Suspend'
32
Calling 'getFunctionTypeFromAsyncSuspend'
474
475 Function *NewF =
476 Function::Create(FnTy, GlobalValue::LinkageTypes::InternalLinkage,
477 OrigF.getName() + Suffix);
478 NewF->addParamAttr(0, Attribute::NonNull);
479
480 // For the async lowering ABI we can't guarantee that the context argument is
481 // not access via a different pointer not based on the argument.
482 if (Shape.ABI != coro::ABI::Async)
483 NewF->addParamAttr(0, Attribute::NoAlias);
484
485 M->getFunctionList().insert(InsertBefore, NewF);
486
487 return NewF;
488}
489
490/// Replace uses of the active llvm.coro.suspend.retcon/async call with the
491/// arguments to the continuation function.
492///
493/// This assumes that the builder has a meaningful insertion point.
494void CoroCloner::replaceRetconOrAsyncSuspendUses() {
495 assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce ||((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce
|| Shape.ABI == coro::ABI::Async) ? static_cast<void> (
0) : __assert_fail ("Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 496, __PRETTY_FUNCTION__))
496 Shape.ABI == coro::ABI::Async)((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce
|| Shape.ABI == coro::ABI::Async) ? static_cast<void> (
0) : __assert_fail ("Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 496, __PRETTY_FUNCTION__))
;
497
498 auto NewS = VMap[ActiveSuspend];
499 if (NewS->use_empty()) return;
500
501 // Copy out all the continuation arguments after the buffer pointer into
502 // an easily-indexed data structure for convenience.
503 SmallVector<Value*, 8> Args;
504 // The async ABI includes all arguments -- including the first argument.
505 bool IsAsyncABI = Shape.ABI == coro::ABI::Async;
506 for (auto I = IsAsyncABI ? NewF->arg_begin() : std::next(NewF->arg_begin()),
507 E = NewF->arg_end();
508 I != E; ++I)
509 Args.push_back(&*I);
510
511 // If the suspend returns a single scalar value, we can just do a simple
512 // replacement.
513 if (!isa<StructType>(NewS->getType())) {
514 assert(Args.size() == 1)((Args.size() == 1) ? static_cast<void> (0) : __assert_fail
("Args.size() == 1", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 514, __PRETTY_FUNCTION__))
;
515 NewS->replaceAllUsesWith(Args.front());
516 return;
517 }
518
519 // Try to peephole extracts of an aggregate return.
520 for (auto UI = NewS->use_begin(), UE = NewS->use_end(); UI != UE; ) {
521 auto EVI = dyn_cast<ExtractValueInst>((UI++)->getUser());
522 if (!EVI || EVI->getNumIndices() != 1)
523 continue;
524
525 EVI->replaceAllUsesWith(Args[EVI->getIndices().front()]);
526 EVI->eraseFromParent();
527 }
528
529 // If we have no remaining uses, we're done.
530 if (NewS->use_empty()) return;
531
532 // Otherwise, we need to create an aggregate.
533 Value *Agg = UndefValue::get(NewS->getType());
534 for (size_t I = 0, E = Args.size(); I != E; ++I)
535 Agg = Builder.CreateInsertValue(Agg, Args[I], I);
536
537 NewS->replaceAllUsesWith(Agg);
538}
539
540void CoroCloner::replaceCoroSuspends() {
541 Value *SuspendResult;
542
543 switch (Shape.ABI) {
544 // In switch lowering, replace coro.suspend with the appropriate value
545 // for the type of function we're extracting.
546 // Replacing coro.suspend with (0) will result in control flow proceeding to
547 // a resume label associated with a suspend point, replacing it with (1) will
548 // result in control flow proceeding to a cleanup label associated with this
549 // suspend point.
550 case coro::ABI::Switch:
551 SuspendResult = Builder.getInt8(isSwitchDestroyFunction() ? 1 : 0);
552 break;
553
554 // In async lowering there are no uses of the result.
555 case coro::ABI::Async:
556 return;
557
558 // In returned-continuation lowering, the arguments from earlier
559 // continuations are theoretically arbitrary, and they should have been
560 // spilled.
561 case coro::ABI::RetconOnce:
562 case coro::ABI::Retcon:
563 return;
564 }
565
566 for (AnyCoroSuspendInst *CS : Shape.CoroSuspends) {
567 // The active suspend was handled earlier.
568 if (CS == ActiveSuspend) continue;
569
570 auto *MappedCS = cast<AnyCoroSuspendInst>(VMap[CS]);
571 MappedCS->replaceAllUsesWith(SuspendResult);
572 MappedCS->eraseFromParent();
573 }
574}
575
576void CoroCloner::replaceCoroEnds() {
577 for (AnyCoroEndInst *CE : Shape.CoroEnds) {
578 // We use a null call graph because there's no call graph node for
579 // the cloned function yet. We'll just be rebuilding that later.
580 auto *NewCE = cast<AnyCoroEndInst>(VMap[CE]);
581 replaceCoroEnd(NewCE, Shape, NewFramePtr, /*in resume*/ true, nullptr);
582 }
583}
584
585static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape,
586 ValueToValueMapTy *VMap) {
587 if (Shape.ABI == coro::ABI::Async && Shape.CoroSuspends.empty())
588 return;
589 Value *CachedSlot = nullptr;
590 auto getSwiftErrorSlot = [&](Type *ValueTy) -> Value * {
591 if (CachedSlot) {
592 assert(CachedSlot->getType()->getPointerElementType() == ValueTy &&((CachedSlot->getType()->getPointerElementType() == ValueTy
&& "multiple swifterror slots in function with different types"
) ? static_cast<void> (0) : __assert_fail ("CachedSlot->getType()->getPointerElementType() == ValueTy && \"multiple swifterror slots in function with different types\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 593, __PRETTY_FUNCTION__))
593 "multiple swifterror slots in function with different types")((CachedSlot->getType()->getPointerElementType() == ValueTy
&& "multiple swifterror slots in function with different types"
) ? static_cast<void> (0) : __assert_fail ("CachedSlot->getType()->getPointerElementType() == ValueTy && \"multiple swifterror slots in function with different types\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 593, __PRETTY_FUNCTION__))
;
594 return CachedSlot;
595 }
596
597 // Check if the function has a swifterror argument.
598 for (auto &Arg : F.args()) {
599 if (Arg.isSwiftError()) {
600 CachedSlot = &Arg;
601 assert(Arg.getType()->getPointerElementType() == ValueTy &&((Arg.getType()->getPointerElementType() == ValueTy &&
"swifterror argument does not have expected type") ? static_cast
<void> (0) : __assert_fail ("Arg.getType()->getPointerElementType() == ValueTy && \"swifterror argument does not have expected type\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 602, __PRETTY_FUNCTION__))
602 "swifterror argument does not have expected type")((Arg.getType()->getPointerElementType() == ValueTy &&
"swifterror argument does not have expected type") ? static_cast
<void> (0) : __assert_fail ("Arg.getType()->getPointerElementType() == ValueTy && \"swifterror argument does not have expected type\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 602, __PRETTY_FUNCTION__))
;
603 return &Arg;
604 }
605 }
606
607 // Create a swifterror alloca.
608 IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHIOrDbg());
609 auto Alloca = Builder.CreateAlloca(ValueTy);
610 Alloca->setSwiftError(true);
611
612 CachedSlot = Alloca;
613 return Alloca;
614 };
615
616 for (CallInst *Op : Shape.SwiftErrorOps) {
617 auto MappedOp = VMap ? cast<CallInst>((*VMap)[Op]) : Op;
618 IRBuilder<> Builder(MappedOp);
619
620 // If there are no arguments, this is a 'get' operation.
621 Value *MappedResult;
622 if (Op->getNumArgOperands() == 0) {
623 auto ValueTy = Op->getType();
624 auto Slot = getSwiftErrorSlot(ValueTy);
625 MappedResult = Builder.CreateLoad(ValueTy, Slot);
626 } else {
627 assert(Op->getNumArgOperands() == 1)((Op->getNumArgOperands() == 1) ? static_cast<void> (
0) : __assert_fail ("Op->getNumArgOperands() == 1", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 627, __PRETTY_FUNCTION__))
;
628 auto Value = MappedOp->getArgOperand(0);
629 auto ValueTy = Value->getType();
630 auto Slot = getSwiftErrorSlot(ValueTy);
631 Builder.CreateStore(Value, Slot);
632 MappedResult = Slot;
633 }
634
635 MappedOp->replaceAllUsesWith(MappedResult);
636 MappedOp->eraseFromParent();
637 }
638
639 // If we're updating the original function, we've invalidated SwiftErrorOps.
640 if (VMap == nullptr) {
641 Shape.SwiftErrorOps.clear();
642 }
643}
644
645void CoroCloner::replaceSwiftErrorOps() {
646 ::replaceSwiftErrorOps(*NewF, Shape, &VMap);
647}
648
649void CoroCloner::salvageDebugInfo() {
650 SmallVector<DbgDeclareInst *, 8> Worklist;
651 SmallDenseMap<llvm::Value *, llvm::AllocaInst *, 4> DbgPtrAllocaCache;
652 for (auto &BB : *NewF)
653 for (auto &I : BB)
654 if (auto *DDI = dyn_cast<DbgDeclareInst>(&I))
655 Worklist.push_back(DDI);
656 for (DbgDeclareInst *DDI : Worklist)
657 coro::salvageDebugInfo(DbgPtrAllocaCache, DDI);
658
659 // Remove all salvaged dbg.declare intrinsics that became
660 // either unreachable or stale due to the CoroSplit transformation.
661 auto IsUnreachableBlock = [&](BasicBlock *BB) {
662 return BB->hasNPredecessors(0) && BB != &NewF->getEntryBlock();
663 };
664 for (DbgDeclareInst *DDI : Worklist) {
665 if (IsUnreachableBlock(DDI->getParent()))
666 DDI->eraseFromParent();
667 else if (dyn_cast_or_null<AllocaInst>(DDI->getAddress())) {
668 // Count all non-debuginfo uses in reachable blocks.
669 unsigned Uses = 0;
670 for (auto *User : DDI->getAddress()->users())
671 if (auto *I = dyn_cast<Instruction>(User))
672 if (!isa<AllocaInst>(I) && !IsUnreachableBlock(I->getParent()))
673 ++Uses;
674 if (!Uses)
675 DDI->eraseFromParent();
676 }
677 }
678}
679
680void CoroCloner::replaceEntryBlock() {
681 // In the original function, the AllocaSpillBlock is a block immediately
682 // following the allocation of the frame object which defines GEPs for
683 // all the allocas that have been moved into the frame, and it ends by
684 // branching to the original beginning of the coroutine. Make this
685 // the entry block of the cloned function.
686 auto *Entry = cast<BasicBlock>(VMap[Shape.AllocaSpillBlock]);
687 auto *OldEntry = &NewF->getEntryBlock();
688 Entry->setName("entry" + Suffix);
689 Entry->moveBefore(OldEntry);
690 Entry->getTerminator()->eraseFromParent();
691
692 // Clear all predecessors of the new entry block. There should be
693 // exactly one predecessor, which we created when splitting out
694 // AllocaSpillBlock to begin with.
695 assert(Entry->hasOneUse())((Entry->hasOneUse()) ? static_cast<void> (0) : __assert_fail
("Entry->hasOneUse()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 695, __PRETTY_FUNCTION__))
;
696 auto BranchToEntry = cast<BranchInst>(Entry->user_back());
697 assert(BranchToEntry->isUnconditional())((BranchToEntry->isUnconditional()) ? static_cast<void>
(0) : __assert_fail ("BranchToEntry->isUnconditional()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 697, __PRETTY_FUNCTION__))
;
698 Builder.SetInsertPoint(BranchToEntry);
699 Builder.CreateUnreachable();
700 BranchToEntry->eraseFromParent();
701
702 // Branch from the entry to the appropriate place.
703 Builder.SetInsertPoint(Entry);
704 switch (Shape.ABI) {
705 case coro::ABI::Switch: {
706 // In switch-lowering, we built a resume-entry block in the original
707 // function. Make the entry block branch to this.
708 auto *SwitchBB =
709 cast<BasicBlock>(VMap[Shape.SwitchLowering.ResumeEntryBlock]);
710 Builder.CreateBr(SwitchBB);
711 break;
712 }
713 case coro::ABI::Async:
714 case coro::ABI::Retcon:
715 case coro::ABI::RetconOnce: {
716 // In continuation ABIs, we want to branch to immediately after the
717 // active suspend point. Earlier phases will have put the suspend in its
718 // own basic block, so just thread our jump directly to its successor.
719 assert((Shape.ABI == coro::ABI::Async &&(((Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst
>(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape
.ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst
>(ActiveSuspend))) ? static_cast<void> (0) : __assert_fail
("(Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst>(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst>(ActiveSuspend))"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 723, __PRETTY_FUNCTION__))
720 isa<CoroSuspendAsyncInst>(ActiveSuspend)) ||(((Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst
>(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape
.ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst
>(ActiveSuspend))) ? static_cast<void> (0) : __assert_fail
("(Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst>(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst>(ActiveSuspend))"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 723, __PRETTY_FUNCTION__))
721 ((Shape.ABI == coro::ABI::Retcon ||(((Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst
>(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape
.ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst
>(ActiveSuspend))) ? static_cast<void> (0) : __assert_fail
("(Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst>(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst>(ActiveSuspend))"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 723, __PRETTY_FUNCTION__))
722 Shape.ABI == coro::ABI::RetconOnce) &&(((Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst
>(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape
.ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst
>(ActiveSuspend))) ? static_cast<void> (0) : __assert_fail
("(Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst>(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst>(ActiveSuspend))"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 723, __PRETTY_FUNCTION__))
723 isa<CoroSuspendRetconInst>(ActiveSuspend)))(((Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst
>(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape
.ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst
>(ActiveSuspend))) ? static_cast<void> (0) : __assert_fail
("(Shape.ABI == coro::ABI::Async && isa<CoroSuspendAsyncInst>(ActiveSuspend)) || ((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce) && isa<CoroSuspendRetconInst>(ActiveSuspend))"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 723, __PRETTY_FUNCTION__))
;
724 auto *MappedCS = cast<AnyCoroSuspendInst>(VMap[ActiveSuspend]);
725 auto Branch = cast<BranchInst>(MappedCS->getNextNode());
726 assert(Branch->isUnconditional())((Branch->isUnconditional()) ? static_cast<void> (0)
: __assert_fail ("Branch->isUnconditional()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 726, __PRETTY_FUNCTION__))
;
727 Builder.CreateBr(Branch->getSuccessor(0));
728 break;
729 }
730 }
731
732 // Any static alloca that's still being used but not reachable from the new
733 // entry needs to be moved to the new entry.
734 Function *F = OldEntry->getParent();
735 DominatorTree DT{*F};
736 for (auto IT = inst_begin(F), End = inst_end(F); IT != End;) {
737 Instruction &I = *IT++;
738 auto *Alloca = dyn_cast<AllocaInst>(&I);
739 if (!Alloca || I.use_empty())
740 continue;
741 if (DT.isReachableFromEntry(I.getParent()) ||
742 !isa<ConstantInt>(Alloca->getArraySize()))
743 continue;
744 I.moveBefore(*Entry, Entry->getFirstInsertionPt());
745 }
746}
747
748/// Derive the value of the new frame pointer.
749Value *CoroCloner::deriveNewFramePointer() {
750 // Builder should be inserting to the front of the new entry block.
751
752 switch (Shape.ABI) {
753 // In switch-lowering, the argument is the frame pointer.
754 case coro::ABI::Switch:
755 return &*NewF->arg_begin();
756 // In async-lowering, one of the arguments is an async context as determined
757 // by the `llvm.coro.id.async` intrinsic. We can retrieve the async context of
758 // the resume function from the async context projection function associated
759 // with the active suspend. The frame is located as a tail to the async
760 // context header.
761 case coro::ABI::Async: {
762 auto *ActiveAsyncSuspend = cast<CoroSuspendAsyncInst>(ActiveSuspend);
763 auto *CalleeContext =
764 NewF->getArg(ActiveAsyncSuspend->getStorageArgumentIndex());
765 auto *FramePtrTy = Shape.FrameTy->getPointerTo();
766 auto *ProjectionFunc =
767 ActiveAsyncSuspend->getAsyncContextProjectionFunction();
768 auto DbgLoc =
769 cast<CoroSuspendAsyncInst>(VMap[ActiveSuspend])->getDebugLoc();
770 // Calling i8* (i8*)
771 auto *CallerContext = Builder.CreateCall(
772 cast<FunctionType>(ProjectionFunc->getType()->getPointerElementType()),
773 ProjectionFunc, CalleeContext);
774 CallerContext->setCallingConv(ProjectionFunc->getCallingConv());
775 CallerContext->setDebugLoc(DbgLoc);
776 // The frame is located after the async_context header.
777 auto &Context = Builder.getContext();
778 auto *FramePtrAddr = Builder.CreateConstInBoundsGEP1_32(
779 Type::getInt8Ty(Context), CallerContext,
780 Shape.AsyncLowering.FrameOffset, "async.ctx.frameptr");
781 // Inline the projection function.
782 InlineFunctionInfo InlineInfo;
783 auto InlineRes = InlineFunction(*CallerContext, InlineInfo);
784 assert(InlineRes.isSuccess())((InlineRes.isSuccess()) ? static_cast<void> (0) : __assert_fail
("InlineRes.isSuccess()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 784, __PRETTY_FUNCTION__))
;
785 (void)InlineRes;
786 return Builder.CreateBitCast(FramePtrAddr, FramePtrTy);
787 }
788 // In continuation-lowering, the argument is the opaque storage.
789 case coro::ABI::Retcon:
790 case coro::ABI::RetconOnce: {
791 Argument *NewStorage = &*NewF->arg_begin();
792 auto FramePtrTy = Shape.FrameTy->getPointerTo();
793
794 // If the storage is inline, just bitcast to the storage to the frame type.
795 if (Shape.RetconLowering.IsFrameInlineInStorage)
796 return Builder.CreateBitCast(NewStorage, FramePtrTy);
797
798 // Otherwise, load the real frame from the opaque storage.
799 auto FramePtrPtr =
800 Builder.CreateBitCast(NewStorage, FramePtrTy->getPointerTo());
801 return Builder.CreateLoad(FramePtrTy, FramePtrPtr);
802 }
803 }
804 llvm_unreachable("bad ABI")::llvm::llvm_unreachable_internal("bad ABI", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 804)
;
805}
806
807static void addFramePointerAttrs(AttributeList &Attrs, LLVMContext &Context,
808 unsigned ParamIndex,
809 uint64_t Size, Align Alignment) {
810 AttrBuilder ParamAttrs;
811 ParamAttrs.addAttribute(Attribute::NonNull);
812 ParamAttrs.addAttribute(Attribute::NoAlias);
813 ParamAttrs.addAlignmentAttr(Alignment);
814 ParamAttrs.addDereferenceableAttr(Size);
815 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
816}
817
818/// Clone the body of the original function into a resume function of
819/// some sort.
820void CoroCloner::create() {
821 // Create the new function if we don't already have one.
822 if (!NewF) {
25
Assuming field 'NewF' is null
26
Taking true branch
823 NewF = createCloneDeclaration(OrigF, Shape, Suffix,
28
Calling 'createCloneDeclaration'
824 OrigF.getParent()->end(), ActiveSuspend);
27
Passing null pointer value via 5th parameter 'ActiveSuspend'
825 }
826
827 // Replace all args with undefs. The buildCoroutineFrame algorithm already
828 // rewritten access to the args that occurs after suspend points with loads
829 // and stores to/from the coroutine frame.
830 for (Argument &A : OrigF.args())
831 VMap[&A] = UndefValue::get(A.getType());
832
833 SmallVector<ReturnInst *, 4> Returns;
834
835 // Ignore attempts to change certain attributes of the function.
836 // TODO: maybe there should be a way to suppress this during cloning?
837 auto savedVisibility = NewF->getVisibility();
838 auto savedUnnamedAddr = NewF->getUnnamedAddr();
839 auto savedDLLStorageClass = NewF->getDLLStorageClass();
840
841 // NewF's linkage (which CloneFunctionInto does *not* change) might not
842 // be compatible with the visibility of OrigF (which it *does* change),
843 // so protect against that.
844 auto savedLinkage = NewF->getLinkage();
845 NewF->setLinkage(llvm::GlobalValue::ExternalLinkage);
846
847 CloneFunctionInto(NewF, &OrigF, VMap,
848 CloneFunctionChangeType::LocalChangesOnly, Returns);
849 // For async functions / continuations, adjust the scope line of the
850 // clone to the line number of the suspend point. The scope line is
851 // associated with all pre-prologue instructions. This avoids a jump
852 // in the linetable from the function declaration to the suspend point.
853 if (DISubprogram *SP = NewF->getSubprogram()) {
854 assert(SP != OrigF.getSubprogram() && SP->isDistinct())((SP != OrigF.getSubprogram() && SP->isDistinct())
? static_cast<void> (0) : __assert_fail ("SP != OrigF.getSubprogram() && SP->isDistinct()"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 854, __PRETTY_FUNCTION__))
;
855 if (ActiveSuspend)
856 if (auto DL = ActiveSuspend->getDebugLoc())
857 SP->setScopeLine(DL->getLine());
858 }
859
860 NewF->setLinkage(savedLinkage);
861 NewF->setVisibility(savedVisibility);
862 NewF->setUnnamedAddr(savedUnnamedAddr);
863 NewF->setDLLStorageClass(savedDLLStorageClass);
864
865 auto &Context = NewF->getContext();
866
867 // Replace the attributes of the new function:
868 auto OrigAttrs = NewF->getAttributes();
869 auto NewAttrs = AttributeList();
870
871 switch (Shape.ABI) {
872 case coro::ABI::Switch:
873 // Bootstrap attributes by copying function attributes from the
874 // original function. This should include optimization settings and so on.
875 NewAttrs = NewAttrs.addAttributes(Context, AttributeList::FunctionIndex,
876 OrigAttrs.getFnAttributes());
877
878 addFramePointerAttrs(NewAttrs, Context, 0,
879 Shape.FrameSize, Shape.FrameAlign);
880 break;
881 case coro::ABI::Async:
882 break;
883 case coro::ABI::Retcon:
884 case coro::ABI::RetconOnce:
885 // If we have a continuation prototype, just use its attributes,
886 // full-stop.
887 NewAttrs = Shape.RetconLowering.ResumePrototype->getAttributes();
888
889 addFramePointerAttrs(NewAttrs, Context, 0,
890 Shape.getRetconCoroId()->getStorageSize(),
891 Shape.getRetconCoroId()->getStorageAlignment());
892 break;
893 }
894
895 switch (Shape.ABI) {
896 // In these ABIs, the cloned functions always return 'void', and the
897 // existing return sites are meaningless. Note that for unique
898 // continuations, this includes the returns associated with suspends;
899 // this is fine because we can't suspend twice.
900 case coro::ABI::Switch:
901 case coro::ABI::RetconOnce:
902 // Remove old returns.
903 for (ReturnInst *Return : Returns)
904 changeToUnreachable(Return, /*UseLLVMTrap=*/false);
905 break;
906
907 // With multi-suspend continuations, we'll already have eliminated the
908 // original returns and inserted returns before all the suspend points,
909 // so we want to leave any returns in place.
910 case coro::ABI::Retcon:
911 break;
912 // Async lowering will insert musttail call functions at all suspend points
913 // followed by a return.
914 // Don't change returns to unreachable because that will trip up the verifier.
915 // These returns should be unreachable from the clone.
916 case coro::ABI::Async:
917 break;
918 }
919
920 NewF->setAttributes(NewAttrs);
921 NewF->setCallingConv(Shape.getResumeFunctionCC());
922
923 // Set up the new entry block.
924 replaceEntryBlock();
925
926 Builder.SetInsertPoint(&NewF->getEntryBlock().front());
927 NewFramePtr = deriveNewFramePointer();
928
929 // Remap frame pointer.
930 Value *OldFramePtr = VMap[Shape.FramePtr];
931 NewFramePtr->takeName(OldFramePtr);
932 OldFramePtr->replaceAllUsesWith(NewFramePtr);
933
934 // Remap vFrame pointer.
935 auto *NewVFrame = Builder.CreateBitCast(
936 NewFramePtr, Type::getInt8PtrTy(Builder.getContext()), "vFrame");
937 Value *OldVFrame = cast<Value>(VMap[Shape.CoroBegin]);
938 OldVFrame->replaceAllUsesWith(NewVFrame);
939
940 switch (Shape.ABI) {
941 case coro::ABI::Switch:
942 // Rewrite final suspend handling as it is not done via switch (allows to
943 // remove final case from the switch, since it is undefined behavior to
944 // resume the coroutine suspended at the final suspend point.
945 if (Shape.SwitchLowering.HasFinalSuspend)
946 handleFinalSuspend();
947 break;
948 case coro::ABI::Async:
949 case coro::ABI::Retcon:
950 case coro::ABI::RetconOnce:
951 // Replace uses of the active suspend with the corresponding
952 // continuation-function arguments.
953 assert(ActiveSuspend != nullptr &&((ActiveSuspend != nullptr && "no active suspend when lowering a continuation-style coroutine"
) ? static_cast<void> (0) : __assert_fail ("ActiveSuspend != nullptr && \"no active suspend when lowering a continuation-style coroutine\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 954, __PRETTY_FUNCTION__))
954 "no active suspend when lowering a continuation-style coroutine")((ActiveSuspend != nullptr && "no active suspend when lowering a continuation-style coroutine"
) ? static_cast<void> (0) : __assert_fail ("ActiveSuspend != nullptr && \"no active suspend when lowering a continuation-style coroutine\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 954, __PRETTY_FUNCTION__))
;
955 replaceRetconOrAsyncSuspendUses();
956 break;
957 }
958
959 // Handle suspends.
960 replaceCoroSuspends();
961
962 // Handle swifterror.
963 replaceSwiftErrorOps();
964
965 // Remove coro.end intrinsics.
966 replaceCoroEnds();
967
968 // Salvage debug info that points into the coroutine frame.
969 salvageDebugInfo();
970
971 // Eliminate coro.free from the clones, replacing it with 'null' in cleanup,
972 // to suppress deallocation code.
973 if (Shape.ABI == coro::ABI::Switch)
974 coro::replaceCoroFree(cast<CoroIdInst>(VMap[Shape.CoroBegin->getId()]),
975 /*Elide=*/ FKind == CoroCloner::Kind::SwitchCleanup);
976}
977
978// Create a resume clone by cloning the body of the original function, setting
979// new entry block and replacing coro.suspend an appropriate value to force
980// resume or cleanup pass for every suspend point.
981static Function *createClone(Function &F, const Twine &Suffix,
982 coro::Shape &Shape, CoroCloner::Kind FKind) {
983 CoroCloner Cloner(F, Suffix, Shape, FKind);
19
Calling constructor for 'CoroCloner'
23
Returning from constructor for 'CoroCloner'
984 Cloner.create();
24
Calling 'CoroCloner::create'
985 return Cloner.getFunction();
986}
987
988/// Remove calls to llvm.coro.end in the original function.
989static void removeCoroEnds(const coro::Shape &Shape, CallGraph *CG) {
990 for (auto End : Shape.CoroEnds) {
991 replaceCoroEnd(End, Shape, Shape.FramePtr, /*in resume*/ false, CG);
992 }
993}
994
995static void updateAsyncFuncPointerContextSize(coro::Shape &Shape) {
996 assert(Shape.ABI == coro::ABI::Async)((Shape.ABI == coro::ABI::Async) ? static_cast<void> (0
) : __assert_fail ("Shape.ABI == coro::ABI::Async", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 996, __PRETTY_FUNCTION__))
;
997
998 auto *FuncPtrStruct = cast<ConstantStruct>(
999 Shape.AsyncLowering.AsyncFuncPointer->getInitializer());
1000 auto *OrigRelativeFunOffset = FuncPtrStruct->getOperand(0);
1001 auto *OrigContextSize = FuncPtrStruct->getOperand(1);
1002 auto *NewContextSize = ConstantInt::get(OrigContextSize->getType(),
1003 Shape.AsyncLowering.ContextSize);
1004 auto *NewFuncPtrStruct = ConstantStruct::get(
1005 FuncPtrStruct->getType(), OrigRelativeFunOffset, NewContextSize);
1006
1007 Shape.AsyncLowering.AsyncFuncPointer->setInitializer(NewFuncPtrStruct);
1008}
1009
1010static void replaceFrameSize(coro::Shape &Shape) {
1011 if (Shape.ABI == coro::ABI::Async)
1012 updateAsyncFuncPointerContextSize(Shape);
1013
1014 if (Shape.CoroSizes.empty())
1015 return;
1016
1017 // In the same function all coro.sizes should have the same result type.
1018 auto *SizeIntrin = Shape.CoroSizes.back();
1019 Module *M = SizeIntrin->getModule();
1020 const DataLayout &DL = M->getDataLayout();
1021 auto Size = DL.getTypeAllocSize(Shape.FrameTy);
1022 auto *SizeConstant = ConstantInt::get(SizeIntrin->getType(), Size);
1023
1024 for (CoroSizeInst *CS : Shape.CoroSizes) {
1025 CS->replaceAllUsesWith(SizeConstant);
1026 CS->eraseFromParent();
1027 }
1028}
1029
1030// Create a global constant array containing pointers to functions provided and
1031// set Info parameter of CoroBegin to point at this constant. Example:
1032//
1033// @f.resumers = internal constant [2 x void(%f.frame*)*]
1034// [void(%f.frame*)* @f.resume, void(%f.frame*)* @f.destroy]
1035// define void @f() {
1036// ...
1037// call i8* @llvm.coro.begin(i8* null, i32 0, i8* null,
1038// i8* bitcast([2 x void(%f.frame*)*] * @f.resumers to i8*))
1039//
1040// Assumes that all the functions have the same signature.
1041static void setCoroInfo(Function &F, coro::Shape &Shape,
1042 ArrayRef<Function *> Fns) {
1043 // This only works under the switch-lowering ABI because coro elision
1044 // only works on the switch-lowering ABI.
1045 assert(Shape.ABI == coro::ABI::Switch)((Shape.ABI == coro::ABI::Switch) ? static_cast<void> (
0) : __assert_fail ("Shape.ABI == coro::ABI::Switch", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 1045, __PRETTY_FUNCTION__))
;
1046
1047 SmallVector<Constant *, 4> Args(Fns.begin(), Fns.end());
1048 assert(!Args.empty())((!Args.empty()) ? static_cast<void> (0) : __assert_fail
("!Args.empty()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 1048, __PRETTY_FUNCTION__))
;
1049 Function *Part = *Fns.begin();
1050 Module *M = Part->getParent();
1051 auto *ArrTy = ArrayType::get(Part->getType(), Args.size());
1052
1053 auto *ConstVal = ConstantArray::get(ArrTy, Args);
1054 auto *GV = new GlobalVariable(*M, ConstVal->getType(), /*isConstant=*/true,
1055 GlobalVariable::PrivateLinkage, ConstVal,
1056 F.getName() + Twine(".resumers"));
1057
1058 // Update coro.begin instruction to refer to this constant.
1059 LLVMContext &C = F.getContext();
1060 auto *BC = ConstantExpr::getPointerCast(GV, Type::getInt8PtrTy(C));
1061 Shape.getSwitchCoroId()->setInfo(BC);
1062}
1063
1064// Store addresses of Resume/Destroy/Cleanup functions in the coroutine frame.
1065static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn,
1066 Function *DestroyFn, Function *CleanupFn) {
1067 assert(Shape.ABI == coro::ABI::Switch)((Shape.ABI == coro::ABI::Switch) ? static_cast<void> (
0) : __assert_fail ("Shape.ABI == coro::ABI::Switch", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 1067, __PRETTY_FUNCTION__))
;
1068
1069 IRBuilder<> Builder(Shape.FramePtr->getNextNode());
1070 auto *ResumeAddr = Builder.CreateStructGEP(
1071 Shape.FrameTy, Shape.FramePtr, coro::Shape::SwitchFieldIndex::Resume,
1072 "resume.addr");
1073 Builder.CreateStore(ResumeFn, ResumeAddr);
1074
1075 Value *DestroyOrCleanupFn = DestroyFn;
1076
1077 CoroIdInst *CoroId = Shape.getSwitchCoroId();
1078 if (CoroAllocInst *CA = CoroId->getCoroAlloc()) {
1079 // If there is a CoroAlloc and it returns false (meaning we elide the
1080 // allocation, use CleanupFn instead of DestroyFn).
1081 DestroyOrCleanupFn = Builder.CreateSelect(CA, DestroyFn, CleanupFn);
1082 }
1083
1084 auto *DestroyAddr = Builder.CreateStructGEP(
1085 Shape.FrameTy, Shape.FramePtr, coro::Shape::SwitchFieldIndex::Destroy,
1086 "destroy.addr");
1087 Builder.CreateStore(DestroyOrCleanupFn, DestroyAddr);
1088}
1089
1090static void postSplitCleanup(Function &F) {
1091 removeUnreachableBlocks(F);
1092
1093 // For now, we do a mandatory verification step because we don't
1094 // entirely trust this pass. Note that we don't want to add a verifier
1095 // pass to FPM below because it will also verify all the global data.
1096 if (verifyFunction(F, &errs()))
1097 report_fatal_error("Broken function");
1098
1099 legacy::FunctionPassManager FPM(F.getParent());
1100
1101 FPM.add(createSCCPPass());
1102 FPM.add(createCFGSimplificationPass());
1103 FPM.add(createEarlyCSEPass());
1104 FPM.add(createCFGSimplificationPass());
1105
1106 FPM.doInitialization();
1107 FPM.run(F);
1108 FPM.doFinalization();
1109}
1110
1111// Assuming we arrived at the block NewBlock from Prev instruction, store
1112// PHI's incoming values in the ResolvedValues map.
1113static void
1114scanPHIsAndUpdateValueMap(Instruction *Prev, BasicBlock *NewBlock,
1115 DenseMap<Value *, Value *> &ResolvedValues) {
1116 auto *PrevBB = Prev->getParent();
1117 for (PHINode &PN : NewBlock->phis()) {
1118 auto V = PN.getIncomingValueForBlock(PrevBB);
1119 // See if we already resolved it.
1120 auto VI = ResolvedValues.find(V);
1121 if (VI != ResolvedValues.end())
1122 V = VI->second;
1123 // Remember the value.
1124 ResolvedValues[&PN] = V;
1125 }
1126}
1127
1128// Replace a sequence of branches leading to a ret, with a clone of a ret
1129// instruction. Suspend instruction represented by a switch, track the PHI
1130// values and select the correct case successor when possible.
1131static bool simplifyTerminatorLeadingToRet(Instruction *InitialInst) {
1132 DenseMap<Value *, Value *> ResolvedValues;
1133 BasicBlock *UnconditionalSucc = nullptr;
1134
1135 Instruction *I = InitialInst;
1136 while (I->isTerminator() ||
1137 (isa<CmpInst>(I) && I->getNextNode()->isTerminator())) {
1138 if (isa<ReturnInst>(I)) {
1139 if (I != InitialInst) {
1140 // If InitialInst is an unconditional branch,
1141 // remove PHI values that come from basic block of InitialInst
1142 if (UnconditionalSucc)
1143 UnconditionalSucc->removePredecessor(InitialInst->getParent(), true);
1144 ReplaceInstWithInst(InitialInst, I->clone());
1145 }
1146 return true;
1147 }
1148 if (auto *BR = dyn_cast<BranchInst>(I)) {
1149 if (BR->isUnconditional()) {
1150 BasicBlock *BB = BR->getSuccessor(0);
1151 if (I == InitialInst)
1152 UnconditionalSucc = BB;
1153 scanPHIsAndUpdateValueMap(I, BB, ResolvedValues);
1154 I = BB->getFirstNonPHIOrDbgOrLifetime();
1155 continue;
1156 }
1157 } else if (auto *CondCmp = dyn_cast<CmpInst>(I)) {
1158 auto *BR = dyn_cast<BranchInst>(I->getNextNode());
1159 if (BR && BR->isConditional() && CondCmp == BR->getCondition()) {
1160 // If the case number of suspended switch instruction is reduced to
1161 // 1, then it is simplified to CmpInst in llvm::ConstantFoldTerminator.
1162 // And the comparsion looks like : %cond = icmp eq i8 %V, constant.
1163 ConstantInt *CondConst = dyn_cast<ConstantInt>(CondCmp->getOperand(1));
1164 if (CondConst && CondCmp->getPredicate() == CmpInst::ICMP_EQ) {
1165 Value *V = CondCmp->getOperand(0);
1166 auto it = ResolvedValues.find(V);
1167 if (it != ResolvedValues.end())
1168 V = it->second;
1169
1170 if (ConstantInt *Cond0 = dyn_cast<ConstantInt>(V)) {
1171 BasicBlock *BB = Cond0->equalsInt(CondConst->getZExtValue())
1172 ? BR->getSuccessor(0)
1173 : BR->getSuccessor(1);
1174 scanPHIsAndUpdateValueMap(I, BB, ResolvedValues);
1175 I = BB->getFirstNonPHIOrDbgOrLifetime();
1176 continue;
1177 }
1178 }
1179 }
1180 } else if (auto *SI = dyn_cast<SwitchInst>(I)) {
1181 Value *V = SI->getCondition();
1182 auto it = ResolvedValues.find(V);
1183 if (it != ResolvedValues.end())
1184 V = it->second;
1185 if (ConstantInt *Cond = dyn_cast<ConstantInt>(V)) {
1186 BasicBlock *BB = SI->findCaseValue(Cond)->getCaseSuccessor();
1187 scanPHIsAndUpdateValueMap(I, BB, ResolvedValues);
1188 I = BB->getFirstNonPHIOrDbgOrLifetime();
1189 continue;
1190 }
1191 }
1192 return false;
1193 }
1194 return false;
1195}
1196
1197// Check whether CI obeys the rules of musttail attribute.
1198static bool shouldBeMustTail(const CallInst &CI, const Function &F) {
1199 if (CI.isInlineAsm())
1200 return false;
1201
1202 // Match prototypes and calling conventions of resume function.
1203 FunctionType *CalleeTy = CI.getFunctionType();
1204 if (!CalleeTy->getReturnType()->isVoidTy() || (CalleeTy->getNumParams() != 1))
1205 return false;
1206
1207 Type *CalleeParmTy = CalleeTy->getParamType(0);
1208 if (!CalleeParmTy->isPointerTy() ||
1209 (CalleeParmTy->getPointerAddressSpace() != 0))
1210 return false;
1211
1212 if (CI.getCallingConv() != F.getCallingConv())
1213 return false;
1214
1215 // CI should not has any ABI-impacting function attributes.
1216 static const Attribute::AttrKind ABIAttrs[] = {
1217 Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca,
1218 Attribute::Preallocated, Attribute::InReg, Attribute::Returned,
1219 Attribute::SwiftSelf, Attribute::SwiftError};
1220 AttributeList Attrs = CI.getAttributes();
1221 for (auto AK : ABIAttrs)
1222 if (Attrs.hasParamAttribute(0, AK))
1223 return false;
1224
1225 return true;
1226}
1227
1228// Add musttail to any resume instructions that is immediately followed by a
1229// suspend (i.e. ret). We do this even in -O0 to support guaranteed tail call
1230// for symmetrical coroutine control transfer (C++ Coroutines TS extension).
1231// This transformation is done only in the resume part of the coroutine that has
1232// identical signature and calling convention as the coro.resume call.
1233static void addMustTailToCoroResumes(Function &F) {
1234 bool changed = false;
1235
1236 // Collect potential resume instructions.
1237 SmallVector<CallInst *, 4> Resumes;
1238 for (auto &I : instructions(F))
1239 if (auto *Call = dyn_cast<CallInst>(&I))
1240 if (shouldBeMustTail(*Call, F))
1241 Resumes.push_back(Call);
1242
1243 // Set musttail on those that are followed by a ret instruction.
1244 for (CallInst *Call : Resumes)
1245 if (simplifyTerminatorLeadingToRet(Call->getNextNode())) {
1246 Call->setTailCallKind(CallInst::TCK_MustTail);
1247 changed = true;
1248 }
1249
1250 if (changed)
1251 removeUnreachableBlocks(F);
1252}
1253
1254// Coroutine has no suspend points. Remove heap allocation for the coroutine
1255// frame if possible.
1256static void handleNoSuspendCoroutine(coro::Shape &Shape) {
1257 auto *CoroBegin = Shape.CoroBegin;
1258 auto *CoroId = CoroBegin->getId();
1259 auto *AllocInst = CoroId->getCoroAlloc();
1260 switch (Shape.ABI) {
1261 case coro::ABI::Switch: {
1262 auto SwitchId = cast<CoroIdInst>(CoroId);
1263 coro::replaceCoroFree(SwitchId, /*Elide=*/AllocInst != nullptr);
1264 if (AllocInst) {
1265 IRBuilder<> Builder(AllocInst);
1266 auto *Frame = Builder.CreateAlloca(Shape.FrameTy);
1267 Frame->setAlignment(Shape.FrameAlign);
1268 auto *VFrame = Builder.CreateBitCast(Frame, Builder.getInt8PtrTy());
1269 AllocInst->replaceAllUsesWith(Builder.getFalse());
1270 AllocInst->eraseFromParent();
1271 CoroBegin->replaceAllUsesWith(VFrame);
1272 } else {
1273 CoroBegin->replaceAllUsesWith(CoroBegin->getMem());
1274 }
1275 break;
1276 }
1277 case coro::ABI::Async:
1278 case coro::ABI::Retcon:
1279 case coro::ABI::RetconOnce:
1280 CoroBegin->replaceAllUsesWith(UndefValue::get(CoroBegin->getType()));
1281 break;
1282 }
1283
1284 CoroBegin->eraseFromParent();
1285}
1286
1287// SimplifySuspendPoint needs to check that there is no calls between
1288// coro_save and coro_suspend, since any of the calls may potentially resume
1289// the coroutine and if that is the case we cannot eliminate the suspend point.
1290static bool hasCallsInBlockBetween(Instruction *From, Instruction *To) {
1291 for (Instruction *I = From; I != To; I = I->getNextNode()) {
1292 // Assume that no intrinsic can resume the coroutine.
1293 if (isa<IntrinsicInst>(I))
1294 continue;
1295
1296 if (isa<CallBase>(I))
1297 return true;
1298 }
1299 return false;
1300}
1301
1302static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB) {
1303 SmallPtrSet<BasicBlock *, 8> Set;
1304 SmallVector<BasicBlock *, 8> Worklist;
1305
1306 Set.insert(SaveBB);
1307 Worklist.push_back(ResDesBB);
1308
1309 // Accumulate all blocks between SaveBB and ResDesBB. Because CoroSaveIntr
1310 // returns a token consumed by suspend instruction, all blocks in between
1311 // will have to eventually hit SaveBB when going backwards from ResDesBB.
1312 while (!Worklist.empty()) {
1313 auto *BB = Worklist.pop_back_val();
1314 Set.insert(BB);
1315 for (auto *Pred : predecessors(BB))
1316 if (Set.count(Pred) == 0)
1317 Worklist.push_back(Pred);
1318 }
1319
1320 // SaveBB and ResDesBB are checked separately in hasCallsBetween.
1321 Set.erase(SaveBB);
1322 Set.erase(ResDesBB);
1323
1324 for (auto *BB : Set)
1325 if (hasCallsInBlockBetween(BB->getFirstNonPHI(), nullptr))
1326 return true;
1327
1328 return false;
1329}
1330
1331static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy) {
1332 auto *SaveBB = Save->getParent();
1333 auto *ResumeOrDestroyBB = ResumeOrDestroy->getParent();
1334
1335 if (SaveBB == ResumeOrDestroyBB)
1336 return hasCallsInBlockBetween(Save->getNextNode(), ResumeOrDestroy);
1337
1338 // Any calls from Save to the end of the block?
1339 if (hasCallsInBlockBetween(Save->getNextNode(), nullptr))
1340 return true;
1341
1342 // Any calls from begging of the block up to ResumeOrDestroy?
1343 if (hasCallsInBlockBetween(ResumeOrDestroyBB->getFirstNonPHI(),
1344 ResumeOrDestroy))
1345 return true;
1346
1347 // Any calls in all of the blocks between SaveBB and ResumeOrDestroyBB?
1348 if (hasCallsInBlocksBetween(SaveBB, ResumeOrDestroyBB))
1349 return true;
1350
1351 return false;
1352}
1353
1354// If a SuspendIntrin is preceded by Resume or Destroy, we can eliminate the
1355// suspend point and replace it with nornal control flow.
1356static bool simplifySuspendPoint(CoroSuspendInst *Suspend,
1357 CoroBeginInst *CoroBegin) {
1358 Instruction *Prev = Suspend->getPrevNode();
1359 if (!Prev) {
1360 auto *Pred = Suspend->getParent()->getSinglePredecessor();
1361 if (!Pred)
1362 return false;
1363 Prev = Pred->getTerminator();
1364 }
1365
1366 CallBase *CB = dyn_cast<CallBase>(Prev);
1367 if (!CB)
1368 return false;
1369
1370 auto *Callee = CB->getCalledOperand()->stripPointerCasts();
1371
1372 // See if the callsite is for resumption or destruction of the coroutine.
1373 auto *SubFn = dyn_cast<CoroSubFnInst>(Callee);
1374 if (!SubFn)
1375 return false;
1376
1377 // Does not refer to the current coroutine, we cannot do anything with it.
1378 if (SubFn->getFrame() != CoroBegin)
1379 return false;
1380
1381 // See if the transformation is safe. Specifically, see if there are any
1382 // calls in between Save and CallInstr. They can potenitally resume the
1383 // coroutine rendering this optimization unsafe.
1384 auto *Save = Suspend->getCoroSave();
1385 if (hasCallsBetween(Save, CB))
1386 return false;
1387
1388 // Replace llvm.coro.suspend with the value that results in resumption over
1389 // the resume or cleanup path.
1390 Suspend->replaceAllUsesWith(SubFn->getRawIndex());
1391 Suspend->eraseFromParent();
1392 Save->eraseFromParent();
1393
1394 // No longer need a call to coro.resume or coro.destroy.
1395 if (auto *Invoke = dyn_cast<InvokeInst>(CB)) {
1396 BranchInst::Create(Invoke->getNormalDest(), Invoke);
1397 }
1398
1399 // Grab the CalledValue from CB before erasing the CallInstr.
1400 auto *CalledValue = CB->getCalledOperand();
1401 CB->eraseFromParent();
1402
1403 // If no more users remove it. Usually it is a bitcast of SubFn.
1404 if (CalledValue != SubFn && CalledValue->user_empty())
1405 if (auto *I = dyn_cast<Instruction>(CalledValue))
1406 I->eraseFromParent();
1407
1408 // Now we are good to remove SubFn.
1409 if (SubFn->user_empty())
1410 SubFn->eraseFromParent();
1411
1412 return true;
1413}
1414
1415// Remove suspend points that are simplified.
1416static void simplifySuspendPoints(coro::Shape &Shape) {
1417 // Currently, the only simplification we do is switch-lowering-specific.
1418 if (Shape.ABI != coro::ABI::Switch)
1419 return;
1420
1421 auto &S = Shape.CoroSuspends;
1422 size_t I = 0, N = S.size();
1423 if (N == 0)
1424 return;
1425 while (true) {
1426 auto SI = cast<CoroSuspendInst>(S[I]);
1427 // Leave final.suspend to handleFinalSuspend since it is undefined behavior
1428 // to resume a coroutine suspended at the final suspend point.
1429 if (!SI->isFinal() && simplifySuspendPoint(SI, Shape.CoroBegin)) {
1430 if (--N == I)
1431 break;
1432 std::swap(S[I], S[N]);
1433 continue;
1434 }
1435 if (++I == N)
1436 break;
1437 }
1438 S.resize(N);
1439}
1440
1441static void splitSwitchCoroutine(Function &F, coro::Shape &Shape,
1442 SmallVectorImpl<Function *> &Clones) {
1443 assert(Shape.ABI == coro::ABI::Switch)((Shape.ABI == coro::ABI::Switch) ? static_cast<void> (
0) : __assert_fail ("Shape.ABI == coro::ABI::Switch", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 1443, __PRETTY_FUNCTION__))
;
17
'?' condition is true
1444
1445 createResumeEntryBlock(F, Shape);
1446 auto ResumeClone = createClone(F, ".resume", Shape,
18
Calling 'createClone'
1447 CoroCloner::Kind::SwitchResume);
1448 auto DestroyClone = createClone(F, ".destroy", Shape,
1449 CoroCloner::Kind::SwitchUnwind);
1450 auto CleanupClone = createClone(F, ".cleanup", Shape,
1451 CoroCloner::Kind::SwitchCleanup);
1452
1453 postSplitCleanup(*ResumeClone);
1454 postSplitCleanup(*DestroyClone);
1455 postSplitCleanup(*CleanupClone);
1456
1457 addMustTailToCoroResumes(*ResumeClone);
1458
1459 // Store addresses resume/destroy/cleanup functions in the coroutine frame.
1460 updateCoroFrame(Shape, ResumeClone, DestroyClone, CleanupClone);
1461
1462 assert(Clones.empty())((Clones.empty()) ? static_cast<void> (0) : __assert_fail
("Clones.empty()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 1462, __PRETTY_FUNCTION__))
;
1463 Clones.push_back(ResumeClone);
1464 Clones.push_back(DestroyClone);
1465 Clones.push_back(CleanupClone);
1466
1467 // Create a constant array referring to resume/destroy/clone functions pointed
1468 // by the last argument of @llvm.coro.info, so that CoroElide pass can
1469 // determined correct function to call.
1470 setCoroInfo(F, Shape, Clones);
1471}
1472
1473static void replaceAsyncResumeFunction(CoroSuspendAsyncInst *Suspend,
1474 Value *Continuation) {
1475 auto *ResumeIntrinsic = Suspend->getResumeFunction();
1476 auto &Context = Suspend->getParent()->getParent()->getContext();
1477 auto *Int8PtrTy = Type::getInt8PtrTy(Context);
1478
1479 IRBuilder<> Builder(ResumeIntrinsic);
1480 auto *Val = Builder.CreateBitOrPointerCast(Continuation, Int8PtrTy);
1481 ResumeIntrinsic->replaceAllUsesWith(Val);
1482 ResumeIntrinsic->eraseFromParent();
1483 Suspend->setOperand(CoroSuspendAsyncInst::ResumeFunctionArg,
1484 UndefValue::get(Int8PtrTy));
1485}
1486
1487/// Coerce the arguments in \p FnArgs according to \p FnTy in \p CallArgs.
1488static void coerceArguments(IRBuilder<> &Builder, FunctionType *FnTy,
1489 ArrayRef<Value *> FnArgs,
1490 SmallVectorImpl<Value *> &CallArgs) {
1491 size_t ArgIdx = 0;
1492 for (auto paramTy : FnTy->params()) {
1493 assert(ArgIdx < FnArgs.size())((ArgIdx < FnArgs.size()) ? static_cast<void> (0) : __assert_fail
("ArgIdx < FnArgs.size()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 1493, __PRETTY_FUNCTION__))
;
1494 if (paramTy != FnArgs[ArgIdx]->getType())
1495 CallArgs.push_back(
1496 Builder.CreateBitOrPointerCast(FnArgs[ArgIdx], paramTy));
1497 else
1498 CallArgs.push_back(FnArgs[ArgIdx]);
1499 ++ArgIdx;
1500 }
1501}
1502
1503CallInst *coro::createMustTailCall(DebugLoc Loc, Function *MustTailCallFn,
1504 ArrayRef<Value *> Arguments,
1505 IRBuilder<> &Builder) {
1506 auto *FnTy =
1507 cast<FunctionType>(MustTailCallFn->getType()->getPointerElementType());
1508 // Coerce the arguments, llvm optimizations seem to ignore the types in
1509 // vaarg functions and throws away casts in optimized mode.
1510 SmallVector<Value *, 8> CallArgs;
1511 coerceArguments(Builder, FnTy, Arguments, CallArgs);
1512
1513 auto *TailCall = Builder.CreateCall(FnTy, MustTailCallFn, CallArgs);
1514 TailCall->setTailCallKind(CallInst::TCK_MustTail);
1515 TailCall->setDebugLoc(Loc);
1516 TailCall->setCallingConv(MustTailCallFn->getCallingConv());
1517 return TailCall;
1518}
1519
1520static void splitAsyncCoroutine(Function &F, coro::Shape &Shape,
1521 SmallVectorImpl<Function *> &Clones) {
1522 assert(Shape.ABI == coro::ABI::Async)((Shape.ABI == coro::ABI::Async) ? static_cast<void> (0
) : __assert_fail ("Shape.ABI == coro::ABI::Async", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 1522, __PRETTY_FUNCTION__))
;
1523 assert(Clones.empty())((Clones.empty()) ? static_cast<void> (0) : __assert_fail
("Clones.empty()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 1523, __PRETTY_FUNCTION__))
;
1524 // Reset various things that the optimizer might have decided it
1525 // "knows" about the coroutine function due to not seeing a return.
1526 F.removeFnAttr(Attribute::NoReturn);
1527 F.removeAttribute(AttributeList::ReturnIndex, Attribute::NoAlias);
1528 F.removeAttribute(AttributeList::ReturnIndex, Attribute::NonNull);
1529
1530 auto &Context = F.getContext();
1531 auto *Int8PtrTy = Type::getInt8PtrTy(Context);
1532
1533 auto *Id = cast<CoroIdAsyncInst>(Shape.CoroBegin->getId());
1534 IRBuilder<> Builder(Id);
1535
1536 auto *FramePtr = Id->getStorage();
1537 FramePtr = Builder.CreateBitOrPointerCast(FramePtr, Int8PtrTy);
1538 FramePtr = Builder.CreateConstInBoundsGEP1_32(
1539 Type::getInt8Ty(Context), FramePtr, Shape.AsyncLowering.FrameOffset,
1540 "async.ctx.frameptr");
1541
1542 // Map all uses of llvm.coro.begin to the allocated frame pointer.
1543 {
1544 // Make sure we don't invalidate Shape.FramePtr.
1545 TrackingVH<Instruction> Handle(Shape.FramePtr);
1546 Shape.CoroBegin->replaceAllUsesWith(FramePtr);
1547 Shape.FramePtr = Handle.getValPtr();
1548 }
1549
1550 // Create all the functions in order after the main function.
1551 auto NextF = std::next(F.getIterator());
1552
1553 // Create a continuation function for each of the suspend points.
1554 Clones.reserve(Shape.CoroSuspends.size());
1555 for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) {
1556 auto *Suspend = cast<CoroSuspendAsyncInst>(Shape.CoroSuspends[Idx]);
1557
1558 // Create the clone declaration.
1559 auto *Continuation = createCloneDeclaration(
1560 F, Shape, ".resume." + Twine(Idx), NextF, Suspend);
1561 Clones.push_back(Continuation);
1562
1563 // Insert a branch to a new return block immediately before the suspend
1564 // point.
1565 auto *SuspendBB = Suspend->getParent();
1566 auto *NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);
1567 auto *Branch = cast<BranchInst>(SuspendBB->getTerminator());
1568
1569 // Place it before the first suspend.
1570 auto *ReturnBB =
1571 BasicBlock::Create(F.getContext(), "coro.return", &F, NewSuspendBB);
1572 Branch->setSuccessor(0, ReturnBB);
1573
1574 IRBuilder<> Builder(ReturnBB);
1575
1576 // Insert the call to the tail call function and inline it.
1577 auto *Fn = Suspend->getMustTailCallFunction();
1578 SmallVector<Value *, 8> Args(Suspend->args());
1579 auto FnArgs = ArrayRef<Value *>(Args).drop_front(
1580 CoroSuspendAsyncInst::MustTailCallFuncArg + 1);
1581 auto *TailCall =
1582 coro::createMustTailCall(Suspend->getDebugLoc(), Fn, FnArgs, Builder);
1583 Builder.CreateRetVoid();
1584 InlineFunctionInfo FnInfo;
1585 auto InlineRes = InlineFunction(*TailCall, FnInfo);
1586 assert(InlineRes.isSuccess() && "Expected inlining to succeed")((InlineRes.isSuccess() && "Expected inlining to succeed"
) ? static_cast<void> (0) : __assert_fail ("InlineRes.isSuccess() && \"Expected inlining to succeed\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 1586, __PRETTY_FUNCTION__))
;
1587 (void)InlineRes;
1588
1589 // Replace the lvm.coro.async.resume intrisic call.
1590 replaceAsyncResumeFunction(Suspend, Continuation);
1591 }
1592
1593 assert(Clones.size() == Shape.CoroSuspends.size())((Clones.size() == Shape.CoroSuspends.size()) ? static_cast<
void> (0) : __assert_fail ("Clones.size() == Shape.CoroSuspends.size()"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 1593, __PRETTY_FUNCTION__))
;
1594 for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) {
1595 auto *Suspend = Shape.CoroSuspends[Idx];
1596 auto *Clone = Clones[Idx];
1597
1598 CoroCloner(F, "resume." + Twine(Idx), Shape, Clone, Suspend).create();
1599 }
1600}
1601
1602static void splitRetconCoroutine(Function &F, coro::Shape &Shape,
1603 SmallVectorImpl<Function *> &Clones) {
1604 assert(Shape.ABI == coro::ABI::Retcon ||((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce
) ? static_cast<void> (0) : __assert_fail ("Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 1605, __PRETTY_FUNCTION__))
1605 Shape.ABI == coro::ABI::RetconOnce)((Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce
) ? static_cast<void> (0) : __assert_fail ("Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 1605, __PRETTY_FUNCTION__))
;
1606 assert(Clones.empty())((Clones.empty()) ? static_cast<void> (0) : __assert_fail
("Clones.empty()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 1606, __PRETTY_FUNCTION__))
;
1607
1608 // Reset various things that the optimizer might have decided it
1609 // "knows" about the coroutine function due to not seeing a return.
1610 F.removeFnAttr(Attribute::NoReturn);
1611 F.removeAttribute(AttributeList::ReturnIndex, Attribute::NoAlias);
1612 F.removeAttribute(AttributeList::ReturnIndex, Attribute::NonNull);
1613
1614 // Allocate the frame.
1615 auto *Id = cast<AnyCoroIdRetconInst>(Shape.CoroBegin->getId());
1616 Value *RawFramePtr;
1617 if (Shape.RetconLowering.IsFrameInlineInStorage) {
1618 RawFramePtr = Id->getStorage();
1619 } else {
1620 IRBuilder<> Builder(Id);
1621
1622 // Determine the size of the frame.
1623 const DataLayout &DL = F.getParent()->getDataLayout();
1624 auto Size = DL.getTypeAllocSize(Shape.FrameTy);
1625
1626 // Allocate. We don't need to update the call graph node because we're
1627 // going to recompute it from scratch after splitting.
1628 // FIXME: pass the required alignment
1629 RawFramePtr = Shape.emitAlloc(Builder, Builder.getInt64(Size), nullptr);
1630 RawFramePtr =
1631 Builder.CreateBitCast(RawFramePtr, Shape.CoroBegin->getType());
1632
1633 // Stash the allocated frame pointer in the continuation storage.
1634 auto Dest = Builder.CreateBitCast(Id->getStorage(),
1635 RawFramePtr->getType()->getPointerTo());
1636 Builder.CreateStore(RawFramePtr, Dest);
1637 }
1638
1639 // Map all uses of llvm.coro.begin to the allocated frame pointer.
1640 {
1641 // Make sure we don't invalidate Shape.FramePtr.
1642 TrackingVH<Instruction> Handle(Shape.FramePtr);
1643 Shape.CoroBegin->replaceAllUsesWith(RawFramePtr);
1644 Shape.FramePtr = Handle.getValPtr();
1645 }
1646
1647 // Create a unique return block.
1648 BasicBlock *ReturnBB = nullptr;
1649 SmallVector<PHINode *, 4> ReturnPHIs;
1650
1651 // Create all the functions in order after the main function.
1652 auto NextF = std::next(F.getIterator());
1653
1654 // Create a continuation function for each of the suspend points.
1655 Clones.reserve(Shape.CoroSuspends.size());
1656 for (size_t i = 0, e = Shape.CoroSuspends.size(); i != e; ++i) {
1657 auto Suspend = cast<CoroSuspendRetconInst>(Shape.CoroSuspends[i]);
1658
1659 // Create the clone declaration.
1660 auto Continuation =
1661 createCloneDeclaration(F, Shape, ".resume." + Twine(i), NextF, nullptr);
1662 Clones.push_back(Continuation);
1663
1664 // Insert a branch to the unified return block immediately before
1665 // the suspend point.
1666 auto SuspendBB = Suspend->getParent();
1667 auto NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);
1668 auto Branch = cast<BranchInst>(SuspendBB->getTerminator());
1669
1670 // Create the unified return block.
1671 if (!ReturnBB) {
1672 // Place it before the first suspend.
1673 ReturnBB = BasicBlock::Create(F.getContext(), "coro.return", &F,
1674 NewSuspendBB);
1675 Shape.RetconLowering.ReturnBlock = ReturnBB;
1676
1677 IRBuilder<> Builder(ReturnBB);
1678
1679 // Create PHIs for all the return values.
1680 assert(ReturnPHIs.empty())((ReturnPHIs.empty()) ? static_cast<void> (0) : __assert_fail
("ReturnPHIs.empty()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 1680, __PRETTY_FUNCTION__))
;
1681
1682 // First, the continuation.
1683 ReturnPHIs.push_back(Builder.CreatePHI(Continuation->getType(),
1684 Shape.CoroSuspends.size()));
1685
1686 // Next, all the directly-yielded values.
1687 for (auto ResultTy : Shape.getRetconResultTypes())
1688 ReturnPHIs.push_back(Builder.CreatePHI(ResultTy,
1689 Shape.CoroSuspends.size()));
1690
1691 // Build the return value.
1692 auto RetTy = F.getReturnType();
1693
1694 // Cast the continuation value if necessary.
1695 // We can't rely on the types matching up because that type would
1696 // have to be infinite.
1697 auto CastedContinuationTy =
1698 (ReturnPHIs.size() == 1 ? RetTy : RetTy->getStructElementType(0));
1699 auto *CastedContinuation =
1700 Builder.CreateBitCast(ReturnPHIs[0], CastedContinuationTy);
1701
1702 Value *RetV;
1703 if (ReturnPHIs.size() == 1) {
1704 RetV = CastedContinuation;
1705 } else {
1706 RetV = UndefValue::get(RetTy);
1707 RetV = Builder.CreateInsertValue(RetV, CastedContinuation, 0);
1708 for (size_t I = 1, E = ReturnPHIs.size(); I != E; ++I)
1709 RetV = Builder.CreateInsertValue(RetV, ReturnPHIs[I], I);
1710 }
1711
1712 Builder.CreateRet(RetV);
1713 }
1714
1715 // Branch to the return block.
1716 Branch->setSuccessor(0, ReturnBB);
1717 ReturnPHIs[0]->addIncoming(Continuation, SuspendBB);
1718 size_t NextPHIIndex = 1;
1719 for (auto &VUse : Suspend->value_operands())
1720 ReturnPHIs[NextPHIIndex++]->addIncoming(&*VUse, SuspendBB);
1721 assert(NextPHIIndex == ReturnPHIs.size())((NextPHIIndex == ReturnPHIs.size()) ? static_cast<void>
(0) : __assert_fail ("NextPHIIndex == ReturnPHIs.size()", "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 1721, __PRETTY_FUNCTION__))
;
1722 }
1723
1724 assert(Clones.size() == Shape.CoroSuspends.size())((Clones.size() == Shape.CoroSuspends.size()) ? static_cast<
void> (0) : __assert_fail ("Clones.size() == Shape.CoroSuspends.size()"
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 1724, __PRETTY_FUNCTION__))
;
1725 for (size_t i = 0, e = Shape.CoroSuspends.size(); i != e; ++i) {
1726 auto Suspend = Shape.CoroSuspends[i];
1727 auto Clone = Clones[i];
1728
1729 CoroCloner(F, "resume." + Twine(i), Shape, Clone, Suspend).create();
1730 }
1731}
1732
1733namespace {
1734 class PrettyStackTraceFunction : public PrettyStackTraceEntry {
1735 Function &F;
1736 public:
1737 PrettyStackTraceFunction(Function &F) : F(F) {}
1738 void print(raw_ostream &OS) const override {
1739 OS << "While splitting coroutine ";
1740 F.printAsOperand(OS, /*print type*/ false, F.getParent());
1741 OS << "\n";
1742 }
1743 };
1744}
1745
1746static coro::Shape splitCoroutine(Function &F,
1747 SmallVectorImpl<Function *> &Clones,
1748 bool ReuseFrameSlot) {
1749 PrettyStackTraceFunction prettyStackTrace(F);
1750
1751 // The suspend-crossing algorithm in buildCoroutineFrame get tripped
1752 // up by uses in unreachable blocks, so remove them as a first pass.
1753 removeUnreachableBlocks(F);
1754
1755 coro::Shape Shape(F, ReuseFrameSlot);
1756 if (!Shape.CoroBegin)
12
Assuming field 'CoroBegin' is non-null
13
Taking false branch
1757 return Shape;
1758
1759 simplifySuspendPoints(Shape);
1760 buildCoroutineFrame(F, Shape);
1761 replaceFrameSize(Shape);
1762
1763 // If there are no suspend points, no split required, just remove
1764 // the allocation and deallocation blocks, they are not needed.
1765 if (Shape.CoroSuspends.empty()) {
14
Taking false branch
1766 handleNoSuspendCoroutine(Shape);
1767 } else {
1768 switch (Shape.ABI) {
15
Control jumps to 'case Switch:' at line 1769
1769 case coro::ABI::Switch:
1770 splitSwitchCoroutine(F, Shape, Clones);
16
Calling 'splitSwitchCoroutine'
1771 break;
1772 case coro::ABI::Async:
1773 splitAsyncCoroutine(F, Shape, Clones);
1774 break;
1775 case coro::ABI::Retcon:
1776 case coro::ABI::RetconOnce:
1777 splitRetconCoroutine(F, Shape, Clones);
1778 break;
1779 }
1780 }
1781
1782 // Replace all the swifterror operations in the original function.
1783 // This invalidates SwiftErrorOps in the Shape.
1784 replaceSwiftErrorOps(F, Shape, nullptr);
1785
1786 return Shape;
1787}
1788
1789static void
1790updateCallGraphAfterCoroutineSplit(Function &F, const coro::Shape &Shape,
1791 const SmallVectorImpl<Function *> &Clones,
1792 CallGraph &CG, CallGraphSCC &SCC) {
1793 if (!Shape.CoroBegin)
1794 return;
1795
1796 removeCoroEnds(Shape, &CG);
1797 postSplitCleanup(F);
1798
1799 // Update call graph and add the functions we created to the SCC.
1800 coro::updateCallGraph(F, Clones, CG, SCC);
1801}
1802
1803static void updateCallGraphAfterCoroutineSplit(
1804 LazyCallGraph::Node &N, const coro::Shape &Shape,
1805 const SmallVectorImpl<Function *> &Clones, LazyCallGraph::SCC &C,
1806 LazyCallGraph &CG, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR,
1807 FunctionAnalysisManager &FAM) {
1808 if (!Shape.CoroBegin)
1809 return;
1810
1811 for (llvm::AnyCoroEndInst *End : Shape.CoroEnds) {
1812 auto &Context = End->getContext();
1813 End->replaceAllUsesWith(ConstantInt::getFalse(Context));
1814 End->eraseFromParent();
1815 }
1816
1817 if (!Clones.empty()) {
1818 switch (Shape.ABI) {
1819 case coro::ABI::Switch:
1820 // Each clone in the Switch lowering is independent of the other clones.
1821 // Let the LazyCallGraph know about each one separately.
1822 for (Function *Clone : Clones)
1823 CG.addSplitFunction(N.getFunction(), *Clone);
1824 break;
1825 case coro::ABI::Async:
1826 case coro::ABI::Retcon:
1827 case coro::ABI::RetconOnce:
1828 // Each clone in the Async/Retcon lowering references of the other clones.
1829 // Let the LazyCallGraph know about all of them at once.
1830 if (!Clones.empty())
1831 CG.addSplitRefRecursiveFunctions(N.getFunction(), Clones);
1832 break;
1833 }
1834
1835 // Let the CGSCC infra handle the changes to the original function.
1836 updateCGAndAnalysisManagerForCGSCCPass(CG, C, N, AM, UR, FAM);
1837 }
1838
1839 // Do some cleanup and let the CGSCC infra see if we've cleaned up any edges
1840 // to the split functions.
1841 postSplitCleanup(N.getFunction());
1842 updateCGAndAnalysisManagerForFunctionPass(CG, C, N, AM, UR, FAM);
1843}
1844
1845// When we see the coroutine the first time, we insert an indirect call to a
1846// devirt trigger function and mark the coroutine that it is now ready for
1847// split.
1848// Async lowering uses this after it has split the function to restart the
1849// pipeline.
1850static void prepareForSplit(Function &F, CallGraph &CG,
1851 bool MarkForAsyncRestart = false) {
1852 Module &M = *F.getParent();
1853 LLVMContext &Context = F.getContext();
1854#ifndef NDEBUG
1855 Function *DevirtFn = M.getFunction(CORO_DEVIRT_TRIGGER_FN"coro.devirt.trigger");
1856 assert(DevirtFn && "coro.devirt.trigger function not found")((DevirtFn && "coro.devirt.trigger function not found"
) ? static_cast<void> (0) : __assert_fail ("DevirtFn && \"coro.devirt.trigger function not found\""
, "/build/llvm-toolchain-snapshot-13~++20210308111132+66e3a4abe99c/llvm/lib/Transforms/Coroutines/CoroSplit.cpp"
, 1856, __PRETTY_FUNCTION__))
;
1857#endif
1858
1859 F.addFnAttr(CORO_PRESPLIT_ATTR"coroutine.presplit", MarkForAsyncRestart
1860 ? ASYNC_RESTART_AFTER_SPLIT"2"
1861 : PREPARED_FOR_SPLIT"1");
1862
1863 // Insert an indirect call sequence that will be devirtualized by CoroElide
1864 // pass:
1865 // %0 = call i8* @llvm.coro.subfn.addr(i8* null, i8 -1)
1866 // %1 = bitcast i8* %0 to void(i8*)*
1867 // call void %1(i8* null)
1868 coro::LowererBase Lowerer(M);
1869 Instruction *InsertPt =
1870 MarkForAsyncRestart ? F.getEntryBlock().getFirstNonPHIOrDbgOrLifetime()
1871 : F.getEntryBlock().getTerminator();
1872 auto *Null = ConstantPointerNull::get(Type::getInt8PtrTy(Context));
1873 auto *DevirtFnAddr =
1874 Lowerer.makeSubFnCall(Null, CoroSubFnInst::RestartTrigger, InsertPt);
1875 FunctionType *FnTy = FunctionType::get(Type::getVoidTy(Context),
1876 {Type::getInt8PtrTy(Context)}, false);
1877 auto *IndirectCall = CallInst::Create(FnTy, DevirtFnAddr, Null, "", InsertPt);
1878
1879 // Update CG graph with an indirect call we just added.
1880 CG[&F]->addCalledFunction(IndirectCall, CG.getCallsExternalNode());
1881}
1882
1883// Make sure that there is a devirtualization trigger function that the
1884// coro-split pass uses to force a restart of the CGSCC pipeline. If the devirt
1885// trigger function is not found, we will create one and add it to the current
1886// SCC.
1887static void createDevirtTriggerFunc(CallGraph &CG, CallGraphSCC &SCC) {
1888 Module &M = CG.getModule();
1889 if (M.getFunction(CORO_DEVIRT_TRIGGER_FN"coro.devirt.trigger"))
1890 return;
1891
1892 LLVMContext &C = M.getContext();
1893 auto *FnTy = FunctionType::get(Type::getVoidTy(C), Type::getInt8PtrTy(C),
1894 /*isVarArg=*/false);
1895 Function *DevirtFn =
1896 Function::Create(FnTy, GlobalValue::LinkageTypes::PrivateLinkage,
1897 CORO_DEVIRT_TRIGGER_FN"coro.devirt.trigger", &M);
1898 DevirtFn->addFnAttr(Attribute::AlwaysInline);
1899 auto *Entry = BasicBlock::Create(C, "entry", DevirtFn);
1900 ReturnInst::Create(C, Entry);
1901
1902 auto *Node = CG.getOrInsertFunction(DevirtFn);
1903
1904 SmallVector<CallGraphNode *, 8> Nodes(SCC.begin(), SCC.end());
1905 Nodes.push_back(Node);
1906 SCC.initialize(Nodes);
1907}
1908
1909/// Replace a call to llvm.coro.prepare.retcon.
1910static void replacePrepare(CallInst *Prepare, LazyCallGraph &CG,
1911 LazyCallGraph::SCC &C) {
1912 auto CastFn = Prepare->getArgOperand(0); // as an i8*
1913 auto Fn = CastFn->stripPointerCasts(); // as its original type
1914
1915 // Attempt to peephole this pattern:
1916 // %0 = bitcast [[TYPE]] @some_function to i8*
1917 // %1 = call @llvm.coro.prepare.retcon(i8* %0)
1918 // %2 = bitcast %1 to [[TYPE]]
1919 // ==>
1920 // %2 = @some_function
1921 for (auto UI = Prepare->use_begin(), UE = Prepare->use_end(); UI != UE;) {
1922 // Look for bitcasts back to the original function type.
1923 auto *Cast = dyn_cast<BitCastInst>((UI++)->getUser());
1924 if (!Cast || Cast->getType() != Fn->getType())
1925 continue;
1926
1927 // Replace and remove the cast.
1928 Cast->replaceAllUsesWith(Fn);
1929 Cast->eraseFromParent();
1930 }
1931
1932 // Replace any remaining uses with the function as an i8*.
1933 // This can never directly be a callee, so we don't need to update CG.
1934 Prepare->replaceAllUsesWith(CastFn);
1935 Prepare->eraseFromParent();
1936
1937 // Kill dead bitcasts.
1938 while (auto *Cast = dyn_cast<BitCastInst>(CastFn)) {
1939 if (!Cast->use_empty())
1940 break;
1941 CastFn = Cast->getOperand(0);
1942 Cast->eraseFromParent();
1943 }
1944}
1945/// Replace a call to llvm.coro.prepare.retcon.
1946static void replacePrepare(CallInst *Prepare, CallGraph &CG) {
1947 auto CastFn = Prepare->getArgOperand(0); // as an i8*
1948 auto Fn = CastFn->stripPointerCasts(); // as its original type
1949
1950 // Find call graph nodes for the preparation.
1951 CallGraphNode *PrepareUserNode = nullptr, *FnNode = nullptr;
1952 if (auto ConcreteFn = dyn_cast<Function>(Fn)) {
1953 PrepareUserNode = CG[Prepare->getFunction()];
1954 FnNode = CG[ConcreteFn];
1955 }
1956
1957 // Attempt to peephole this pattern:
1958 // %0 = bitcast [[TYPE]] @some_function to i8*
1959 // %1 = call @llvm.coro.prepare.retcon(i8* %0)
1960 // %2 = bitcast %1 to [[TYPE]]
1961 // ==>
1962 // %2 = @some_function
1963 for (auto UI = Prepare->use_begin(), UE = Prepare->use_end();
1964 UI != UE; ) {
1965 // Look for bitcasts back to the original function type.
1966 auto *Cast = dyn_cast<BitCastInst>((UI++)->getUser());
1967 if (!Cast || Cast->getType() != Fn->getType()) continue;
1968
1969 // Check whether the replacement will introduce new direct calls.
1970 // If so, we'll need to update the call graph.
1971 if (PrepareUserNode) {
1972 for (auto &Use : Cast->uses()) {
1973 if (auto *CB = dyn_cast<CallBase>(Use.getUser())) {
1974 if (!CB->isCallee(&Use))
1975 continue;
1976 PrepareUserNode->removeCallEdgeFor(*CB);
1977 PrepareUserNode->addCalledFunction(CB, FnNode);
1978 }
1979 }
1980 }
1981
1982 // Replace and remove the cast.
1983 Cast->replaceAllUsesWith(Fn);
1984 Cast->eraseFromParent();
1985 }
1986
1987 // Replace any remaining uses with the function as an i8*.
1988 // This can never directly be a callee, so we don't need to update CG.
1989 Prepare->replaceAllUsesWith(CastFn);
1990 Prepare->eraseFromParent();
1991
1992 // Kill dead bitcasts.
1993 while (auto *Cast = dyn_cast<BitCastInst>(CastFn)) {
1994 if (!Cast->use_empty()) break;
1995 CastFn = Cast->getOperand(0);
1996 Cast->eraseFromParent();
1997 }
1998}
1999
2000static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG,
2001 LazyCallGraph::SCC &C) {
2002 bool Changed = false;
2003 for (auto PI = PrepareFn->use_begin(), PE = PrepareFn->use_end(); PI != PE;) {
2004 // Intrinsics can only be used in calls.
2005 auto *Prepare = cast<CallInst>((PI++)->getUser());
2006 replacePrepare(Prepare, CG, C);
2007 Changed = true;
2008 }
2009
2010 return Changed;
2011}
2012
2013/// Remove calls to llvm.coro.prepare.retcon, a barrier meant to prevent
2014/// IPO from operating on calls to a retcon coroutine before it's been
2015/// split. This is only safe to do after we've split all retcon
2016/// coroutines in the module. We can do that this in this pass because
2017/// this pass does promise to split all retcon coroutines (as opposed to
2018/// switch coroutines, which are lowered in multiple stages).
2019static bool replaceAllPrepares(Function *PrepareFn, CallGraph &CG) {
2020 bool Changed = false;
2021 for (auto PI = PrepareFn->use_begin(), PE = PrepareFn->use_end();
2022 PI != PE; ) {
2023 // Intrinsics can only be used in calls.
2024 auto *Prepare = cast<CallInst>((PI++)->getUser());
2025 replacePrepare(Prepare, CG);
2026 Changed = true;
2027 }
2028
2029 return Changed;
2030}
2031
2032static bool declaresCoroSplitIntrinsics(const Module &M) {
2033 return coro::declaresIntrinsics(M, {"llvm.coro.begin",
2034 "llvm.coro.prepare.retcon",
2035 "llvm.coro.prepare.async"});
2036}
2037
2038static void addPrepareFunction(const Module &M,
2039 SmallVectorImpl<Function *> &Fns,
2040 StringRef Name) {
2041 auto *PrepareFn = M.getFunction(Name);
2042 if (PrepareFn && !PrepareFn->use_empty())
2043 Fns.push_back(PrepareFn);
2044}
2045
2046PreservedAnalyses CoroSplitPass::run(LazyCallGraph::SCC &C,
2047 CGSCCAnalysisManager &AM,
2048 LazyCallGraph &CG, CGSCCUpdateResult &UR) {
2049 // NB: One invariant of a valid LazyCallGraph::SCC is that it must contain a
2050 // non-zero number of nodes, so we assume that here and grab the first
2051 // node's function's module.
2052 Module &M = *C.begin()->getFunction().getParent();
2053 auto &FAM =
2054 AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager();
2055
2056 if (!declaresCoroSplitIntrinsics(M))
2057 return PreservedAnalyses::all();
2058
2059 // Check for uses of llvm.coro.prepare.retcon/async.
2060 SmallVector<Function *, 2> PrepareFns;
2061 addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.retcon");
2062 addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.async");
2063
2064 // Find coroutines for processing.
2065 SmallVector<LazyCallGraph::Node *, 4> Coroutines;
2066 for (LazyCallGraph::Node &N : C)
2067 if (N.getFunction().hasFnAttribute(CORO_PRESPLIT_ATTR"coroutine.presplit"))
2068 Coroutines.push_back(&N);
2069
2070 if (Coroutines.empty() && PrepareFns.empty())
2071 return PreservedAnalyses::all();
2072
2073 if (Coroutines.empty()) {
2074 for (auto *PrepareFn : PrepareFns) {
2075 replaceAllPrepares(PrepareFn, CG, C);
2076 }
2077 }
2078
2079 // Split all the coroutines.
2080 for (LazyCallGraph::Node *N : Coroutines) {
2081 Function &F = N->getFunction();
2082 Attribute Attr = F.getFnAttribute(CORO_PRESPLIT_ATTR"coroutine.presplit");
2083 StringRef Value = Attr.getValueAsString();
2084 LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F.getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("coro-split")) { dbgs() << "CoroSplit: Processing coroutine '"
<< F.getName() << "' state: " << Value <<
"\n"; } } while (false)
2085 << "' state: " << Value << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("coro-split")) { dbgs() << "CoroSplit: Processing coroutine '"
<< F.getName() << "' state: " << Value <<
"\n"; } } while (false)
;
2086 if (Value == UNPREPARED_FOR_SPLIT"0") {
2087 // Enqueue a second iteration of the CGSCC pipeline on this SCC.
2088 UR.CWorklist.insert(&C);
2089 F.addFnAttr(CORO_PRESPLIT_ATTR"coroutine.presplit", PREPARED_FOR_SPLIT"1");
2090 continue;
2091 }
2092 F.removeFnAttr(CORO_PRESPLIT_ATTR"coroutine.presplit");
2093
2094 SmallVector<Function *, 4> Clones;
2095 const coro::Shape Shape = splitCoroutine(F, Clones, ReuseFrameSlot);
2096 updateCallGraphAfterCoroutineSplit(*N, Shape, Clones, C, CG, AM, UR, FAM);
2097
2098 if ((Shape.ABI == coro::ABI::Async || Shape.ABI == coro::ABI::Retcon ||
2099 Shape.ABI == coro::ABI::RetconOnce) &&
2100 !Shape.CoroSuspends.empty()) {
2101 // Run the CGSCC pipeline on the newly split functions.
2102 // All clones will be in the same RefSCC, so choose a random clone.
2103 UR.RCWorklist.insert(CG.lookupRefSCC(CG.get(*Clones[0])));
2104 }
2105 }
2106
2107 if (!PrepareFns.empty()) {
2108 for (auto *PrepareFn : PrepareFns) {
2109 replaceAllPrepares(PrepareFn, CG, C);
2110 }
2111 }
2112
2113 return PreservedAnalyses::none();
2114}
2115
2116namespace {
2117
2118// We present a coroutine to LLVM as an ordinary function with suspension
2119// points marked up with intrinsics. We let the optimizer party on the coroutine
2120// as a single function for as long as possible. Shortly before the coroutine is
2121// eligible to be inlined into its callers, we split up the coroutine into parts
2122// corresponding to initial, resume and destroy invocations of the coroutine,
2123// add them to the current SCC and restart the IPO pipeline to optimize the
2124// coroutine subfunctions we extracted before proceeding to the caller of the
2125// coroutine.
2126struct CoroSplitLegacy : public CallGraphSCCPass {
2127 static char ID; // Pass identification, replacement for typeid
2128
2129 CoroSplitLegacy(bool ReuseFrameSlot = false)
2130 : CallGraphSCCPass(ID), ReuseFrameSlot(ReuseFrameSlot) {
2131 initializeCoroSplitLegacyPass(*PassRegistry::getPassRegistry());
2132 }
2133
2134 bool Run = false;
2135 bool ReuseFrameSlot;
2136
2137 // A coroutine is identified by the presence of coro.begin intrinsic, if
2138 // we don't have any, this pass has nothing to do.
2139 bool doInitialization(CallGraph &CG) override {
2140 Run = declaresCoroSplitIntrinsics(CG.getModule());
2141 return CallGraphSCCPass::doInitialization(CG);
2142 }
2143
2144 bool runOnSCC(CallGraphSCC &SCC) override {
2145 if (!Run)
1
Assuming field 'Run' is true
2
Taking false branch
2146 return false;
2147
2148 // Check for uses of llvm.coro.prepare.retcon.
2149 SmallVector<Function *, 2> PrepareFns;
2150 auto &M = SCC.getCallGraph().getModule();
2151 addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.retcon");
2152 addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.async");
2153
2154 // Find coroutines for processing.
2155 SmallVector<Function *, 4> Coroutines;
2156 for (CallGraphNode *CGN : SCC)
2157 if (auto *F = CGN->getFunction())
2158 if (F->hasFnAttribute(CORO_PRESPLIT_ATTR"coroutine.presplit"))
2159 Coroutines.push_back(F);
2160
2161 if (Coroutines.empty() && PrepareFns.empty())
2162 return false;
2163
2164 CallGraph &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph();
2165
2166 if (Coroutines.empty()) {
3
Taking false branch
2167 bool Changed = false;
2168 for (auto *PrepareFn : PrepareFns)
2169 Changed |= replaceAllPrepares(PrepareFn, CG);
2170 return Changed;
2171 }
2172
2173 createDevirtTriggerFunc(CG, SCC);
2174
2175 // Split all the coroutines.
2176 for (Function *F : Coroutines) {
4
Assuming '__begin2' is not equal to '__end2'
2177 Attribute Attr = F->getFnAttribute(CORO_PRESPLIT_ATTR"coroutine.presplit");
2178 StringRef Value = Attr.getValueAsString();
2179 LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("coro-split")) { dbgs() << "CoroSplit: Processing coroutine '"
<< F->getName() << "' state: " << Value
<< "\n"; } } while (false)
5
Assuming 'DebugFlag' is false
6
Loop condition is false. Exiting loop
2180 << "' state: " << Value << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("coro-split")) { dbgs() << "CoroSplit: Processing coroutine '"
<< F->getName() << "' state: " << Value
<< "\n"; } } while (false)
;
2181 // Async lowering marks coroutines to trigger a restart of the pipeline
2182 // after it has split them.
2183 if (Value == ASYNC_RESTART_AFTER_SPLIT"2") {
7
Assuming the condition is false
8
Taking false branch
2184 F->removeFnAttr(CORO_PRESPLIT_ATTR"coroutine.presplit");
2185 continue;
2186 }
2187 if (Value == UNPREPARED_FOR_SPLIT"0") {
9
Assuming the condition is false
10
Taking false branch
2188 prepareForSplit(*F, CG);
2189 continue;
2190 }
2191 F->removeFnAttr(CORO_PRESPLIT_ATTR"coroutine.presplit");
2192
2193 SmallVector<Function *, 4> Clones;
2194 const coro::Shape Shape = splitCoroutine(*F, Clones, ReuseFrameSlot);
11
Calling 'splitCoroutine'
2195 updateCallGraphAfterCoroutineSplit(*F, Shape, Clones, CG, SCC);
2196 if (Shape.ABI == coro::ABI::Async) {
2197 // Restart SCC passes.
2198 // Mark function for CoroElide pass. It will devirtualize causing a
2199 // restart of the SCC pipeline.
2200 prepareForSplit(*F, CG, true /*MarkForAsyncRestart*/);
2201 }
2202 }
2203
2204 for (auto *PrepareFn : PrepareFns)
2205 replaceAllPrepares(PrepareFn, CG);
2206
2207 return true;
2208 }
2209
2210 void getAnalysisUsage(AnalysisUsage &AU) const override {
2211 CallGraphSCCPass::getAnalysisUsage(AU);
2212 }
2213
2214 StringRef getPassName() const override { return "Coroutine Splitting"; }
2215};
2216
2217} // end anonymous namespace
2218
2219char CoroSplitLegacy::ID = 0;
2220
2221INITIALIZE_PASS_BEGIN(static void *initializeCoroSplitLegacyPassOnce(PassRegistry &
Registry) {
2222 CoroSplitLegacy, "coro-split",static void *initializeCoroSplitLegacyPassOnce(PassRegistry &
Registry) {
2223 "Split coroutine into a set of functions driving its state machine", false,static void *initializeCoroSplitLegacyPassOnce(PassRegistry &
Registry) {
2224 false)static void *initializeCoroSplitLegacyPassOnce(PassRegistry &
Registry) {
2225INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass)initializeCallGraphWrapperPassPass(Registry);
2226INITIALIZE_PASS_END(PassInfo *PI = new PassInfo( "Split coroutine into a set of functions driving its state machine"
, "coro-split", &CoroSplitLegacy::ID, PassInfo::NormalCtor_t
(callDefaultCtor<CoroSplitLegacy>), false, false); Registry
.registerPass(*PI, true); return PI; } static llvm::once_flag
InitializeCoroSplitLegacyPassFlag; void llvm::initializeCoroSplitLegacyPass
(PassRegistry &Registry) { llvm::call_once(InitializeCoroSplitLegacyPassFlag
, initializeCoroSplitLegacyPassOnce, std::ref(Registry)); }
2227 CoroSplitLegacy, "coro-split",PassInfo *PI = new PassInfo( "Split coroutine into a set of functions driving its state machine"
, "coro-split", &CoroSplitLegacy::ID, PassInfo::NormalCtor_t
(callDefaultCtor<CoroSplitLegacy>), false, false); Registry
.registerPass(*PI, true); return PI; } static llvm::once_flag
InitializeCoroSplitLegacyPassFlag; void llvm::initializeCoroSplitLegacyPass
(PassRegistry &Registry) { llvm::call_once(InitializeCoroSplitLegacyPassFlag
, initializeCoroSplitLegacyPassOnce, std::ref(Registry)); }
2228 "Split coroutine into a set of functions driving its state machine", false,PassInfo *PI = new PassInfo( "Split coroutine into a set of functions driving its state machine"
, "coro-split", &CoroSplitLegacy::ID, PassInfo::NormalCtor_t
(callDefaultCtor<CoroSplitLegacy>), false, false); Registry
.registerPass(*PI, true); return PI; } static llvm::once_flag
InitializeCoroSplitLegacyPassFlag; void llvm::initializeCoroSplitLegacyPass
(PassRegistry &Registry) { llvm::call_once(InitializeCoroSplitLegacyPassFlag
, initializeCoroSplitLegacyPassOnce, std::ref(Registry)); }
2229 false)PassInfo *PI = new PassInfo( "Split coroutine into a set of functions driving its state machine"
, "coro-split", &CoroSplitLegacy::ID, PassInfo::NormalCtor_t
(callDefaultCtor<CoroSplitLegacy>), false, false); Registry
.registerPass(*PI, true); return PI; } static llvm::once_flag
InitializeCoroSplitLegacyPassFlag; void llvm::initializeCoroSplitLegacyPass
(PassRegistry &Registry) { llvm::call_once(InitializeCoroSplitLegacyPassFlag
, initializeCoroSplitLegacyPassOnce, std::ref(Registry)); }
2230
2231Pass *llvm::createCoroSplitLegacyPass(bool ReuseFrameSlot) {
2232 return new CoroSplitLegacy(ReuseFrameSlot);
2233}