LLVM 19.0.0git
CoroSplit.cpp
Go to the documentation of this file.
1//===- CoroSplit.cpp - Converts a coroutine into a state machine ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// This pass builds the coroutine frame and outlines resume and destroy parts
9// of the coroutine into separate functions.
10//
11// We present a coroutine to an LLVM as an ordinary function with suspension
12// points marked up with intrinsics. We let the optimizer party on the coroutine
13// as a single function for as long as possible. Shortly before the coroutine is
14// eligible to be inlined into its callers, we split up the coroutine into parts
15// corresponding to an initial, resume and destroy invocations of the coroutine,
16// add them to the current SCC and restart the IPO pipeline to optimize the
17// coroutine subfunctions we extracted before proceeding to the caller of the
18// coroutine.
19//===----------------------------------------------------------------------===//
20
22#include "CoroInstr.h"
23#include "CoroInternal.h"
24#include "llvm/ADT/DenseMap.h"
28#include "llvm/ADT/StringRef.h"
29#include "llvm/ADT/Twine.h"
30#include "llvm/Analysis/CFG.h"
37#include "llvm/IR/Argument.h"
38#include "llvm/IR/Attributes.h"
39#include "llvm/IR/BasicBlock.h"
40#include "llvm/IR/CFG.h"
41#include "llvm/IR/CallingConv.h"
42#include "llvm/IR/Constants.h"
43#include "llvm/IR/DataLayout.h"
45#include "llvm/IR/Dominators.h"
46#include "llvm/IR/Function.h"
47#include "llvm/IR/GlobalValue.h"
49#include "llvm/IR/IRBuilder.h"
51#include "llvm/IR/InstrTypes.h"
52#include "llvm/IR/Instruction.h"
55#include "llvm/IR/LLVMContext.h"
56#include "llvm/IR/Module.h"
57#include "llvm/IR/Type.h"
58#include "llvm/IR/Value.h"
59#include "llvm/IR/Verifier.h"
61#include "llvm/Support/Debug.h"
70#include <cassert>
71#include <cstddef>
72#include <cstdint>
73#include <initializer_list>
74#include <iterator>
75
76using namespace llvm;
77
78#define DEBUG_TYPE "coro-split"
79
80namespace {
81
82/// A little helper class for building
83class CoroCloner {
84public:
85 enum class Kind {
86 /// The shared resume function for a switch lowering.
87 SwitchResume,
88
89 /// The shared unwind function for a switch lowering.
90 SwitchUnwind,
91
92 /// The shared cleanup function for a switch lowering.
93 SwitchCleanup,
94
95 /// An individual continuation function.
96 Continuation,
97
98 /// An async resume function.
99 Async,
100 };
101
102private:
103 Function &OrigF;
104 Function *NewF;
105 const Twine &Suffix;
106 coro::Shape &Shape;
107 Kind FKind;
109 IRBuilder<> Builder;
110 Value *NewFramePtr = nullptr;
111
112 /// The active suspend instruction; meaningful only for continuation and async
113 /// ABIs.
114 AnyCoroSuspendInst *ActiveSuspend = nullptr;
115
117
118public:
119 /// Create a cloner for a switch lowering.
120 CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
121 Kind FKind, TargetTransformInfo &TTI)
122 : OrigF(OrigF), NewF(nullptr), Suffix(Suffix), Shape(Shape), FKind(FKind),
123 Builder(OrigF.getContext()), TTI(TTI) {
124 assert(Shape.ABI == coro::ABI::Switch);
125 }
126
127 /// Create a cloner for a continuation lowering.
128 CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
129 Function *NewF, AnyCoroSuspendInst *ActiveSuspend,
131 : OrigF(OrigF), NewF(NewF), Suffix(Suffix), Shape(Shape),
132 FKind(Shape.ABI == coro::ABI::Async ? Kind::Async : Kind::Continuation),
133 Builder(OrigF.getContext()), ActiveSuspend(ActiveSuspend), TTI(TTI) {
134 assert(Shape.ABI == coro::ABI::Retcon ||
135 Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async);
136 assert(NewF && "need existing function for continuation");
137 assert(ActiveSuspend && "need active suspend point for continuation");
138 }
139
140 Function *getFunction() const {
141 assert(NewF != nullptr && "declaration not yet set");
142 return NewF;
143 }
144
145 void create();
146
147private:
148 bool isSwitchDestroyFunction() {
149 switch (FKind) {
150 case Kind::Async:
151 case Kind::Continuation:
152 case Kind::SwitchResume:
153 return false;
154 case Kind::SwitchUnwind:
155 case Kind::SwitchCleanup:
156 return true;
157 }
158 llvm_unreachable("Unknown CoroCloner::Kind enum");
159 }
160
161 void replaceEntryBlock();
162 Value *deriveNewFramePointer();
163 void replaceRetconOrAsyncSuspendUses();
164 void replaceCoroSuspends();
165 void replaceCoroEnds();
167 void salvageDebugInfo();
168 void handleFinalSuspend();
169};
170
171} // end anonymous namespace
172
173// FIXME:
174// Lower the intrinisc in CoroEarly phase if coroutine frame doesn't escape
175// and it is known that other transformations, for example, sanitizers
176// won't lead to incorrect code.
178 coro::Shape &Shape) {
179 auto Wrapper = CB->getWrapperFunction();
180 auto Awaiter = CB->getAwaiter();
181 auto FramePtr = CB->getFrame();
182
183 Builder.SetInsertPoint(CB);
184
185 CallBase *NewCall = nullptr;
186 // await_suspend has only 2 parameters, awaiter and handle.
187 // Copy parameter attributes from the intrinsic call, but remove the last,
188 // because the last parameter now becomes the function that is being called.
189 AttributeList NewAttributes =
191
192 if (auto Invoke = dyn_cast<InvokeInst>(CB)) {
193 auto WrapperInvoke =
194 Builder.CreateInvoke(Wrapper, Invoke->getNormalDest(),
195 Invoke->getUnwindDest(), {Awaiter, FramePtr});
196
197 WrapperInvoke->setCallingConv(Invoke->getCallingConv());
198 std::copy(Invoke->bundle_op_info_begin(), Invoke->bundle_op_info_end(),
199 WrapperInvoke->bundle_op_info_begin());
200 WrapperInvoke->setAttributes(NewAttributes);
201 WrapperInvoke->setDebugLoc(Invoke->getDebugLoc());
202 NewCall = WrapperInvoke;
203 } else if (auto Call = dyn_cast<CallInst>(CB)) {
204 auto WrapperCall = Builder.CreateCall(Wrapper, {Awaiter, FramePtr});
205
206 WrapperCall->setAttributes(NewAttributes);
207 WrapperCall->setDebugLoc(Call->getDebugLoc());
208 NewCall = WrapperCall;
209 } else {
210 llvm_unreachable("Unexpected coro_await_suspend invocation method");
211 }
212
213 if (CB->getCalledFunction()->getIntrinsicID() ==
214 Intrinsic::coro_await_suspend_handle) {
215 // Follow the lowered await_suspend call above with a lowered resume call
216 // to the returned coroutine.
217 if (auto *Invoke = dyn_cast<InvokeInst>(CB)) {
218 // If the await_suspend call is an invoke, we continue in the next block.
219 Builder.SetInsertPoint(Invoke->getNormalDest()->getFirstInsertionPt());
220 }
221
222 coro::LowererBase LB(*Wrapper->getParent());
223 auto *ResumeAddr = LB.makeSubFnCall(NewCall, CoroSubFnInst::ResumeIndex,
224 &*Builder.GetInsertPoint());
225
226 LLVMContext &Ctx = Builder.getContext();
227 FunctionType *ResumeTy = FunctionType::get(
228 Type::getVoidTy(Ctx), PointerType::getUnqual(Ctx), false);
229 auto *ResumeCall = Builder.CreateCall(ResumeTy, ResumeAddr, {NewCall});
231
232 // We can't insert the 'ret' instruction and adjust the cc until the
233 // function has been split, so remember this for later.
234 Shape.SymmetricTransfers.push_back(ResumeCall);
235
236 NewCall = ResumeCall;
237 }
238
239 CB->replaceAllUsesWith(NewCall);
240 CB->eraseFromParent();
241}
242
244 IRBuilder<> Builder(F.getContext());
245 for (auto *AWS : Shape.CoroAwaitSuspends)
246 lowerAwaitSuspend(Builder, AWS, Shape);
247}
248
250 const coro::Shape &Shape, Value *FramePtr,
251 CallGraph *CG) {
252 assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce);
254 return;
255
256 Shape.emitDealloc(Builder, FramePtr, CG);
257}
258
259/// Replace an llvm.coro.end.async.
260/// Will inline the must tail call function call if there is one.
261/// \returns true if cleanup of the coro.end block is needed, false otherwise.
263 IRBuilder<> Builder(End);
264
265 auto *EndAsync = dyn_cast<CoroAsyncEndInst>(End);
266 if (!EndAsync) {
267 Builder.CreateRetVoid();
268 return true /*needs cleanup of coro.end block*/;
269 }
270
271 auto *MustTailCallFunc = EndAsync->getMustTailCallFunction();
272 if (!MustTailCallFunc) {
273 Builder.CreateRetVoid();
274 return true /*needs cleanup of coro.end block*/;
275 }
276
277 // Move the must tail call from the predecessor block into the end block.
278 auto *CoroEndBlock = End->getParent();
279 auto *MustTailCallFuncBlock = CoroEndBlock->getSinglePredecessor();
280 assert(MustTailCallFuncBlock && "Must have a single predecessor block");
281 auto It = MustTailCallFuncBlock->getTerminator()->getIterator();
282 auto *MustTailCall = cast<CallInst>(&*std::prev(It));
283 CoroEndBlock->splice(End->getIterator(), MustTailCallFuncBlock,
284 MustTailCall->getIterator());
285
286 // Insert the return instruction.
287 Builder.SetInsertPoint(End);
288 Builder.CreateRetVoid();
289 InlineFunctionInfo FnInfo;
290
291 // Remove the rest of the block, by splitting it into an unreachable block.
292 auto *BB = End->getParent();
293 BB->splitBasicBlock(End);
294 BB->getTerminator()->eraseFromParent();
295
296 auto InlineRes = InlineFunction(*MustTailCall, FnInfo);
297 assert(InlineRes.isSuccess() && "Expected inlining to succeed");
298 (void)InlineRes;
299
300 // We have cleaned up the coro.end block above.
301 return false;
302}
303
304/// Replace a non-unwind call to llvm.coro.end.
306 const coro::Shape &Shape, Value *FramePtr,
307 bool InResume, CallGraph *CG) {
308 // Start inserting right before the coro.end.
309 IRBuilder<> Builder(End);
310
311 // Create the return instruction.
312 switch (Shape.ABI) {
313 // The cloned functions in switch-lowering always return void.
314 case coro::ABI::Switch:
315 assert(!cast<CoroEndInst>(End)->hasResults() &&
316 "switch coroutine should not return any values");
317 // coro.end doesn't immediately end the coroutine in the main function
318 // in this lowering, because we need to deallocate the coroutine.
319 if (!InResume)
320 return;
321 Builder.CreateRetVoid();
322 break;
323
324 // In async lowering this returns.
325 case coro::ABI::Async: {
326 bool CoroEndBlockNeedsCleanup = replaceCoroEndAsync(End);
327 if (!CoroEndBlockNeedsCleanup)
328 return;
329 break;
330 }
331
332 // In unique continuation lowering, the continuations always return void.
333 // But we may have implicitly allocated storage.
334 case coro::ABI::RetconOnce: {
335 maybeFreeRetconStorage(Builder, Shape, FramePtr, CG);
336 auto *CoroEnd = cast<CoroEndInst>(End);
337 auto *RetTy = Shape.getResumeFunctionType()->getReturnType();
338
339 if (!CoroEnd->hasResults()) {
340 assert(RetTy->isVoidTy());
341 Builder.CreateRetVoid();
342 break;
343 }
344
345 auto *CoroResults = CoroEnd->getResults();
346 unsigned NumReturns = CoroResults->numReturns();
347
348 if (auto *RetStructTy = dyn_cast<StructType>(RetTy)) {
349 assert(RetStructTy->getNumElements() == NumReturns &&
350 "numbers of returns should match resume function singature");
351 Value *ReturnValue = UndefValue::get(RetStructTy);
352 unsigned Idx = 0;
353 for (Value *RetValEl : CoroResults->return_values())
354 ReturnValue = Builder.CreateInsertValue(ReturnValue, RetValEl, Idx++);
355 Builder.CreateRet(ReturnValue);
356 } else if (NumReturns == 0) {
357 assert(RetTy->isVoidTy());
358 Builder.CreateRetVoid();
359 } else {
360 assert(NumReturns == 1);
361 Builder.CreateRet(*CoroResults->retval_begin());
362 }
363 CoroResults->replaceAllUsesWith(
364 ConstantTokenNone::get(CoroResults->getContext()));
365 CoroResults->eraseFromParent();
366 break;
367 }
368
369 // In non-unique continuation lowering, we signal completion by returning
370 // a null continuation.
371 case coro::ABI::Retcon: {
372 assert(!cast<CoroEndInst>(End)->hasResults() &&
373 "retcon coroutine should not return any values");
374 maybeFreeRetconStorage(Builder, Shape, FramePtr, CG);
375 auto RetTy = Shape.getResumeFunctionType()->getReturnType();
376 auto RetStructTy = dyn_cast<StructType>(RetTy);
377 PointerType *ContinuationTy =
378 cast<PointerType>(RetStructTy ? RetStructTy->getElementType(0) : RetTy);
379
380 Value *ReturnValue = ConstantPointerNull::get(ContinuationTy);
381 if (RetStructTy) {
382 ReturnValue = Builder.CreateInsertValue(UndefValue::get(RetStructTy),
383 ReturnValue, 0);
384 }
385 Builder.CreateRet(ReturnValue);
386 break;
387 }
388 }
389
390 // Remove the rest of the block, by splitting it into an unreachable block.
391 auto *BB = End->getParent();
392 BB->splitBasicBlock(End);
393 BB->getTerminator()->eraseFromParent();
394}
395
396// Mark a coroutine as done, which implies that the coroutine is finished and
397// never get resumed.
398//
399// In resume-switched ABI, the done state is represented by storing zero in
400// ResumeFnAddr.
401//
402// NOTE: We couldn't omit the argument `FramePtr`. It is necessary because the
403// pointer to the frame in splitted function is not stored in `Shape`.
404static void markCoroutineAsDone(IRBuilder<> &Builder, const coro::Shape &Shape,
405 Value *FramePtr) {
406 assert(
407 Shape.ABI == coro::ABI::Switch &&
408 "markCoroutineAsDone is only supported for Switch-Resumed ABI for now.");
409 auto *GepIndex = Builder.CreateStructGEP(
411 "ResumeFn.addr");
412 auto *NullPtr = ConstantPointerNull::get(cast<PointerType>(
414 Builder.CreateStore(NullPtr, GepIndex);
415
416 // If the coroutine don't have unwind coro end, we could omit the store to
417 // the final suspend point since we could infer the coroutine is suspended
418 // at the final suspend point by the nullness of ResumeFnAddr.
419 // However, we can't skip it if the coroutine have unwind coro end. Since
420 // the coroutine reaches unwind coro end is considered suspended at the
421 // final suspend point (the ResumeFnAddr is null) but in fact the coroutine
422 // didn't complete yet. We need the IndexVal for the final suspend point
423 // to make the states clear.
426 assert(cast<CoroSuspendInst>(Shape.CoroSuspends.back())->isFinal() &&
427 "The final suspend should only live in the last position of "
428 "CoroSuspends.");
429 ConstantInt *IndexVal = Shape.getIndex(Shape.CoroSuspends.size() - 1);
430 auto *FinalIndex = Builder.CreateStructGEP(
431 Shape.FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
432
433 Builder.CreateStore(IndexVal, FinalIndex);
434 }
435}
436
437/// Replace an unwind call to llvm.coro.end.
439 Value *FramePtr, bool InResume,
440 CallGraph *CG) {
441 IRBuilder<> Builder(End);
442
443 switch (Shape.ABI) {
444 // In switch-lowering, this does nothing in the main function.
445 case coro::ABI::Switch: {
446 // In C++'s specification, the coroutine should be marked as done
447 // if promise.unhandled_exception() throws. The frontend will
448 // call coro.end(true) along this path.
449 //
450 // FIXME: We should refactor this once there is other language
451 // which uses Switch-Resumed style other than C++.
452 markCoroutineAsDone(Builder, Shape, FramePtr);
453 if (!InResume)
454 return;
455 break;
456 }
457 // In async lowering this does nothing.
458 case coro::ABI::Async:
459 break;
460 // In continuation-lowering, this frees the continuation storage.
461 case coro::ABI::Retcon:
462 case coro::ABI::RetconOnce:
463 maybeFreeRetconStorage(Builder, Shape, FramePtr, CG);
464 break;
465 }
466
467 // If coro.end has an associated bundle, add cleanupret instruction.
468 if (auto Bundle = End->getOperandBundle(LLVMContext::OB_funclet)) {
469 auto *FromPad = cast<CleanupPadInst>(Bundle->Inputs[0]);
470 auto *CleanupRet = Builder.CreateCleanupRet(FromPad, nullptr);
471 End->getParent()->splitBasicBlock(End);
472 CleanupRet->getParent()->getTerminator()->eraseFromParent();
473 }
474}
475
476static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape,
477 Value *FramePtr, bool InResume, CallGraph *CG) {
478 if (End->isUnwind())
479 replaceUnwindCoroEnd(End, Shape, FramePtr, InResume, CG);
480 else
481 replaceFallthroughCoroEnd(End, Shape, FramePtr, InResume, CG);
482
483 auto &Context = End->getContext();
484 End->replaceAllUsesWith(InResume ? ConstantInt::getTrue(Context)
485 : ConstantInt::getFalse(Context));
486 End->eraseFromParent();
487}
488
489// In the resume function, we remove the last case (when coro::Shape is built,
490// the final suspend point (if present) is always the last element of
491// CoroSuspends array) since it is an undefined behavior to resume a coroutine
492// suspended at the final suspend point.
493// In the destroy function, if it isn't possible that the ResumeFnAddr is NULL
494// and the coroutine doesn't suspend at the final suspend point actually (this
495// is possible since the coroutine is considered suspended at the final suspend
496// point if promise.unhandled_exception() exits via an exception), we can
497// remove the last case.
498void CoroCloner::handleFinalSuspend() {
499 assert(Shape.ABI == coro::ABI::Switch &&
500 Shape.SwitchLowering.HasFinalSuspend);
501
502 if (isSwitchDestroyFunction() && Shape.SwitchLowering.HasUnwindCoroEnd)
503 return;
504
505 auto *Switch = cast<SwitchInst>(VMap[Shape.SwitchLowering.ResumeSwitch]);
506 auto FinalCaseIt = std::prev(Switch->case_end());
507 BasicBlock *ResumeBB = FinalCaseIt->getCaseSuccessor();
508 Switch->removeCase(FinalCaseIt);
509 if (isSwitchDestroyFunction()) {
510 BasicBlock *OldSwitchBB = Switch->getParent();
511 auto *NewSwitchBB = OldSwitchBB->splitBasicBlock(Switch, "Switch");
512 Builder.SetInsertPoint(OldSwitchBB->getTerminator());
513
514 if (NewF->isCoroOnlyDestroyWhenComplete()) {
515 // When the coroutine can only be destroyed when complete, we don't need
516 // to generate code for other cases.
517 Builder.CreateBr(ResumeBB);
518 } else {
519 auto *GepIndex = Builder.CreateStructGEP(
520 Shape.FrameTy, NewFramePtr, coro::Shape::SwitchFieldIndex::Resume,
521 "ResumeFn.addr");
522 auto *Load =
523 Builder.CreateLoad(Shape.getSwitchResumePointerType(), GepIndex);
524 auto *Cond = Builder.CreateIsNull(Load);
525 Builder.CreateCondBr(Cond, ResumeBB, NewSwitchBB);
526 }
527 OldSwitchBB->getTerminator()->eraseFromParent();
528 }
529}
530
531static FunctionType *
533 auto *AsyncSuspend = cast<CoroSuspendAsyncInst>(Suspend);
534 auto *StructTy = cast<StructType>(AsyncSuspend->getType());
535 auto &Context = Suspend->getParent()->getParent()->getContext();
536 auto *VoidTy = Type::getVoidTy(Context);
537 return FunctionType::get(VoidTy, StructTy->elements(), false);
538}
539
541 const Twine &Suffix,
542 Module::iterator InsertBefore,
543 AnyCoroSuspendInst *ActiveSuspend) {
544 Module *M = OrigF.getParent();
545 auto *FnTy = (Shape.ABI != coro::ABI::Async)
546 ? Shape.getResumeFunctionType()
547 : getFunctionTypeFromAsyncSuspend(ActiveSuspend);
548
549 Function *NewF =
550 Function::Create(FnTy, GlobalValue::LinkageTypes::InternalLinkage,
551 OrigF.getName() + Suffix);
552
553 M->getFunctionList().insert(InsertBefore, NewF);
554
555 return NewF;
556}
557
558/// Replace uses of the active llvm.coro.suspend.retcon/async call with the
559/// arguments to the continuation function.
560///
561/// This assumes that the builder has a meaningful insertion point.
562void CoroCloner::replaceRetconOrAsyncSuspendUses() {
563 assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce ||
564 Shape.ABI == coro::ABI::Async);
565
566 auto NewS = VMap[ActiveSuspend];
567 if (NewS->use_empty())
568 return;
569
570 // Copy out all the continuation arguments after the buffer pointer into
571 // an easily-indexed data structure for convenience.
573 // The async ABI includes all arguments -- including the first argument.
574 bool IsAsyncABI = Shape.ABI == coro::ABI::Async;
575 for (auto I = IsAsyncABI ? NewF->arg_begin() : std::next(NewF->arg_begin()),
576 E = NewF->arg_end();
577 I != E; ++I)
578 Args.push_back(&*I);
579
580 // If the suspend returns a single scalar value, we can just do a simple
581 // replacement.
582 if (!isa<StructType>(NewS->getType())) {
583 assert(Args.size() == 1);
584 NewS->replaceAllUsesWith(Args.front());
585 return;
586 }
587
588 // Try to peephole extracts of an aggregate return.
589 for (Use &U : llvm::make_early_inc_range(NewS->uses())) {
590 auto *EVI = dyn_cast<ExtractValueInst>(U.getUser());
591 if (!EVI || EVI->getNumIndices() != 1)
592 continue;
593
594 EVI->replaceAllUsesWith(Args[EVI->getIndices().front()]);
595 EVI->eraseFromParent();
596 }
597
598 // If we have no remaining uses, we're done.
599 if (NewS->use_empty())
600 return;
601
602 // Otherwise, we need to create an aggregate.
603 Value *Agg = PoisonValue::get(NewS->getType());
604 for (size_t I = 0, E = Args.size(); I != E; ++I)
605 Agg = Builder.CreateInsertValue(Agg, Args[I], I);
606
607 NewS->replaceAllUsesWith(Agg);
608}
609
610void CoroCloner::replaceCoroSuspends() {
611 Value *SuspendResult;
612
613 switch (Shape.ABI) {
614 // In switch lowering, replace coro.suspend with the appropriate value
615 // for the type of function we're extracting.
616 // Replacing coro.suspend with (0) will result in control flow proceeding to
617 // a resume label associated with a suspend point, replacing it with (1) will
618 // result in control flow proceeding to a cleanup label associated with this
619 // suspend point.
620 case coro::ABI::Switch:
621 SuspendResult = Builder.getInt8(isSwitchDestroyFunction() ? 1 : 0);
622 break;
623
624 // In async lowering there are no uses of the result.
625 case coro::ABI::Async:
626 return;
627
628 // In returned-continuation lowering, the arguments from earlier
629 // continuations are theoretically arbitrary, and they should have been
630 // spilled.
631 case coro::ABI::RetconOnce:
632 case coro::ABI::Retcon:
633 return;
634 }
635
636 for (AnyCoroSuspendInst *CS : Shape.CoroSuspends) {
637 // The active suspend was handled earlier.
638 if (CS == ActiveSuspend)
639 continue;
640
641 auto *MappedCS = cast<AnyCoroSuspendInst>(VMap[CS]);
642 MappedCS->replaceAllUsesWith(SuspendResult);
643 MappedCS->eraseFromParent();
644 }
645}
646
647void CoroCloner::replaceCoroEnds() {
648 for (AnyCoroEndInst *CE : Shape.CoroEnds) {
649 // We use a null call graph because there's no call graph node for
650 // the cloned function yet. We'll just be rebuilding that later.
651 auto *NewCE = cast<AnyCoroEndInst>(VMap[CE]);
652 replaceCoroEnd(NewCE, Shape, NewFramePtr, /*in resume*/ true, nullptr);
653 }
654}
655
657 ValueToValueMapTy *VMap) {
658 if (Shape.ABI == coro::ABI::Async && Shape.CoroSuspends.empty())
659 return;
660 Value *CachedSlot = nullptr;
661 auto getSwiftErrorSlot = [&](Type *ValueTy) -> Value * {
662 if (CachedSlot)
663 return CachedSlot;
664
665 // Check if the function has a swifterror argument.
666 for (auto &Arg : F.args()) {
667 if (Arg.isSwiftError()) {
668 CachedSlot = &Arg;
669 return &Arg;
670 }
671 }
672
673 // Create a swifterror alloca.
674 IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHIOrDbg());
675 auto Alloca = Builder.CreateAlloca(ValueTy);
676 Alloca->setSwiftError(true);
677
678 CachedSlot = Alloca;
679 return Alloca;
680 };
681
682 for (CallInst *Op : Shape.SwiftErrorOps) {
683 auto MappedOp = VMap ? cast<CallInst>((*VMap)[Op]) : Op;
684 IRBuilder<> Builder(MappedOp);
685
686 // If there are no arguments, this is a 'get' operation.
687 Value *MappedResult;
688 if (Op->arg_empty()) {
689 auto ValueTy = Op->getType();
690 auto Slot = getSwiftErrorSlot(ValueTy);
691 MappedResult = Builder.CreateLoad(ValueTy, Slot);
692 } else {
693 assert(Op->arg_size() == 1);
694 auto Value = MappedOp->getArgOperand(0);
695 auto ValueTy = Value->getType();
696 auto Slot = getSwiftErrorSlot(ValueTy);
697 Builder.CreateStore(Value, Slot);
698 MappedResult = Slot;
699 }
700
701 MappedOp->replaceAllUsesWith(MappedResult);
702 MappedOp->eraseFromParent();
703 }
704
705 // If we're updating the original function, we've invalidated SwiftErrorOps.
706 if (VMap == nullptr) {
707 Shape.SwiftErrorOps.clear();
708 }
709}
710
711/// Returns all DbgVariableIntrinsic in F.
712static std::pair<SmallVector<DbgVariableIntrinsic *, 8>,
716 SmallVector<DbgVariableRecord *> DbgVariableRecords;
717 for (auto &I : instructions(F)) {
718 for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
719 DbgVariableRecords.push_back(&DVR);
720 if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I))
721 Intrinsics.push_back(DVI);
722 }
723 return {Intrinsics, DbgVariableRecords};
724}
725
726void CoroCloner::replaceSwiftErrorOps() {
727 ::replaceSwiftErrorOps(*NewF, Shape, &VMap);
728}
729
730void CoroCloner::salvageDebugInfo() {
731 auto [Worklist, DbgVariableRecords] = collectDbgVariableIntrinsics(*NewF);
733
734 // Only 64-bit ABIs have a register we can refer to with the entry value.
735 bool UseEntryValue =
736 llvm::Triple(OrigF.getParent()->getTargetTriple()).isArch64Bit();
737 for (DbgVariableIntrinsic *DVI : Worklist)
738 coro::salvageDebugInfo(ArgToAllocaMap, *DVI, Shape.OptimizeFrame,
739 UseEntryValue);
740 for (DbgVariableRecord *DVR : DbgVariableRecords)
741 coro::salvageDebugInfo(ArgToAllocaMap, *DVR, Shape.OptimizeFrame,
742 UseEntryValue);
743
744 // Remove all salvaged dbg.declare intrinsics that became
745 // either unreachable or stale due to the CoroSplit transformation.
746 DominatorTree DomTree(*NewF);
747 auto IsUnreachableBlock = [&](BasicBlock *BB) {
748 return !isPotentiallyReachable(&NewF->getEntryBlock(), BB, nullptr,
749 &DomTree);
750 };
751 auto RemoveOne = [&](auto *DVI) {
752 if (IsUnreachableBlock(DVI->getParent()))
753 DVI->eraseFromParent();
754 else if (isa_and_nonnull<AllocaInst>(DVI->getVariableLocationOp(0))) {
755 // Count all non-debuginfo uses in reachable blocks.
756 unsigned Uses = 0;
757 for (auto *User : DVI->getVariableLocationOp(0)->users())
758 if (auto *I = dyn_cast<Instruction>(User))
759 if (!isa<AllocaInst>(I) && !IsUnreachableBlock(I->getParent()))
760 ++Uses;
761 if (!Uses)
762 DVI->eraseFromParent();
763 }
764 };
765 for_each(Worklist, RemoveOne);
766 for_each(DbgVariableRecords, RemoveOne);
767}
768
769void CoroCloner::replaceEntryBlock() {
770 // In the original function, the AllocaSpillBlock is a block immediately
771 // following the allocation of the frame object which defines GEPs for
772 // all the allocas that have been moved into the frame, and it ends by
773 // branching to the original beginning of the coroutine. Make this
774 // the entry block of the cloned function.
775 auto *Entry = cast<BasicBlock>(VMap[Shape.AllocaSpillBlock]);
776 auto *OldEntry = &NewF->getEntryBlock();
777 Entry->setName("entry" + Suffix);
778 Entry->moveBefore(OldEntry);
779 Entry->getTerminator()->eraseFromParent();
780
781 // Clear all predecessors of the new entry block. There should be
782 // exactly one predecessor, which we created when splitting out
783 // AllocaSpillBlock to begin with.
784 assert(Entry->hasOneUse());
785 auto BranchToEntry = cast<BranchInst>(Entry->user_back());
786 assert(BranchToEntry->isUnconditional());
787 Builder.SetInsertPoint(BranchToEntry);
788 Builder.CreateUnreachable();
789 BranchToEntry->eraseFromParent();
790
791 // Branch from the entry to the appropriate place.
792 Builder.SetInsertPoint(Entry);
793 switch (Shape.ABI) {
794 case coro::ABI::Switch: {
795 // In switch-lowering, we built a resume-entry block in the original
796 // function. Make the entry block branch to this.
797 auto *SwitchBB =
798 cast<BasicBlock>(VMap[Shape.SwitchLowering.ResumeEntryBlock]);
799 Builder.CreateBr(SwitchBB);
800 break;
801 }
802 case coro::ABI::Async:
803 case coro::ABI::Retcon:
804 case coro::ABI::RetconOnce: {
805 // In continuation ABIs, we want to branch to immediately after the
806 // active suspend point. Earlier phases will have put the suspend in its
807 // own basic block, so just thread our jump directly to its successor.
808 assert((Shape.ABI == coro::ABI::Async &&
809 isa<CoroSuspendAsyncInst>(ActiveSuspend)) ||
810 ((Shape.ABI == coro::ABI::Retcon ||
811 Shape.ABI == coro::ABI::RetconOnce) &&
812 isa<CoroSuspendRetconInst>(ActiveSuspend)));
813 auto *MappedCS = cast<AnyCoroSuspendInst>(VMap[ActiveSuspend]);
814 auto Branch = cast<BranchInst>(MappedCS->getNextNode());
815 assert(Branch->isUnconditional());
816 Builder.CreateBr(Branch->getSuccessor(0));
817 break;
818 }
819 }
820
821 // Any static alloca that's still being used but not reachable from the new
822 // entry needs to be moved to the new entry.
823 Function *F = OldEntry->getParent();
824 DominatorTree DT{*F};
826 auto *Alloca = dyn_cast<AllocaInst>(&I);
827 if (!Alloca || I.use_empty())
828 continue;
829 if (DT.isReachableFromEntry(I.getParent()) ||
830 !isa<ConstantInt>(Alloca->getArraySize()))
831 continue;
832 I.moveBefore(*Entry, Entry->getFirstInsertionPt());
833 }
834}
835
836/// Derive the value of the new frame pointer.
837Value *CoroCloner::deriveNewFramePointer() {
838 // Builder should be inserting to the front of the new entry block.
839
840 switch (Shape.ABI) {
841 // In switch-lowering, the argument is the frame pointer.
842 case coro::ABI::Switch:
843 return &*NewF->arg_begin();
844 // In async-lowering, one of the arguments is an async context as determined
845 // by the `llvm.coro.id.async` intrinsic. We can retrieve the async context of
846 // the resume function from the async context projection function associated
847 // with the active suspend. The frame is located as a tail to the async
848 // context header.
849 case coro::ABI::Async: {
850 auto *ActiveAsyncSuspend = cast<CoroSuspendAsyncInst>(ActiveSuspend);
851 auto ContextIdx = ActiveAsyncSuspend->getStorageArgumentIndex() & 0xff;
852 auto *CalleeContext = NewF->getArg(ContextIdx);
853 auto *ProjectionFunc =
854 ActiveAsyncSuspend->getAsyncContextProjectionFunction();
855 auto DbgLoc =
856 cast<CoroSuspendAsyncInst>(VMap[ActiveSuspend])->getDebugLoc();
857 // Calling i8* (i8*)
858 auto *CallerContext = Builder.CreateCall(ProjectionFunc->getFunctionType(),
859 ProjectionFunc, CalleeContext);
860 CallerContext->setCallingConv(ProjectionFunc->getCallingConv());
861 CallerContext->setDebugLoc(DbgLoc);
862 // The frame is located after the async_context header.
863 auto &Context = Builder.getContext();
864 auto *FramePtrAddr = Builder.CreateConstInBoundsGEP1_32(
865 Type::getInt8Ty(Context), CallerContext,
866 Shape.AsyncLowering.FrameOffset, "async.ctx.frameptr");
867 // Inline the projection function.
869 auto InlineRes = InlineFunction(*CallerContext, InlineInfo);
870 assert(InlineRes.isSuccess());
871 (void)InlineRes;
872 return FramePtrAddr;
873 }
874 // In continuation-lowering, the argument is the opaque storage.
875 case coro::ABI::Retcon:
876 case coro::ABI::RetconOnce: {
877 Argument *NewStorage = &*NewF->arg_begin();
878 auto FramePtrTy = PointerType::getUnqual(Shape.FrameTy->getContext());
879
880 // If the storage is inline, just bitcast to the storage to the frame type.
881 if (Shape.RetconLowering.IsFrameInlineInStorage)
882 return NewStorage;
883
884 // Otherwise, load the real frame from the opaque storage.
885 return Builder.CreateLoad(FramePtrTy, NewStorage);
886 }
887 }
888 llvm_unreachable("bad ABI");
889}
890
891static void addFramePointerAttrs(AttributeList &Attrs, LLVMContext &Context,
892 unsigned ParamIndex, uint64_t Size,
893 Align Alignment, bool NoAlias) {
894 AttrBuilder ParamAttrs(Context);
895 ParamAttrs.addAttribute(Attribute::NonNull);
896 ParamAttrs.addAttribute(Attribute::NoUndef);
897
898 if (NoAlias)
899 ParamAttrs.addAttribute(Attribute::NoAlias);
900
901 ParamAttrs.addAlignmentAttr(Alignment);
902 ParamAttrs.addDereferenceableAttr(Size);
903 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
904}
905
906static void addAsyncContextAttrs(AttributeList &Attrs, LLVMContext &Context,
907 unsigned ParamIndex) {
908 AttrBuilder ParamAttrs(Context);
909 ParamAttrs.addAttribute(Attribute::SwiftAsync);
910 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
911}
912
913static void addSwiftSelfAttrs(AttributeList &Attrs, LLVMContext &Context,
914 unsigned ParamIndex) {
915 AttrBuilder ParamAttrs(Context);
916 ParamAttrs.addAttribute(Attribute::SwiftSelf);
917 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
918}
919
920/// Clone the body of the original function into a resume function of
921/// some sort.
922void CoroCloner::create() {
923 // Create the new function if we don't already have one.
924 if (!NewF) {
925 NewF = createCloneDeclaration(OrigF, Shape, Suffix,
926 OrigF.getParent()->end(), ActiveSuspend);
927 }
928
929 // Replace all args with dummy instructions. If an argument is the old frame
930 // pointer, the dummy will be replaced by the new frame pointer once it is
931 // computed below. Uses of all other arguments should have already been
932 // rewritten by buildCoroutineFrame() to use loads/stores on the coroutine
933 // frame.
935 for (Argument &A : OrigF.args()) {
936 DummyArgs.push_back(new FreezeInst(PoisonValue::get(A.getType())));
937 VMap[&A] = DummyArgs.back();
938 }
939
941
942 // Ignore attempts to change certain attributes of the function.
943 // TODO: maybe there should be a way to suppress this during cloning?
944 auto savedVisibility = NewF->getVisibility();
945 auto savedUnnamedAddr = NewF->getUnnamedAddr();
946 auto savedDLLStorageClass = NewF->getDLLStorageClass();
947
948 // NewF's linkage (which CloneFunctionInto does *not* change) might not
949 // be compatible with the visibility of OrigF (which it *does* change),
950 // so protect against that.
951 auto savedLinkage = NewF->getLinkage();
952 NewF->setLinkage(llvm::GlobalValue::ExternalLinkage);
953
954 CloneFunctionInto(NewF, &OrigF, VMap,
955 CloneFunctionChangeType::LocalChangesOnly, Returns);
956
957 auto &Context = NewF->getContext();
958
959 // For async functions / continuations, adjust the scope line of the
960 // clone to the line number of the suspend point. However, only
961 // adjust the scope line when the files are the same. This ensures
962 // line number and file name belong together. The scope line is
963 // associated with all pre-prologue instructions. This avoids a jump
964 // in the linetable from the function declaration to the suspend point.
965 if (DISubprogram *SP = NewF->getSubprogram()) {
966 assert(SP != OrigF.getSubprogram() && SP->isDistinct());
967 if (ActiveSuspend)
968 if (auto DL = ActiveSuspend->getDebugLoc())
969 if (SP->getFile() == DL->getFile())
970 SP->setScopeLine(DL->getLine());
971 // Update the linkage name to reflect the modified symbol name. It
972 // is necessary to update the linkage name in Swift, since the
973 // mangling changes for resume functions. It might also be the
974 // right thing to do in C++, but due to a limitation in LLVM's
975 // AsmPrinter we can only do this if the function doesn't have an
976 // abstract specification, since the DWARF backend expects the
977 // abstract specification to contain the linkage name and asserts
978 // that they are identical.
979 if (SP->getUnit() &&
980 SP->getUnit()->getSourceLanguage() == dwarf::DW_LANG_Swift) {
981 SP->replaceLinkageName(MDString::get(Context, NewF->getName()));
982 if (auto *Decl = SP->getDeclaration()) {
983 auto *NewDecl = DISubprogram::get(
984 Decl->getContext(), Decl->getScope(), Decl->getName(),
985 NewF->getName(), Decl->getFile(), Decl->getLine(), Decl->getType(),
986 Decl->getScopeLine(), Decl->getContainingType(),
987 Decl->getVirtualIndex(), Decl->getThisAdjustment(),
988 Decl->getFlags(), Decl->getSPFlags(), Decl->getUnit(),
989 Decl->getTemplateParams(), nullptr, Decl->getRetainedNodes(),
990 Decl->getThrownTypes(), Decl->getAnnotations(),
991 Decl->getTargetFuncName());
992 SP->replaceDeclaration(NewDecl);
993 }
994 }
995 }
996
997 NewF->setLinkage(savedLinkage);
998 NewF->setVisibility(savedVisibility);
999 NewF->setUnnamedAddr(savedUnnamedAddr);
1000 NewF->setDLLStorageClass(savedDLLStorageClass);
1001 // The function sanitizer metadata needs to match the signature of the
1002 // function it is being attached to. However this does not hold for split
1003 // functions here. Thus remove the metadata for split functions.
1004 if (Shape.ABI == coro::ABI::Switch &&
1005 NewF->hasMetadata(LLVMContext::MD_func_sanitize))
1006 NewF->eraseMetadata(LLVMContext::MD_func_sanitize);
1007
1008 // Replace the attributes of the new function:
1009 auto OrigAttrs = NewF->getAttributes();
1010 auto NewAttrs = AttributeList();
1011
1012 switch (Shape.ABI) {
1013 case coro::ABI::Switch:
1014 // Bootstrap attributes by copying function attributes from the
1015 // original function. This should include optimization settings and so on.
1016 NewAttrs = NewAttrs.addFnAttributes(
1017 Context, AttrBuilder(Context, OrigAttrs.getFnAttrs()));
1018
1019 addFramePointerAttrs(NewAttrs, Context, 0, Shape.FrameSize,
1020 Shape.FrameAlign, /*NoAlias=*/false);
1021 break;
1022 case coro::ABI::Async: {
1023 auto *ActiveAsyncSuspend = cast<CoroSuspendAsyncInst>(ActiveSuspend);
1024 if (OrigF.hasParamAttribute(Shape.AsyncLowering.ContextArgNo,
1025 Attribute::SwiftAsync)) {
1026 uint32_t ArgAttributeIndices =
1027 ActiveAsyncSuspend->getStorageArgumentIndex();
1028 auto ContextArgIndex = ArgAttributeIndices & 0xff;
1029 addAsyncContextAttrs(NewAttrs, Context, ContextArgIndex);
1030
1031 // `swiftasync` must preceed `swiftself` so 0 is not a valid index for
1032 // `swiftself`.
1033 auto SwiftSelfIndex = ArgAttributeIndices >> 8;
1034 if (SwiftSelfIndex)
1035 addSwiftSelfAttrs(NewAttrs, Context, SwiftSelfIndex);
1036 }
1037
1038 // Transfer the original function's attributes.
1039 auto FnAttrs = OrigF.getAttributes().getFnAttrs();
1040 NewAttrs = NewAttrs.addFnAttributes(Context, AttrBuilder(Context, FnAttrs));
1041 break;
1042 }
1043 case coro::ABI::Retcon:
1044 case coro::ABI::RetconOnce:
1045 // If we have a continuation prototype, just use its attributes,
1046 // full-stop.
1047 NewAttrs = Shape.RetconLowering.ResumePrototype->getAttributes();
1048
1049 /// FIXME: Is it really good to add the NoAlias attribute?
1050 addFramePointerAttrs(NewAttrs, Context, 0,
1051 Shape.getRetconCoroId()->getStorageSize(),
1052 Shape.getRetconCoroId()->getStorageAlignment(),
1053 /*NoAlias=*/true);
1054
1055 break;
1056 }
1057
1058 switch (Shape.ABI) {
1059 // In these ABIs, the cloned functions always return 'void', and the
1060 // existing return sites are meaningless. Note that for unique
1061 // continuations, this includes the returns associated with suspends;
1062 // this is fine because we can't suspend twice.
1063 case coro::ABI::Switch:
1064 case coro::ABI::RetconOnce:
1065 // Remove old returns.
1066 for (ReturnInst *Return : Returns)
1067 changeToUnreachable(Return);
1068 break;
1069
1070 // With multi-suspend continuations, we'll already have eliminated the
1071 // original returns and inserted returns before all the suspend points,
1072 // so we want to leave any returns in place.
1073 case coro::ABI::Retcon:
1074 break;
1075 // Async lowering will insert musttail call functions at all suspend points
1076 // followed by a return.
1077 // Don't change returns to unreachable because that will trip up the verifier.
1078 // These returns should be unreachable from the clone.
1079 case coro::ABI::Async:
1080 break;
1081 }
1082
1083 NewF->setAttributes(NewAttrs);
1084 NewF->setCallingConv(Shape.getResumeFunctionCC());
1085
1086 // Set up the new entry block.
1087 replaceEntryBlock();
1088
1089 // Turn symmetric transfers into musttail calls.
1090 for (CallInst *ResumeCall : Shape.SymmetricTransfers) {
1091 ResumeCall = cast<CallInst>(VMap[ResumeCall]);
1092 if (TTI.supportsTailCallFor(ResumeCall)) {
1093 // FIXME: Could we support symmetric transfer effectively without
1094 // musttail?
1096 }
1097
1098 // Put a 'ret void' after the call, and split any remaining instructions to
1099 // an unreachable block.
1100 BasicBlock *BB = ResumeCall->getParent();
1101 BB->splitBasicBlock(ResumeCall->getNextNode());
1102 Builder.SetInsertPoint(BB->getTerminator());
1103 Builder.CreateRetVoid();
1105 }
1106
1107 Builder.SetInsertPoint(&NewF->getEntryBlock().front());
1108 NewFramePtr = deriveNewFramePointer();
1109
1110 // Remap frame pointer.
1111 Value *OldFramePtr = VMap[Shape.FramePtr];
1112 NewFramePtr->takeName(OldFramePtr);
1113 OldFramePtr->replaceAllUsesWith(NewFramePtr);
1114
1115 // Remap vFrame pointer.
1116 auto *NewVFrame = Builder.CreateBitCast(
1117 NewFramePtr, PointerType::getUnqual(Builder.getContext()), "vFrame");
1118 Value *OldVFrame = cast<Value>(VMap[Shape.CoroBegin]);
1119 if (OldVFrame != NewVFrame)
1120 OldVFrame->replaceAllUsesWith(NewVFrame);
1121
1122 // All uses of the arguments should have been resolved by this point,
1123 // so we can safely remove the dummy values.
1124 for (Instruction *DummyArg : DummyArgs) {
1125 DummyArg->replaceAllUsesWith(PoisonValue::get(DummyArg->getType()));
1126 DummyArg->deleteValue();
1127 }
1128
1129 switch (Shape.ABI) {
1130 case coro::ABI::Switch:
1131 // Rewrite final suspend handling as it is not done via switch (allows to
1132 // remove final case from the switch, since it is undefined behavior to
1133 // resume the coroutine suspended at the final suspend point.
1134 if (Shape.SwitchLowering.HasFinalSuspend)
1135 handleFinalSuspend();
1136 break;
1137 case coro::ABI::Async:
1138 case coro::ABI::Retcon:
1139 case coro::ABI::RetconOnce:
1140 // Replace uses of the active suspend with the corresponding
1141 // continuation-function arguments.
1142 assert(ActiveSuspend != nullptr &&
1143 "no active suspend when lowering a continuation-style coroutine");
1144 replaceRetconOrAsyncSuspendUses();
1145 break;
1146 }
1147
1148 // Handle suspends.
1149 replaceCoroSuspends();
1150
1151 // Handle swifterror.
1153
1154 // Remove coro.end intrinsics.
1155 replaceCoroEnds();
1156
1157 // Salvage debug info that points into the coroutine frame.
1159
1160 // Eliminate coro.free from the clones, replacing it with 'null' in cleanup,
1161 // to suppress deallocation code.
1162 if (Shape.ABI == coro::ABI::Switch)
1163 coro::replaceCoroFree(cast<CoroIdInst>(VMap[Shape.CoroBegin->getId()]),
1164 /*Elide=*/FKind == CoroCloner::Kind::SwitchCleanup);
1165}
1166
1168 assert(Shape.ABI == coro::ABI::Async);
1169
1170 auto *FuncPtrStruct = cast<ConstantStruct>(
1172 auto *OrigRelativeFunOffset = FuncPtrStruct->getOperand(0);
1173 auto *OrigContextSize = FuncPtrStruct->getOperand(1);
1174 auto *NewContextSize = ConstantInt::get(OrigContextSize->getType(),
1176 auto *NewFuncPtrStruct = ConstantStruct::get(
1177 FuncPtrStruct->getType(), OrigRelativeFunOffset, NewContextSize);
1178
1179 Shape.AsyncLowering.AsyncFuncPointer->setInitializer(NewFuncPtrStruct);
1180}
1181
1183 if (Shape.ABI == coro::ABI::Async)
1185
1186 for (CoroAlignInst *CA : Shape.CoroAligns) {
1188 ConstantInt::get(CA->getType(), Shape.FrameAlign.value()));
1189 CA->eraseFromParent();
1190 }
1191
1192 if (Shape.CoroSizes.empty())
1193 return;
1194
1195 // In the same function all coro.sizes should have the same result type.
1196 auto *SizeIntrin = Shape.CoroSizes.back();
1197 Module *M = SizeIntrin->getModule();
1198 const DataLayout &DL = M->getDataLayout();
1199 auto Size = DL.getTypeAllocSize(Shape.FrameTy);
1200 auto *SizeConstant = ConstantInt::get(SizeIntrin->getType(), Size);
1201
1202 for (CoroSizeInst *CS : Shape.CoroSizes) {
1203 CS->replaceAllUsesWith(SizeConstant);
1204 CS->eraseFromParent();
1205 }
1206}
1207
1210
1211#ifndef NDEBUG
1212 // For now, we do a mandatory verification step because we don't
1213 // entirely trust this pass. Note that we don't want to add a verifier
1214 // pass to FPM below because it will also verify all the global data.
1215 if (verifyFunction(F, &errs()))
1216 report_fatal_error("Broken function");
1217#endif
1218}
1219
1220// Coroutine has no suspend points. Remove heap allocation for the coroutine
1221// frame if possible.
1223 auto *CoroBegin = Shape.CoroBegin;
1224 auto *CoroId = CoroBegin->getId();
1225 auto *AllocInst = CoroId->getCoroAlloc();
1226 switch (Shape.ABI) {
1227 case coro::ABI::Switch: {
1228 auto SwitchId = cast<CoroIdInst>(CoroId);
1229 coro::replaceCoroFree(SwitchId, /*Elide=*/AllocInst != nullptr);
1230 if (AllocInst) {
1231 IRBuilder<> Builder(AllocInst);
1232 auto *Frame = Builder.CreateAlloca(Shape.FrameTy);
1233 Frame->setAlignment(Shape.FrameAlign);
1234 AllocInst->replaceAllUsesWith(Builder.getFalse());
1235 AllocInst->eraseFromParent();
1236 CoroBegin->replaceAllUsesWith(Frame);
1237 } else {
1238 CoroBegin->replaceAllUsesWith(CoroBegin->getMem());
1239 }
1240
1241 break;
1242 }
1243 case coro::ABI::Async:
1244 case coro::ABI::Retcon:
1245 case coro::ABI::RetconOnce:
1246 CoroBegin->replaceAllUsesWith(UndefValue::get(CoroBegin->getType()));
1247 break;
1248 }
1249
1250 CoroBegin->eraseFromParent();
1251}
1252
1253// SimplifySuspendPoint needs to check that there is no calls between
1254// coro_save and coro_suspend, since any of the calls may potentially resume
1255// the coroutine and if that is the case we cannot eliminate the suspend point.
1257 for (Instruction *I = From; I != To; I = I->getNextNode()) {
1258 // Assume that no intrinsic can resume the coroutine.
1259 if (isa<IntrinsicInst>(I))
1260 continue;
1261
1262 if (isa<CallBase>(I))
1263 return true;
1264 }
1265 return false;
1266}
1267
1268static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB) {
1271
1272 Set.insert(SaveBB);
1273 Worklist.push_back(ResDesBB);
1274
1275 // Accumulate all blocks between SaveBB and ResDesBB. Because CoroSaveIntr
1276 // returns a token consumed by suspend instruction, all blocks in between
1277 // will have to eventually hit SaveBB when going backwards from ResDesBB.
1278 while (!Worklist.empty()) {
1279 auto *BB = Worklist.pop_back_val();
1280 Set.insert(BB);
1281 for (auto *Pred : predecessors(BB))
1282 if (!Set.contains(Pred))
1283 Worklist.push_back(Pred);
1284 }
1285
1286 // SaveBB and ResDesBB are checked separately in hasCallsBetween.
1287 Set.erase(SaveBB);
1288 Set.erase(ResDesBB);
1289
1290 for (auto *BB : Set)
1291 if (hasCallsInBlockBetween(BB->getFirstNonPHI(), nullptr))
1292 return true;
1293
1294 return false;
1295}
1296
1297static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy) {
1298 auto *SaveBB = Save->getParent();
1299 auto *ResumeOrDestroyBB = ResumeOrDestroy->getParent();
1300
1301 if (SaveBB == ResumeOrDestroyBB)
1302 return hasCallsInBlockBetween(Save->getNextNode(), ResumeOrDestroy);
1303
1304 // Any calls from Save to the end of the block?
1305 if (hasCallsInBlockBetween(Save->getNextNode(), nullptr))
1306 return true;
1307
1308 // Any calls from begging of the block up to ResumeOrDestroy?
1309 if (hasCallsInBlockBetween(ResumeOrDestroyBB->getFirstNonPHI(),
1310 ResumeOrDestroy))
1311 return true;
1312
1313 // Any calls in all of the blocks between SaveBB and ResumeOrDestroyBB?
1314 if (hasCallsInBlocksBetween(SaveBB, ResumeOrDestroyBB))
1315 return true;
1316
1317 return false;
1318}
1319
1320// If a SuspendIntrin is preceded by Resume or Destroy, we can eliminate the
1321// suspend point and replace it with nornal control flow.
1323 CoroBeginInst *CoroBegin) {
1324 Instruction *Prev = Suspend->getPrevNode();
1325 if (!Prev) {
1326 auto *Pred = Suspend->getParent()->getSinglePredecessor();
1327 if (!Pred)
1328 return false;
1329 Prev = Pred->getTerminator();
1330 }
1331
1332 CallBase *CB = dyn_cast<CallBase>(Prev);
1333 if (!CB)
1334 return false;
1335
1336 auto *Callee = CB->getCalledOperand()->stripPointerCasts();
1337
1338 // See if the callsite is for resumption or destruction of the coroutine.
1339 auto *SubFn = dyn_cast<CoroSubFnInst>(Callee);
1340 if (!SubFn)
1341 return false;
1342
1343 // Does not refer to the current coroutine, we cannot do anything with it.
1344 if (SubFn->getFrame() != CoroBegin)
1345 return false;
1346
1347 // See if the transformation is safe. Specifically, see if there are any
1348 // calls in between Save and CallInstr. They can potenitally resume the
1349 // coroutine rendering this optimization unsafe.
1350 auto *Save = Suspend->getCoroSave();
1351 if (hasCallsBetween(Save, CB))
1352 return false;
1353
1354 // Replace llvm.coro.suspend with the value that results in resumption over
1355 // the resume or cleanup path.
1356 Suspend->replaceAllUsesWith(SubFn->getRawIndex());
1357 Suspend->eraseFromParent();
1358 Save->eraseFromParent();
1359
1360 // No longer need a call to coro.resume or coro.destroy.
1361 if (auto *Invoke = dyn_cast<InvokeInst>(CB)) {
1362 BranchInst::Create(Invoke->getNormalDest(), Invoke->getIterator());
1363 }
1364
1365 // Grab the CalledValue from CB before erasing the CallInstr.
1366 auto *CalledValue = CB->getCalledOperand();
1367 CB->eraseFromParent();
1368
1369 // If no more users remove it. Usually it is a bitcast of SubFn.
1370 if (CalledValue != SubFn && CalledValue->user_empty())
1371 if (auto *I = dyn_cast<Instruction>(CalledValue))
1372 I->eraseFromParent();
1373
1374 // Now we are good to remove SubFn.
1375 if (SubFn->user_empty())
1376 SubFn->eraseFromParent();
1377
1378 return true;
1379}
1380
1381// Remove suspend points that are simplified.
1383 // Currently, the only simplification we do is switch-lowering-specific.
1384 if (Shape.ABI != coro::ABI::Switch)
1385 return;
1386
1387 auto &S = Shape.CoroSuspends;
1388 size_t I = 0, N = S.size();
1389 if (N == 0)
1390 return;
1391
1392 size_t ChangedFinalIndex = std::numeric_limits<size_t>::max();
1393 while (true) {
1394 auto SI = cast<CoroSuspendInst>(S[I]);
1395 // Leave final.suspend to handleFinalSuspend since it is undefined behavior
1396 // to resume a coroutine suspended at the final suspend point.
1397 if (!SI->isFinal() && simplifySuspendPoint(SI, Shape.CoroBegin)) {
1398 if (--N == I)
1399 break;
1400
1401 std::swap(S[I], S[N]);
1402
1403 if (cast<CoroSuspendInst>(S[I])->isFinal()) {
1405 ChangedFinalIndex = I;
1406 }
1407
1408 continue;
1409 }
1410 if (++I == N)
1411 break;
1412 }
1413 S.resize(N);
1414
1415 // Maintain final.suspend in case final suspend was swapped.
1416 // Due to we requrie the final suspend to be the last element of CoroSuspends.
1417 if (ChangedFinalIndex < N) {
1418 assert(cast<CoroSuspendInst>(S[ChangedFinalIndex])->isFinal());
1419 std::swap(S[ChangedFinalIndex], S.back());
1420 }
1421}
1422
1423namespace {
1424
1425struct SwitchCoroutineSplitter {
1426 static void split(Function &F, coro::Shape &Shape,
1429 assert(Shape.ABI == coro::ABI::Switch);
1430
1431 createResumeEntryBlock(F, Shape);
1432 auto *ResumeClone =
1433 createClone(F, ".resume", Shape, CoroCloner::Kind::SwitchResume, TTI);
1434 auto *DestroyClone =
1435 createClone(F, ".destroy", Shape, CoroCloner::Kind::SwitchUnwind, TTI);
1436 auto *CleanupClone =
1437 createClone(F, ".cleanup", Shape, CoroCloner::Kind::SwitchCleanup, TTI);
1438
1439 postSplitCleanup(*ResumeClone);
1440 postSplitCleanup(*DestroyClone);
1441 postSplitCleanup(*CleanupClone);
1442
1443 // Store addresses resume/destroy/cleanup functions in the coroutine frame.
1444 updateCoroFrame(Shape, ResumeClone, DestroyClone, CleanupClone);
1445
1446 assert(Clones.empty());
1447 Clones.push_back(ResumeClone);
1448 Clones.push_back(DestroyClone);
1449 Clones.push_back(CleanupClone);
1450
1451 // Create a constant array referring to resume/destroy/clone functions
1452 // pointed by the last argument of @llvm.coro.info, so that CoroElide pass
1453 // can determined correct function to call.
1454 setCoroInfo(F, Shape, Clones);
1455 }
1456
1457private:
1458 // Create a resume clone by cloning the body of the original function, setting
1459 // new entry block and replacing coro.suspend an appropriate value to force
1460 // resume or cleanup pass for every suspend point.
1461 static Function *createClone(Function &F, const Twine &Suffix,
1462 coro::Shape &Shape, CoroCloner::Kind FKind,
1464 CoroCloner Cloner(F, Suffix, Shape, FKind, TTI);
1465 Cloner.create();
1466 return Cloner.getFunction();
1467 }
1468
1469 // Create an entry block for a resume function with a switch that will jump to
1470 // suspend points.
1471 static void createResumeEntryBlock(Function &F, coro::Shape &Shape) {
1472 LLVMContext &C = F.getContext();
1473
1474 // resume.entry:
1475 // %index.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32
1476 // 0, i32 2 % index = load i32, i32* %index.addr switch i32 %index, label
1477 // %unreachable [
1478 // i32 0, label %resume.0
1479 // i32 1, label %resume.1
1480 // ...
1481 // ]
1482
1483 auto *NewEntry = BasicBlock::Create(C, "resume.entry", &F);
1484 auto *UnreachBB = BasicBlock::Create(C, "unreachable", &F);
1485
1486 IRBuilder<> Builder(NewEntry);
1487 auto *FramePtr = Shape.FramePtr;
1488 auto *FrameTy = Shape.FrameTy;
1489 auto *GepIndex = Builder.CreateStructGEP(
1490 FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
1491 auto *Index = Builder.CreateLoad(Shape.getIndexType(), GepIndex, "index");
1492 auto *Switch =
1493 Builder.CreateSwitch(Index, UnreachBB, Shape.CoroSuspends.size());
1495
1496 size_t SuspendIndex = 0;
1497 for (auto *AnyS : Shape.CoroSuspends) {
1498 auto *S = cast<CoroSuspendInst>(AnyS);
1499 ConstantInt *IndexVal = Shape.getIndex(SuspendIndex);
1500
1501 // Replace CoroSave with a store to Index:
1502 // %index.addr = getelementptr %f.frame... (index field number)
1503 // store i32 %IndexVal, i32* %index.addr1
1504 auto *Save = S->getCoroSave();
1505 Builder.SetInsertPoint(Save);
1506 if (S->isFinal()) {
1507 // The coroutine should be marked done if it reaches the final suspend
1508 // point.
1509 markCoroutineAsDone(Builder, Shape, FramePtr);
1510 } else {
1511 auto *GepIndex = Builder.CreateStructGEP(
1512 FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
1513 Builder.CreateStore(IndexVal, GepIndex);
1514 }
1515
1516 Save->replaceAllUsesWith(ConstantTokenNone::get(C));
1517 Save->eraseFromParent();
1518
1519 // Split block before and after coro.suspend and add a jump from an entry
1520 // switch:
1521 //
1522 // whateverBB:
1523 // whatever
1524 // %0 = call i8 @llvm.coro.suspend(token none, i1 false)
1525 // switch i8 %0, label %suspend[i8 0, label %resume
1526 // i8 1, label %cleanup]
1527 // becomes:
1528 //
1529 // whateverBB:
1530 // whatever
1531 // br label %resume.0.landing
1532 //
1533 // resume.0: ; <--- jump from the switch in the resume.entry
1534 // %0 = tail call i8 @llvm.coro.suspend(token none, i1 false)
1535 // br label %resume.0.landing
1536 //
1537 // resume.0.landing:
1538 // %1 = phi i8[-1, %whateverBB], [%0, %resume.0]
1539 // switch i8 % 1, label %suspend [i8 0, label %resume
1540 // i8 1, label %cleanup]
1541
1542 auto *SuspendBB = S->getParent();
1543 auto *ResumeBB =
1544 SuspendBB->splitBasicBlock(S, "resume." + Twine(SuspendIndex));
1545 auto *LandingBB = ResumeBB->splitBasicBlock(
1546 S->getNextNode(), ResumeBB->getName() + Twine(".landing"));
1547 Switch->addCase(IndexVal, ResumeBB);
1548
1549 cast<BranchInst>(SuspendBB->getTerminator())->setSuccessor(0, LandingBB);
1550 auto *PN = PHINode::Create(Builder.getInt8Ty(), 2, "");
1551 PN->insertBefore(LandingBB->begin());
1552 S->replaceAllUsesWith(PN);
1553 PN->addIncoming(Builder.getInt8(-1), SuspendBB);
1554 PN->addIncoming(S, ResumeBB);
1555
1556 ++SuspendIndex;
1557 }
1558
1559 Builder.SetInsertPoint(UnreachBB);
1560 Builder.CreateUnreachable();
1561
1562 Shape.SwitchLowering.ResumeEntryBlock = NewEntry;
1563 }
1564
1565 // Store addresses of Resume/Destroy/Cleanup functions in the coroutine frame.
1566 static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn,
1567 Function *DestroyFn, Function *CleanupFn) {
1568 IRBuilder<> Builder(&*Shape.getInsertPtAfterFramePtr());
1569
1570 auto *ResumeAddr = Builder.CreateStructGEP(
1572 "resume.addr");
1573 Builder.CreateStore(ResumeFn, ResumeAddr);
1574
1575 Value *DestroyOrCleanupFn = DestroyFn;
1576
1577 CoroIdInst *CoroId = Shape.getSwitchCoroId();
1578 if (CoroAllocInst *CA = CoroId->getCoroAlloc()) {
1579 // If there is a CoroAlloc and it returns false (meaning we elide the
1580 // allocation, use CleanupFn instead of DestroyFn).
1581 DestroyOrCleanupFn = Builder.CreateSelect(CA, DestroyFn, CleanupFn);
1582 }
1583
1584 auto *DestroyAddr = Builder.CreateStructGEP(
1586 "destroy.addr");
1587 Builder.CreateStore(DestroyOrCleanupFn, DestroyAddr);
1588 }
1589
1590 // Create a global constant array containing pointers to functions provided
1591 // and set Info parameter of CoroBegin to point at this constant. Example:
1592 //
1593 // @f.resumers = internal constant [2 x void(%f.frame*)*]
1594 // [void(%f.frame*)* @f.resume, void(%f.frame*)*
1595 // @f.destroy]
1596 // define void @f() {
1597 // ...
1598 // call i8* @llvm.coro.begin(i8* null, i32 0, i8* null,
1599 // i8* bitcast([2 x void(%f.frame*)*] * @f.resumers to
1600 // i8*))
1601 //
1602 // Assumes that all the functions have the same signature.
1603 static void setCoroInfo(Function &F, coro::Shape &Shape,
1605 // This only works under the switch-lowering ABI because coro elision
1606 // only works on the switch-lowering ABI.
1608 assert(!Args.empty());
1609 Function *Part = *Fns.begin();
1610 Module *M = Part->getParent();
1611 auto *ArrTy = ArrayType::get(Part->getType(), Args.size());
1612
1613 auto *ConstVal = ConstantArray::get(ArrTy, Args);
1614 auto *GV = new GlobalVariable(*M, ConstVal->getType(), /*isConstant=*/true,
1615 GlobalVariable::PrivateLinkage, ConstVal,
1616 F.getName() + Twine(".resumers"));
1617
1618 // Update coro.begin instruction to refer to this constant.
1619 LLVMContext &C = F.getContext();
1620 auto *BC = ConstantExpr::getPointerCast(GV, PointerType::getUnqual(C));
1621 Shape.getSwitchCoroId()->setInfo(BC);
1622 }
1623};
1624
1625} // namespace
1626
1628 Value *Continuation) {
1629 auto *ResumeIntrinsic = Suspend->getResumeFunction();
1630 auto &Context = Suspend->getParent()->getParent()->getContext();
1631 auto *Int8PtrTy = PointerType::getUnqual(Context);
1632
1633 IRBuilder<> Builder(ResumeIntrinsic);
1634 auto *Val = Builder.CreateBitOrPointerCast(Continuation, Int8PtrTy);
1635 ResumeIntrinsic->replaceAllUsesWith(Val);
1636 ResumeIntrinsic->eraseFromParent();
1638 UndefValue::get(Int8PtrTy));
1639}
1640
1641/// Coerce the arguments in \p FnArgs according to \p FnTy in \p CallArgs.
1642static void coerceArguments(IRBuilder<> &Builder, FunctionType *FnTy,
1643 ArrayRef<Value *> FnArgs,
1644 SmallVectorImpl<Value *> &CallArgs) {
1645 size_t ArgIdx = 0;
1646 for (auto *paramTy : FnTy->params()) {
1647 assert(ArgIdx < FnArgs.size());
1648 if (paramTy != FnArgs[ArgIdx]->getType())
1649 CallArgs.push_back(
1650 Builder.CreateBitOrPointerCast(FnArgs[ArgIdx], paramTy));
1651 else
1652 CallArgs.push_back(FnArgs[ArgIdx]);
1653 ++ArgIdx;
1654 }
1655}
1656
1660 IRBuilder<> &Builder) {
1661 auto *FnTy = MustTailCallFn->getFunctionType();
1662 // Coerce the arguments, llvm optimizations seem to ignore the types in
1663 // vaarg functions and throws away casts in optimized mode.
1664 SmallVector<Value *, 8> CallArgs;
1665 coerceArguments(Builder, FnTy, Arguments, CallArgs);
1666
1667 auto *TailCall = Builder.CreateCall(FnTy, MustTailCallFn, CallArgs);
1668 // Skip targets which don't support tail call.
1669 if (TTI.supportsTailCallFor(TailCall)) {
1670 TailCall->setTailCallKind(CallInst::TCK_MustTail);
1671 }
1672 TailCall->setDebugLoc(Loc);
1673 TailCall->setCallingConv(MustTailCallFn->getCallingConv());
1674 return TailCall;
1675}
1676
1680 assert(Shape.ABI == coro::ABI::Async);
1681 assert(Clones.empty());
1682 // Reset various things that the optimizer might have decided it
1683 // "knows" about the coroutine function due to not seeing a return.
1684 F.removeFnAttr(Attribute::NoReturn);
1685 F.removeRetAttr(Attribute::NoAlias);
1686 F.removeRetAttr(Attribute::NonNull);
1687
1688 auto &Context = F.getContext();
1689 auto *Int8PtrTy = PointerType::getUnqual(Context);
1690
1691 auto *Id = cast<CoroIdAsyncInst>(Shape.CoroBegin->getId());
1692 IRBuilder<> Builder(Id);
1693
1694 auto *FramePtr = Id->getStorage();
1695 FramePtr = Builder.CreateBitOrPointerCast(FramePtr, Int8PtrTy);
1698 "async.ctx.frameptr");
1699
1700 // Map all uses of llvm.coro.begin to the allocated frame pointer.
1701 {
1702 // Make sure we don't invalidate Shape.FramePtr.
1703 TrackingVH<Value> Handle(Shape.FramePtr);
1705 Shape.FramePtr = Handle.getValPtr();
1706 }
1707
1708 // Create all the functions in order after the main function.
1709 auto NextF = std::next(F.getIterator());
1710
1711 // Create a continuation function for each of the suspend points.
1712 Clones.reserve(Shape.CoroSuspends.size());
1713 for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) {
1714 auto *Suspend = cast<CoroSuspendAsyncInst>(Shape.CoroSuspends[Idx]);
1715
1716 // Create the clone declaration.
1717 auto ResumeNameSuffix = ".resume.";
1718 auto ProjectionFunctionName =
1719 Suspend->getAsyncContextProjectionFunction()->getName();
1720 bool UseSwiftMangling = false;
1721 if (ProjectionFunctionName == "__swift_async_resume_project_context") {
1722 ResumeNameSuffix = "TQ";
1723 UseSwiftMangling = true;
1724 } else if (ProjectionFunctionName == "__swift_async_resume_get_context") {
1725 ResumeNameSuffix = "TY";
1726 UseSwiftMangling = true;
1727 }
1728 auto *Continuation = createCloneDeclaration(
1729 F, Shape,
1730 UseSwiftMangling ? ResumeNameSuffix + Twine(Idx) + "_"
1731 : ResumeNameSuffix + Twine(Idx),
1732 NextF, Suspend);
1733 Clones.push_back(Continuation);
1734
1735 // Insert a branch to a new return block immediately before the suspend
1736 // point.
1737 auto *SuspendBB = Suspend->getParent();
1738 auto *NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);
1739 auto *Branch = cast<BranchInst>(SuspendBB->getTerminator());
1740
1741 // Place it before the first suspend.
1742 auto *ReturnBB =
1743 BasicBlock::Create(F.getContext(), "coro.return", &F, NewSuspendBB);
1744 Branch->setSuccessor(0, ReturnBB);
1745
1746 IRBuilder<> Builder(ReturnBB);
1747
1748 // Insert the call to the tail call function and inline it.
1749 auto *Fn = Suspend->getMustTailCallFunction();
1750 SmallVector<Value *, 8> Args(Suspend->args());
1751 auto FnArgs = ArrayRef<Value *>(Args).drop_front(
1753 auto *TailCall = coro::createMustTailCall(Suspend->getDebugLoc(), Fn, TTI,
1754 FnArgs, Builder);
1755 Builder.CreateRetVoid();
1756 InlineFunctionInfo FnInfo;
1757 (void)InlineFunction(*TailCall, FnInfo);
1758
1759 // Replace the lvm.coro.async.resume intrisic call.
1760 replaceAsyncResumeFunction(Suspend, Continuation);
1761 }
1762
1763 assert(Clones.size() == Shape.CoroSuspends.size());
1764 for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) {
1765 auto *Suspend = Shape.CoroSuspends[Idx];
1766 auto *Clone = Clones[Idx];
1767
1768 CoroCloner(F, "resume." + Twine(Idx), Shape, Clone, Suspend, TTI).create();
1769 }
1770}
1771
1775 assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce);
1776 assert(Clones.empty());
1777
1778 // Reset various things that the optimizer might have decided it
1779 // "knows" about the coroutine function due to not seeing a return.
1780 F.removeFnAttr(Attribute::NoReturn);
1781 F.removeRetAttr(Attribute::NoAlias);
1782 F.removeRetAttr(Attribute::NonNull);
1783
1784 // Allocate the frame.
1785 auto *Id = cast<AnyCoroIdRetconInst>(Shape.CoroBegin->getId());
1786 Value *RawFramePtr;
1788 RawFramePtr = Id->getStorage();
1789 } else {
1790 IRBuilder<> Builder(Id);
1791
1792 // Determine the size of the frame.
1793 const DataLayout &DL = F.getParent()->getDataLayout();
1794 auto Size = DL.getTypeAllocSize(Shape.FrameTy);
1795
1796 // Allocate. We don't need to update the call graph node because we're
1797 // going to recompute it from scratch after splitting.
1798 // FIXME: pass the required alignment
1799 RawFramePtr = Shape.emitAlloc(Builder, Builder.getInt64(Size), nullptr);
1800 RawFramePtr =
1801 Builder.CreateBitCast(RawFramePtr, Shape.CoroBegin->getType());
1802
1803 // Stash the allocated frame pointer in the continuation storage.
1804 Builder.CreateStore(RawFramePtr, Id->getStorage());
1805 }
1806
1807 // Map all uses of llvm.coro.begin to the allocated frame pointer.
1808 {
1809 // Make sure we don't invalidate Shape.FramePtr.
1810 TrackingVH<Value> Handle(Shape.FramePtr);
1811 Shape.CoroBegin->replaceAllUsesWith(RawFramePtr);
1812 Shape.FramePtr = Handle.getValPtr();
1813 }
1814
1815 // Create a unique return block.
1816 BasicBlock *ReturnBB = nullptr;
1817 SmallVector<PHINode *, 4> ReturnPHIs;
1818
1819 // Create all the functions in order after the main function.
1820 auto NextF = std::next(F.getIterator());
1821
1822 // Create a continuation function for each of the suspend points.
1823 Clones.reserve(Shape.CoroSuspends.size());
1824 for (size_t i = 0, e = Shape.CoroSuspends.size(); i != e; ++i) {
1825 auto Suspend = cast<CoroSuspendRetconInst>(Shape.CoroSuspends[i]);
1826
1827 // Create the clone declaration.
1828 auto Continuation =
1829 createCloneDeclaration(F, Shape, ".resume." + Twine(i), NextF, nullptr);
1830 Clones.push_back(Continuation);
1831
1832 // Insert a branch to the unified return block immediately before
1833 // the suspend point.
1834 auto SuspendBB = Suspend->getParent();
1835 auto NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);
1836 auto Branch = cast<BranchInst>(SuspendBB->getTerminator());
1837
1838 // Create the unified return block.
1839 if (!ReturnBB) {
1840 // Place it before the first suspend.
1841 ReturnBB =
1842 BasicBlock::Create(F.getContext(), "coro.return", &F, NewSuspendBB);
1843 Shape.RetconLowering.ReturnBlock = ReturnBB;
1844
1845 IRBuilder<> Builder(ReturnBB);
1846
1847 // Create PHIs for all the return values.
1848 assert(ReturnPHIs.empty());
1849
1850 // First, the continuation.
1851 ReturnPHIs.push_back(Builder.CreatePHI(Continuation->getType(),
1852 Shape.CoroSuspends.size()));
1853
1854 // Next, all the directly-yielded values.
1855 for (auto *ResultTy : Shape.getRetconResultTypes())
1856 ReturnPHIs.push_back(
1857 Builder.CreatePHI(ResultTy, Shape.CoroSuspends.size()));
1858
1859 // Build the return value.
1860 auto RetTy = F.getReturnType();
1861
1862 // Cast the continuation value if necessary.
1863 // We can't rely on the types matching up because that type would
1864 // have to be infinite.
1865 auto CastedContinuationTy =
1866 (ReturnPHIs.size() == 1 ? RetTy : RetTy->getStructElementType(0));
1867 auto *CastedContinuation =
1868 Builder.CreateBitCast(ReturnPHIs[0], CastedContinuationTy);
1869
1870 Value *RetV;
1871 if (ReturnPHIs.size() == 1) {
1872 RetV = CastedContinuation;
1873 } else {
1874 RetV = PoisonValue::get(RetTy);
1875 RetV = Builder.CreateInsertValue(RetV, CastedContinuation, 0);
1876 for (size_t I = 1, E = ReturnPHIs.size(); I != E; ++I)
1877 RetV = Builder.CreateInsertValue(RetV, ReturnPHIs[I], I);
1878 }
1879
1880 Builder.CreateRet(RetV);
1881 }
1882
1883 // Branch to the return block.
1884 Branch->setSuccessor(0, ReturnBB);
1885 ReturnPHIs[0]->addIncoming(Continuation, SuspendBB);
1886 size_t NextPHIIndex = 1;
1887 for (auto &VUse : Suspend->value_operands())
1888 ReturnPHIs[NextPHIIndex++]->addIncoming(&*VUse, SuspendBB);
1889 assert(NextPHIIndex == ReturnPHIs.size());
1890 }
1891
1892 assert(Clones.size() == Shape.CoroSuspends.size());
1893 for (size_t i = 0, e = Shape.CoroSuspends.size(); i != e; ++i) {
1894 auto Suspend = Shape.CoroSuspends[i];
1895 auto Clone = Clones[i];
1896
1897 CoroCloner(F, "resume." + Twine(i), Shape, Clone, Suspend, TTI).create();
1898 }
1899}
1900
1901namespace {
1902class PrettyStackTraceFunction : public PrettyStackTraceEntry {
1903 Function &F;
1904
1905public:
1906 PrettyStackTraceFunction(Function &F) : F(F) {}
1907 void print(raw_ostream &OS) const override {
1908 OS << "While splitting coroutine ";
1909 F.printAsOperand(OS, /*print type*/ false, F.getParent());
1910 OS << "\n";
1911 }
1912};
1913} // namespace
1914
1915static coro::Shape
1917 TargetTransformInfo &TTI, bool OptimizeFrame,
1918 std::function<bool(Instruction &)> MaterializableCallback) {
1919 PrettyStackTraceFunction prettyStackTrace(F);
1920
1921 // The suspend-crossing algorithm in buildCoroutineFrame get tripped
1922 // up by uses in unreachable blocks, so remove them as a first pass.
1924
1925 coro::Shape Shape(F, OptimizeFrame);
1926 if (!Shape.CoroBegin)
1927 return Shape;
1928
1929 lowerAwaitSuspends(F, Shape);
1930
1931 simplifySuspendPoints(Shape);
1932 buildCoroutineFrame(F, Shape, TTI, MaterializableCallback);
1934
1935 // If there are no suspend points, no split required, just remove
1936 // the allocation and deallocation blocks, they are not needed.
1937 if (Shape.CoroSuspends.empty()) {
1939 } else {
1940 switch (Shape.ABI) {
1941 case coro::ABI::Switch:
1942 SwitchCoroutineSplitter::split(F, Shape, Clones, TTI);
1943 break;
1944 case coro::ABI::Async:
1945 splitAsyncCoroutine(F, Shape, Clones, TTI);
1946 break;
1947 case coro::ABI::Retcon:
1948 case coro::ABI::RetconOnce:
1949 splitRetconCoroutine(F, Shape, Clones, TTI);
1950 break;
1951 }
1952 }
1953
1954 // Replace all the swifterror operations in the original function.
1955 // This invalidates SwiftErrorOps in the Shape.
1956 replaceSwiftErrorOps(F, Shape, nullptr);
1957
1958 // Salvage debug intrinsics that point into the coroutine frame in the
1959 // original function. The Cloner has already salvaged debug info in the new
1960 // coroutine funclets.
1962 auto [DbgInsts, DbgVariableRecords] = collectDbgVariableIntrinsics(F);
1963 for (auto *DDI : DbgInsts)
1964 coro::salvageDebugInfo(ArgToAllocaMap, *DDI, Shape.OptimizeFrame,
1965 false /*UseEntryValue*/);
1966 for (DbgVariableRecord *DVR : DbgVariableRecords)
1967 coro::salvageDebugInfo(ArgToAllocaMap, *DVR, Shape.OptimizeFrame,
1968 false /*UseEntryValue*/);
1969 return Shape;
1970}
1971
1972/// Remove calls to llvm.coro.end in the original function.
1973static void removeCoroEnds(const coro::Shape &Shape) {
1974 for (auto *End : Shape.CoroEnds) {
1975 replaceCoroEnd(End, Shape, Shape.FramePtr, /*in resume*/ false, nullptr);
1976 }
1977}
1978
1980 LazyCallGraph::Node &N, const coro::Shape &Shape,
1984 if (!Shape.CoroBegin)
1985 return;
1986
1987 if (Shape.ABI != coro::ABI::Switch)
1988 removeCoroEnds(Shape);
1989 else {
1990 for (llvm::AnyCoroEndInst *End : Shape.CoroEnds) {
1991 auto &Context = End->getContext();
1992 End->replaceAllUsesWith(ConstantInt::getFalse(Context));
1993 End->eraseFromParent();
1994 }
1995 }
1996
1997 if (!Clones.empty()) {
1998 switch (Shape.ABI) {
1999 case coro::ABI::Switch:
2000 // Each clone in the Switch lowering is independent of the other clones.
2001 // Let the LazyCallGraph know about each one separately.
2002 for (Function *Clone : Clones)
2003 CG.addSplitFunction(N.getFunction(), *Clone);
2004 break;
2005 case coro::ABI::Async:
2006 case coro::ABI::Retcon:
2007 case coro::ABI::RetconOnce:
2008 // Each clone in the Async/Retcon lowering references of the other clones.
2009 // Let the LazyCallGraph know about all of them at once.
2010 if (!Clones.empty())
2011 CG.addSplitRefRecursiveFunctions(N.getFunction(), Clones);
2012 break;
2013 }
2014
2015 // Let the CGSCC infra handle the changes to the original function.
2017 }
2018
2019 // Do some cleanup and let the CGSCC infra see if we've cleaned up any edges
2020 // to the split functions.
2021 postSplitCleanup(N.getFunction());
2023}
2024
2025/// Replace a call to llvm.coro.prepare.retcon.
2026static void replacePrepare(CallInst *Prepare, LazyCallGraph &CG,
2028 auto CastFn = Prepare->getArgOperand(0); // as an i8*
2029 auto Fn = CastFn->stripPointerCasts(); // as its original type
2030
2031 // Attempt to peephole this pattern:
2032 // %0 = bitcast [[TYPE]] @some_function to i8*
2033 // %1 = call @llvm.coro.prepare.retcon(i8* %0)
2034 // %2 = bitcast %1 to [[TYPE]]
2035 // ==>
2036 // %2 = @some_function
2037 for (Use &U : llvm::make_early_inc_range(Prepare->uses())) {
2038 // Look for bitcasts back to the original function type.
2039 auto *Cast = dyn_cast<BitCastInst>(U.getUser());
2040 if (!Cast || Cast->getType() != Fn->getType())
2041 continue;
2042
2043 // Replace and remove the cast.
2044 Cast->replaceAllUsesWith(Fn);
2045 Cast->eraseFromParent();
2046 }
2047
2048 // Replace any remaining uses with the function as an i8*.
2049 // This can never directly be a callee, so we don't need to update CG.
2050 Prepare->replaceAllUsesWith(CastFn);
2051 Prepare->eraseFromParent();
2052
2053 // Kill dead bitcasts.
2054 while (auto *Cast = dyn_cast<BitCastInst>(CastFn)) {
2055 if (!Cast->use_empty())
2056 break;
2057 CastFn = Cast->getOperand(0);
2058 Cast->eraseFromParent();
2059 }
2060}
2061
2062static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG,
2064 bool Changed = false;
2065 for (Use &P : llvm::make_early_inc_range(PrepareFn->uses())) {
2066 // Intrinsics can only be used in calls.
2067 auto *Prepare = cast<CallInst>(P.getUser());
2068 replacePrepare(Prepare, CG, C);
2069 Changed = true;
2070 }
2071
2072 return Changed;
2073}
2074
2075static void addPrepareFunction(const Module &M,
2077 StringRef Name) {
2078 auto *PrepareFn = M.getFunction(Name);
2079 if (PrepareFn && !PrepareFn->use_empty())
2080 Fns.push_back(PrepareFn);
2081}
2082
2084 : MaterializableCallback(coro::defaultMaterializable),
2085 OptimizeFrame(OptimizeFrame) {}
2086
2090 // NB: One invariant of a valid LazyCallGraph::SCC is that it must contain a
2091 // non-zero number of nodes, so we assume that here and grab the first
2092 // node's function's module.
2093 Module &M = *C.begin()->getFunction().getParent();
2094 auto &FAM =
2095 AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager();
2096
2097 // Check for uses of llvm.coro.prepare.retcon/async.
2098 SmallVector<Function *, 2> PrepareFns;
2099 addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.retcon");
2100 addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.async");
2101
2102 // Find coroutines for processing.
2104 for (LazyCallGraph::Node &N : C)
2105 if (N.getFunction().isPresplitCoroutine())
2106 Coroutines.push_back(&N);
2107
2108 if (Coroutines.empty() && PrepareFns.empty())
2109 return PreservedAnalyses::all();
2110
2111 if (Coroutines.empty()) {
2112 for (auto *PrepareFn : PrepareFns) {
2113 replaceAllPrepares(PrepareFn, CG, C);
2114 }
2115 }
2116
2117 // Split all the coroutines.
2118 for (LazyCallGraph::Node *N : Coroutines) {
2119 Function &F = N->getFunction();
2120 LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F.getName()
2121 << "\n");
2122 F.setSplittedCoroutine();
2123
2126 const coro::Shape Shape =
2129 updateCallGraphAfterCoroutineSplit(*N, Shape, Clones, C, CG, AM, UR, FAM);
2130
2131 ORE.emit([&]() {
2132 return OptimizationRemark(DEBUG_TYPE, "CoroSplit", &F)
2133 << "Split '" << ore::NV("function", F.getName())
2134 << "' (frame_size=" << ore::NV("frame_size", Shape.FrameSize)
2135 << ", align=" << ore::NV("align", Shape.FrameAlign.value()) << ")";
2136 });
2137
2138 if (!Shape.CoroSuspends.empty()) {
2139 // Run the CGSCC pipeline on the original and newly split functions.
2140 UR.CWorklist.insert(&C);
2141 for (Function *Clone : Clones)
2142 UR.CWorklist.insert(CG.lookupSCC(CG.get(*Clone)));
2143 }
2144 }
2145
2146 if (!PrepareFns.empty()) {
2147 for (auto *PrepareFn : PrepareFns) {
2148 replaceAllPrepares(PrepareFn, CG, C);
2149 }
2150 }
2151
2152 return PreservedAnalyses::none();
2153}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
AMDGPU Lower Kernel Arguments
Expand Atomic instructions
This file contains the simple types necessary to represent the attributes associated with functions a...
BlockVerifier::State From
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file provides interfaces used to manipulate a call graph, regardless if it is a "old style" Call...
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Remove calls to llvm coro end in the original static function void removeCoroEnds(const coro::Shape &Shape)
Definition: CoroSplit.cpp:1973
static void addSwiftSelfAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)
Definition: CoroSplit.cpp:913
static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy)
Definition: CoroSplit.cpp:1297
static std::pair< SmallVector< DbgVariableIntrinsic *, 8 >, SmallVector< DbgVariableRecord * > > collectDbgVariableIntrinsics(Function &F)
Returns all DbgVariableIntrinsic in F.
Definition: CoroSplit.cpp:714
static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape, ValueToValueMapTy *VMap)
Definition: CoroSplit.cpp:656
static void addAsyncContextAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)
Definition: CoroSplit.cpp:906
static void maybeFreeRetconStorage(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr, CallGraph *CG)
Definition: CoroSplit.cpp:249
static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB)
Definition: CoroSplit.cpp:1268
static Function * createCloneDeclaration(Function &OrigF, coro::Shape &Shape, const Twine &Suffix, Module::iterator InsertBefore, AnyCoroSuspendInst *ActiveSuspend)
Definition: CoroSplit.cpp:540
static FunctionType * getFunctionTypeFromAsyncSuspend(AnyCoroSuspendInst *Suspend)
Definition: CoroSplit.cpp:532
static void addPrepareFunction(const Module &M, SmallVectorImpl< Function * > &Fns, StringRef Name)
Definition: CoroSplit.cpp:2075
static void updateCallGraphAfterCoroutineSplit(LazyCallGraph::Node &N, const coro::Shape &Shape, const SmallVectorImpl< Function * > &Clones, LazyCallGraph::SCC &C, LazyCallGraph &CG, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Definition: CoroSplit.cpp:1979
static void simplifySuspendPoints(coro::Shape &Shape)
Definition: CoroSplit.cpp:1382
static void addFramePointerAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex, uint64_t Size, Align Alignment, bool NoAlias)
Definition: CoroSplit.cpp:891
static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG, LazyCallGraph::SCC &C)
Definition: CoroSplit.cpp:2062
static void replaceFallthroughCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Replace a non-unwind call to llvm.coro.end.
Definition: CoroSplit.cpp:305
static void replaceFrameSizeAndAlignment(coro::Shape &Shape)
Definition: CoroSplit.cpp:1182
static bool replaceCoroEndAsync(AnyCoroEndInst *End)
Replace an llvm.coro.end.async.
Definition: CoroSplit.cpp:262
static void splitRetconCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI)
Definition: CoroSplit.cpp:1772
Replace a call to llvm coro prepare static retcon void replacePrepare(CallInst *Prepare, LazyCallGraph &CG, LazyCallGraph::SCC &C)
Definition: CoroSplit.cpp:2026
static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Replace an unwind call to llvm.coro.end.
Definition: CoroSplit.cpp:438
static bool simplifySuspendPoint(CoroSuspendInst *Suspend, CoroBeginInst *CoroBegin)
Definition: CoroSplit.cpp:1322
static bool hasCallsInBlockBetween(Instruction *From, Instruction *To)
Definition: CoroSplit.cpp:1256
static void markCoroutineAsDone(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr)
Definition: CoroSplit.cpp:404
static void updateAsyncFuncPointerContextSize(coro::Shape &Shape)
Definition: CoroSplit.cpp:1167
static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Definition: CoroSplit.cpp:476
static void lowerAwaitSuspend(IRBuilder<> &Builder, CoroAwaitSuspendInst *CB, coro::Shape &Shape)
Definition: CoroSplit.cpp:177
static void lowerAwaitSuspends(Function &F, coro::Shape &Shape)
Definition: CoroSplit.cpp:243
static void handleNoSuspendCoroutine(coro::Shape &Shape)
Definition: CoroSplit.cpp:1222
static coro::Shape splitCoroutine(Function &F, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI, bool OptimizeFrame, std::function< bool(Instruction &)> MaterializableCallback)
Definition: CoroSplit.cpp:1916
static void postSplitCleanup(Function &F)
Definition: CoroSplit.cpp:1208
static void splitAsyncCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI)
Definition: CoroSplit.cpp:1677
Coerce the arguments in p FnArgs according to p FnTy in p static CallArgs void coerceArguments(IRBuilder<> &Builder, FunctionType *FnTy, ArrayRef< Value * > FnArgs, SmallVectorImpl< Value * > &CallArgs)
Definition: CoroSplit.cpp:1642
static void replaceAsyncResumeFunction(CoroSuspendAsyncInst *Suspend, Value *Continuation)
Definition: CoroSplit.cpp:1627
static Error split(StringRef Str, char Separator, std::pair< StringRef, StringRef > &Split)
Checked version of split, to ensure mandatory subparts.
Definition: DataLayout.cpp:235
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseMap class.
This file contains constants used for implementing Dwarf debug support.
std::string Name
uint64_t Size
bool End
Definition: ELF_riscv.cpp:480
static Function * getFunction(Constant *C)
Definition: Evaluator.cpp:236
@ InlineInfo
Rewrite Partial Register Uses
#define DEBUG_TYPE
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Implements a lazy call graph analysis and related passes for the new pass manager.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Module.h This file contains the declarations for the Module class.
#define P(N)
FunctionAnalysisManager FAM
This file provides a priority worklist.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
raw_pwrite_stream & OS
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:40
This pass exposes codegen information to IR-level passes.
static const unsigned FramePtr
void setSwiftError(bool V)
Specify whether this alloca is used to represent a swifterror.
Definition: Instructions.h:148
void setAlignment(Align Align)
Definition: Instructions.h:125
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:239
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:391
CoroAllocInst * getCoroAlloc()
Definition: CoroInstr.h:117
This class represents an incoming formal argument to a Function.
Definition: Argument.h:31
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
Definition: ArrayRef.h:204
iterator end() const
Definition: ArrayRef.h:154
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
iterator begin() const
Definition: ArrayRef.h:153
AttrBuilder & addAlignmentAttr(MaybeAlign Align)
This turns an alignment into the form used internally in Attribute.
AttrBuilder & addAttribute(Attribute::AttrKind Val)
Add an attribute to the builder.
AttrBuilder & addDereferenceableAttr(uint64_t Bytes)
This turns the number of dereferenceable bytes into the form used internally in Attribute.
AttributeList removeParamAttributes(LLVMContext &C, unsigned ArgNo, const AttributeMask &AttrsToRemove) const
Remove the specified attribute at the specified arg index from this attribute list.
Definition: Attributes.h:725
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:361
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:201
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
Definition: BasicBlock.cpp:571
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:223
static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1236
void setCallingConv(CallingConv::ID CC)
Definition: InstrTypes.h:1527
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1465
Value * getCalledOperand() const
Definition: InstrTypes.h:1458
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
Definition: InstrTypes.h:1546
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1410
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1542
The basic data container for the call graph of a Module of IR.
Definition: CallGraph.h:72
This class represents a function call, abstracting a target machine's calling convention.
void setTailCallKind(TailCallKind TCK)
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1292
static Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition: Constants.cpp:2177
This is the shared class of boolean and integer constants.
Definition: Constants.h:81
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:850
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:857
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1762
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1357
static ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition: Constants.cpp:1500
This represents the llvm.coro.align instruction.
Definition: CoroInstr.h:634
This represents the llvm.coro.alloc instruction.
Definition: CoroInstr.h:70
This represents the llvm.coro.await.suspend.{void,bool,handle} instructions.
Definition: CoroInstr.h:85
Value * getFrame() const
Definition: CoroInstr.h:91
Value * getAwaiter() const
Definition: CoroInstr.h:89
Function * getWrapperFunction() const
Definition: CoroInstr.h:93
This class represents the llvm.coro.begin instruction.
Definition: CoroInstr.h:451
AnyCoroIdInst * getId() const
Definition: CoroInstr.h:455
This represents the llvm.coro.id instruction.
Definition: CoroInstr.h:146
void setInfo(Constant *C)
Definition: CoroInstr.h:213
This represents the llvm.coro.size instruction.
Definition: CoroInstr.h:622
This represents the llvm.coro.suspend.async instruction.
Definition: CoroInstr.h:556
CoroAsyncResumeInst * getResumeFunction() const
Definition: CoroInstr.h:577
This represents the llvm.coro.suspend instruction.
Definition: CoroInstr.h:524
CoroSaveInst * getCoroSave() const
Definition: CoroInstr.h:528
DISubprogram * getSubprogram() const
Get the subprogram for this scope.
Subprogram description.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
This is the common base class for debug info intrinsics for variables.
Record of a variable value-assignment, aka a non instruction representation of the dbg....
A debug info location.
Definition: DebugLoc.h:33
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:162
This class represents a freeze function that returns random concrete value if an operand is either a ...
A proxy from a FunctionAnalysisManager to an SCC.
Type * getReturnType() const
Definition: DerivedTypes.h:124
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:164
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:202
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:232
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:264
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:358
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:655
PointerType * getType() const
Global values are always pointers.
Definition: GlobalValue.h:293
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:51
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
void setInitializer(Constant *InitVal)
setInitializer - Sets the initializer for this global variable, removing any existing initializer if ...
Definition: Globals.cpp:481
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
Definition: IRBuilder.h:1771
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Definition: IRBuilder.h:2521
InvokeInst * CreateInvoke(FunctionType *Ty, Value *Callee, BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef< Value * > Args, ArrayRef< OperandBundleDef > OpBundles, const Twine &Name="")
Create an invoke instruction.
Definition: IRBuilder.h:1156
BasicBlock::iterator GetInsertPoint() const
Definition: IRBuilder.h:173
Value * CreateStructGEP(Type *Ty, Value *Ptr, unsigned Idx, const Twine &Name="")
Definition: IRBuilder.h:1969
Value * CreateConstInBoundsGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Definition: IRBuilder.h:1887
CleanupReturnInst * CreateCleanupRet(CleanupPadInst *CleanupPad, BasicBlock *UnwindBB=nullptr)
Definition: IRBuilder.h:1234
ReturnInst * CreateRet(Value *V)
Create a 'ret <val>' instruction.
Definition: IRBuilder.h:1093
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Definition: IRBuilder.h:489
Value * CreateBitOrPointerCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2203
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Definition: IRBuilder.h:2395
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2125
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Definition: IRBuilder.h:1788
LLVMContext & getContext() const
Definition: IRBuilder.h:174
ReturnInst * CreateRetVoid()
Create a 'ret void' instruction.
Definition: IRBuilder.h:1088
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Definition: IRBuilder.h:1801
ConstantInt * getFalse()
Get the constant value for i1 false.
Definition: IRBuilder.h:469
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Definition: IRBuilder.h:178
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2410
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2664
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
Definition: Cloning.h:202
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:87
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
A node in the call graph.
An SCC of the call graph.
A lazily constructed view of the call graph of a module.
void addSplitFunction(Function &OriginalFunction, Function &NewFunction)
Add a new function split/outlined from an existing function.
void addSplitRefRecursiveFunctions(Function &OriginalFunction, ArrayRef< Function * > NewFunctions)
Add new ref-recursive functions split/outlined from an existing function.
Node & get(Function &F)
Get a graph node for a given function, scanning it to populate the graph data as necessary.
SCC * lookupSCC(Node &N) const
Lookup a function's SCC in the graph.
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:600
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
FunctionListType::iterator iterator
The Function iterators.
Definition: Module.h:90
Diagnostic information for applied optimization remarks.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1814
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:109
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:115
PrettyStackTraceEntry - This class is used to represent a frame of the "pretty" stack trace that is d...
virtual void print(raw_ostream &OS) const =0
print - Emit information about this stack frame to OS.
Return a value (possibly void), from a function.
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
Definition: SmallPtrSet.h:356
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
bool contains(ConstPtrType Ptr) const
Definition: SmallPtrSet.h:366
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void reserve(size_type N)
Definition: SmallVector.h:676
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Type * getTypeAtIndex(const Value *V) const
Given an index value into the type, return the type of the element.
Definition: Type.cpp:612
Analysis pass providing the TargetTransformInfo.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
bool supportsTailCallFor(const CallBase *CB) const
If target supports tail call on CB.
Value handle that tracks a Value across RAUW.
Definition: ValueHandle.h:331
ValueTy * getValPtr() const
Definition: ValueHandle.h:335
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
bool isArch64Bit() const
Test whether the architecture is 64-bit.
Definition: Triple.cpp:1661
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt8Ty(LLVMContext &C)
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1795
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
void setOperand(unsigned i, Value *Val)
Definition: User.h:174
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:534
iterator_range< user_iterator > users()
Definition: Value.h:421
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:693
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1074
iterator_range< use_iterator > uses()
Definition: Value.h:376
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:383
const ParentTy * getParent() const
Definition: ilist_node.h:32
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:353
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
@ Entry
Definition: COFF.h:811
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
void salvageDebugInfo(SmallDenseMap< Argument *, AllocaInst *, 4 > &ArgToAllocaMap, DbgVariableIntrinsic &DVI, bool OptimizeFrame, bool IsEntryPoint)
Attempts to rewrite the location operand of debug intrinsics in terms of the coroutine frame pointer,...
Definition: CoroFrame.cpp:2950
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
CallInst * createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, TargetTransformInfo &TTI, ArrayRef< Value * > Arguments, IRBuilder<> &)
Definition: CoroSplit.cpp:1657
void replaceCoroFree(CoroIdInst *CoroId, bool Elide)
Definition: Coroutines.cpp:128
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
UnaryFunction for_each(R &&Range, UnaryFunction F)
Provide wrappers to std::for_each which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1715
bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
Definition: Verifier.cpp:7095
void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
Definition: Utils.cpp:1665
LazyCallGraph::SCC & updateCGAndAnalysisManagerForFunctionPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a function pass.
LazyCallGraph::SCC & updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a CGSCC pass.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:656
@ Async
"Asynchronous" unwind tables (instr precise)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
Definition: Local.cpp:2839
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
DWARFExpression::Operation Op
InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)
This function inlines the called function into the basic block of the caller.
void CloneFunctionInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, CloneFunctionChangeType Changes, SmallVectorImpl< ReturnInst * > &Returns, const char *NameSuffix="", ClonedCodeInfo *CodeInfo=nullptr, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Clone OldFunc into NewFunc, transforming the old arguments into references to VMap values.
auto predecessors(const MachineBasicBlock *BB)
static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)
Filter the DbgRecord range to DbgVariableRecord types only and downcast.
bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Remove all blocks that can not be reached from the function's entry.
Definition: Local.cpp:3204
bool isPotentiallyReachable(const Instruction *From, const Instruction *To, const SmallPtrSetImpl< BasicBlock * > *ExclusionSet=nullptr, const DominatorTree *DT=nullptr, const LoopInfo *LI=nullptr)
Determine whether instruction 'To' is reachable from 'From', without passing through any blocks in Ex...
Definition: CFG.cpp:281
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
SmallPriorityWorklist< LazyCallGraph::SCC *, 1 > & CWorklist
Worklist of the SCCs queued for processing.
const std::function< bool(Instruction &)> MaterializableCallback
Definition: CoroSplit.h:25
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)
Definition: CoroSplit.cpp:2087
CoroSplitPass(bool OptimizeFrame=false)
Definition: CoroSplit.cpp:2083
CallInst * makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt)
Definition: Coroutines.cpp:50
SmallVector< CallInst *, 2 > SymmetricTransfers
Definition: CoroInternal.h:88
SmallVector< CoroAwaitSuspendInst *, 4 > CoroAwaitSuspends
Definition: CoroInternal.h:87
AsyncLoweringStorage AsyncLowering
Definition: CoroInternal.h:151
FunctionType * getResumeFunctionType() const
Definition: CoroInternal.h:190
IntegerType * getIndexType() const
Definition: CoroInternal.h:175
StructType * FrameTy
Definition: CoroInternal.h:107
CoroIdInst * getSwitchCoroId() const
Definition: CoroInternal.h:154
SmallVector< CoroSizeInst *, 2 > CoroSizes
Definition: CoroInternal.h:83
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
Definition: CoroInternal.h:85
Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const
Allocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:456
SmallVector< CallInst *, 2 > SwiftErrorOps
Definition: CoroInternal.h:86
ConstantInt * getIndex(uint64_t Value) const
Definition: CoroInternal.h:180
bool OptimizeFrame
This would only be true if optimization are enabled.
Definition: CoroInternal.h:114
SwitchLoweringStorage SwitchLowering
Definition: CoroInternal.h:149
CoroBeginInst * CoroBegin
Definition: CoroInternal.h:81
BasicBlock::iterator getInsertPtAfterFramePtr() const
Definition: CoroInternal.h:249
ArrayRef< Type * > getRetconResultTypes() const
Definition: CoroInternal.h:207
void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const
Deallocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:479
RetconLoweringStorage RetconLowering
Definition: CoroInternal.h:150
SmallVector< CoroAlignInst *, 2 > CoroAligns
Definition: CoroInternal.h:84
SmallVector< AnyCoroEndInst *, 4 > CoroEnds
Definition: CoroInternal.h:82
unsigned getSwitchIndexField() const
Definition: CoroInternal.h:170