LLVM 20.0.0git
CoroSplit.cpp
Go to the documentation of this file.
1//===- CoroSplit.cpp - Converts a coroutine into a state machine ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// This pass builds the coroutine frame and outlines resume and destroy parts
9// of the coroutine into separate functions.
10//
11// We present a coroutine to an LLVM as an ordinary function with suspension
12// points marked up with intrinsics. We let the optimizer party on the coroutine
13// as a single function for as long as possible. Shortly before the coroutine is
14// eligible to be inlined into its callers, we split up the coroutine into parts
15// corresponding to an initial, resume and destroy invocations of the coroutine,
16// add them to the current SCC and restart the IPO pipeline to optimize the
17// coroutine subfunctions we extracted before proceeding to the caller of the
18// coroutine.
19//===----------------------------------------------------------------------===//
20
22#include "CoroInstr.h"
23#include "CoroInternal.h"
24#include "llvm/ADT/DenseMap.h"
28#include "llvm/ADT/StringRef.h"
29#include "llvm/ADT/Twine.h"
30#include "llvm/Analysis/CFG.h"
37#include "llvm/IR/Argument.h"
38#include "llvm/IR/Attributes.h"
39#include "llvm/IR/BasicBlock.h"
40#include "llvm/IR/CFG.h"
41#include "llvm/IR/CallingConv.h"
42#include "llvm/IR/Constants.h"
43#include "llvm/IR/DataLayout.h"
45#include "llvm/IR/Dominators.h"
46#include "llvm/IR/Function.h"
47#include "llvm/IR/GlobalValue.h"
49#include "llvm/IR/IRBuilder.h"
51#include "llvm/IR/InstrTypes.h"
52#include "llvm/IR/Instruction.h"
55#include "llvm/IR/LLVMContext.h"
56#include "llvm/IR/Module.h"
57#include "llvm/IR/Type.h"
58#include "llvm/IR/Value.h"
59#include "llvm/IR/Verifier.h"
61#include "llvm/Support/Debug.h"
70#include <cassert>
71#include <cstddef>
72#include <cstdint>
73#include <initializer_list>
74#include <iterator>
75
76using namespace llvm;
77
78#define DEBUG_TYPE "coro-split"
79
80namespace {
81
82/// A little helper class for building
83class CoroCloner {
84public:
85 enum class Kind {
86 /// The shared resume function for a switch lowering.
87 SwitchResume,
88
89 /// The shared unwind function for a switch lowering.
90 SwitchUnwind,
91
92 /// The shared cleanup function for a switch lowering.
93 SwitchCleanup,
94
95 /// An individual continuation function.
96 Continuation,
97
98 /// An async resume function.
99 Async,
100 };
101
102private:
103 Function &OrigF;
104 Function *NewF;
105 const Twine &Suffix;
106 coro::Shape &Shape;
107 Kind FKind;
109 IRBuilder<> Builder;
110 Value *NewFramePtr = nullptr;
111
112 /// The active suspend instruction; meaningful only for continuation and async
113 /// ABIs.
114 AnyCoroSuspendInst *ActiveSuspend = nullptr;
115
117
118public:
119 /// Create a cloner for a switch lowering.
120 CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
121 Kind FKind, TargetTransformInfo &TTI)
122 : OrigF(OrigF), NewF(nullptr), Suffix(Suffix), Shape(Shape), FKind(FKind),
123 Builder(OrigF.getContext()), TTI(TTI) {
124 assert(Shape.ABI == coro::ABI::Switch);
125 }
126
127 /// Create a cloner for a continuation lowering.
128 CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
129 Function *NewF, AnyCoroSuspendInst *ActiveSuspend,
131 : OrigF(OrigF), NewF(NewF), Suffix(Suffix), Shape(Shape),
132 FKind(Shape.ABI == coro::ABI::Async ? Kind::Async : Kind::Continuation),
133 Builder(OrigF.getContext()), ActiveSuspend(ActiveSuspend), TTI(TTI) {
134 assert(Shape.ABI == coro::ABI::Retcon ||
135 Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async);
136 assert(NewF && "need existing function for continuation");
137 assert(ActiveSuspend && "need active suspend point for continuation");
138 }
139
140 Function *getFunction() const {
141 assert(NewF != nullptr && "declaration not yet set");
142 return NewF;
143 }
144
145 void create();
146
147private:
148 bool isSwitchDestroyFunction() {
149 switch (FKind) {
150 case Kind::Async:
151 case Kind::Continuation:
152 case Kind::SwitchResume:
153 return false;
154 case Kind::SwitchUnwind:
155 case Kind::SwitchCleanup:
156 return true;
157 }
158 llvm_unreachable("Unknown CoroCloner::Kind enum");
159 }
160
161 void replaceEntryBlock();
162 Value *deriveNewFramePointer();
163 void replaceRetconOrAsyncSuspendUses();
164 void replaceCoroSuspends();
165 void replaceCoroEnds();
167 void salvageDebugInfo();
168 void handleFinalSuspend();
169};
170
171} // end anonymous namespace
172
173// FIXME:
174// Lower the intrinisc in CoroEarly phase if coroutine frame doesn't escape
175// and it is known that other transformations, for example, sanitizers
176// won't lead to incorrect code.
178 coro::Shape &Shape) {
179 auto Wrapper = CB->getWrapperFunction();
180 auto Awaiter = CB->getAwaiter();
181 auto FramePtr = CB->getFrame();
182
183 Builder.SetInsertPoint(CB);
184
185 CallBase *NewCall = nullptr;
186 // await_suspend has only 2 parameters, awaiter and handle.
187 // Copy parameter attributes from the intrinsic call, but remove the last,
188 // because the last parameter now becomes the function that is being called.
189 AttributeList NewAttributes =
191
192 if (auto Invoke = dyn_cast<InvokeInst>(CB)) {
193 auto WrapperInvoke =
194 Builder.CreateInvoke(Wrapper, Invoke->getNormalDest(),
195 Invoke->getUnwindDest(), {Awaiter, FramePtr});
196
197 WrapperInvoke->setCallingConv(Invoke->getCallingConv());
198 std::copy(Invoke->bundle_op_info_begin(), Invoke->bundle_op_info_end(),
199 WrapperInvoke->bundle_op_info_begin());
200 WrapperInvoke->setAttributes(NewAttributes);
201 WrapperInvoke->setDebugLoc(Invoke->getDebugLoc());
202 NewCall = WrapperInvoke;
203 } else if (auto Call = dyn_cast<CallInst>(CB)) {
204 auto WrapperCall = Builder.CreateCall(Wrapper, {Awaiter, FramePtr});
205
206 WrapperCall->setAttributes(NewAttributes);
207 WrapperCall->setDebugLoc(Call->getDebugLoc());
208 NewCall = WrapperCall;
209 } else {
210 llvm_unreachable("Unexpected coro_await_suspend invocation method");
211 }
212
213 if (CB->getCalledFunction()->getIntrinsicID() ==
214 Intrinsic::coro_await_suspend_handle) {
215 // Follow the lowered await_suspend call above with a lowered resume call
216 // to the returned coroutine.
217 if (auto *Invoke = dyn_cast<InvokeInst>(CB)) {
218 // If the await_suspend call is an invoke, we continue in the next block.
219 Builder.SetInsertPoint(Invoke->getNormalDest()->getFirstInsertionPt());
220 }
221
222 coro::LowererBase LB(*Wrapper->getParent());
223 auto *ResumeAddr = LB.makeSubFnCall(NewCall, CoroSubFnInst::ResumeIndex,
224 &*Builder.GetInsertPoint());
225
226 LLVMContext &Ctx = Builder.getContext();
227 FunctionType *ResumeTy = FunctionType::get(
228 Type::getVoidTy(Ctx), PointerType::getUnqual(Ctx), false);
229 auto *ResumeCall = Builder.CreateCall(ResumeTy, ResumeAddr, {NewCall});
231
232 // We can't insert the 'ret' instruction and adjust the cc until the
233 // function has been split, so remember this for later.
234 Shape.SymmetricTransfers.push_back(ResumeCall);
235
236 NewCall = ResumeCall;
237 }
238
239 CB->replaceAllUsesWith(NewCall);
240 CB->eraseFromParent();
241}
242
244 IRBuilder<> Builder(F.getContext());
245 for (auto *AWS : Shape.CoroAwaitSuspends)
246 lowerAwaitSuspend(Builder, AWS, Shape);
247}
248
250 const coro::Shape &Shape, Value *FramePtr,
251 CallGraph *CG) {
252 assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce);
254 return;
255
256 Shape.emitDealloc(Builder, FramePtr, CG);
257}
258
259/// Replace an llvm.coro.end.async.
260/// Will inline the must tail call function call if there is one.
261/// \returns true if cleanup of the coro.end block is needed, false otherwise.
263 IRBuilder<> Builder(End);
264
265 auto *EndAsync = dyn_cast<CoroAsyncEndInst>(End);
266 if (!EndAsync) {
267 Builder.CreateRetVoid();
268 return true /*needs cleanup of coro.end block*/;
269 }
270
271 auto *MustTailCallFunc = EndAsync->getMustTailCallFunction();
272 if (!MustTailCallFunc) {
273 Builder.CreateRetVoid();
274 return true /*needs cleanup of coro.end block*/;
275 }
276
277 // Move the must tail call from the predecessor block into the end block.
278 auto *CoroEndBlock = End->getParent();
279 auto *MustTailCallFuncBlock = CoroEndBlock->getSinglePredecessor();
280 assert(MustTailCallFuncBlock && "Must have a single predecessor block");
281 auto It = MustTailCallFuncBlock->getTerminator()->getIterator();
282 auto *MustTailCall = cast<CallInst>(&*std::prev(It));
283 CoroEndBlock->splice(End->getIterator(), MustTailCallFuncBlock,
284 MustTailCall->getIterator());
285
286 // Insert the return instruction.
287 Builder.SetInsertPoint(End);
288 Builder.CreateRetVoid();
289 InlineFunctionInfo FnInfo;
290
291 // Remove the rest of the block, by splitting it into an unreachable block.
292 auto *BB = End->getParent();
293 BB->splitBasicBlock(End);
294 BB->getTerminator()->eraseFromParent();
295
296 auto InlineRes = InlineFunction(*MustTailCall, FnInfo);
297 assert(InlineRes.isSuccess() && "Expected inlining to succeed");
298 (void)InlineRes;
299
300 // We have cleaned up the coro.end block above.
301 return false;
302}
303
304/// Replace a non-unwind call to llvm.coro.end.
306 const coro::Shape &Shape, Value *FramePtr,
307 bool InResume, CallGraph *CG) {
308 // Start inserting right before the coro.end.
309 IRBuilder<> Builder(End);
310
311 // Create the return instruction.
312 switch (Shape.ABI) {
313 // The cloned functions in switch-lowering always return void.
314 case coro::ABI::Switch:
315 assert(!cast<CoroEndInst>(End)->hasResults() &&
316 "switch coroutine should not return any values");
317 // coro.end doesn't immediately end the coroutine in the main function
318 // in this lowering, because we need to deallocate the coroutine.
319 if (!InResume)
320 return;
321 Builder.CreateRetVoid();
322 break;
323
324 // In async lowering this returns.
325 case coro::ABI::Async: {
326 bool CoroEndBlockNeedsCleanup = replaceCoroEndAsync(End);
327 if (!CoroEndBlockNeedsCleanup)
328 return;
329 break;
330 }
331
332 // In unique continuation lowering, the continuations always return void.
333 // But we may have implicitly allocated storage.
334 case coro::ABI::RetconOnce: {
335 maybeFreeRetconStorage(Builder, Shape, FramePtr, CG);
336 auto *CoroEnd = cast<CoroEndInst>(End);
337 auto *RetTy = Shape.getResumeFunctionType()->getReturnType();
338
339 if (!CoroEnd->hasResults()) {
340 assert(RetTy->isVoidTy());
341 Builder.CreateRetVoid();
342 break;
343 }
344
345 auto *CoroResults = CoroEnd->getResults();
346 unsigned NumReturns = CoroResults->numReturns();
347
348 if (auto *RetStructTy = dyn_cast<StructType>(RetTy)) {
349 assert(RetStructTy->getNumElements() == NumReturns &&
350 "numbers of returns should match resume function singature");
351 Value *ReturnValue = UndefValue::get(RetStructTy);
352 unsigned Idx = 0;
353 for (Value *RetValEl : CoroResults->return_values())
354 ReturnValue = Builder.CreateInsertValue(ReturnValue, RetValEl, Idx++);
355 Builder.CreateRet(ReturnValue);
356 } else if (NumReturns == 0) {
357 assert(RetTy->isVoidTy());
358 Builder.CreateRetVoid();
359 } else {
360 assert(NumReturns == 1);
361 Builder.CreateRet(*CoroResults->retval_begin());
362 }
363 CoroResults->replaceAllUsesWith(
364 ConstantTokenNone::get(CoroResults->getContext()));
365 CoroResults->eraseFromParent();
366 break;
367 }
368
369 // In non-unique continuation lowering, we signal completion by returning
370 // a null continuation.
371 case coro::ABI::Retcon: {
372 assert(!cast<CoroEndInst>(End)->hasResults() &&
373 "retcon coroutine should not return any values");
374 maybeFreeRetconStorage(Builder, Shape, FramePtr, CG);
375 auto RetTy = Shape.getResumeFunctionType()->getReturnType();
376 auto RetStructTy = dyn_cast<StructType>(RetTy);
377 PointerType *ContinuationTy =
378 cast<PointerType>(RetStructTy ? RetStructTy->getElementType(0) : RetTy);
379
380 Value *ReturnValue = ConstantPointerNull::get(ContinuationTy);
381 if (RetStructTy) {
382 ReturnValue = Builder.CreateInsertValue(UndefValue::get(RetStructTy),
383 ReturnValue, 0);
384 }
385 Builder.CreateRet(ReturnValue);
386 break;
387 }
388 }
389
390 // Remove the rest of the block, by splitting it into an unreachable block.
391 auto *BB = End->getParent();
392 BB->splitBasicBlock(End);
393 BB->getTerminator()->eraseFromParent();
394}
395
396// Mark a coroutine as done, which implies that the coroutine is finished and
397// never get resumed.
398//
399// In resume-switched ABI, the done state is represented by storing zero in
400// ResumeFnAddr.
401//
402// NOTE: We couldn't omit the argument `FramePtr`. It is necessary because the
403// pointer to the frame in splitted function is not stored in `Shape`.
404static void markCoroutineAsDone(IRBuilder<> &Builder, const coro::Shape &Shape,
405 Value *FramePtr) {
406 assert(
407 Shape.ABI == coro::ABI::Switch &&
408 "markCoroutineAsDone is only supported for Switch-Resumed ABI for now.");
409 auto *GepIndex = Builder.CreateStructGEP(
411 "ResumeFn.addr");
412 auto *NullPtr = ConstantPointerNull::get(cast<PointerType>(
414 Builder.CreateStore(NullPtr, GepIndex);
415
416 // If the coroutine don't have unwind coro end, we could omit the store to
417 // the final suspend point since we could infer the coroutine is suspended
418 // at the final suspend point by the nullness of ResumeFnAddr.
419 // However, we can't skip it if the coroutine have unwind coro end. Since
420 // the coroutine reaches unwind coro end is considered suspended at the
421 // final suspend point (the ResumeFnAddr is null) but in fact the coroutine
422 // didn't complete yet. We need the IndexVal for the final suspend point
423 // to make the states clear.
426 assert(cast<CoroSuspendInst>(Shape.CoroSuspends.back())->isFinal() &&
427 "The final suspend should only live in the last position of "
428 "CoroSuspends.");
429 ConstantInt *IndexVal = Shape.getIndex(Shape.CoroSuspends.size() - 1);
430 auto *FinalIndex = Builder.CreateStructGEP(
431 Shape.FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
432
433 Builder.CreateStore(IndexVal, FinalIndex);
434 }
435}
436
437/// Replace an unwind call to llvm.coro.end.
439 Value *FramePtr, bool InResume,
440 CallGraph *CG) {
441 IRBuilder<> Builder(End);
442
443 switch (Shape.ABI) {
444 // In switch-lowering, this does nothing in the main function.
445 case coro::ABI::Switch: {
446 // In C++'s specification, the coroutine should be marked as done
447 // if promise.unhandled_exception() throws. The frontend will
448 // call coro.end(true) along this path.
449 //
450 // FIXME: We should refactor this once there is other language
451 // which uses Switch-Resumed style other than C++.
452 markCoroutineAsDone(Builder, Shape, FramePtr);
453 if (!InResume)
454 return;
455 break;
456 }
457 // In async lowering this does nothing.
458 case coro::ABI::Async:
459 break;
460 // In continuation-lowering, this frees the continuation storage.
461 case coro::ABI::Retcon:
462 case coro::ABI::RetconOnce:
463 maybeFreeRetconStorage(Builder, Shape, FramePtr, CG);
464 break;
465 }
466
467 // If coro.end has an associated bundle, add cleanupret instruction.
468 if (auto Bundle = End->getOperandBundle(LLVMContext::OB_funclet)) {
469 auto *FromPad = cast<CleanupPadInst>(Bundle->Inputs[0]);
470 auto *CleanupRet = Builder.CreateCleanupRet(FromPad, nullptr);
471 End->getParent()->splitBasicBlock(End);
472 CleanupRet->getParent()->getTerminator()->eraseFromParent();
473 }
474}
475
476static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape,
477 Value *FramePtr, bool InResume, CallGraph *CG) {
478 if (End->isUnwind())
479 replaceUnwindCoroEnd(End, Shape, FramePtr, InResume, CG);
480 else
481 replaceFallthroughCoroEnd(End, Shape, FramePtr, InResume, CG);
482
483 auto &Context = End->getContext();
484 End->replaceAllUsesWith(InResume ? ConstantInt::getTrue(Context)
485 : ConstantInt::getFalse(Context));
486 End->eraseFromParent();
487}
488
489// In the resume function, we remove the last case (when coro::Shape is built,
490// the final suspend point (if present) is always the last element of
491// CoroSuspends array) since it is an undefined behavior to resume a coroutine
492// suspended at the final suspend point.
493// In the destroy function, if it isn't possible that the ResumeFnAddr is NULL
494// and the coroutine doesn't suspend at the final suspend point actually (this
495// is possible since the coroutine is considered suspended at the final suspend
496// point if promise.unhandled_exception() exits via an exception), we can
497// remove the last case.
498void CoroCloner::handleFinalSuspend() {
499 assert(Shape.ABI == coro::ABI::Switch &&
500 Shape.SwitchLowering.HasFinalSuspend);
501
502 if (isSwitchDestroyFunction() && Shape.SwitchLowering.HasUnwindCoroEnd)
503 return;
504
505 auto *Switch = cast<SwitchInst>(VMap[Shape.SwitchLowering.ResumeSwitch]);
506 auto FinalCaseIt = std::prev(Switch->case_end());
507 BasicBlock *ResumeBB = FinalCaseIt->getCaseSuccessor();
508 Switch->removeCase(FinalCaseIt);
509 if (isSwitchDestroyFunction()) {
510 BasicBlock *OldSwitchBB = Switch->getParent();
511 auto *NewSwitchBB = OldSwitchBB->splitBasicBlock(Switch, "Switch");
512 Builder.SetInsertPoint(OldSwitchBB->getTerminator());
513
514 if (NewF->isCoroOnlyDestroyWhenComplete()) {
515 // When the coroutine can only be destroyed when complete, we don't need
516 // to generate code for other cases.
517 Builder.CreateBr(ResumeBB);
518 } else {
519 auto *GepIndex = Builder.CreateStructGEP(
520 Shape.FrameTy, NewFramePtr, coro::Shape::SwitchFieldIndex::Resume,
521 "ResumeFn.addr");
522 auto *Load =
523 Builder.CreateLoad(Shape.getSwitchResumePointerType(), GepIndex);
524 auto *Cond = Builder.CreateIsNull(Load);
525 Builder.CreateCondBr(Cond, ResumeBB, NewSwitchBB);
526 }
527 OldSwitchBB->getTerminator()->eraseFromParent();
528 }
529}
530
531static FunctionType *
533 auto *AsyncSuspend = cast<CoroSuspendAsyncInst>(Suspend);
534 auto *StructTy = cast<StructType>(AsyncSuspend->getType());
535 auto &Context = Suspend->getParent()->getParent()->getContext();
536 auto *VoidTy = Type::getVoidTy(Context);
537 return FunctionType::get(VoidTy, StructTy->elements(), false);
538}
539
541 const Twine &Suffix,
542 Module::iterator InsertBefore,
543 AnyCoroSuspendInst *ActiveSuspend) {
544 Module *M = OrigF.getParent();
545 auto *FnTy = (Shape.ABI != coro::ABI::Async)
546 ? Shape.getResumeFunctionType()
547 : getFunctionTypeFromAsyncSuspend(ActiveSuspend);
548
549 Function *NewF =
550 Function::Create(FnTy, GlobalValue::LinkageTypes::InternalLinkage,
551 OrigF.getName() + Suffix);
552
553 M->getFunctionList().insert(InsertBefore, NewF);
554
555 return NewF;
556}
557
558/// Replace uses of the active llvm.coro.suspend.retcon/async call with the
559/// arguments to the continuation function.
560///
561/// This assumes that the builder has a meaningful insertion point.
562void CoroCloner::replaceRetconOrAsyncSuspendUses() {
563 assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce ||
564 Shape.ABI == coro::ABI::Async);
565
566 auto NewS = VMap[ActiveSuspend];
567 if (NewS->use_empty())
568 return;
569
570 // Copy out all the continuation arguments after the buffer pointer into
571 // an easily-indexed data structure for convenience.
573 // The async ABI includes all arguments -- including the first argument.
574 bool IsAsyncABI = Shape.ABI == coro::ABI::Async;
575 for (auto I = IsAsyncABI ? NewF->arg_begin() : std::next(NewF->arg_begin()),
576 E = NewF->arg_end();
577 I != E; ++I)
578 Args.push_back(&*I);
579
580 // If the suspend returns a single scalar value, we can just do a simple
581 // replacement.
582 if (!isa<StructType>(NewS->getType())) {
583 assert(Args.size() == 1);
584 NewS->replaceAllUsesWith(Args.front());
585 return;
586 }
587
588 // Try to peephole extracts of an aggregate return.
589 for (Use &U : llvm::make_early_inc_range(NewS->uses())) {
590 auto *EVI = dyn_cast<ExtractValueInst>(U.getUser());
591 if (!EVI || EVI->getNumIndices() != 1)
592 continue;
593
594 EVI->replaceAllUsesWith(Args[EVI->getIndices().front()]);
595 EVI->eraseFromParent();
596 }
597
598 // If we have no remaining uses, we're done.
599 if (NewS->use_empty())
600 return;
601
602 // Otherwise, we need to create an aggregate.
603 Value *Agg = PoisonValue::get(NewS->getType());
604 for (size_t I = 0, E = Args.size(); I != E; ++I)
605 Agg = Builder.CreateInsertValue(Agg, Args[I], I);
606
607 NewS->replaceAllUsesWith(Agg);
608}
609
610void CoroCloner::replaceCoroSuspends() {
611 Value *SuspendResult;
612
613 switch (Shape.ABI) {
614 // In switch lowering, replace coro.suspend with the appropriate value
615 // for the type of function we're extracting.
616 // Replacing coro.suspend with (0) will result in control flow proceeding to
617 // a resume label associated with a suspend point, replacing it with (1) will
618 // result in control flow proceeding to a cleanup label associated with this
619 // suspend point.
620 case coro::ABI::Switch:
621 SuspendResult = Builder.getInt8(isSwitchDestroyFunction() ? 1 : 0);
622 break;
623
624 // In async lowering there are no uses of the result.
625 case coro::ABI::Async:
626 return;
627
628 // In returned-continuation lowering, the arguments from earlier
629 // continuations are theoretically arbitrary, and they should have been
630 // spilled.
631 case coro::ABI::RetconOnce:
632 case coro::ABI::Retcon:
633 return;
634 }
635
636 for (AnyCoroSuspendInst *CS : Shape.CoroSuspends) {
637 // The active suspend was handled earlier.
638 if (CS == ActiveSuspend)
639 continue;
640
641 auto *MappedCS = cast<AnyCoroSuspendInst>(VMap[CS]);
642 MappedCS->replaceAllUsesWith(SuspendResult);
643 MappedCS->eraseFromParent();
644 }
645}
646
647void CoroCloner::replaceCoroEnds() {
648 for (AnyCoroEndInst *CE : Shape.CoroEnds) {
649 // We use a null call graph because there's no call graph node for
650 // the cloned function yet. We'll just be rebuilding that later.
651 auto *NewCE = cast<AnyCoroEndInst>(VMap[CE]);
652 replaceCoroEnd(NewCE, Shape, NewFramePtr, /*in resume*/ true, nullptr);
653 }
654}
655
657 ValueToValueMapTy *VMap) {
658 if (Shape.ABI == coro::ABI::Async && Shape.CoroSuspends.empty())
659 return;
660 Value *CachedSlot = nullptr;
661 auto getSwiftErrorSlot = [&](Type *ValueTy) -> Value * {
662 if (CachedSlot)
663 return CachedSlot;
664
665 // Check if the function has a swifterror argument.
666 for (auto &Arg : F.args()) {
667 if (Arg.isSwiftError()) {
668 CachedSlot = &Arg;
669 return &Arg;
670 }
671 }
672
673 // Create a swifterror alloca.
674 IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHIOrDbg());
675 auto Alloca = Builder.CreateAlloca(ValueTy);
676 Alloca->setSwiftError(true);
677
678 CachedSlot = Alloca;
679 return Alloca;
680 };
681
682 for (CallInst *Op : Shape.SwiftErrorOps) {
683 auto MappedOp = VMap ? cast<CallInst>((*VMap)[Op]) : Op;
684 IRBuilder<> Builder(MappedOp);
685
686 // If there are no arguments, this is a 'get' operation.
687 Value *MappedResult;
688 if (Op->arg_empty()) {
689 auto ValueTy = Op->getType();
690 auto Slot = getSwiftErrorSlot(ValueTy);
691 MappedResult = Builder.CreateLoad(ValueTy, Slot);
692 } else {
693 assert(Op->arg_size() == 1);
694 auto Value = MappedOp->getArgOperand(0);
695 auto ValueTy = Value->getType();
696 auto Slot = getSwiftErrorSlot(ValueTy);
697 Builder.CreateStore(Value, Slot);
698 MappedResult = Slot;
699 }
700
701 MappedOp->replaceAllUsesWith(MappedResult);
702 MappedOp->eraseFromParent();
703 }
704
705 // If we're updating the original function, we've invalidated SwiftErrorOps.
706 if (VMap == nullptr) {
707 Shape.SwiftErrorOps.clear();
708 }
709}
710
711/// Returns all DbgVariableIntrinsic in F.
712static std::pair<SmallVector<DbgVariableIntrinsic *, 8>,
716 SmallVector<DbgVariableRecord *> DbgVariableRecords;
717 for (auto &I : instructions(F)) {
718 for (DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange()))
719 DbgVariableRecords.push_back(&DVR);
720 if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I))
721 Intrinsics.push_back(DVI);
722 }
723 return {Intrinsics, DbgVariableRecords};
724}
725
726void CoroCloner::replaceSwiftErrorOps() {
727 ::replaceSwiftErrorOps(*NewF, Shape, &VMap);
728}
729
730void CoroCloner::salvageDebugInfo() {
731 auto [Worklist, DbgVariableRecords] = collectDbgVariableIntrinsics(*NewF);
733
734 // Only 64-bit ABIs have a register we can refer to with the entry value.
735 bool UseEntryValue =
736 llvm::Triple(OrigF.getParent()->getTargetTriple()).isArch64Bit();
737 for (DbgVariableIntrinsic *DVI : Worklist)
738 coro::salvageDebugInfo(ArgToAllocaMap, *DVI, UseEntryValue);
739 for (DbgVariableRecord *DVR : DbgVariableRecords)
740 coro::salvageDebugInfo(ArgToAllocaMap, *DVR, UseEntryValue);
741
742 // Remove all salvaged dbg.declare intrinsics that became
743 // either unreachable or stale due to the CoroSplit transformation.
744 DominatorTree DomTree(*NewF);
745 auto IsUnreachableBlock = [&](BasicBlock *BB) {
746 return !isPotentiallyReachable(&NewF->getEntryBlock(), BB, nullptr,
747 &DomTree);
748 };
749 auto RemoveOne = [&](auto *DVI) {
750 if (IsUnreachableBlock(DVI->getParent()))
751 DVI->eraseFromParent();
752 else if (isa_and_nonnull<AllocaInst>(DVI->getVariableLocationOp(0))) {
753 // Count all non-debuginfo uses in reachable blocks.
754 unsigned Uses = 0;
755 for (auto *User : DVI->getVariableLocationOp(0)->users())
756 if (auto *I = dyn_cast<Instruction>(User))
757 if (!isa<AllocaInst>(I) && !IsUnreachableBlock(I->getParent()))
758 ++Uses;
759 if (!Uses)
760 DVI->eraseFromParent();
761 }
762 };
763 for_each(Worklist, RemoveOne);
764 for_each(DbgVariableRecords, RemoveOne);
765}
766
767void CoroCloner::replaceEntryBlock() {
768 // In the original function, the AllocaSpillBlock is a block immediately
769 // following the allocation of the frame object which defines GEPs for
770 // all the allocas that have been moved into the frame, and it ends by
771 // branching to the original beginning of the coroutine. Make this
772 // the entry block of the cloned function.
773 auto *Entry = cast<BasicBlock>(VMap[Shape.AllocaSpillBlock]);
774 auto *OldEntry = &NewF->getEntryBlock();
775 Entry->setName("entry" + Suffix);
776 Entry->moveBefore(OldEntry);
777 Entry->getTerminator()->eraseFromParent();
778
779 // Clear all predecessors of the new entry block. There should be
780 // exactly one predecessor, which we created when splitting out
781 // AllocaSpillBlock to begin with.
782 assert(Entry->hasOneUse());
783 auto BranchToEntry = cast<BranchInst>(Entry->user_back());
784 assert(BranchToEntry->isUnconditional());
785 Builder.SetInsertPoint(BranchToEntry);
786 Builder.CreateUnreachable();
787 BranchToEntry->eraseFromParent();
788
789 // Branch from the entry to the appropriate place.
790 Builder.SetInsertPoint(Entry);
791 switch (Shape.ABI) {
792 case coro::ABI::Switch: {
793 // In switch-lowering, we built a resume-entry block in the original
794 // function. Make the entry block branch to this.
795 auto *SwitchBB =
796 cast<BasicBlock>(VMap[Shape.SwitchLowering.ResumeEntryBlock]);
797 Builder.CreateBr(SwitchBB);
798 break;
799 }
800 case coro::ABI::Async:
801 case coro::ABI::Retcon:
802 case coro::ABI::RetconOnce: {
803 // In continuation ABIs, we want to branch to immediately after the
804 // active suspend point. Earlier phases will have put the suspend in its
805 // own basic block, so just thread our jump directly to its successor.
806 assert((Shape.ABI == coro::ABI::Async &&
807 isa<CoroSuspendAsyncInst>(ActiveSuspend)) ||
808 ((Shape.ABI == coro::ABI::Retcon ||
809 Shape.ABI == coro::ABI::RetconOnce) &&
810 isa<CoroSuspendRetconInst>(ActiveSuspend)));
811 auto *MappedCS = cast<AnyCoroSuspendInst>(VMap[ActiveSuspend]);
812 auto Branch = cast<BranchInst>(MappedCS->getNextNode());
813 assert(Branch->isUnconditional());
814 Builder.CreateBr(Branch->getSuccessor(0));
815 break;
816 }
817 }
818
819 // Any static alloca that's still being used but not reachable from the new
820 // entry needs to be moved to the new entry.
821 Function *F = OldEntry->getParent();
822 DominatorTree DT{*F};
824 auto *Alloca = dyn_cast<AllocaInst>(&I);
825 if (!Alloca || I.use_empty())
826 continue;
827 if (DT.isReachableFromEntry(I.getParent()) ||
828 !isa<ConstantInt>(Alloca->getArraySize()))
829 continue;
830 I.moveBefore(*Entry, Entry->getFirstInsertionPt());
831 }
832}
833
834/// Derive the value of the new frame pointer.
835Value *CoroCloner::deriveNewFramePointer() {
836 // Builder should be inserting to the front of the new entry block.
837
838 switch (Shape.ABI) {
839 // In switch-lowering, the argument is the frame pointer.
840 case coro::ABI::Switch:
841 return &*NewF->arg_begin();
842 // In async-lowering, one of the arguments is an async context as determined
843 // by the `llvm.coro.id.async` intrinsic. We can retrieve the async context of
844 // the resume function from the async context projection function associated
845 // with the active suspend. The frame is located as a tail to the async
846 // context header.
847 case coro::ABI::Async: {
848 auto *ActiveAsyncSuspend = cast<CoroSuspendAsyncInst>(ActiveSuspend);
849 auto ContextIdx = ActiveAsyncSuspend->getStorageArgumentIndex() & 0xff;
850 auto *CalleeContext = NewF->getArg(ContextIdx);
851 auto *ProjectionFunc =
852 ActiveAsyncSuspend->getAsyncContextProjectionFunction();
853 auto DbgLoc =
854 cast<CoroSuspendAsyncInst>(VMap[ActiveSuspend])->getDebugLoc();
855 // Calling i8* (i8*)
856 auto *CallerContext = Builder.CreateCall(ProjectionFunc->getFunctionType(),
857 ProjectionFunc, CalleeContext);
858 CallerContext->setCallingConv(ProjectionFunc->getCallingConv());
859 CallerContext->setDebugLoc(DbgLoc);
860 // The frame is located after the async_context header.
861 auto &Context = Builder.getContext();
862 auto *FramePtrAddr = Builder.CreateConstInBoundsGEP1_32(
863 Type::getInt8Ty(Context), CallerContext,
864 Shape.AsyncLowering.FrameOffset, "async.ctx.frameptr");
865 // Inline the projection function.
867 auto InlineRes = InlineFunction(*CallerContext, InlineInfo);
868 assert(InlineRes.isSuccess());
869 (void)InlineRes;
870 return FramePtrAddr;
871 }
872 // In continuation-lowering, the argument is the opaque storage.
873 case coro::ABI::Retcon:
874 case coro::ABI::RetconOnce: {
875 Argument *NewStorage = &*NewF->arg_begin();
876 auto FramePtrTy = PointerType::getUnqual(Shape.FrameTy->getContext());
877
878 // If the storage is inline, just bitcast to the storage to the frame type.
879 if (Shape.RetconLowering.IsFrameInlineInStorage)
880 return NewStorage;
881
882 // Otherwise, load the real frame from the opaque storage.
883 return Builder.CreateLoad(FramePtrTy, NewStorage);
884 }
885 }
886 llvm_unreachable("bad ABI");
887}
888
889static void addFramePointerAttrs(AttributeList &Attrs, LLVMContext &Context,
890 unsigned ParamIndex, uint64_t Size,
891 Align Alignment, bool NoAlias) {
892 AttrBuilder ParamAttrs(Context);
893 ParamAttrs.addAttribute(Attribute::NonNull);
894 ParamAttrs.addAttribute(Attribute::NoUndef);
895
896 if (NoAlias)
897 ParamAttrs.addAttribute(Attribute::NoAlias);
898
899 ParamAttrs.addAlignmentAttr(Alignment);
900 ParamAttrs.addDereferenceableAttr(Size);
901 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
902}
903
904static void addAsyncContextAttrs(AttributeList &Attrs, LLVMContext &Context,
905 unsigned ParamIndex) {
906 AttrBuilder ParamAttrs(Context);
907 ParamAttrs.addAttribute(Attribute::SwiftAsync);
908 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
909}
910
911static void addSwiftSelfAttrs(AttributeList &Attrs, LLVMContext &Context,
912 unsigned ParamIndex) {
913 AttrBuilder ParamAttrs(Context);
914 ParamAttrs.addAttribute(Attribute::SwiftSelf);
915 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
916}
917
918/// Clone the body of the original function into a resume function of
919/// some sort.
920void CoroCloner::create() {
921 // Create the new function if we don't already have one.
922 if (!NewF) {
923 NewF = createCloneDeclaration(OrigF, Shape, Suffix,
924 OrigF.getParent()->end(), ActiveSuspend);
925 }
926
927 // Replace all args with dummy instructions. If an argument is the old frame
928 // pointer, the dummy will be replaced by the new frame pointer once it is
929 // computed below. Uses of all other arguments should have already been
930 // rewritten by buildCoroutineFrame() to use loads/stores on the coroutine
931 // frame.
933 for (Argument &A : OrigF.args()) {
934 DummyArgs.push_back(new FreezeInst(PoisonValue::get(A.getType())));
935 VMap[&A] = DummyArgs.back();
936 }
937
939
940 // Ignore attempts to change certain attributes of the function.
941 // TODO: maybe there should be a way to suppress this during cloning?
942 auto savedVisibility = NewF->getVisibility();
943 auto savedUnnamedAddr = NewF->getUnnamedAddr();
944 auto savedDLLStorageClass = NewF->getDLLStorageClass();
945
946 // NewF's linkage (which CloneFunctionInto does *not* change) might not
947 // be compatible with the visibility of OrigF (which it *does* change),
948 // so protect against that.
949 auto savedLinkage = NewF->getLinkage();
950 NewF->setLinkage(llvm::GlobalValue::ExternalLinkage);
951
952 CloneFunctionInto(NewF, &OrigF, VMap,
953 CloneFunctionChangeType::LocalChangesOnly, Returns);
954
955 auto &Context = NewF->getContext();
956
957 // For async functions / continuations, adjust the scope line of the
958 // clone to the line number of the suspend point. However, only
959 // adjust the scope line when the files are the same. This ensures
960 // line number and file name belong together. The scope line is
961 // associated with all pre-prologue instructions. This avoids a jump
962 // in the linetable from the function declaration to the suspend point.
963 if (DISubprogram *SP = NewF->getSubprogram()) {
964 assert(SP != OrigF.getSubprogram() && SP->isDistinct());
965 if (ActiveSuspend)
966 if (auto DL = ActiveSuspend->getDebugLoc())
967 if (SP->getFile() == DL->getFile())
968 SP->setScopeLine(DL->getLine());
969 // Update the linkage name to reflect the modified symbol name. It
970 // is necessary to update the linkage name in Swift, since the
971 // mangling changes for resume functions. It might also be the
972 // right thing to do in C++, but due to a limitation in LLVM's
973 // AsmPrinter we can only do this if the function doesn't have an
974 // abstract specification, since the DWARF backend expects the
975 // abstract specification to contain the linkage name and asserts
976 // that they are identical.
977 if (SP->getUnit() &&
978 SP->getUnit()->getSourceLanguage() == dwarf::DW_LANG_Swift) {
979 SP->replaceLinkageName(MDString::get(Context, NewF->getName()));
980 if (auto *Decl = SP->getDeclaration()) {
981 auto *NewDecl = DISubprogram::get(
982 Decl->getContext(), Decl->getScope(), Decl->getName(),
983 NewF->getName(), Decl->getFile(), Decl->getLine(), Decl->getType(),
984 Decl->getScopeLine(), Decl->getContainingType(),
985 Decl->getVirtualIndex(), Decl->getThisAdjustment(),
986 Decl->getFlags(), Decl->getSPFlags(), Decl->getUnit(),
987 Decl->getTemplateParams(), nullptr, Decl->getRetainedNodes(),
988 Decl->getThrownTypes(), Decl->getAnnotations(),
989 Decl->getTargetFuncName());
990 SP->replaceDeclaration(NewDecl);
991 }
992 }
993 }
994
995 NewF->setLinkage(savedLinkage);
996 NewF->setVisibility(savedVisibility);
997 NewF->setUnnamedAddr(savedUnnamedAddr);
998 NewF->setDLLStorageClass(savedDLLStorageClass);
999 // The function sanitizer metadata needs to match the signature of the
1000 // function it is being attached to. However this does not hold for split
1001 // functions here. Thus remove the metadata for split functions.
1002 if (Shape.ABI == coro::ABI::Switch &&
1003 NewF->hasMetadata(LLVMContext::MD_func_sanitize))
1004 NewF->eraseMetadata(LLVMContext::MD_func_sanitize);
1005
1006 // Replace the attributes of the new function:
1007 auto OrigAttrs = NewF->getAttributes();
1008 auto NewAttrs = AttributeList();
1009
1010 switch (Shape.ABI) {
1011 case coro::ABI::Switch:
1012 // Bootstrap attributes by copying function attributes from the
1013 // original function. This should include optimization settings and so on.
1014 NewAttrs = NewAttrs.addFnAttributes(
1015 Context, AttrBuilder(Context, OrigAttrs.getFnAttrs()));
1016
1017 addFramePointerAttrs(NewAttrs, Context, 0, Shape.FrameSize,
1018 Shape.FrameAlign, /*NoAlias=*/false);
1019 break;
1020 case coro::ABI::Async: {
1021 auto *ActiveAsyncSuspend = cast<CoroSuspendAsyncInst>(ActiveSuspend);
1022 if (OrigF.hasParamAttribute(Shape.AsyncLowering.ContextArgNo,
1023 Attribute::SwiftAsync)) {
1024 uint32_t ArgAttributeIndices =
1025 ActiveAsyncSuspend->getStorageArgumentIndex();
1026 auto ContextArgIndex = ArgAttributeIndices & 0xff;
1027 addAsyncContextAttrs(NewAttrs, Context, ContextArgIndex);
1028
1029 // `swiftasync` must preceed `swiftself` so 0 is not a valid index for
1030 // `swiftself`.
1031 auto SwiftSelfIndex = ArgAttributeIndices >> 8;
1032 if (SwiftSelfIndex)
1033 addSwiftSelfAttrs(NewAttrs, Context, SwiftSelfIndex);
1034 }
1035
1036 // Transfer the original function's attributes.
1037 auto FnAttrs = OrigF.getAttributes().getFnAttrs();
1038 NewAttrs = NewAttrs.addFnAttributes(Context, AttrBuilder(Context, FnAttrs));
1039 break;
1040 }
1041 case coro::ABI::Retcon:
1042 case coro::ABI::RetconOnce:
1043 // If we have a continuation prototype, just use its attributes,
1044 // full-stop.
1045 NewAttrs = Shape.RetconLowering.ResumePrototype->getAttributes();
1046
1047 /// FIXME: Is it really good to add the NoAlias attribute?
1048 addFramePointerAttrs(NewAttrs, Context, 0,
1049 Shape.getRetconCoroId()->getStorageSize(),
1050 Shape.getRetconCoroId()->getStorageAlignment(),
1051 /*NoAlias=*/true);
1052
1053 break;
1054 }
1055
1056 switch (Shape.ABI) {
1057 // In these ABIs, the cloned functions always return 'void', and the
1058 // existing return sites are meaningless. Note that for unique
1059 // continuations, this includes the returns associated with suspends;
1060 // this is fine because we can't suspend twice.
1061 case coro::ABI::Switch:
1062 case coro::ABI::RetconOnce:
1063 // Remove old returns.
1064 for (ReturnInst *Return : Returns)
1065 changeToUnreachable(Return);
1066 break;
1067
1068 // With multi-suspend continuations, we'll already have eliminated the
1069 // original returns and inserted returns before all the suspend points,
1070 // so we want to leave any returns in place.
1071 case coro::ABI::Retcon:
1072 break;
1073 // Async lowering will insert musttail call functions at all suspend points
1074 // followed by a return.
1075 // Don't change returns to unreachable because that will trip up the verifier.
1076 // These returns should be unreachable from the clone.
1077 case coro::ABI::Async:
1078 break;
1079 }
1080
1081 NewF->setAttributes(NewAttrs);
1082 NewF->setCallingConv(Shape.getResumeFunctionCC());
1083
1084 // Set up the new entry block.
1085 replaceEntryBlock();
1086
1087 // Turn symmetric transfers into musttail calls.
1088 for (CallInst *ResumeCall : Shape.SymmetricTransfers) {
1089 ResumeCall = cast<CallInst>(VMap[ResumeCall]);
1090 if (TTI.supportsTailCallFor(ResumeCall)) {
1091 // FIXME: Could we support symmetric transfer effectively without
1092 // musttail?
1094 }
1095
1096 // Put a 'ret void' after the call, and split any remaining instructions to
1097 // an unreachable block.
1098 BasicBlock *BB = ResumeCall->getParent();
1099 BB->splitBasicBlock(ResumeCall->getNextNode());
1100 Builder.SetInsertPoint(BB->getTerminator());
1101 Builder.CreateRetVoid();
1103 }
1104
1105 Builder.SetInsertPoint(&NewF->getEntryBlock().front());
1106 NewFramePtr = deriveNewFramePointer();
1107
1108 // Remap frame pointer.
1109 Value *OldFramePtr = VMap[Shape.FramePtr];
1110 NewFramePtr->takeName(OldFramePtr);
1111 OldFramePtr->replaceAllUsesWith(NewFramePtr);
1112
1113 // Remap vFrame pointer.
1114 auto *NewVFrame = Builder.CreateBitCast(
1115 NewFramePtr, PointerType::getUnqual(Builder.getContext()), "vFrame");
1116 Value *OldVFrame = cast<Value>(VMap[Shape.CoroBegin]);
1117 if (OldVFrame != NewVFrame)
1118 OldVFrame->replaceAllUsesWith(NewVFrame);
1119
1120 // All uses of the arguments should have been resolved by this point,
1121 // so we can safely remove the dummy values.
1122 for (Instruction *DummyArg : DummyArgs) {
1123 DummyArg->replaceAllUsesWith(PoisonValue::get(DummyArg->getType()));
1124 DummyArg->deleteValue();
1125 }
1126
1127 switch (Shape.ABI) {
1128 case coro::ABI::Switch:
1129 // Rewrite final suspend handling as it is not done via switch (allows to
1130 // remove final case from the switch, since it is undefined behavior to
1131 // resume the coroutine suspended at the final suspend point.
1132 if (Shape.SwitchLowering.HasFinalSuspend)
1133 handleFinalSuspend();
1134 break;
1135 case coro::ABI::Async:
1136 case coro::ABI::Retcon:
1137 case coro::ABI::RetconOnce:
1138 // Replace uses of the active suspend with the corresponding
1139 // continuation-function arguments.
1140 assert(ActiveSuspend != nullptr &&
1141 "no active suspend when lowering a continuation-style coroutine");
1142 replaceRetconOrAsyncSuspendUses();
1143 break;
1144 }
1145
1146 // Handle suspends.
1147 replaceCoroSuspends();
1148
1149 // Handle swifterror.
1151
1152 // Remove coro.end intrinsics.
1153 replaceCoroEnds();
1154
1155 // Salvage debug info that points into the coroutine frame.
1157
1158 // Eliminate coro.free from the clones, replacing it with 'null' in cleanup,
1159 // to suppress deallocation code.
1160 if (Shape.ABI == coro::ABI::Switch)
1161 coro::replaceCoroFree(cast<CoroIdInst>(VMap[Shape.CoroBegin->getId()]),
1162 /*Elide=*/FKind == CoroCloner::Kind::SwitchCleanup);
1163}
1164
1166 assert(Shape.ABI == coro::ABI::Async);
1167
1168 auto *FuncPtrStruct = cast<ConstantStruct>(
1170 auto *OrigRelativeFunOffset = FuncPtrStruct->getOperand(0);
1171 auto *OrigContextSize = FuncPtrStruct->getOperand(1);
1172 auto *NewContextSize = ConstantInt::get(OrigContextSize->getType(),
1174 auto *NewFuncPtrStruct = ConstantStruct::get(
1175 FuncPtrStruct->getType(), OrigRelativeFunOffset, NewContextSize);
1176
1177 Shape.AsyncLowering.AsyncFuncPointer->setInitializer(NewFuncPtrStruct);
1178}
1179
1181 if (Shape.ABI == coro::ABI::Async)
1183
1184 for (CoroAlignInst *CA : Shape.CoroAligns) {
1186 ConstantInt::get(CA->getType(), Shape.FrameAlign.value()));
1187 CA->eraseFromParent();
1188 }
1189
1190 if (Shape.CoroSizes.empty())
1191 return;
1192
1193 // In the same function all coro.sizes should have the same result type.
1194 auto *SizeIntrin = Shape.CoroSizes.back();
1195 Module *M = SizeIntrin->getModule();
1196 const DataLayout &DL = M->getDataLayout();
1197 auto Size = DL.getTypeAllocSize(Shape.FrameTy);
1198 auto *SizeConstant = ConstantInt::get(SizeIntrin->getType(), Size);
1199
1200 for (CoroSizeInst *CS : Shape.CoroSizes) {
1201 CS->replaceAllUsesWith(SizeConstant);
1202 CS->eraseFromParent();
1203 }
1204}
1205
1208
1209#ifndef NDEBUG
1210 // For now, we do a mandatory verification step because we don't
1211 // entirely trust this pass. Note that we don't want to add a verifier
1212 // pass to FPM below because it will also verify all the global data.
1213 if (verifyFunction(F, &errs()))
1214 report_fatal_error("Broken function");
1215#endif
1216}
1217
1218// Coroutine has no suspend points. Remove heap allocation for the coroutine
1219// frame if possible.
1221 auto *CoroBegin = Shape.CoroBegin;
1222 auto *CoroId = CoroBegin->getId();
1223 auto *AllocInst = CoroId->getCoroAlloc();
1224 switch (Shape.ABI) {
1225 case coro::ABI::Switch: {
1226 auto SwitchId = cast<CoroIdInst>(CoroId);
1227 coro::replaceCoroFree(SwitchId, /*Elide=*/AllocInst != nullptr);
1228 if (AllocInst) {
1229 IRBuilder<> Builder(AllocInst);
1230 auto *Frame = Builder.CreateAlloca(Shape.FrameTy);
1231 Frame->setAlignment(Shape.FrameAlign);
1232 AllocInst->replaceAllUsesWith(Builder.getFalse());
1233 AllocInst->eraseFromParent();
1234 CoroBegin->replaceAllUsesWith(Frame);
1235 } else {
1236 CoroBegin->replaceAllUsesWith(CoroBegin->getMem());
1237 }
1238
1239 break;
1240 }
1241 case coro::ABI::Async:
1242 case coro::ABI::Retcon:
1243 case coro::ABI::RetconOnce:
1244 CoroBegin->replaceAllUsesWith(UndefValue::get(CoroBegin->getType()));
1245 break;
1246 }
1247
1248 CoroBegin->eraseFromParent();
1249 Shape.CoroBegin = nullptr;
1250}
1251
1252// SimplifySuspendPoint needs to check that there is no calls between
1253// coro_save and coro_suspend, since any of the calls may potentially resume
1254// the coroutine and if that is the case we cannot eliminate the suspend point.
1256 for (Instruction *I = From; I != To; I = I->getNextNode()) {
1257 // Assume that no intrinsic can resume the coroutine.
1258 if (isa<IntrinsicInst>(I))
1259 continue;
1260
1261 if (isa<CallBase>(I))
1262 return true;
1263 }
1264 return false;
1265}
1266
1267static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB) {
1270
1271 Set.insert(SaveBB);
1272 Worklist.push_back(ResDesBB);
1273
1274 // Accumulate all blocks between SaveBB and ResDesBB. Because CoroSaveIntr
1275 // returns a token consumed by suspend instruction, all blocks in between
1276 // will have to eventually hit SaveBB when going backwards from ResDesBB.
1277 while (!Worklist.empty()) {
1278 auto *BB = Worklist.pop_back_val();
1279 Set.insert(BB);
1280 for (auto *Pred : predecessors(BB))
1281 if (!Set.contains(Pred))
1282 Worklist.push_back(Pred);
1283 }
1284
1285 // SaveBB and ResDesBB are checked separately in hasCallsBetween.
1286 Set.erase(SaveBB);
1287 Set.erase(ResDesBB);
1288
1289 for (auto *BB : Set)
1290 if (hasCallsInBlockBetween(BB->getFirstNonPHI(), nullptr))
1291 return true;
1292
1293 return false;
1294}
1295
1296static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy) {
1297 auto *SaveBB = Save->getParent();
1298 auto *ResumeOrDestroyBB = ResumeOrDestroy->getParent();
1299
1300 if (SaveBB == ResumeOrDestroyBB)
1301 return hasCallsInBlockBetween(Save->getNextNode(), ResumeOrDestroy);
1302
1303 // Any calls from Save to the end of the block?
1304 if (hasCallsInBlockBetween(Save->getNextNode(), nullptr))
1305 return true;
1306
1307 // Any calls from begging of the block up to ResumeOrDestroy?
1308 if (hasCallsInBlockBetween(ResumeOrDestroyBB->getFirstNonPHI(),
1309 ResumeOrDestroy))
1310 return true;
1311
1312 // Any calls in all of the blocks between SaveBB and ResumeOrDestroyBB?
1313 if (hasCallsInBlocksBetween(SaveBB, ResumeOrDestroyBB))
1314 return true;
1315
1316 return false;
1317}
1318
1319// If a SuspendIntrin is preceded by Resume or Destroy, we can eliminate the
1320// suspend point and replace it with nornal control flow.
1322 CoroBeginInst *CoroBegin) {
1323 Instruction *Prev = Suspend->getPrevNode();
1324 if (!Prev) {
1325 auto *Pred = Suspend->getParent()->getSinglePredecessor();
1326 if (!Pred)
1327 return false;
1328 Prev = Pred->getTerminator();
1329 }
1330
1331 CallBase *CB = dyn_cast<CallBase>(Prev);
1332 if (!CB)
1333 return false;
1334
1335 auto *Callee = CB->getCalledOperand()->stripPointerCasts();
1336
1337 // See if the callsite is for resumption or destruction of the coroutine.
1338 auto *SubFn = dyn_cast<CoroSubFnInst>(Callee);
1339 if (!SubFn)
1340 return false;
1341
1342 // Does not refer to the current coroutine, we cannot do anything with it.
1343 if (SubFn->getFrame() != CoroBegin)
1344 return false;
1345
1346 // See if the transformation is safe. Specifically, see if there are any
1347 // calls in between Save and CallInstr. They can potenitally resume the
1348 // coroutine rendering this optimization unsafe.
1349 auto *Save = Suspend->getCoroSave();
1350 if (hasCallsBetween(Save, CB))
1351 return false;
1352
1353 // Replace llvm.coro.suspend with the value that results in resumption over
1354 // the resume or cleanup path.
1355 Suspend->replaceAllUsesWith(SubFn->getRawIndex());
1356 Suspend->eraseFromParent();
1357 Save->eraseFromParent();
1358
1359 // No longer need a call to coro.resume or coro.destroy.
1360 if (auto *Invoke = dyn_cast<InvokeInst>(CB)) {
1361 BranchInst::Create(Invoke->getNormalDest(), Invoke->getIterator());
1362 }
1363
1364 // Grab the CalledValue from CB before erasing the CallInstr.
1365 auto *CalledValue = CB->getCalledOperand();
1366 CB->eraseFromParent();
1367
1368 // If no more users remove it. Usually it is a bitcast of SubFn.
1369 if (CalledValue != SubFn && CalledValue->user_empty())
1370 if (auto *I = dyn_cast<Instruction>(CalledValue))
1371 I->eraseFromParent();
1372
1373 // Now we are good to remove SubFn.
1374 if (SubFn->user_empty())
1375 SubFn->eraseFromParent();
1376
1377 return true;
1378}
1379
1380// Remove suspend points that are simplified.
1382 // Currently, the only simplification we do is switch-lowering-specific.
1383 if (Shape.ABI != coro::ABI::Switch)
1384 return;
1385
1386 auto &S = Shape.CoroSuspends;
1387 size_t I = 0, N = S.size();
1388 if (N == 0)
1389 return;
1390
1391 size_t ChangedFinalIndex = std::numeric_limits<size_t>::max();
1392 while (true) {
1393 auto SI = cast<CoroSuspendInst>(S[I]);
1394 // Leave final.suspend to handleFinalSuspend since it is undefined behavior
1395 // to resume a coroutine suspended at the final suspend point.
1396 if (!SI->isFinal() && simplifySuspendPoint(SI, Shape.CoroBegin)) {
1397 if (--N == I)
1398 break;
1399
1400 std::swap(S[I], S[N]);
1401
1402 if (cast<CoroSuspendInst>(S[I])->isFinal()) {
1404 ChangedFinalIndex = I;
1405 }
1406
1407 continue;
1408 }
1409 if (++I == N)
1410 break;
1411 }
1412 S.resize(N);
1413
1414 // Maintain final.suspend in case final suspend was swapped.
1415 // Due to we requrie the final suspend to be the last element of CoroSuspends.
1416 if (ChangedFinalIndex < N) {
1417 assert(cast<CoroSuspendInst>(S[ChangedFinalIndex])->isFinal());
1418 std::swap(S[ChangedFinalIndex], S.back());
1419 }
1420}
1421
1422namespace {
1423
1424struct SwitchCoroutineSplitter {
1425 static void split(Function &F, coro::Shape &Shape,
1428 assert(Shape.ABI == coro::ABI::Switch);
1429
1430 createResumeEntryBlock(F, Shape);
1431 auto *ResumeClone =
1432 createClone(F, ".resume", Shape, CoroCloner::Kind::SwitchResume, TTI);
1433 auto *DestroyClone =
1434 createClone(F, ".destroy", Shape, CoroCloner::Kind::SwitchUnwind, TTI);
1435 auto *CleanupClone =
1436 createClone(F, ".cleanup", Shape, CoroCloner::Kind::SwitchCleanup, TTI);
1437
1438 postSplitCleanup(*ResumeClone);
1439 postSplitCleanup(*DestroyClone);
1440 postSplitCleanup(*CleanupClone);
1441
1442 // Store addresses resume/destroy/cleanup functions in the coroutine frame.
1443 updateCoroFrame(Shape, ResumeClone, DestroyClone, CleanupClone);
1444
1445 assert(Clones.empty());
1446 Clones.push_back(ResumeClone);
1447 Clones.push_back(DestroyClone);
1448 Clones.push_back(CleanupClone);
1449
1450 // Create a constant array referring to resume/destroy/clone functions
1451 // pointed by the last argument of @llvm.coro.info, so that CoroElide pass
1452 // can determined correct function to call.
1453 setCoroInfo(F, Shape, Clones);
1454 }
1455
1456private:
1457 // Create a resume clone by cloning the body of the original function, setting
1458 // new entry block and replacing coro.suspend an appropriate value to force
1459 // resume or cleanup pass for every suspend point.
1460 static Function *createClone(Function &F, const Twine &Suffix,
1461 coro::Shape &Shape, CoroCloner::Kind FKind,
1463 CoroCloner Cloner(F, Suffix, Shape, FKind, TTI);
1464 Cloner.create();
1465 return Cloner.getFunction();
1466 }
1467
1468 // Create an entry block for a resume function with a switch that will jump to
1469 // suspend points.
1470 static void createResumeEntryBlock(Function &F, coro::Shape &Shape) {
1471 LLVMContext &C = F.getContext();
1472
1473 // resume.entry:
1474 // %index.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32
1475 // 0, i32 2 % index = load i32, i32* %index.addr switch i32 %index, label
1476 // %unreachable [
1477 // i32 0, label %resume.0
1478 // i32 1, label %resume.1
1479 // ...
1480 // ]
1481
1482 auto *NewEntry = BasicBlock::Create(C, "resume.entry", &F);
1483 auto *UnreachBB = BasicBlock::Create(C, "unreachable", &F);
1484
1485 IRBuilder<> Builder(NewEntry);
1486 auto *FramePtr = Shape.FramePtr;
1487 auto *FrameTy = Shape.FrameTy;
1488 auto *GepIndex = Builder.CreateStructGEP(
1489 FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
1490 auto *Index = Builder.CreateLoad(Shape.getIndexType(), GepIndex, "index");
1491 auto *Switch =
1492 Builder.CreateSwitch(Index, UnreachBB, Shape.CoroSuspends.size());
1494
1495 size_t SuspendIndex = 0;
1496 for (auto *AnyS : Shape.CoroSuspends) {
1497 auto *S = cast<CoroSuspendInst>(AnyS);
1498 ConstantInt *IndexVal = Shape.getIndex(SuspendIndex);
1499
1500 // Replace CoroSave with a store to Index:
1501 // %index.addr = getelementptr %f.frame... (index field number)
1502 // store i32 %IndexVal, i32* %index.addr1
1503 auto *Save = S->getCoroSave();
1504 Builder.SetInsertPoint(Save);
1505 if (S->isFinal()) {
1506 // The coroutine should be marked done if it reaches the final suspend
1507 // point.
1508 markCoroutineAsDone(Builder, Shape, FramePtr);
1509 } else {
1510 auto *GepIndex = Builder.CreateStructGEP(
1511 FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
1512 Builder.CreateStore(IndexVal, GepIndex);
1513 }
1514
1515 Save->replaceAllUsesWith(ConstantTokenNone::get(C));
1516 Save->eraseFromParent();
1517
1518 // Split block before and after coro.suspend and add a jump from an entry
1519 // switch:
1520 //
1521 // whateverBB:
1522 // whatever
1523 // %0 = call i8 @llvm.coro.suspend(token none, i1 false)
1524 // switch i8 %0, label %suspend[i8 0, label %resume
1525 // i8 1, label %cleanup]
1526 // becomes:
1527 //
1528 // whateverBB:
1529 // whatever
1530 // br label %resume.0.landing
1531 //
1532 // resume.0: ; <--- jump from the switch in the resume.entry
1533 // %0 = tail call i8 @llvm.coro.suspend(token none, i1 false)
1534 // br label %resume.0.landing
1535 //
1536 // resume.0.landing:
1537 // %1 = phi i8[-1, %whateverBB], [%0, %resume.0]
1538 // switch i8 % 1, label %suspend [i8 0, label %resume
1539 // i8 1, label %cleanup]
1540
1541 auto *SuspendBB = S->getParent();
1542 auto *ResumeBB =
1543 SuspendBB->splitBasicBlock(S, "resume." + Twine(SuspendIndex));
1544 auto *LandingBB = ResumeBB->splitBasicBlock(
1545 S->getNextNode(), ResumeBB->getName() + Twine(".landing"));
1546 Switch->addCase(IndexVal, ResumeBB);
1547
1548 cast<BranchInst>(SuspendBB->getTerminator())->setSuccessor(0, LandingBB);
1549 auto *PN = PHINode::Create(Builder.getInt8Ty(), 2, "");
1550 PN->insertBefore(LandingBB->begin());
1551 S->replaceAllUsesWith(PN);
1552 PN->addIncoming(Builder.getInt8(-1), SuspendBB);
1553 PN->addIncoming(S, ResumeBB);
1554
1555 ++SuspendIndex;
1556 }
1557
1558 Builder.SetInsertPoint(UnreachBB);
1559 Builder.CreateUnreachable();
1560
1561 Shape.SwitchLowering.ResumeEntryBlock = NewEntry;
1562 }
1563
1564 // Store addresses of Resume/Destroy/Cleanup functions in the coroutine frame.
1565 static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn,
1566 Function *DestroyFn, Function *CleanupFn) {
1567 IRBuilder<> Builder(&*Shape.getInsertPtAfterFramePtr());
1568
1569 auto *ResumeAddr = Builder.CreateStructGEP(
1571 "resume.addr");
1572 Builder.CreateStore(ResumeFn, ResumeAddr);
1573
1574 Value *DestroyOrCleanupFn = DestroyFn;
1575
1576 CoroIdInst *CoroId = Shape.getSwitchCoroId();
1577 if (CoroAllocInst *CA = CoroId->getCoroAlloc()) {
1578 // If there is a CoroAlloc and it returns false (meaning we elide the
1579 // allocation, use CleanupFn instead of DestroyFn).
1580 DestroyOrCleanupFn = Builder.CreateSelect(CA, DestroyFn, CleanupFn);
1581 }
1582
1583 auto *DestroyAddr = Builder.CreateStructGEP(
1585 "destroy.addr");
1586 Builder.CreateStore(DestroyOrCleanupFn, DestroyAddr);
1587 }
1588
1589 // Create a global constant array containing pointers to functions provided
1590 // and set Info parameter of CoroBegin to point at this constant. Example:
1591 //
1592 // @f.resumers = internal constant [2 x void(%f.frame*)*]
1593 // [void(%f.frame*)* @f.resume, void(%f.frame*)*
1594 // @f.destroy]
1595 // define void @f() {
1596 // ...
1597 // call i8* @llvm.coro.begin(i8* null, i32 0, i8* null,
1598 // i8* bitcast([2 x void(%f.frame*)*] * @f.resumers to
1599 // i8*))
1600 //
1601 // Assumes that all the functions have the same signature.
1602 static void setCoroInfo(Function &F, coro::Shape &Shape,
1604 // This only works under the switch-lowering ABI because coro elision
1605 // only works on the switch-lowering ABI.
1607 assert(!Args.empty());
1608 Function *Part = *Fns.begin();
1609 Module *M = Part->getParent();
1610 auto *ArrTy = ArrayType::get(Part->getType(), Args.size());
1611
1612 auto *ConstVal = ConstantArray::get(ArrTy, Args);
1613 auto *GV = new GlobalVariable(*M, ConstVal->getType(), /*isConstant=*/true,
1614 GlobalVariable::PrivateLinkage, ConstVal,
1615 F.getName() + Twine(".resumers"));
1616
1617 // Update coro.begin instruction to refer to this constant.
1618 LLVMContext &C = F.getContext();
1619 auto *BC = ConstantExpr::getPointerCast(GV, PointerType::getUnqual(C));
1620 Shape.getSwitchCoroId()->setInfo(BC);
1621 }
1622};
1623
1624} // namespace
1625
1627 Value *Continuation) {
1628 auto *ResumeIntrinsic = Suspend->getResumeFunction();
1629 auto &Context = Suspend->getParent()->getParent()->getContext();
1630 auto *Int8PtrTy = PointerType::getUnqual(Context);
1631
1632 IRBuilder<> Builder(ResumeIntrinsic);
1633 auto *Val = Builder.CreateBitOrPointerCast(Continuation, Int8PtrTy);
1634 ResumeIntrinsic->replaceAllUsesWith(Val);
1635 ResumeIntrinsic->eraseFromParent();
1637 UndefValue::get(Int8PtrTy));
1638}
1639
1640/// Coerce the arguments in \p FnArgs according to \p FnTy in \p CallArgs.
1641static void coerceArguments(IRBuilder<> &Builder, FunctionType *FnTy,
1642 ArrayRef<Value *> FnArgs,
1643 SmallVectorImpl<Value *> &CallArgs) {
1644 size_t ArgIdx = 0;
1645 for (auto *paramTy : FnTy->params()) {
1646 assert(ArgIdx < FnArgs.size());
1647 if (paramTy != FnArgs[ArgIdx]->getType())
1648 CallArgs.push_back(
1649 Builder.CreateBitOrPointerCast(FnArgs[ArgIdx], paramTy));
1650 else
1651 CallArgs.push_back(FnArgs[ArgIdx]);
1652 ++ArgIdx;
1653 }
1654}
1655
1659 IRBuilder<> &Builder) {
1660 auto *FnTy = MustTailCallFn->getFunctionType();
1661 // Coerce the arguments, llvm optimizations seem to ignore the types in
1662 // vaarg functions and throws away casts in optimized mode.
1663 SmallVector<Value *, 8> CallArgs;
1664 coerceArguments(Builder, FnTy, Arguments, CallArgs);
1665
1666 auto *TailCall = Builder.CreateCall(FnTy, MustTailCallFn, CallArgs);
1667 // Skip targets which don't support tail call.
1668 if (TTI.supportsTailCallFor(TailCall)) {
1669 TailCall->setTailCallKind(CallInst::TCK_MustTail);
1670 }
1671 TailCall->setDebugLoc(Loc);
1672 TailCall->setCallingConv(MustTailCallFn->getCallingConv());
1673 return TailCall;
1674}
1675
1679 assert(Shape.ABI == coro::ABI::Async);
1680 assert(Clones.empty());
1681 // Reset various things that the optimizer might have decided it
1682 // "knows" about the coroutine function due to not seeing a return.
1683 F.removeFnAttr(Attribute::NoReturn);
1684 F.removeRetAttr(Attribute::NoAlias);
1685 F.removeRetAttr(Attribute::NonNull);
1686
1687 auto &Context = F.getContext();
1688 auto *Int8PtrTy = PointerType::getUnqual(Context);
1689
1690 auto *Id = cast<CoroIdAsyncInst>(Shape.CoroBegin->getId());
1691 IRBuilder<> Builder(Id);
1692
1693 auto *FramePtr = Id->getStorage();
1694 FramePtr = Builder.CreateBitOrPointerCast(FramePtr, Int8PtrTy);
1697 "async.ctx.frameptr");
1698
1699 // Map all uses of llvm.coro.begin to the allocated frame pointer.
1700 {
1701 // Make sure we don't invalidate Shape.FramePtr.
1702 TrackingVH<Value> Handle(Shape.FramePtr);
1704 Shape.FramePtr = Handle.getValPtr();
1705 }
1706
1707 // Create all the functions in order after the main function.
1708 auto NextF = std::next(F.getIterator());
1709
1710 // Create a continuation function for each of the suspend points.
1711 Clones.reserve(Shape.CoroSuspends.size());
1712 for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) {
1713 auto *Suspend = cast<CoroSuspendAsyncInst>(Shape.CoroSuspends[Idx]);
1714
1715 // Create the clone declaration.
1716 auto ResumeNameSuffix = ".resume.";
1717 auto ProjectionFunctionName =
1718 Suspend->getAsyncContextProjectionFunction()->getName();
1719 bool UseSwiftMangling = false;
1720 if (ProjectionFunctionName == "__swift_async_resume_project_context") {
1721 ResumeNameSuffix = "TQ";
1722 UseSwiftMangling = true;
1723 } else if (ProjectionFunctionName == "__swift_async_resume_get_context") {
1724 ResumeNameSuffix = "TY";
1725 UseSwiftMangling = true;
1726 }
1727 auto *Continuation = createCloneDeclaration(
1728 F, Shape,
1729 UseSwiftMangling ? ResumeNameSuffix + Twine(Idx) + "_"
1730 : ResumeNameSuffix + Twine(Idx),
1731 NextF, Suspend);
1732 Clones.push_back(Continuation);
1733
1734 // Insert a branch to a new return block immediately before the suspend
1735 // point.
1736 auto *SuspendBB = Suspend->getParent();
1737 auto *NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);
1738 auto *Branch = cast<BranchInst>(SuspendBB->getTerminator());
1739
1740 // Place it before the first suspend.
1741 auto *ReturnBB =
1742 BasicBlock::Create(F.getContext(), "coro.return", &F, NewSuspendBB);
1743 Branch->setSuccessor(0, ReturnBB);
1744
1745 IRBuilder<> Builder(ReturnBB);
1746
1747 // Insert the call to the tail call function and inline it.
1748 auto *Fn = Suspend->getMustTailCallFunction();
1749 SmallVector<Value *, 8> Args(Suspend->args());
1750 auto FnArgs = ArrayRef<Value *>(Args).drop_front(
1752 auto *TailCall = coro::createMustTailCall(Suspend->getDebugLoc(), Fn, TTI,
1753 FnArgs, Builder);
1754 Builder.CreateRetVoid();
1755 InlineFunctionInfo FnInfo;
1756 (void)InlineFunction(*TailCall, FnInfo);
1757
1758 // Replace the lvm.coro.async.resume intrisic call.
1759 replaceAsyncResumeFunction(Suspend, Continuation);
1760 }
1761
1762 assert(Clones.size() == Shape.CoroSuspends.size());
1763 for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) {
1764 auto *Suspend = Shape.CoroSuspends[Idx];
1765 auto *Clone = Clones[Idx];
1766
1767 CoroCloner(F, "resume." + Twine(Idx), Shape, Clone, Suspend, TTI).create();
1768 }
1769}
1770
1774 assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce);
1775 assert(Clones.empty());
1776
1777 // Reset various things that the optimizer might have decided it
1778 // "knows" about the coroutine function due to not seeing a return.
1779 F.removeFnAttr(Attribute::NoReturn);
1780 F.removeRetAttr(Attribute::NoAlias);
1781 F.removeRetAttr(Attribute::NonNull);
1782
1783 // Allocate the frame.
1784 auto *Id = cast<AnyCoroIdRetconInst>(Shape.CoroBegin->getId());
1785 Value *RawFramePtr;
1787 RawFramePtr = Id->getStorage();
1788 } else {
1789 IRBuilder<> Builder(Id);
1790
1791 // Determine the size of the frame.
1792 const DataLayout &DL = F.getDataLayout();
1793 auto Size = DL.getTypeAllocSize(Shape.FrameTy);
1794
1795 // Allocate. We don't need to update the call graph node because we're
1796 // going to recompute it from scratch after splitting.
1797 // FIXME: pass the required alignment
1798 RawFramePtr = Shape.emitAlloc(Builder, Builder.getInt64(Size), nullptr);
1799 RawFramePtr =
1800 Builder.CreateBitCast(RawFramePtr, Shape.CoroBegin->getType());
1801
1802 // Stash the allocated frame pointer in the continuation storage.
1803 Builder.CreateStore(RawFramePtr, Id->getStorage());
1804 }
1805
1806 // Map all uses of llvm.coro.begin to the allocated frame pointer.
1807 {
1808 // Make sure we don't invalidate Shape.FramePtr.
1809 TrackingVH<Value> Handle(Shape.FramePtr);
1810 Shape.CoroBegin->replaceAllUsesWith(RawFramePtr);
1811 Shape.FramePtr = Handle.getValPtr();
1812 }
1813
1814 // Create a unique return block.
1815 BasicBlock *ReturnBB = nullptr;
1816 SmallVector<PHINode *, 4> ReturnPHIs;
1817
1818 // Create all the functions in order after the main function.
1819 auto NextF = std::next(F.getIterator());
1820
1821 // Create a continuation function for each of the suspend points.
1822 Clones.reserve(Shape.CoroSuspends.size());
1823 for (size_t i = 0, e = Shape.CoroSuspends.size(); i != e; ++i) {
1824 auto Suspend = cast<CoroSuspendRetconInst>(Shape.CoroSuspends[i]);
1825
1826 // Create the clone declaration.
1827 auto Continuation =
1828 createCloneDeclaration(F, Shape, ".resume." + Twine(i), NextF, nullptr);
1829 Clones.push_back(Continuation);
1830
1831 // Insert a branch to the unified return block immediately before
1832 // the suspend point.
1833 auto SuspendBB = Suspend->getParent();
1834 auto NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);
1835 auto Branch = cast<BranchInst>(SuspendBB->getTerminator());
1836
1837 // Create the unified return block.
1838 if (!ReturnBB) {
1839 // Place it before the first suspend.
1840 ReturnBB =
1841 BasicBlock::Create(F.getContext(), "coro.return", &F, NewSuspendBB);
1842 Shape.RetconLowering.ReturnBlock = ReturnBB;
1843
1844 IRBuilder<> Builder(ReturnBB);
1845
1846 // Create PHIs for all the return values.
1847 assert(ReturnPHIs.empty());
1848
1849 // First, the continuation.
1850 ReturnPHIs.push_back(Builder.CreatePHI(Continuation->getType(),
1851 Shape.CoroSuspends.size()));
1852
1853 // Next, all the directly-yielded values.
1854 for (auto *ResultTy : Shape.getRetconResultTypes())
1855 ReturnPHIs.push_back(
1856 Builder.CreatePHI(ResultTy, Shape.CoroSuspends.size()));
1857
1858 // Build the return value.
1859 auto RetTy = F.getReturnType();
1860
1861 // Cast the continuation value if necessary.
1862 // We can't rely on the types matching up because that type would
1863 // have to be infinite.
1864 auto CastedContinuationTy =
1865 (ReturnPHIs.size() == 1 ? RetTy : RetTy->getStructElementType(0));
1866 auto *CastedContinuation =
1867 Builder.CreateBitCast(ReturnPHIs[0], CastedContinuationTy);
1868
1869 Value *RetV;
1870 if (ReturnPHIs.size() == 1) {
1871 RetV = CastedContinuation;
1872 } else {
1873 RetV = PoisonValue::get(RetTy);
1874 RetV = Builder.CreateInsertValue(RetV, CastedContinuation, 0);
1875 for (size_t I = 1, E = ReturnPHIs.size(); I != E; ++I)
1876 RetV = Builder.CreateInsertValue(RetV, ReturnPHIs[I], I);
1877 }
1878
1879 Builder.CreateRet(RetV);
1880 }
1881
1882 // Branch to the return block.
1883 Branch->setSuccessor(0, ReturnBB);
1884 ReturnPHIs[0]->addIncoming(Continuation, SuspendBB);
1885 size_t NextPHIIndex = 1;
1886 for (auto &VUse : Suspend->value_operands())
1887 ReturnPHIs[NextPHIIndex++]->addIncoming(&*VUse, SuspendBB);
1888 assert(NextPHIIndex == ReturnPHIs.size());
1889 }
1890
1891 assert(Clones.size() == Shape.CoroSuspends.size());
1892 for (size_t i = 0, e = Shape.CoroSuspends.size(); i != e; ++i) {
1893 auto Suspend = Shape.CoroSuspends[i];
1894 auto Clone = Clones[i];
1895
1896 CoroCloner(F, "resume." + Twine(i), Shape, Clone, Suspend, TTI).create();
1897 }
1898}
1899
1900namespace {
1901class PrettyStackTraceFunction : public PrettyStackTraceEntry {
1902 Function &F;
1903
1904public:
1905 PrettyStackTraceFunction(Function &F) : F(F) {}
1906 void print(raw_ostream &OS) const override {
1907 OS << "While splitting coroutine ";
1908 F.printAsOperand(OS, /*print type*/ false, F.getParent());
1909 OS << "\n";
1910 }
1911};
1912} // namespace
1913
1914static coro::Shape
1916 TargetTransformInfo &TTI, bool OptimizeFrame,
1917 std::function<bool(Instruction &)> MaterializableCallback) {
1918 PrettyStackTraceFunction prettyStackTrace(F);
1919
1920 // The suspend-crossing algorithm in buildCoroutineFrame get tripped
1921 // up by uses in unreachable blocks, so remove them as a first pass.
1923
1924 coro::Shape Shape(F, OptimizeFrame);
1925 if (!Shape.CoroBegin)
1926 return Shape;
1927
1928 lowerAwaitSuspends(F, Shape);
1929
1930 simplifySuspendPoints(Shape);
1931 buildCoroutineFrame(F, Shape, TTI, MaterializableCallback);
1933
1934 // If there are no suspend points, no split required, just remove
1935 // the allocation and deallocation blocks, they are not needed.
1936 if (Shape.CoroSuspends.empty()) {
1938 } else {
1939 switch (Shape.ABI) {
1940 case coro::ABI::Switch:
1941 SwitchCoroutineSplitter::split(F, Shape, Clones, TTI);
1942 break;
1943 case coro::ABI::Async:
1944 splitAsyncCoroutine(F, Shape, Clones, TTI);
1945 break;
1946 case coro::ABI::Retcon:
1947 case coro::ABI::RetconOnce:
1948 splitRetconCoroutine(F, Shape, Clones, TTI);
1949 break;
1950 }
1951 }
1952
1953 // Replace all the swifterror operations in the original function.
1954 // This invalidates SwiftErrorOps in the Shape.
1955 replaceSwiftErrorOps(F, Shape, nullptr);
1956
1957 // Salvage debug intrinsics that point into the coroutine frame in the
1958 // original function. The Cloner has already salvaged debug info in the new
1959 // coroutine funclets.
1961 auto [DbgInsts, DbgVariableRecords] = collectDbgVariableIntrinsics(F);
1962 for (auto *DDI : DbgInsts)
1963 coro::salvageDebugInfo(ArgToAllocaMap, *DDI, false /*UseEntryValue*/);
1964 for (DbgVariableRecord *DVR : DbgVariableRecords)
1965 coro::salvageDebugInfo(ArgToAllocaMap, *DVR, false /*UseEntryValue*/);
1966 return Shape;
1967}
1968
1969/// Remove calls to llvm.coro.end in the original function.
1971 if (Shape.ABI != coro::ABI::Switch) {
1972 for (auto *End : Shape.CoroEnds) {
1973 replaceCoroEnd(End, Shape, Shape.FramePtr, /*in resume*/ false, nullptr);
1974 }
1975 } else {
1976 for (llvm::AnyCoroEndInst *End : Shape.CoroEnds) {
1977 auto &Context = End->getContext();
1978 End->replaceAllUsesWith(ConstantInt::getFalse(Context));
1979 End->eraseFromParent();
1980 }
1981 }
1982}
1983
1985 LazyCallGraph::Node &N, const coro::Shape &Shape,
1989
1990 if (!Clones.empty()) {
1991 switch (Shape.ABI) {
1992 case coro::ABI::Switch:
1993 // Each clone in the Switch lowering is independent of the other clones.
1994 // Let the LazyCallGraph know about each one separately.
1995 for (Function *Clone : Clones)
1996 CG.addSplitFunction(N.getFunction(), *Clone);
1997 break;
1998 case coro::ABI::Async:
1999 case coro::ABI::Retcon:
2000 case coro::ABI::RetconOnce:
2001 // Each clone in the Async/Retcon lowering references of the other clones.
2002 // Let the LazyCallGraph know about all of them at once.
2003 if (!Clones.empty())
2004 CG.addSplitRefRecursiveFunctions(N.getFunction(), Clones);
2005 break;
2006 }
2007
2008 // Let the CGSCC infra handle the changes to the original function.
2010 }
2011
2012 // Do some cleanup and let the CGSCC infra see if we've cleaned up any edges
2013 // to the split functions.
2014 postSplitCleanup(N.getFunction());
2016}
2017
2018/// Replace a call to llvm.coro.prepare.retcon.
2019static void replacePrepare(CallInst *Prepare, LazyCallGraph &CG,
2021 auto CastFn = Prepare->getArgOperand(0); // as an i8*
2022 auto Fn = CastFn->stripPointerCasts(); // as its original type
2023
2024 // Attempt to peephole this pattern:
2025 // %0 = bitcast [[TYPE]] @some_function to i8*
2026 // %1 = call @llvm.coro.prepare.retcon(i8* %0)
2027 // %2 = bitcast %1 to [[TYPE]]
2028 // ==>
2029 // %2 = @some_function
2030 for (Use &U : llvm::make_early_inc_range(Prepare->uses())) {
2031 // Look for bitcasts back to the original function type.
2032 auto *Cast = dyn_cast<BitCastInst>(U.getUser());
2033 if (!Cast || Cast->getType() != Fn->getType())
2034 continue;
2035
2036 // Replace and remove the cast.
2037 Cast->replaceAllUsesWith(Fn);
2038 Cast->eraseFromParent();
2039 }
2040
2041 // Replace any remaining uses with the function as an i8*.
2042 // This can never directly be a callee, so we don't need to update CG.
2043 Prepare->replaceAllUsesWith(CastFn);
2044 Prepare->eraseFromParent();
2045
2046 // Kill dead bitcasts.
2047 while (auto *Cast = dyn_cast<BitCastInst>(CastFn)) {
2048 if (!Cast->use_empty())
2049 break;
2050 CastFn = Cast->getOperand(0);
2051 Cast->eraseFromParent();
2052 }
2053}
2054
2055static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG,
2057 bool Changed = false;
2058 for (Use &P : llvm::make_early_inc_range(PrepareFn->uses())) {
2059 // Intrinsics can only be used in calls.
2060 auto *Prepare = cast<CallInst>(P.getUser());
2061 replacePrepare(Prepare, CG, C);
2062 Changed = true;
2063 }
2064
2065 return Changed;
2066}
2067
2068static void addPrepareFunction(const Module &M,
2070 StringRef Name) {
2071 auto *PrepareFn = M.getFunction(Name);
2072 if (PrepareFn && !PrepareFn->use_empty())
2073 Fns.push_back(PrepareFn);
2074}
2075
2077 : MaterializableCallback(coro::defaultMaterializable),
2078 OptimizeFrame(OptimizeFrame) {}
2079
2083 // NB: One invariant of a valid LazyCallGraph::SCC is that it must contain a
2084 // non-zero number of nodes, so we assume that here and grab the first
2085 // node's function's module.
2086 Module &M = *C.begin()->getFunction().getParent();
2087 auto &FAM =
2088 AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager();
2089
2090 // Check for uses of llvm.coro.prepare.retcon/async.
2091 SmallVector<Function *, 2> PrepareFns;
2092 addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.retcon");
2093 addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.async");
2094
2095 // Find coroutines for processing.
2097 for (LazyCallGraph::Node &N : C)
2098 if (N.getFunction().isPresplitCoroutine())
2099 Coroutines.push_back(&N);
2100
2101 if (Coroutines.empty() && PrepareFns.empty())
2102 return PreservedAnalyses::all();
2103
2104 // Split all the coroutines.
2105 for (LazyCallGraph::Node *N : Coroutines) {
2106 Function &F = N->getFunction();
2107 LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F.getName()
2108 << "\n");
2109 F.setSplittedCoroutine();
2110
2113 const coro::Shape Shape =
2117 updateCallGraphAfterCoroutineSplit(*N, Shape, Clones, C, CG, AM, UR, FAM);
2118
2119 ORE.emit([&]() {
2120 return OptimizationRemark(DEBUG_TYPE, "CoroSplit", &F)
2121 << "Split '" << ore::NV("function", F.getName())
2122 << "' (frame_size=" << ore::NV("frame_size", Shape.FrameSize)
2123 << ", align=" << ore::NV("align", Shape.FrameAlign.value()) << ")";
2124 });
2125
2126 if (!Shape.CoroSuspends.empty()) {
2127 // Run the CGSCC pipeline on the original and newly split functions.
2128 UR.CWorklist.insert(&C);
2129 for (Function *Clone : Clones)
2130 UR.CWorklist.insert(CG.lookupSCC(CG.get(*Clone)));
2131 }
2132 }
2133
2134 for (auto *PrepareFn : PrepareFns) {
2135 replaceAllPrepares(PrepareFn, CG, C);
2136 }
2137
2138 return PreservedAnalyses::none();
2139}
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
AMDGPU Lower Kernel Arguments
static MachineBasicBlock * split(MachineBasicBlock::iterator I)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Expand Atomic instructions
This file contains the simple types necessary to represent the attributes associated with functions a...
BlockVerifier::State From
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file provides interfaces used to manipulate a call graph, regardless if it is a "old style" Call...
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static void addSwiftSelfAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)
Definition: CoroSplit.cpp:911
static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy)
Definition: CoroSplit.cpp:1296
static std::pair< SmallVector< DbgVariableIntrinsic *, 8 >, SmallVector< DbgVariableRecord * > > collectDbgVariableIntrinsics(Function &F)
Returns all DbgVariableIntrinsic in F.
Definition: CoroSplit.cpp:714
static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape, ValueToValueMapTy *VMap)
Definition: CoroSplit.cpp:656
static void addAsyncContextAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)
Definition: CoroSplit.cpp:904
static void maybeFreeRetconStorage(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr, CallGraph *CG)
Definition: CoroSplit.cpp:249
static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB)
Definition: CoroSplit.cpp:1267
static Function * createCloneDeclaration(Function &OrigF, coro::Shape &Shape, const Twine &Suffix, Module::iterator InsertBefore, AnyCoroSuspendInst *ActiveSuspend)
Definition: CoroSplit.cpp:540
Remove calls to llvm coro end in the original static function void removeCoroEndsFromRampFunction(const coro::Shape &Shape)
Definition: CoroSplit.cpp:1970
static FunctionType * getFunctionTypeFromAsyncSuspend(AnyCoroSuspendInst *Suspend)
Definition: CoroSplit.cpp:532
static void addPrepareFunction(const Module &M, SmallVectorImpl< Function * > &Fns, StringRef Name)
Definition: CoroSplit.cpp:2068
static void updateCallGraphAfterCoroutineSplit(LazyCallGraph::Node &N, const coro::Shape &Shape, const SmallVectorImpl< Function * > &Clones, LazyCallGraph::SCC &C, LazyCallGraph &CG, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Definition: CoroSplit.cpp:1984
static void simplifySuspendPoints(coro::Shape &Shape)
Definition: CoroSplit.cpp:1381
static void addFramePointerAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex, uint64_t Size, Align Alignment, bool NoAlias)
Definition: CoroSplit.cpp:889
static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG, LazyCallGraph::SCC &C)
Definition: CoroSplit.cpp:2055
static void replaceFallthroughCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Replace a non-unwind call to llvm.coro.end.
Definition: CoroSplit.cpp:305
static void replaceFrameSizeAndAlignment(coro::Shape &Shape)
Definition: CoroSplit.cpp:1180
static bool replaceCoroEndAsync(AnyCoroEndInst *End)
Replace an llvm.coro.end.async.
Definition: CoroSplit.cpp:262
static void splitRetconCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI)
Definition: CoroSplit.cpp:1771
Replace a call to llvm coro prepare static retcon void replacePrepare(CallInst *Prepare, LazyCallGraph &CG, LazyCallGraph::SCC &C)
Definition: CoroSplit.cpp:2019
static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Replace an unwind call to llvm.coro.end.
Definition: CoroSplit.cpp:438
static bool simplifySuspendPoint(CoroSuspendInst *Suspend, CoroBeginInst *CoroBegin)
Definition: CoroSplit.cpp:1321
static bool hasCallsInBlockBetween(Instruction *From, Instruction *To)
Definition: CoroSplit.cpp:1255
static void markCoroutineAsDone(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr)
Definition: CoroSplit.cpp:404
static void updateAsyncFuncPointerContextSize(coro::Shape &Shape)
Definition: CoroSplit.cpp:1165
static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Definition: CoroSplit.cpp:476
static void lowerAwaitSuspend(IRBuilder<> &Builder, CoroAwaitSuspendInst *CB, coro::Shape &Shape)
Definition: CoroSplit.cpp:177
static void lowerAwaitSuspends(Function &F, coro::Shape &Shape)
Definition: CoroSplit.cpp:243
static void handleNoSuspendCoroutine(coro::Shape &Shape)
Definition: CoroSplit.cpp:1220
static coro::Shape splitCoroutine(Function &F, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI, bool OptimizeFrame, std::function< bool(Instruction &)> MaterializableCallback)
Definition: CoroSplit.cpp:1915
static void postSplitCleanup(Function &F)
Definition: CoroSplit.cpp:1206
static void splitAsyncCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI)
Definition: CoroSplit.cpp:1676
Coerce the arguments in p FnArgs according to p FnTy in p static CallArgs void coerceArguments(IRBuilder<> &Builder, FunctionType *FnTy, ArrayRef< Value * > FnArgs, SmallVectorImpl< Value * > &CallArgs)
Definition: CoroSplit.cpp:1641
static void replaceAsyncResumeFunction(CoroSuspendAsyncInst *Suspend, Value *Continuation)
Definition: CoroSplit.cpp:1626
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseMap class.
This file contains constants used for implementing Dwarf debug support.
std::string Name
uint64_t Size
bool End
Definition: ELF_riscv.cpp:480
static Function * getFunction(Constant *C)
Definition: Evaluator.cpp:236
@ InlineInfo
Rewrite Partial Register Uses
#define DEBUG_TYPE
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Implements a lazy call graph analysis and related passes for the new pass manager.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Module.h This file contains the declarations for the Module class.
#define P(N)
FunctionAnalysisManager FAM
This file provides a priority worklist.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
raw_pwrite_stream & OS
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:40
This pass exposes codegen information to IR-level passes.
static const unsigned FramePtr
void setSwiftError(bool V)
Specify whether this alloca is used to represent a swifterror.
Definition: Instructions.h:149
void setAlignment(Align Align)
Definition: Instructions.h:126
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:253
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:405
CoroAllocInst * getCoroAlloc()
Definition: CoroInstr.h:117
This class represents an incoming formal argument to a Function.
Definition: Argument.h:31
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
Definition: ArrayRef.h:204
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
iterator begin() const
Definition: ArrayRef.h:153
AttrBuilder & addAlignmentAttr(MaybeAlign Align)
This turns an alignment into the form used internally in Attribute.
AttrBuilder & addAttribute(Attribute::AttrKind Val)
Add an attribute to the builder.
AttrBuilder & addDereferenceableAttr(uint64_t Bytes)
This turns the number of dereferenceable bytes into the form used internally in Attribute.
AttributeList removeParamAttributes(LLVMContext &C, unsigned ArgNo, const AttributeMask &AttrsToRemove) const
Remove the specified attribute at the specified arg index from this attribute list.
Definition: Attributes.h:725
LLVM Basic Block Representation.
Definition: BasicBlock.h:61
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:367
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:212
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
Definition: BasicBlock.cpp:577
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:239
static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1236
void setCallingConv(CallingConv::ID CC)
Definition: InstrTypes.h:1527
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1465
Value * getCalledOperand() const
Definition: InstrTypes.h:1458
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
Definition: InstrTypes.h:1546
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1410
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1542
The basic data container for the call graph of a Module of IR.
Definition: CallGraph.h:71
This class represents a function call, abstracting a target machine's calling convention.
void setTailCallKind(TailCallKind TCK)
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1292
static Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition: Constants.cpp:2227
This is the shared class of boolean and integer constants.
Definition: Constants.h:81
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:850
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:857
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1800
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1357
static ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition: Constants.cpp:1500
This represents the llvm.coro.align instruction.
Definition: CoroInstr.h:634
This represents the llvm.coro.alloc instruction.
Definition: CoroInstr.h:70
This represents the llvm.coro.await.suspend.{void,bool,handle} instructions.
Definition: CoroInstr.h:85
Value * getFrame() const
Definition: CoroInstr.h:91
Value * getAwaiter() const
Definition: CoroInstr.h:89
Function * getWrapperFunction() const
Definition: CoroInstr.h:93
This class represents the llvm.coro.begin instruction.
Definition: CoroInstr.h:451
AnyCoroIdInst * getId() const
Definition: CoroInstr.h:455
This represents the llvm.coro.id instruction.
Definition: CoroInstr.h:146
void setInfo(Constant *C)
Definition: CoroInstr.h:213
This represents the llvm.coro.size instruction.
Definition: CoroInstr.h:622
This represents the llvm.coro.suspend.async instruction.
Definition: CoroInstr.h:556
CoroAsyncResumeInst * getResumeFunction() const
Definition: CoroInstr.h:577
This represents the llvm.coro.suspend instruction.
Definition: CoroInstr.h:524
CoroSaveInst * getCoroSave() const
Definition: CoroInstr.h:528
DISubprogram * getSubprogram() const
Get the subprogram for this scope.
Subprogram description.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:63
This is the common base class for debug info intrinsics for variables.
Record of a variable value-assignment, aka a non instruction representation of the dbg....
A debug info location.
Definition: DebugLoc.h:33
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:162
This class represents a freeze function that returns random concrete value if an operand is either a ...
A proxy from a FunctionAnalysisManager to an SCC.
Type * getReturnType() const
Definition: DerivedTypes.h:124
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:172
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:214
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Definition: Function.h:249
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:281
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:380
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:656
PointerType * getType() const
Global values are always pointers.
Definition: GlobalValue.h:294
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:52
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
void setInitializer(Constant *InitVal)
setInitializer - Sets the initializer for this global variable, removing any existing initializer if ...
Definition: Globals.cpp:485
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
Definition: IRBuilder.h:1790
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Definition: IRBuilder.h:2543
InvokeInst * CreateInvoke(FunctionType *Ty, Value *Callee, BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef< Value * > Args, ArrayRef< OperandBundleDef > OpBundles, const Twine &Name="")
Create an invoke instruction.
Definition: IRBuilder.h:1175
BasicBlock::iterator GetInsertPoint() const
Definition: IRBuilder.h:172
Value * CreateStructGEP(Type *Ty, Value *Ptr, unsigned Idx, const Twine &Name="")
Definition: IRBuilder.h:1989
Value * CreateConstInBoundsGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Definition: IRBuilder.h:1906
CleanupReturnInst * CreateCleanupRet(CleanupPadInst *CleanupPad, BasicBlock *UnwindBB=nullptr)
Definition: IRBuilder.h:1253
ReturnInst * CreateRet(Value *V)
Create a 'ret <val>' instruction.
Definition: IRBuilder.h:1112
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Definition: IRBuilder.h:488
Value * CreateBitOrPointerCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2225
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Definition: IRBuilder.h:2417
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2147
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Definition: IRBuilder.h:1807
LLVMContext & getContext() const
Definition: IRBuilder.h:173
ReturnInst * CreateRetVoid()
Create a 'ret void' instruction.
Definition: IRBuilder.h:1107
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Definition: IRBuilder.h:1820
ConstantInt * getFalse()
Get the constant value for i1 false.
Definition: IRBuilder.h:468
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Definition: IRBuilder.h:177
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2432
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2686
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
Definition: Cloning.h:203
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:92
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
A node in the call graph.
An SCC of the call graph.
A lazily constructed view of the call graph of a module.
void addSplitFunction(Function &OriginalFunction, Function &NewFunction)
Add a new function split/outlined from an existing function.
void addSplitRefRecursiveFunctions(Function &OriginalFunction, ArrayRef< Function * > NewFunctions)
Add new ref-recursive functions split/outlined from an existing function.
Node & get(Function &F)
Get a graph node for a given function, scanning it to populate the graph data as necessary.
SCC * lookupSCC(Node &N) const
Lookup a function's SCC in the graph.
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:606
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
FunctionListType::iterator iterator
The Function iterators.
Definition: Module.h:90
Diagnostic information for applied optimization remarks.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1852
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:111
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: Analysis.h:114
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:117
PrettyStackTraceEntry - This class is used to represent a frame of the "pretty" stack trace that is d...
virtual void print(raw_ostream &OS) const =0
print - Emit information about this stack frame to OS.
Return a value (possibly void), from a function.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:502
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void reserve(size_type N)
Definition: SmallVector.h:676
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Type * getTypeAtIndex(const Value *V) const
Given an index value into the type, return the type of the element.
Definition: Type.cpp:600
Analysis pass providing the TargetTransformInfo.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
bool supportsTailCallFor(const CallBase *CB) const
If target supports tail call on CB.
Value handle that tracks a Value across RAUW.
Definition: ValueHandle.h:331
ValueTy * getValPtr() const
Definition: ValueHandle.h:335
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
bool isArch64Bit() const
Test whether the architecture is 64-bit.
Definition: Triple.cpp:1651
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt8Ty(LLVMContext &C)
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1833
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
void setOperand(unsigned i, Value *Val)
Definition: User.h:174
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:534
iterator_range< user_iterator > users()
Definition: Value.h:421
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:694
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1075
iterator_range< use_iterator > uses()
Definition: Value.h:376
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:383
const ParentTy * getParent() const
Definition: ilist_node.h:32
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:353
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
@ Entry
Definition: COFF.h:826
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
void salvageDebugInfo(SmallDenseMap< Argument *, AllocaInst *, 4 > &ArgToAllocaMap, DbgVariableIntrinsic &DVI, bool IsEntryPoint)
Attempts to rewrite the location operand of debug intrinsics in terms of the coroutine frame pointer,...
Definition: CoroFrame.cpp:2930
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
CallInst * createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, TargetTransformInfo &TTI, ArrayRef< Value * > Arguments, IRBuilder<> &)
Definition: CoroSplit.cpp:1656
void replaceCoroFree(CoroIdInst *CoroId, bool Elide)
Definition: Coroutines.cpp:128
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
UnaryFunction for_each(R &&Range, UnaryFunction F)
Provide wrappers to std::for_each which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1715
bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
Definition: Verifier.cpp:7137
void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
Definition: Utils.cpp:1678
LazyCallGraph::SCC & updateCGAndAnalysisManagerForFunctionPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a function pass.
LazyCallGraph::SCC & updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a CGSCC pass.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:656
@ Async
"Asynchronous" unwind tables (instr precise)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
Definition: Local.cpp:2837
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
DWARFExpression::Operation Op
InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)
This function inlines the called function into the basic block of the caller.
void CloneFunctionInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, CloneFunctionChangeType Changes, SmallVectorImpl< ReturnInst * > &Returns, const char *NameSuffix="", ClonedCodeInfo *CodeInfo=nullptr, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Clone OldFunc into NewFunc, transforming the old arguments into references to VMap values.
auto predecessors(const MachineBasicBlock *BB)
static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)
Filter the DbgRecord range to DbgVariableRecord types only and downcast.
bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Remove all blocks that can not be reached from the function's entry.
Definition: Local.cpp:3202
bool isPotentiallyReachable(const Instruction *From, const Instruction *To, const SmallPtrSetImpl< BasicBlock * > *ExclusionSet=nullptr, const DominatorTree *DT=nullptr, const LoopInfo *LI=nullptr)
Determine whether instruction 'To' is reachable from 'From', without passing through any blocks in Ex...
Definition: CFG.cpp:281
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
SmallPriorityWorklist< LazyCallGraph::SCC *, 1 > & CWorklist
Worklist of the SCCs queued for processing.
const std::function< bool(Instruction &)> MaterializableCallback
Definition: CoroSplit.h:25
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)
Definition: CoroSplit.cpp:2080
CoroSplitPass(bool OptimizeFrame=false)
Definition: CoroSplit.cpp:2076
CallInst * makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt)
Definition: Coroutines.cpp:50
SmallVector< CallInst *, 2 > SymmetricTransfers
Definition: CoroInternal.h:88
SmallVector< CoroAwaitSuspendInst *, 4 > CoroAwaitSuspends
Definition: CoroInternal.h:87
AsyncLoweringStorage AsyncLowering
Definition: CoroInternal.h:151
FunctionType * getResumeFunctionType() const
Definition: CoroInternal.h:190
IntegerType * getIndexType() const
Definition: CoroInternal.h:175
StructType * FrameTy
Definition: CoroInternal.h:107
CoroIdInst * getSwitchCoroId() const
Definition: CoroInternal.h:154
SmallVector< CoroSizeInst *, 2 > CoroSizes
Definition: CoroInternal.h:83
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
Definition: CoroInternal.h:85
Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const
Allocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:456
SmallVector< CallInst *, 2 > SwiftErrorOps
Definition: CoroInternal.h:86
ConstantInt * getIndex(uint64_t Value) const
Definition: CoroInternal.h:180
SwitchLoweringStorage SwitchLowering
Definition: CoroInternal.h:149
CoroBeginInst * CoroBegin
Definition: CoroInternal.h:81
BasicBlock::iterator getInsertPtAfterFramePtr() const
Definition: CoroInternal.h:249
ArrayRef< Type * > getRetconResultTypes() const
Definition: CoroInternal.h:207
void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const
Deallocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:479
RetconLoweringStorage RetconLowering
Definition: CoroInternal.h:150
SmallVector< CoroAlignInst *, 2 > CoroAligns
Definition: CoroInternal.h:84
SmallVector< AnyCoroEndInst *, 4 > CoroEnds
Definition: CoroInternal.h:82
unsigned getSwitchIndexField() const
Definition: CoroInternal.h:170