LLVM 17.0.0git
CoroSplit.cpp
Go to the documentation of this file.
1//===- CoroSplit.cpp - Converts a coroutine into a state machine ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// This pass builds the coroutine frame and outlines resume and destroy parts
9// of the coroutine into separate functions.
10//
11// We present a coroutine to an LLVM as an ordinary function with suspension
12// points marked up with intrinsics. We let the optimizer party on the coroutine
13// as a single function for as long as possible. Shortly before the coroutine is
14// eligible to be inlined into its callers, we split up the coroutine into parts
15// corresponding to an initial, resume and destroy invocations of the coroutine,
16// add them to the current SCC and restart the IPO pipeline to optimize the
17// coroutine subfunctions we extracted before proceeding to the caller of the
18// coroutine.
19//===----------------------------------------------------------------------===//
20
22#include "CoroInstr.h"
23#include "CoroInternal.h"
24#include "llvm/ADT/DenseMap.h"
28#include "llvm/ADT/StringRef.h"
29#include "llvm/ADT/Twine.h"
30#include "llvm/Analysis/CFG.h"
36#include "llvm/IR/Argument.h"
37#include "llvm/IR/Attributes.h"
38#include "llvm/IR/BasicBlock.h"
39#include "llvm/IR/CFG.h"
40#include "llvm/IR/CallingConv.h"
41#include "llvm/IR/Constants.h"
42#include "llvm/IR/DataLayout.h"
44#include "llvm/IR/Dominators.h"
45#include "llvm/IR/Function.h"
46#include "llvm/IR/GlobalValue.h"
48#include "llvm/IR/IRBuilder.h"
50#include "llvm/IR/InstrTypes.h"
51#include "llvm/IR/Instruction.h"
54#include "llvm/IR/LLVMContext.h"
55#include "llvm/IR/Module.h"
56#include "llvm/IR/Type.h"
57#include "llvm/IR/Value.h"
58#include "llvm/IR/Verifier.h"
60#include "llvm/Support/Debug.h"
69#include <cassert>
70#include <cstddef>
71#include <cstdint>
72#include <initializer_list>
73#include <iterator>
74
75using namespace llvm;
76
77#define DEBUG_TYPE "coro-split"
78
79namespace {
80
81/// A little helper class for building
82class CoroCloner {
83public:
84 enum class Kind {
85 /// The shared resume function for a switch lowering.
86 SwitchResume,
87
88 /// The shared unwind function for a switch lowering.
89 SwitchUnwind,
90
91 /// The shared cleanup function for a switch lowering.
92 SwitchCleanup,
93
94 /// An individual continuation function.
95 Continuation,
96
97 /// An async resume function.
98 Async,
99 };
100
101private:
102 Function &OrigF;
103 Function *NewF;
104 const Twine &Suffix;
105 coro::Shape &Shape;
106 Kind FKind;
109 Value *NewFramePtr = nullptr;
110
111 /// The active suspend instruction; meaningful only for continuation and async
112 /// ABIs.
113 AnyCoroSuspendInst *ActiveSuspend = nullptr;
114
115public:
116 /// Create a cloner for a switch lowering.
117 CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
118 Kind FKind)
119 : OrigF(OrigF), NewF(nullptr), Suffix(Suffix), Shape(Shape),
120 FKind(FKind), Builder(OrigF.getContext()) {
121 assert(Shape.ABI == coro::ABI::Switch);
122 }
123
124 /// Create a cloner for a continuation lowering.
125 CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
126 Function *NewF, AnyCoroSuspendInst *ActiveSuspend)
127 : OrigF(OrigF), NewF(NewF), Suffix(Suffix), Shape(Shape),
128 FKind(Shape.ABI == coro::ABI::Async ? Kind::Async : Kind::Continuation),
129 Builder(OrigF.getContext()), ActiveSuspend(ActiveSuspend) {
130 assert(Shape.ABI == coro::ABI::Retcon ||
131 Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async);
132 assert(NewF && "need existing function for continuation");
133 assert(ActiveSuspend && "need active suspend point for continuation");
134 }
135
136 Function *getFunction() const {
137 assert(NewF != nullptr && "declaration not yet set");
138 return NewF;
139 }
140
141 void create();
142
143private:
144 bool isSwitchDestroyFunction() {
145 switch (FKind) {
146 case Kind::Async:
147 case Kind::Continuation:
148 case Kind::SwitchResume:
149 return false;
150 case Kind::SwitchUnwind:
151 case Kind::SwitchCleanup:
152 return true;
153 }
154 llvm_unreachable("Unknown CoroCloner::Kind enum");
155 }
156
157 void replaceEntryBlock();
158 Value *deriveNewFramePointer();
159 void replaceRetconOrAsyncSuspendUses();
160 void replaceCoroSuspends();
161 void replaceCoroEnds();
163 void salvageDebugInfo();
164 void handleFinalSuspend();
165};
166
167} // end anonymous namespace
168
170 const coro::Shape &Shape, Value *FramePtr,
171 CallGraph *CG) {
172 assert(Shape.ABI == coro::ABI::Retcon ||
173 Shape.ABI == coro::ABI::RetconOnce);
175 return;
176
177 Shape.emitDealloc(Builder, FramePtr, CG);
178}
179
180/// Replace an llvm.coro.end.async.
181/// Will inline the must tail call function call if there is one.
182/// \returns true if cleanup of the coro.end block is needed, false otherwise.
184 IRBuilder<> Builder(End);
185
186 auto *EndAsync = dyn_cast<CoroAsyncEndInst>(End);
187 if (!EndAsync) {
188 Builder.CreateRetVoid();
189 return true /*needs cleanup of coro.end block*/;
190 }
191
192 auto *MustTailCallFunc = EndAsync->getMustTailCallFunction();
193 if (!MustTailCallFunc) {
194 Builder.CreateRetVoid();
195 return true /*needs cleanup of coro.end block*/;
196 }
197
198 // Move the must tail call from the predecessor block into the end block.
199 auto *CoroEndBlock = End->getParent();
200 auto *MustTailCallFuncBlock = CoroEndBlock->getSinglePredecessor();
201 assert(MustTailCallFuncBlock && "Must have a single predecessor block");
202 auto It = MustTailCallFuncBlock->getTerminator()->getIterator();
203 auto *MustTailCall = cast<CallInst>(&*std::prev(It));
204 CoroEndBlock->splice(End->getIterator(), MustTailCallFuncBlock,
205 MustTailCall->getIterator());
206
207 // Insert the return instruction.
208 Builder.SetInsertPoint(End);
209 Builder.CreateRetVoid();
210 InlineFunctionInfo FnInfo;
211
212 // Remove the rest of the block, by splitting it into an unreachable block.
213 auto *BB = End->getParent();
214 BB->splitBasicBlock(End);
215 BB->getTerminator()->eraseFromParent();
216
217 auto InlineRes = InlineFunction(*MustTailCall, FnInfo);
218 assert(InlineRes.isSuccess() && "Expected inlining to succeed");
219 (void)InlineRes;
220
221 // We have cleaned up the coro.end block above.
222 return false;
223}
224
225/// Replace a non-unwind call to llvm.coro.end.
227 const coro::Shape &Shape, Value *FramePtr,
228 bool InResume, CallGraph *CG) {
229 // Start inserting right before the coro.end.
230 IRBuilder<> Builder(End);
231
232 // Create the return instruction.
233 switch (Shape.ABI) {
234 // The cloned functions in switch-lowering always return void.
235 case coro::ABI::Switch:
236 // coro.end doesn't immediately end the coroutine in the main function
237 // in this lowering, because we need to deallocate the coroutine.
238 if (!InResume)
239 return;
240 Builder.CreateRetVoid();
241 break;
242
243 // In async lowering this returns.
244 case coro::ABI::Async: {
245 bool CoroEndBlockNeedsCleanup = replaceCoroEndAsync(End);
246 if (!CoroEndBlockNeedsCleanup)
247 return;
248 break;
249 }
250
251 // In unique continuation lowering, the continuations always return void.
252 // But we may have implicitly allocated storage.
253 case coro::ABI::RetconOnce:
255 Builder.CreateRetVoid();
256 break;
257
258 // In non-unique continuation lowering, we signal completion by returning
259 // a null continuation.
260 case coro::ABI::Retcon: {
262 auto RetTy = Shape.getResumeFunctionType()->getReturnType();
263 auto RetStructTy = dyn_cast<StructType>(RetTy);
264 PointerType *ContinuationTy =
265 cast<PointerType>(RetStructTy ? RetStructTy->getElementType(0) : RetTy);
266
267 Value *ReturnValue = ConstantPointerNull::get(ContinuationTy);
268 if (RetStructTy) {
269 ReturnValue = Builder.CreateInsertValue(UndefValue::get(RetStructTy),
270 ReturnValue, 0);
271 }
272 Builder.CreateRet(ReturnValue);
273 break;
274 }
275 }
276
277 // Remove the rest of the block, by splitting it into an unreachable block.
278 auto *BB = End->getParent();
279 BB->splitBasicBlock(End);
280 BB->getTerminator()->eraseFromParent();
281}
282
283// Mark a coroutine as done, which implies that the coroutine is finished and
284// never get resumed.
285//
286// In resume-switched ABI, the done state is represented by storing zero in
287// ResumeFnAddr.
288//
289// NOTE: We couldn't omit the argument `FramePtr`. It is necessary because the
290// pointer to the frame in splitted function is not stored in `Shape`.
291static void markCoroutineAsDone(IRBuilder<> &Builder, const coro::Shape &Shape,
292 Value *FramePtr) {
293 assert(
294 Shape.ABI == coro::ABI::Switch &&
295 "markCoroutineAsDone is only supported for Switch-Resumed ABI for now.");
296 auto *GepIndex = Builder.CreateStructGEP(
298 "ResumeFn.addr");
299 auto *NullPtr = ConstantPointerNull::get(cast<PointerType>(
301 Builder.CreateStore(NullPtr, GepIndex);
302}
303
304/// Replace an unwind call to llvm.coro.end.
305static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape,
306 Value *FramePtr, bool InResume,
307 CallGraph *CG) {
308 IRBuilder<> Builder(End);
309
310 switch (Shape.ABI) {
311 // In switch-lowering, this does nothing in the main function.
312 case coro::ABI::Switch: {
313 // In C++'s specification, the coroutine should be marked as done
314 // if promise.unhandled_exception() throws. The frontend will
315 // call coro.end(true) along this path.
316 //
317 // FIXME: We should refactor this once there is other language
318 // which uses Switch-Resumed style other than C++.
320 if (!InResume)
321 return;
322 break;
323 }
324 // In async lowering this does nothing.
325 case coro::ABI::Async:
326 break;
327 // In continuation-lowering, this frees the continuation storage.
328 case coro::ABI::Retcon:
329 case coro::ABI::RetconOnce:
331 break;
332 }
333
334 // If coro.end has an associated bundle, add cleanupret instruction.
335 if (auto Bundle = End->getOperandBundle(LLVMContext::OB_funclet)) {
336 auto *FromPad = cast<CleanupPadInst>(Bundle->Inputs[0]);
337 auto *CleanupRet = Builder.CreateCleanupRet(FromPad, nullptr);
338 End->getParent()->splitBasicBlock(End);
339 CleanupRet->getParent()->getTerminator()->eraseFromParent();
340 }
341}
342
343static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape,
344 Value *FramePtr, bool InResume, CallGraph *CG) {
345 if (End->isUnwind())
346 replaceUnwindCoroEnd(End, Shape, FramePtr, InResume, CG);
347 else
348 replaceFallthroughCoroEnd(End, Shape, FramePtr, InResume, CG);
349
350 auto &Context = End->getContext();
351 End->replaceAllUsesWith(InResume ? ConstantInt::getTrue(Context)
353 End->eraseFromParent();
354}
355
356// Create an entry block for a resume function with a switch that will jump to
357// suspend points.
359 assert(Shape.ABI == coro::ABI::Switch);
360 LLVMContext &C = F.getContext();
361
362 // resume.entry:
363 // %index.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0,
364 // i32 2
365 // % index = load i32, i32* %index.addr
366 // switch i32 %index, label %unreachable [
367 // i32 0, label %resume.0
368 // i32 1, label %resume.1
369 // ...
370 // ]
371
372 auto *NewEntry = BasicBlock::Create(C, "resume.entry", &F);
373 auto *UnreachBB = BasicBlock::Create(C, "unreachable", &F);
374
375 IRBuilder<> Builder(NewEntry);
376 auto *FramePtr = Shape.FramePtr;
377 auto *FrameTy = Shape.FrameTy;
378 auto *GepIndex = Builder.CreateStructGEP(
379 FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
380 auto *Index = Builder.CreateLoad(Shape.getIndexType(), GepIndex, "index");
381 auto *Switch =
382 Builder.CreateSwitch(Index, UnreachBB, Shape.CoroSuspends.size());
383 Shape.SwitchLowering.ResumeSwitch = Switch;
384
385 size_t SuspendIndex = 0;
386 for (auto *AnyS : Shape.CoroSuspends) {
387 auto *S = cast<CoroSuspendInst>(AnyS);
388 ConstantInt *IndexVal = Shape.getIndex(SuspendIndex);
389
390 // Replace CoroSave with a store to Index:
391 // %index.addr = getelementptr %f.frame... (index field number)
392 // store i32 %IndexVal, i32* %index.addr1
393 auto *Save = S->getCoroSave();
394 Builder.SetInsertPoint(Save);
395 if (S->isFinal()) {
396 // The coroutine should be marked done if it reaches the final suspend
397 // point.
399 }
400
401 // If the coroutine don't have unwind coro end, we could omit the store to
402 // the final suspend point since we could infer the coroutine is suspended
403 // at the final suspend point by the nullness of ResumeFnAddr.
404 // However, we can't skip it if the coroutine have unwind coro end. Since
405 // the coroutine reaches unwind coro end is considered suspended at the
406 // final suspend point (the ResumeFnAddr is null) but in fact the coroutine
407 // didn't complete yet. We need the IndexVal for the final suspend point
408 // to make the states clear.
409 if (!S->isFinal() || Shape.SwitchLowering.HasUnwindCoroEnd) {
410 auto *GepIndex = Builder.CreateStructGEP(
411 FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
412 Builder.CreateStore(IndexVal, GepIndex);
413 }
414
415 Save->replaceAllUsesWith(ConstantTokenNone::get(C));
416 Save->eraseFromParent();
417
418 // Split block before and after coro.suspend and add a jump from an entry
419 // switch:
420 //
421 // whateverBB:
422 // whatever
423 // %0 = call i8 @llvm.coro.suspend(token none, i1 false)
424 // switch i8 %0, label %suspend[i8 0, label %resume
425 // i8 1, label %cleanup]
426 // becomes:
427 //
428 // whateverBB:
429 // whatever
430 // br label %resume.0.landing
431 //
432 // resume.0: ; <--- jump from the switch in the resume.entry
433 // %0 = tail call i8 @llvm.coro.suspend(token none, i1 false)
434 // br label %resume.0.landing
435 //
436 // resume.0.landing:
437 // %1 = phi i8[-1, %whateverBB], [%0, %resume.0]
438 // switch i8 % 1, label %suspend [i8 0, label %resume
439 // i8 1, label %cleanup]
440
441 auto *SuspendBB = S->getParent();
442 auto *ResumeBB =
443 SuspendBB->splitBasicBlock(S, "resume." + Twine(SuspendIndex));
444 auto *LandingBB = ResumeBB->splitBasicBlock(
445 S->getNextNode(), ResumeBB->getName() + Twine(".landing"));
446 Switch->addCase(IndexVal, ResumeBB);
447
448 cast<BranchInst>(SuspendBB->getTerminator())->setSuccessor(0, LandingBB);
449 auto *PN = PHINode::Create(Builder.getInt8Ty(), 2, "", &LandingBB->front());
450 S->replaceAllUsesWith(PN);
451 PN->addIncoming(Builder.getInt8(-1), SuspendBB);
452 PN->addIncoming(S, ResumeBB);
453
454 ++SuspendIndex;
455 }
456
457 Builder.SetInsertPoint(UnreachBB);
458 Builder.CreateUnreachable();
459
460 Shape.SwitchLowering.ResumeEntryBlock = NewEntry;
461}
462
463// In the resume function, we remove the last case (when coro::Shape is built,
464// the final suspend point (if present) is always the last element of
465// CoroSuspends array) since it is an undefined behavior to resume a coroutine
466// suspended at the final suspend point.
467// In the destroy function, if it isn't possible that the ResumeFnAddr is NULL
468// and the coroutine doesn't suspend at the final suspend point actually (this
469// is possible since the coroutine is considered suspended at the final suspend
470// point if promise.unhandled_exception() exits via an exception), we can
471// remove the last case.
472void CoroCloner::handleFinalSuspend() {
473 assert(Shape.ABI == coro::ABI::Switch &&
474 Shape.SwitchLowering.HasFinalSuspend);
475
476 if (isSwitchDestroyFunction() && Shape.SwitchLowering.HasUnwindCoroEnd)
477 return;
478
479 auto *Switch = cast<SwitchInst>(VMap[Shape.SwitchLowering.ResumeSwitch]);
480 auto FinalCaseIt = std::prev(Switch->case_end());
481 BasicBlock *ResumeBB = FinalCaseIt->getCaseSuccessor();
482 Switch->removeCase(FinalCaseIt);
483 if (isSwitchDestroyFunction()) {
484 BasicBlock *OldSwitchBB = Switch->getParent();
485 auto *NewSwitchBB = OldSwitchBB->splitBasicBlock(Switch, "Switch");
486 Builder.SetInsertPoint(OldSwitchBB->getTerminator());
487 auto *GepIndex = Builder.CreateStructGEP(Shape.FrameTy, NewFramePtr,
489 "ResumeFn.addr");
490 auto *Load = Builder.CreateLoad(Shape.getSwitchResumePointerType(),
491 GepIndex);
492 auto *Cond = Builder.CreateIsNull(Load);
493 Builder.CreateCondBr(Cond, ResumeBB, NewSwitchBB);
494 OldSwitchBB->getTerminator()->eraseFromParent();
495 }
496}
497
498static FunctionType *
500 auto *AsyncSuspend = cast<CoroSuspendAsyncInst>(Suspend);
501 auto *StructTy = cast<StructType>(AsyncSuspend->getType());
502 auto &Context = Suspend->getParent()->getParent()->getContext();
503 auto *VoidTy = Type::getVoidTy(Context);
504 return FunctionType::get(VoidTy, StructTy->elements(), false);
505}
506
508 const Twine &Suffix,
509 Module::iterator InsertBefore,
510 AnyCoroSuspendInst *ActiveSuspend) {
511 Module *M = OrigF.getParent();
512 auto *FnTy = (Shape.ABI != coro::ABI::Async)
513 ? Shape.getResumeFunctionType()
514 : getFunctionTypeFromAsyncSuspend(ActiveSuspend);
515
516 Function *NewF =
517 Function::Create(FnTy, GlobalValue::LinkageTypes::InternalLinkage,
518 OrigF.getName() + Suffix);
519
520 M->getFunctionList().insert(InsertBefore, NewF);
521
522 return NewF;
523}
524
525/// Replace uses of the active llvm.coro.suspend.retcon/async call with the
526/// arguments to the continuation function.
527///
528/// This assumes that the builder has a meaningful insertion point.
529void CoroCloner::replaceRetconOrAsyncSuspendUses() {
530 assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce ||
531 Shape.ABI == coro::ABI::Async);
532
533 auto NewS = VMap[ActiveSuspend];
534 if (NewS->use_empty()) return;
535
536 // Copy out all the continuation arguments after the buffer pointer into
537 // an easily-indexed data structure for convenience.
539 // The async ABI includes all arguments -- including the first argument.
540 bool IsAsyncABI = Shape.ABI == coro::ABI::Async;
541 for (auto I = IsAsyncABI ? NewF->arg_begin() : std::next(NewF->arg_begin()),
542 E = NewF->arg_end();
543 I != E; ++I)
544 Args.push_back(&*I);
545
546 // If the suspend returns a single scalar value, we can just do a simple
547 // replacement.
548 if (!isa<StructType>(NewS->getType())) {
549 assert(Args.size() == 1);
550 NewS->replaceAllUsesWith(Args.front());
551 return;
552 }
553
554 // Try to peephole extracts of an aggregate return.
555 for (Use &U : llvm::make_early_inc_range(NewS->uses())) {
556 auto *EVI = dyn_cast<ExtractValueInst>(U.getUser());
557 if (!EVI || EVI->getNumIndices() != 1)
558 continue;
559
560 EVI->replaceAllUsesWith(Args[EVI->getIndices().front()]);
561 EVI->eraseFromParent();
562 }
563
564 // If we have no remaining uses, we're done.
565 if (NewS->use_empty()) return;
566
567 // Otherwise, we need to create an aggregate.
568 Value *Agg = UndefValue::get(NewS->getType());
569 for (size_t I = 0, E = Args.size(); I != E; ++I)
570 Agg = Builder.CreateInsertValue(Agg, Args[I], I);
571
572 NewS->replaceAllUsesWith(Agg);
573}
574
575void CoroCloner::replaceCoroSuspends() {
576 Value *SuspendResult;
577
578 switch (Shape.ABI) {
579 // In switch lowering, replace coro.suspend with the appropriate value
580 // for the type of function we're extracting.
581 // Replacing coro.suspend with (0) will result in control flow proceeding to
582 // a resume label associated with a suspend point, replacing it with (1) will
583 // result in control flow proceeding to a cleanup label associated with this
584 // suspend point.
585 case coro::ABI::Switch:
586 SuspendResult = Builder.getInt8(isSwitchDestroyFunction() ? 1 : 0);
587 break;
588
589 // In async lowering there are no uses of the result.
590 case coro::ABI::Async:
591 return;
592
593 // In returned-continuation lowering, the arguments from earlier
594 // continuations are theoretically arbitrary, and they should have been
595 // spilled.
596 case coro::ABI::RetconOnce:
597 case coro::ABI::Retcon:
598 return;
599 }
600
601 for (AnyCoroSuspendInst *CS : Shape.CoroSuspends) {
602 // The active suspend was handled earlier.
603 if (CS == ActiveSuspend) continue;
604
605 auto *MappedCS = cast<AnyCoroSuspendInst>(VMap[CS]);
606 MappedCS->replaceAllUsesWith(SuspendResult);
607 MappedCS->eraseFromParent();
608 }
609}
610
611void CoroCloner::replaceCoroEnds() {
612 for (AnyCoroEndInst *CE : Shape.CoroEnds) {
613 // We use a null call graph because there's no call graph node for
614 // the cloned function yet. We'll just be rebuilding that later.
615 auto *NewCE = cast<AnyCoroEndInst>(VMap[CE]);
616 replaceCoroEnd(NewCE, Shape, NewFramePtr, /*in resume*/ true, nullptr);
617 }
618}
619
621 ValueToValueMapTy *VMap) {
622 if (Shape.ABI == coro::ABI::Async && Shape.CoroSuspends.empty())
623 return;
624 Value *CachedSlot = nullptr;
625 auto getSwiftErrorSlot = [&](Type *ValueTy) -> Value * {
626 if (CachedSlot) {
627 assert(cast<PointerType>(CachedSlot->getType())
628 ->isOpaqueOrPointeeTypeMatches(ValueTy) &&
629 "multiple swifterror slots in function with different types");
630 return CachedSlot;
631 }
632
633 // Check if the function has a swifterror argument.
634 for (auto &Arg : F.args()) {
635 if (Arg.isSwiftError()) {
636 CachedSlot = &Arg;
637 assert(cast<PointerType>(Arg.getType())
638 ->isOpaqueOrPointeeTypeMatches(ValueTy) &&
639 "swifterror argument does not have expected type");
640 return &Arg;
641 }
642 }
643
644 // Create a swifterror alloca.
645 IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHIOrDbg());
646 auto Alloca = Builder.CreateAlloca(ValueTy);
647 Alloca->setSwiftError(true);
648
649 CachedSlot = Alloca;
650 return Alloca;
651 };
652
653 for (CallInst *Op : Shape.SwiftErrorOps) {
654 auto MappedOp = VMap ? cast<CallInst>((*VMap)[Op]) : Op;
655 IRBuilder<> Builder(MappedOp);
656
657 // If there are no arguments, this is a 'get' operation.
658 Value *MappedResult;
659 if (Op->arg_empty()) {
660 auto ValueTy = Op->getType();
661 auto Slot = getSwiftErrorSlot(ValueTy);
662 MappedResult = Builder.CreateLoad(ValueTy, Slot);
663 } else {
664 assert(Op->arg_size() == 1);
665 auto Value = MappedOp->getArgOperand(0);
666 auto ValueTy = Value->getType();
667 auto Slot = getSwiftErrorSlot(ValueTy);
668 Builder.CreateStore(Value, Slot);
669 MappedResult = Slot;
670 }
671
672 MappedOp->replaceAllUsesWith(MappedResult);
673 MappedOp->eraseFromParent();
674 }
675
676 // If we're updating the original function, we've invalidated SwiftErrorOps.
677 if (VMap == nullptr) {
678 Shape.SwiftErrorOps.clear();
679 }
680}
681
682void CoroCloner::replaceSwiftErrorOps() {
683 ::replaceSwiftErrorOps(*NewF, Shape, &VMap);
684}
685
686void CoroCloner::salvageDebugInfo() {
689 for (auto &BB : *NewF)
690 for (auto &I : BB)
691 if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I))
692 Worklist.push_back(DVI);
693 for (DbgVariableIntrinsic *DVI : Worklist)
694 coro::salvageDebugInfo(DbgPtrAllocaCache, DVI, Shape.OptimizeFrame);
695
696 // Remove all salvaged dbg.declare intrinsics that became
697 // either unreachable or stale due to the CoroSplit transformation.
698 DominatorTree DomTree(*NewF);
699 auto IsUnreachableBlock = [&](BasicBlock *BB) {
700 return !isPotentiallyReachable(&NewF->getEntryBlock(), BB, nullptr,
701 &DomTree);
702 };
703 for (DbgVariableIntrinsic *DVI : Worklist) {
704 if (IsUnreachableBlock(DVI->getParent()))
705 DVI->eraseFromParent();
706 else if (isa_and_nonnull<AllocaInst>(DVI->getVariableLocationOp(0))) {
707 // Count all non-debuginfo uses in reachable blocks.
708 unsigned Uses = 0;
709 for (auto *User : DVI->getVariableLocationOp(0)->users())
710 if (auto *I = dyn_cast<Instruction>(User))
711 if (!isa<AllocaInst>(I) && !IsUnreachableBlock(I->getParent()))
712 ++Uses;
713 if (!Uses)
714 DVI->eraseFromParent();
715 }
716 }
717}
718
719void CoroCloner::replaceEntryBlock() {
720 // In the original function, the AllocaSpillBlock is a block immediately
721 // following the allocation of the frame object which defines GEPs for
722 // all the allocas that have been moved into the frame, and it ends by
723 // branching to the original beginning of the coroutine. Make this
724 // the entry block of the cloned function.
725 auto *Entry = cast<BasicBlock>(VMap[Shape.AllocaSpillBlock]);
726 auto *OldEntry = &NewF->getEntryBlock();
727 Entry->setName("entry" + Suffix);
728 Entry->moveBefore(OldEntry);
729 Entry->getTerminator()->eraseFromParent();
730
731 // Clear all predecessors of the new entry block. There should be
732 // exactly one predecessor, which we created when splitting out
733 // AllocaSpillBlock to begin with.
734 assert(Entry->hasOneUse());
735 auto BranchToEntry = cast<BranchInst>(Entry->user_back());
736 assert(BranchToEntry->isUnconditional());
737 Builder.SetInsertPoint(BranchToEntry);
738 Builder.CreateUnreachable();
739 BranchToEntry->eraseFromParent();
740
741 // Branch from the entry to the appropriate place.
742 Builder.SetInsertPoint(Entry);
743 switch (Shape.ABI) {
744 case coro::ABI::Switch: {
745 // In switch-lowering, we built a resume-entry block in the original
746 // function. Make the entry block branch to this.
747 auto *SwitchBB =
748 cast<BasicBlock>(VMap[Shape.SwitchLowering.ResumeEntryBlock]);
749 Builder.CreateBr(SwitchBB);
750 break;
751 }
752 case coro::ABI::Async:
753 case coro::ABI::Retcon:
754 case coro::ABI::RetconOnce: {
755 // In continuation ABIs, we want to branch to immediately after the
756 // active suspend point. Earlier phases will have put the suspend in its
757 // own basic block, so just thread our jump directly to its successor.
758 assert((Shape.ABI == coro::ABI::Async &&
759 isa<CoroSuspendAsyncInst>(ActiveSuspend)) ||
760 ((Shape.ABI == coro::ABI::Retcon ||
761 Shape.ABI == coro::ABI::RetconOnce) &&
762 isa<CoroSuspendRetconInst>(ActiveSuspend)));
763 auto *MappedCS = cast<AnyCoroSuspendInst>(VMap[ActiveSuspend]);
764 auto Branch = cast<BranchInst>(MappedCS->getNextNode());
765 assert(Branch->isUnconditional());
766 Builder.CreateBr(Branch->getSuccessor(0));
767 break;
768 }
769 }
770
771 // Any static alloca that's still being used but not reachable from the new
772 // entry needs to be moved to the new entry.
773 Function *F = OldEntry->getParent();
774 DominatorTree DT{*F};
776 auto *Alloca = dyn_cast<AllocaInst>(&I);
777 if (!Alloca || I.use_empty())
778 continue;
779 if (DT.isReachableFromEntry(I.getParent()) ||
780 !isa<ConstantInt>(Alloca->getArraySize()))
781 continue;
782 I.moveBefore(*Entry, Entry->getFirstInsertionPt());
783 }
784}
785
786/// Derive the value of the new frame pointer.
787Value *CoroCloner::deriveNewFramePointer() {
788 // Builder should be inserting to the front of the new entry block.
789
790 switch (Shape.ABI) {
791 // In switch-lowering, the argument is the frame pointer.
792 case coro::ABI::Switch:
793 return &*NewF->arg_begin();
794 // In async-lowering, one of the arguments is an async context as determined
795 // by the `llvm.coro.id.async` intrinsic. We can retrieve the async context of
796 // the resume function from the async context projection function associated
797 // with the active suspend. The frame is located as a tail to the async
798 // context header.
799 case coro::ABI::Async: {
800 auto *ActiveAsyncSuspend = cast<CoroSuspendAsyncInst>(ActiveSuspend);
801 auto ContextIdx = ActiveAsyncSuspend->getStorageArgumentIndex() & 0xff;
802 auto *CalleeContext = NewF->getArg(ContextIdx);
803 auto *FramePtrTy = Shape.FrameTy->getPointerTo();
804 auto *ProjectionFunc =
805 ActiveAsyncSuspend->getAsyncContextProjectionFunction();
806 auto DbgLoc =
807 cast<CoroSuspendAsyncInst>(VMap[ActiveSuspend])->getDebugLoc();
808 // Calling i8* (i8*)
809 auto *CallerContext = Builder.CreateCall(ProjectionFunc->getFunctionType(),
810 ProjectionFunc, CalleeContext);
811 CallerContext->setCallingConv(ProjectionFunc->getCallingConv());
812 CallerContext->setDebugLoc(DbgLoc);
813 // The frame is located after the async_context header.
814 auto &Context = Builder.getContext();
815 auto *FramePtrAddr = Builder.CreateConstInBoundsGEP1_32(
816 Type::getInt8Ty(Context), CallerContext,
817 Shape.AsyncLowering.FrameOffset, "async.ctx.frameptr");
818 // Inline the projection function.
820 auto InlineRes = InlineFunction(*CallerContext, InlineInfo);
821 assert(InlineRes.isSuccess());
822 (void)InlineRes;
823 return Builder.CreateBitCast(FramePtrAddr, FramePtrTy);
824 }
825 // In continuation-lowering, the argument is the opaque storage.
826 case coro::ABI::Retcon:
827 case coro::ABI::RetconOnce: {
828 Argument *NewStorage = &*NewF->arg_begin();
829 auto FramePtrTy = Shape.FrameTy->getPointerTo();
830
831 // If the storage is inline, just bitcast to the storage to the frame type.
832 if (Shape.RetconLowering.IsFrameInlineInStorage)
833 return Builder.CreateBitCast(NewStorage, FramePtrTy);
834
835 // Otherwise, load the real frame from the opaque storage.
836 auto FramePtrPtr =
837 Builder.CreateBitCast(NewStorage, FramePtrTy->getPointerTo());
838 return Builder.CreateLoad(FramePtrTy, FramePtrPtr);
839 }
840 }
841 llvm_unreachable("bad ABI");
842}
843
844static void addFramePointerAttrs(AttributeList &Attrs, LLVMContext &Context,
845 unsigned ParamIndex, uint64_t Size,
846 Align Alignment, bool NoAlias) {
847 AttrBuilder ParamAttrs(Context);
848 ParamAttrs.addAttribute(Attribute::NonNull);
849 ParamAttrs.addAttribute(Attribute::NoUndef);
850
851 if (NoAlias)
852 ParamAttrs.addAttribute(Attribute::NoAlias);
853
854 ParamAttrs.addAlignmentAttr(Alignment);
855 ParamAttrs.addDereferenceableAttr(Size);
856 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
857}
858
859static void addAsyncContextAttrs(AttributeList &Attrs, LLVMContext &Context,
860 unsigned ParamIndex) {
861 AttrBuilder ParamAttrs(Context);
862 ParamAttrs.addAttribute(Attribute::SwiftAsync);
863 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
864}
865
866static void addSwiftSelfAttrs(AttributeList &Attrs, LLVMContext &Context,
867 unsigned ParamIndex) {
868 AttrBuilder ParamAttrs(Context);
869 ParamAttrs.addAttribute(Attribute::SwiftSelf);
870 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
871}
872
873/// Clone the body of the original function into a resume function of
874/// some sort.
875void CoroCloner::create() {
876 // Create the new function if we don't already have one.
877 if (!NewF) {
878 NewF = createCloneDeclaration(OrigF, Shape, Suffix,
879 OrigF.getParent()->end(), ActiveSuspend);
880 }
881
882 // Replace all args with dummy instructions. If an argument is the old frame
883 // pointer, the dummy will be replaced by the new frame pointer once it is
884 // computed below. Uses of all other arguments should have already been
885 // rewritten by buildCoroutineFrame() to use loads/stores on the coroutine
886 // frame.
888 for (Argument &A : OrigF.args()) {
889 DummyArgs.push_back(new FreezeInst(UndefValue::get(A.getType())));
890 VMap[&A] = DummyArgs.back();
891 }
892
894
895 // Ignore attempts to change certain attributes of the function.
896 // TODO: maybe there should be a way to suppress this during cloning?
897 auto savedVisibility = NewF->getVisibility();
898 auto savedUnnamedAddr = NewF->getUnnamedAddr();
899 auto savedDLLStorageClass = NewF->getDLLStorageClass();
900
901 // NewF's linkage (which CloneFunctionInto does *not* change) might not
902 // be compatible with the visibility of OrigF (which it *does* change),
903 // so protect against that.
904 auto savedLinkage = NewF->getLinkage();
905 NewF->setLinkage(llvm::GlobalValue::ExternalLinkage);
906
907 CloneFunctionInto(NewF, &OrigF, VMap,
908 CloneFunctionChangeType::LocalChangesOnly, Returns);
909
910 auto &Context = NewF->getContext();
911
912 // For async functions / continuations, adjust the scope line of the
913 // clone to the line number of the suspend point. However, only
914 // adjust the scope line when the files are the same. This ensures
915 // line number and file name belong together. The scope line is
916 // associated with all pre-prologue instructions. This avoids a jump
917 // in the linetable from the function declaration to the suspend point.
918 if (DISubprogram *SP = NewF->getSubprogram()) {
919 assert(SP != OrigF.getSubprogram() && SP->isDistinct());
920 if (ActiveSuspend)
921 if (auto DL = ActiveSuspend->getDebugLoc())
922 if (SP->getFile() == DL->getFile())
923 SP->setScopeLine(DL->getLine());
924 // Update the linkage name to reflect the modified symbol name. It
925 // is necessary to update the linkage name in Swift, since the
926 // mangling changes for resume functions. It might also be the
927 // right thing to do in C++, but due to a limitation in LLVM's
928 // AsmPrinter we can only do this if the function doesn't have an
929 // abstract specification, since the DWARF backend expects the
930 // abstract specification to contain the linkage name and asserts
931 // that they are identical.
932 if (!SP->getDeclaration() && SP->getUnit() &&
933 SP->getUnit()->getSourceLanguage() == dwarf::DW_LANG_Swift)
934 SP->replaceLinkageName(MDString::get(Context, NewF->getName()));
935 }
936
937 NewF->setLinkage(savedLinkage);
938 NewF->setVisibility(savedVisibility);
939 NewF->setUnnamedAddr(savedUnnamedAddr);
940 NewF->setDLLStorageClass(savedDLLStorageClass);
941 // The function sanitizer metadata needs to match the signature of the
942 // function it is being attached to. However this does not hold for split
943 // functions here. Thus remove the metadata for split functions.
944 if (Shape.ABI == coro::ABI::Switch &&
945 NewF->hasMetadata(LLVMContext::MD_func_sanitize))
946 NewF->eraseMetadata(LLVMContext::MD_func_sanitize);
947
948 // Replace the attributes of the new function:
949 auto OrigAttrs = NewF->getAttributes();
950 auto NewAttrs = AttributeList();
951
952 switch (Shape.ABI) {
953 case coro::ABI::Switch:
954 // Bootstrap attributes by copying function attributes from the
955 // original function. This should include optimization settings and so on.
956 NewAttrs = NewAttrs.addFnAttributes(
957 Context, AttrBuilder(Context, OrigAttrs.getFnAttrs()));
958
959 addFramePointerAttrs(NewAttrs, Context, 0, Shape.FrameSize,
960 Shape.FrameAlign, /*NoAlias=*/false);
961 break;
962 case coro::ABI::Async: {
963 auto *ActiveAsyncSuspend = cast<CoroSuspendAsyncInst>(ActiveSuspend);
964 if (OrigF.hasParamAttribute(Shape.AsyncLowering.ContextArgNo,
965 Attribute::SwiftAsync)) {
966 uint32_t ArgAttributeIndices =
967 ActiveAsyncSuspend->getStorageArgumentIndex();
968 auto ContextArgIndex = ArgAttributeIndices & 0xff;
969 addAsyncContextAttrs(NewAttrs, Context, ContextArgIndex);
970
971 // `swiftasync` must preceed `swiftself` so 0 is not a valid index for
972 // `swiftself`.
973 auto SwiftSelfIndex = ArgAttributeIndices >> 8;
974 if (SwiftSelfIndex)
975 addSwiftSelfAttrs(NewAttrs, Context, SwiftSelfIndex);
976 }
977
978 // Transfer the original function's attributes.
979 auto FnAttrs = OrigF.getAttributes().getFnAttrs();
980 NewAttrs = NewAttrs.addFnAttributes(Context, AttrBuilder(Context, FnAttrs));
981 break;
982 }
983 case coro::ABI::Retcon:
984 case coro::ABI::RetconOnce:
985 // If we have a continuation prototype, just use its attributes,
986 // full-stop.
987 NewAttrs = Shape.RetconLowering.ResumePrototype->getAttributes();
988
989 /// FIXME: Is it really good to add the NoAlias attribute?
990 addFramePointerAttrs(NewAttrs, Context, 0,
991 Shape.getRetconCoroId()->getStorageSize(),
992 Shape.getRetconCoroId()->getStorageAlignment(),
993 /*NoAlias=*/true);
994
995 break;
996 }
997
998 switch (Shape.ABI) {
999 // In these ABIs, the cloned functions always return 'void', and the
1000 // existing return sites are meaningless. Note that for unique
1001 // continuations, this includes the returns associated with suspends;
1002 // this is fine because we can't suspend twice.
1003 case coro::ABI::Switch:
1004 case coro::ABI::RetconOnce:
1005 // Remove old returns.
1006 for (ReturnInst *Return : Returns)
1007 changeToUnreachable(Return);
1008 break;
1009
1010 // With multi-suspend continuations, we'll already have eliminated the
1011 // original returns and inserted returns before all the suspend points,
1012 // so we want to leave any returns in place.
1013 case coro::ABI::Retcon:
1014 break;
1015 // Async lowering will insert musttail call functions at all suspend points
1016 // followed by a return.
1017 // Don't change returns to unreachable because that will trip up the verifier.
1018 // These returns should be unreachable from the clone.
1019 case coro::ABI::Async:
1020 break;
1021 }
1022
1023 NewF->setAttributes(NewAttrs);
1024 NewF->setCallingConv(Shape.getResumeFunctionCC());
1025
1026 // Set up the new entry block.
1027 replaceEntryBlock();
1028
1029 Builder.SetInsertPoint(&NewF->getEntryBlock().front());
1030 NewFramePtr = deriveNewFramePointer();
1031
1032 // Remap frame pointer.
1033 Value *OldFramePtr = VMap[Shape.FramePtr];
1034 NewFramePtr->takeName(OldFramePtr);
1035 OldFramePtr->replaceAllUsesWith(NewFramePtr);
1036
1037 // Remap vFrame pointer.
1038 auto *NewVFrame = Builder.CreateBitCast(
1039 NewFramePtr, Type::getInt8PtrTy(Builder.getContext()), "vFrame");
1040 Value *OldVFrame = cast<Value>(VMap[Shape.CoroBegin]);
1041 if (OldVFrame != NewVFrame)
1042 OldVFrame->replaceAllUsesWith(NewVFrame);
1043
1044 // All uses of the arguments should have been resolved by this point,
1045 // so we can safely remove the dummy values.
1046 for (Instruction *DummyArg : DummyArgs) {
1047 DummyArg->replaceAllUsesWith(UndefValue::get(DummyArg->getType()));
1048 DummyArg->deleteValue();
1049 }
1050
1051 switch (Shape.ABI) {
1052 case coro::ABI::Switch:
1053 // Rewrite final suspend handling as it is not done via switch (allows to
1054 // remove final case from the switch, since it is undefined behavior to
1055 // resume the coroutine suspended at the final suspend point.
1056 if (Shape.SwitchLowering.HasFinalSuspend)
1057 handleFinalSuspend();
1058 break;
1059 case coro::ABI::Async:
1060 case coro::ABI::Retcon:
1061 case coro::ABI::RetconOnce:
1062 // Replace uses of the active suspend with the corresponding
1063 // continuation-function arguments.
1064 assert(ActiveSuspend != nullptr &&
1065 "no active suspend when lowering a continuation-style coroutine");
1066 replaceRetconOrAsyncSuspendUses();
1067 break;
1068 }
1069
1070 // Handle suspends.
1071 replaceCoroSuspends();
1072
1073 // Handle swifterror.
1075
1076 // Remove coro.end intrinsics.
1077 replaceCoroEnds();
1078
1079 // Salvage debug info that points into the coroutine frame.
1081
1082 // Eliminate coro.free from the clones, replacing it with 'null' in cleanup,
1083 // to suppress deallocation code.
1084 if (Shape.ABI == coro::ABI::Switch)
1085 coro::replaceCoroFree(cast<CoroIdInst>(VMap[Shape.CoroBegin->getId()]),
1086 /*Elide=*/ FKind == CoroCloner::Kind::SwitchCleanup);
1087}
1088
1089// Create a resume clone by cloning the body of the original function, setting
1090// new entry block and replacing coro.suspend an appropriate value to force
1091// resume or cleanup pass for every suspend point.
1092static Function *createClone(Function &F, const Twine &Suffix,
1093 coro::Shape &Shape, CoroCloner::Kind FKind) {
1094 CoroCloner Cloner(F, Suffix, Shape, FKind);
1095 Cloner.create();
1096 return Cloner.getFunction();
1097}
1098
1100 assert(Shape.ABI == coro::ABI::Async);
1101
1102 auto *FuncPtrStruct = cast<ConstantStruct>(
1104 auto *OrigRelativeFunOffset = FuncPtrStruct->getOperand(0);
1105 auto *OrigContextSize = FuncPtrStruct->getOperand(1);
1106 auto *NewContextSize = ConstantInt::get(OrigContextSize->getType(),
1108 auto *NewFuncPtrStruct = ConstantStruct::get(
1109 FuncPtrStruct->getType(), OrigRelativeFunOffset, NewContextSize);
1110
1111 Shape.AsyncLowering.AsyncFuncPointer->setInitializer(NewFuncPtrStruct);
1112}
1113
1115 if (Shape.ABI == coro::ABI::Async)
1117
1118 for (CoroAlignInst *CA : Shape.CoroAligns) {
1120 ConstantInt::get(CA->getType(), Shape.FrameAlign.value()));
1121 CA->eraseFromParent();
1122 }
1123
1124 if (Shape.CoroSizes.empty())
1125 return;
1126
1127 // In the same function all coro.sizes should have the same result type.
1128 auto *SizeIntrin = Shape.CoroSizes.back();
1129 Module *M = SizeIntrin->getModule();
1130 const DataLayout &DL = M->getDataLayout();
1131 auto Size = DL.getTypeAllocSize(Shape.FrameTy);
1132 auto *SizeConstant = ConstantInt::get(SizeIntrin->getType(), Size);
1133
1134 for (CoroSizeInst *CS : Shape.CoroSizes) {
1135 CS->replaceAllUsesWith(SizeConstant);
1136 CS->eraseFromParent();
1137 }
1138}
1139
1140// Create a global constant array containing pointers to functions provided and
1141// set Info parameter of CoroBegin to point at this constant. Example:
1142//
1143// @f.resumers = internal constant [2 x void(%f.frame*)*]
1144// [void(%f.frame*)* @f.resume, void(%f.frame*)* @f.destroy]
1145// define void @f() {
1146// ...
1147// call i8* @llvm.coro.begin(i8* null, i32 0, i8* null,
1148// i8* bitcast([2 x void(%f.frame*)*] * @f.resumers to i8*))
1149//
1150// Assumes that all the functions have the same signature.
1151static void setCoroInfo(Function &F, coro::Shape &Shape,
1153 // This only works under the switch-lowering ABI because coro elision
1154 // only works on the switch-lowering ABI.
1155 assert(Shape.ABI == coro::ABI::Switch);
1156
1157 SmallVector<Constant *, 4> Args(Fns.begin(), Fns.end());
1158 assert(!Args.empty());
1159 Function *Part = *Fns.begin();
1160 Module *M = Part->getParent();
1161 auto *ArrTy = ArrayType::get(Part->getType(), Args.size());
1162
1163 auto *ConstVal = ConstantArray::get(ArrTy, Args);
1164 auto *GV = new GlobalVariable(*M, ConstVal->getType(), /*isConstant=*/true,
1165 GlobalVariable::PrivateLinkage, ConstVal,
1166 F.getName() + Twine(".resumers"));
1167
1168 // Update coro.begin instruction to refer to this constant.
1169 LLVMContext &C = F.getContext();
1171 Shape.getSwitchCoroId()->setInfo(BC);
1172}
1173
1174// Store addresses of Resume/Destroy/Cleanup functions in the coroutine frame.
1175static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn,
1176 Function *DestroyFn, Function *CleanupFn) {
1177 assert(Shape.ABI == coro::ABI::Switch);
1178
1180
1181 auto *ResumeAddr = Builder.CreateStructGEP(
1183 "resume.addr");
1184 Builder.CreateStore(ResumeFn, ResumeAddr);
1185
1186 Value *DestroyOrCleanupFn = DestroyFn;
1187
1188 CoroIdInst *CoroId = Shape.getSwitchCoroId();
1189 if (CoroAllocInst *CA = CoroId->getCoroAlloc()) {
1190 // If there is a CoroAlloc and it returns false (meaning we elide the
1191 // allocation, use CleanupFn instead of DestroyFn).
1192 DestroyOrCleanupFn = Builder.CreateSelect(CA, DestroyFn, CleanupFn);
1193 }
1194
1195 auto *DestroyAddr = Builder.CreateStructGEP(
1197 "destroy.addr");
1198 Builder.CreateStore(DestroyOrCleanupFn, DestroyAddr);
1199}
1200
1203
1204#ifndef NDEBUG
1205 // For now, we do a mandatory verification step because we don't
1206 // entirely trust this pass. Note that we don't want to add a verifier
1207 // pass to FPM below because it will also verify all the global data.
1208 if (verifyFunction(F, &errs()))
1209 report_fatal_error("Broken function");
1210#endif
1211}
1212
1213// Assuming we arrived at the block NewBlock from Prev instruction, store
1214// PHI's incoming values in the ResolvedValues map.
1215static void
1217 DenseMap<Value *, Value *> &ResolvedValues) {
1218 auto *PrevBB = Prev->getParent();
1219 for (PHINode &PN : NewBlock->phis()) {
1220 auto V = PN.getIncomingValueForBlock(PrevBB);
1221 // See if we already resolved it.
1222 auto VI = ResolvedValues.find(V);
1223 if (VI != ResolvedValues.end())
1224 V = VI->second;
1225 // Remember the value.
1226 ResolvedValues[&PN] = V;
1227 }
1228}
1229
1230// Replace a sequence of branches leading to a ret, with a clone of a ret
1231// instruction. Suspend instruction represented by a switch, track the PHI
1232// values and select the correct case successor when possible.
1234 DenseMap<Value *, Value *> ResolvedValues;
1235 BasicBlock *UnconditionalSucc = nullptr;
1236 assert(InitialInst->getModule());
1237 const DataLayout &DL = InitialInst->getModule()->getDataLayout();
1238
1239 auto GetFirstValidInstruction = [](Instruction *I) {
1240 while (I) {
1241 // BitCastInst wouldn't generate actual code so that we could skip it.
1242 if (isa<BitCastInst>(I) || I->isDebugOrPseudoInst() ||
1243 I->isLifetimeStartOrEnd())
1244 I = I->getNextNode();
1245 else if (isInstructionTriviallyDead(I))
1246 // Duing we are in the middle of the transformation, we need to erase
1247 // the dead instruction manually.
1248 I = &*I->eraseFromParent();
1249 else
1250 break;
1251 }
1252 return I;
1253 };
1254
1255 auto TryResolveConstant = [&ResolvedValues](Value *V) {
1256 auto It = ResolvedValues.find(V);
1257 if (It != ResolvedValues.end())
1258 V = It->second;
1259 return dyn_cast<ConstantInt>(V);
1260 };
1261
1262 Instruction *I = InitialInst;
1263 while (I->isTerminator() || isa<CmpInst>(I)) {
1264 if (isa<ReturnInst>(I)) {
1265 if (I != InitialInst) {
1266 // If InitialInst is an unconditional branch,
1267 // remove PHI values that come from basic block of InitialInst
1268 if (UnconditionalSucc)
1269 UnconditionalSucc->removePredecessor(InitialInst->getParent(), true);
1270 ReplaceInstWithInst(InitialInst, I->clone());
1271 }
1272 return true;
1273 }
1274 if (auto *BR = dyn_cast<BranchInst>(I)) {
1275 if (BR->isUnconditional()) {
1276 BasicBlock *Succ = BR->getSuccessor(0);
1277 if (I == InitialInst)
1278 UnconditionalSucc = Succ;
1279 scanPHIsAndUpdateValueMap(I, Succ, ResolvedValues);
1280 I = GetFirstValidInstruction(Succ->getFirstNonPHIOrDbgOrLifetime());
1281 continue;
1282 }
1283
1284 BasicBlock *BB = BR->getParent();
1285 // Handle the case the condition of the conditional branch is constant.
1286 // e.g.,
1287 //
1288 // br i1 false, label %cleanup, label %CoroEnd
1289 //
1290 // It is possible during the transformation. We could continue the
1291 // simplifying in this case.
1292 if (ConstantFoldTerminator(BB, /*DeleteDeadConditions=*/true)) {
1293 // Handle this branch in next iteration.
1294 I = BB->getTerminator();
1295 continue;
1296 }
1297 } else if (auto *CondCmp = dyn_cast<CmpInst>(I)) {
1298 // If the case number of suspended switch instruction is reduced to
1299 // 1, then it is simplified to CmpInst in llvm::ConstantFoldTerminator.
1300 auto *BR = dyn_cast<BranchInst>(
1301 GetFirstValidInstruction(CondCmp->getNextNode()));
1302 if (!BR || !BR->isConditional() || CondCmp != BR->getCondition())
1303 return false;
1304
1305 // And the comparsion looks like : %cond = icmp eq i8 %V, constant.
1306 // So we try to resolve constant for the first operand only since the
1307 // second operand should be literal constant by design.
1308 ConstantInt *Cond0 = TryResolveConstant(CondCmp->getOperand(0));
1309 auto *Cond1 = dyn_cast<ConstantInt>(CondCmp->getOperand(1));
1310 if (!Cond0 || !Cond1)
1311 return false;
1312
1313 // Both operands of the CmpInst are Constant. So that we could evaluate
1314 // it immediately to get the destination.
1315 auto *ConstResult =
1316 dyn_cast_or_null<ConstantInt>(ConstantFoldCompareInstOperands(
1317 CondCmp->getPredicate(), Cond0, Cond1, DL));
1318 if (!ConstResult)
1319 return false;
1320
1321 CondCmp->replaceAllUsesWith(ConstResult);
1322 CondCmp->eraseFromParent();
1323
1324 // Handle this branch in next iteration.
1325 I = BR;
1326 continue;
1327 } else if (auto *SI = dyn_cast<SwitchInst>(I)) {
1328 ConstantInt *Cond = TryResolveConstant(SI->getCondition());
1329 if (!Cond)
1330 return false;
1331
1332 BasicBlock *BB = SI->findCaseValue(Cond)->getCaseSuccessor();
1333 scanPHIsAndUpdateValueMap(I, BB, ResolvedValues);
1334 I = GetFirstValidInstruction(BB->getFirstNonPHIOrDbgOrLifetime());
1335 continue;
1336 }
1337
1338 return false;
1339 }
1340 return false;
1341}
1342
1343// Check whether CI obeys the rules of musttail attribute.
1344static bool shouldBeMustTail(const CallInst &CI, const Function &F) {
1345 if (CI.isInlineAsm())
1346 return false;
1347
1348 // Match prototypes and calling conventions of resume function.
1349 FunctionType *CalleeTy = CI.getFunctionType();
1350 if (!CalleeTy->getReturnType()->isVoidTy() || (CalleeTy->getNumParams() != 1))
1351 return false;
1352
1353 Type *CalleeParmTy = CalleeTy->getParamType(0);
1354 if (!CalleeParmTy->isPointerTy() ||
1355 (CalleeParmTy->getPointerAddressSpace() != 0))
1356 return false;
1357
1358 if (CI.getCallingConv() != F.getCallingConv())
1359 return false;
1360
1361 // CI should not has any ABI-impacting function attributes.
1362 static const Attribute::AttrKind ABIAttrs[] = {
1363 Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca,
1364 Attribute::Preallocated, Attribute::InReg, Attribute::Returned,
1365 Attribute::SwiftSelf, Attribute::SwiftError};
1366 AttributeList Attrs = CI.getAttributes();
1367 for (auto AK : ABIAttrs)
1368 if (Attrs.hasParamAttr(0, AK))
1369 return false;
1370
1371 return true;
1372}
1373
1374// Add musttail to any resume instructions that is immediately followed by a
1375// suspend (i.e. ret). We do this even in -O0 to support guaranteed tail call
1376// for symmetrical coroutine control transfer (C++ Coroutines TS extension).
1377// This transformation is done only in the resume part of the coroutine that has
1378// identical signature and calling convention as the coro.resume call.
1380 bool changed = false;
1381
1382 // Collect potential resume instructions.
1384 for (auto &I : instructions(F))
1385 if (auto *Call = dyn_cast<CallInst>(&I))
1386 if (shouldBeMustTail(*Call, F))
1387 Resumes.push_back(Call);
1388
1389 // Set musttail on those that are followed by a ret instruction.
1390 for (CallInst *Call : Resumes)
1391 // Skip targets which don't support tail call on the specific case.
1392 if (TTI.supportsTailCallFor(Call) &&
1393 simplifyTerminatorLeadingToRet(Call->getNextNode())) {
1394 Call->setTailCallKind(CallInst::TCK_MustTail);
1395 changed = true;
1396 }
1397
1398 if (changed)
1400}
1401
1402// Coroutine has no suspend points. Remove heap allocation for the coroutine
1403// frame if possible.
1405 auto *CoroBegin = Shape.CoroBegin;
1406 auto *CoroId = CoroBegin->getId();
1407 auto *AllocInst = CoroId->getCoroAlloc();
1408 switch (Shape.ABI) {
1409 case coro::ABI::Switch: {
1410 auto SwitchId = cast<CoroIdInst>(CoroId);
1411 coro::replaceCoroFree(SwitchId, /*Elide=*/AllocInst != nullptr);
1412 if (AllocInst) {
1413 IRBuilder<> Builder(AllocInst);
1414 auto *Frame = Builder.CreateAlloca(Shape.FrameTy);
1415 Frame->setAlignment(Shape.FrameAlign);
1416 auto *VFrame = Builder.CreateBitCast(Frame, Builder.getInt8PtrTy());
1417 AllocInst->replaceAllUsesWith(Builder.getFalse());
1418 AllocInst->eraseFromParent();
1419 CoroBegin->replaceAllUsesWith(VFrame);
1420 } else {
1421 CoroBegin->replaceAllUsesWith(CoroBegin->getMem());
1422 }
1423
1424 break;
1425 }
1426 case coro::ABI::Async:
1427 case coro::ABI::Retcon:
1428 case coro::ABI::RetconOnce:
1429 CoroBegin->replaceAllUsesWith(UndefValue::get(CoroBegin->getType()));
1430 break;
1431 }
1432
1433 CoroBegin->eraseFromParent();
1434}
1435
1436// SimplifySuspendPoint needs to check that there is no calls between
1437// coro_save and coro_suspend, since any of the calls may potentially resume
1438// the coroutine and if that is the case we cannot eliminate the suspend point.
1440 for (Instruction *I = From; I != To; I = I->getNextNode()) {
1441 // Assume that no intrinsic can resume the coroutine.
1442 if (isa<IntrinsicInst>(I))
1443 continue;
1444
1445 if (isa<CallBase>(I))
1446 return true;
1447 }
1448 return false;
1449}
1450
1451static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB) {
1454
1455 Set.insert(SaveBB);
1456 Worklist.push_back(ResDesBB);
1457
1458 // Accumulate all blocks between SaveBB and ResDesBB. Because CoroSaveIntr
1459 // returns a token consumed by suspend instruction, all blocks in between
1460 // will have to eventually hit SaveBB when going backwards from ResDesBB.
1461 while (!Worklist.empty()) {
1462 auto *BB = Worklist.pop_back_val();
1463 Set.insert(BB);
1464 for (auto *Pred : predecessors(BB))
1465 if (!Set.contains(Pred))
1466 Worklist.push_back(Pred);
1467 }
1468
1469 // SaveBB and ResDesBB are checked separately in hasCallsBetween.
1470 Set.erase(SaveBB);
1471 Set.erase(ResDesBB);
1472
1473 for (auto *BB : Set)
1474 if (hasCallsInBlockBetween(BB->getFirstNonPHI(), nullptr))
1475 return true;
1476
1477 return false;
1478}
1479
1480static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy) {
1481 auto *SaveBB = Save->getParent();
1482 auto *ResumeOrDestroyBB = ResumeOrDestroy->getParent();
1483
1484 if (SaveBB == ResumeOrDestroyBB)
1485 return hasCallsInBlockBetween(Save->getNextNode(), ResumeOrDestroy);
1486
1487 // Any calls from Save to the end of the block?
1488 if (hasCallsInBlockBetween(Save->getNextNode(), nullptr))
1489 return true;
1490
1491 // Any calls from begging of the block up to ResumeOrDestroy?
1492 if (hasCallsInBlockBetween(ResumeOrDestroyBB->getFirstNonPHI(),
1493 ResumeOrDestroy))
1494 return true;
1495
1496 // Any calls in all of the blocks between SaveBB and ResumeOrDestroyBB?
1497 if (hasCallsInBlocksBetween(SaveBB, ResumeOrDestroyBB))
1498 return true;
1499
1500 return false;
1501}
1502
1503// If a SuspendIntrin is preceded by Resume or Destroy, we can eliminate the
1504// suspend point and replace it with nornal control flow.
1506 CoroBeginInst *CoroBegin) {
1507 Instruction *Prev = Suspend->getPrevNode();
1508 if (!Prev) {
1509 auto *Pred = Suspend->getParent()->getSinglePredecessor();
1510 if (!Pred)
1511 return false;
1512 Prev = Pred->getTerminator();
1513 }
1514
1515 CallBase *CB = dyn_cast<CallBase>(Prev);
1516 if (!CB)
1517 return false;
1518
1519 auto *Callee = CB->getCalledOperand()->stripPointerCasts();
1520
1521 // See if the callsite is for resumption or destruction of the coroutine.
1522 auto *SubFn = dyn_cast<CoroSubFnInst>(Callee);
1523 if (!SubFn)
1524 return false;
1525
1526 // Does not refer to the current coroutine, we cannot do anything with it.
1527 if (SubFn->getFrame() != CoroBegin)
1528 return false;
1529
1530 // See if the transformation is safe. Specifically, see if there are any
1531 // calls in between Save and CallInstr. They can potenitally resume the
1532 // coroutine rendering this optimization unsafe.
1533 auto *Save = Suspend->getCoroSave();
1534 if (hasCallsBetween(Save, CB))
1535 return false;
1536
1537 // Replace llvm.coro.suspend with the value that results in resumption over
1538 // the resume or cleanup path.
1539 Suspend->replaceAllUsesWith(SubFn->getRawIndex());
1540 Suspend->eraseFromParent();
1541 Save->eraseFromParent();
1542
1543 // No longer need a call to coro.resume or coro.destroy.
1544 if (auto *Invoke = dyn_cast<InvokeInst>(CB)) {
1545 BranchInst::Create(Invoke->getNormalDest(), Invoke);
1546 }
1547
1548 // Grab the CalledValue from CB before erasing the CallInstr.
1549 auto *CalledValue = CB->getCalledOperand();
1550 CB->eraseFromParent();
1551
1552 // If no more users remove it. Usually it is a bitcast of SubFn.
1553 if (CalledValue != SubFn && CalledValue->user_empty())
1554 if (auto *I = dyn_cast<Instruction>(CalledValue))
1555 I->eraseFromParent();
1556
1557 // Now we are good to remove SubFn.
1558 if (SubFn->user_empty())
1559 SubFn->eraseFromParent();
1560
1561 return true;
1562}
1563
1564// Remove suspend points that are simplified.
1566 // Currently, the only simplification we do is switch-lowering-specific.
1567 if (Shape.ABI != coro::ABI::Switch)
1568 return;
1569
1570 auto &S = Shape.CoroSuspends;
1571 size_t I = 0, N = S.size();
1572 if (N == 0)
1573 return;
1574
1575 size_t ChangedFinalIndex = std::numeric_limits<size_t>::max();
1576 while (true) {
1577 auto SI = cast<CoroSuspendInst>(S[I]);
1578 // Leave final.suspend to handleFinalSuspend since it is undefined behavior
1579 // to resume a coroutine suspended at the final suspend point.
1580 if (!SI->isFinal() && simplifySuspendPoint(SI, Shape.CoroBegin)) {
1581 if (--N == I)
1582 break;
1583
1584 std::swap(S[I], S[N]);
1585
1586 if (cast<CoroSuspendInst>(S[I])->isFinal()) {
1588 ChangedFinalIndex = I;
1589 }
1590
1591 continue;
1592 }
1593 if (++I == N)
1594 break;
1595 }
1596 S.resize(N);
1597
1598 // Maintain final.suspend in case final suspend was swapped.
1599 // Due to we requrie the final suspend to be the last element of CoroSuspends.
1600 if (ChangedFinalIndex < N) {
1601 assert(cast<CoroSuspendInst>(S[ChangedFinalIndex])->isFinal());
1602 std::swap(S[ChangedFinalIndex], S.back());
1603 }
1604}
1605
1609 assert(Shape.ABI == coro::ABI::Switch);
1610
1611 createResumeEntryBlock(F, Shape);
1612 auto ResumeClone = createClone(F, ".resume", Shape,
1613 CoroCloner::Kind::SwitchResume);
1614 auto DestroyClone = createClone(F, ".destroy", Shape,
1615 CoroCloner::Kind::SwitchUnwind);
1616 auto CleanupClone = createClone(F, ".cleanup", Shape,
1617 CoroCloner::Kind::SwitchCleanup);
1618
1619 postSplitCleanup(*ResumeClone);
1620 postSplitCleanup(*DestroyClone);
1621 postSplitCleanup(*CleanupClone);
1622
1623 // Adding musttail call to support symmetric transfer.
1624 // Skip targets which don't support tail call.
1625 //
1626 // FIXME: Could we support symmetric transfer effectively without musttail
1627 // call?
1628 if (TTI.supportsTailCalls())
1629 addMustTailToCoroResumes(*ResumeClone, TTI);
1630
1631 // Store addresses resume/destroy/cleanup functions in the coroutine frame.
1632 updateCoroFrame(Shape, ResumeClone, DestroyClone, CleanupClone);
1633
1634 assert(Clones.empty());
1635 Clones.push_back(ResumeClone);
1636 Clones.push_back(DestroyClone);
1637 Clones.push_back(CleanupClone);
1638
1639 // Create a constant array referring to resume/destroy/clone functions pointed
1640 // by the last argument of @llvm.coro.info, so that CoroElide pass can
1641 // determined correct function to call.
1642 setCoroInfo(F, Shape, Clones);
1643}
1644
1646 Value *Continuation) {
1647 auto *ResumeIntrinsic = Suspend->getResumeFunction();
1648 auto &Context = Suspend->getParent()->getParent()->getContext();
1649 auto *Int8PtrTy = Type::getInt8PtrTy(Context);
1650
1651 IRBuilder<> Builder(ResumeIntrinsic);
1652 auto *Val = Builder.CreateBitOrPointerCast(Continuation, Int8PtrTy);
1653 ResumeIntrinsic->replaceAllUsesWith(Val);
1654 ResumeIntrinsic->eraseFromParent();
1656 UndefValue::get(Int8PtrTy));
1657}
1658
1659/// Coerce the arguments in \p FnArgs according to \p FnTy in \p CallArgs.
1660static void coerceArguments(IRBuilder<> &Builder, FunctionType *FnTy,
1661 ArrayRef<Value *> FnArgs,
1662 SmallVectorImpl<Value *> &CallArgs) {
1663 size_t ArgIdx = 0;
1664 for (auto *paramTy : FnTy->params()) {
1665 assert(ArgIdx < FnArgs.size());
1666 if (paramTy != FnArgs[ArgIdx]->getType())
1667 CallArgs.push_back(
1668 Builder.CreateBitOrPointerCast(FnArgs[ArgIdx], paramTy));
1669 else
1670 CallArgs.push_back(FnArgs[ArgIdx]);
1671 ++ArgIdx;
1672 }
1673}
1674
1677 IRBuilder<> &Builder) {
1678 auto *FnTy = MustTailCallFn->getFunctionType();
1679 // Coerce the arguments, llvm optimizations seem to ignore the types in
1680 // vaarg functions and throws away casts in optimized mode.
1681 SmallVector<Value *, 8> CallArgs;
1682 coerceArguments(Builder, FnTy, Arguments, CallArgs);
1683
1684 auto *TailCall = Builder.CreateCall(FnTy, MustTailCallFn, CallArgs);
1685 TailCall->setTailCallKind(CallInst::TCK_MustTail);
1686 TailCall->setDebugLoc(Loc);
1687 TailCall->setCallingConv(MustTailCallFn->getCallingConv());
1688 return TailCall;
1689}
1690
1693 assert(Shape.ABI == coro::ABI::Async);
1694 assert(Clones.empty());
1695 // Reset various things that the optimizer might have decided it
1696 // "knows" about the coroutine function due to not seeing a return.
1697 F.removeFnAttr(Attribute::NoReturn);
1698 F.removeRetAttr(Attribute::NoAlias);
1699 F.removeRetAttr(Attribute::NonNull);
1700
1701 auto &Context = F.getContext();
1702 auto *Int8PtrTy = Type::getInt8PtrTy(Context);
1703
1704 auto *Id = cast<CoroIdAsyncInst>(Shape.CoroBegin->getId());
1705 IRBuilder<> Builder(Id);
1706
1707 auto *FramePtr = Id->getStorage();
1708 FramePtr = Builder.CreateBitOrPointerCast(FramePtr, Int8PtrTy);
1709 FramePtr = Builder.CreateConstInBoundsGEP1_32(
1711 "async.ctx.frameptr");
1712
1713 // Map all uses of llvm.coro.begin to the allocated frame pointer.
1714 {
1715 // Make sure we don't invalidate Shape.FramePtr.
1716 TrackingVH<Value> Handle(Shape.FramePtr);
1718 Shape.FramePtr = Handle.getValPtr();
1719 }
1720
1721 // Create all the functions in order after the main function.
1722 auto NextF = std::next(F.getIterator());
1723
1724 // Create a continuation function for each of the suspend points.
1725 Clones.reserve(Shape.CoroSuspends.size());
1726 for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) {
1727 auto *Suspend = cast<CoroSuspendAsyncInst>(Shape.CoroSuspends[Idx]);
1728
1729 // Create the clone declaration.
1730 auto ResumeNameSuffix = ".resume.";
1731 auto ProjectionFunctionName =
1732 Suspend->getAsyncContextProjectionFunction()->getName();
1733 bool UseSwiftMangling = false;
1734 if (ProjectionFunctionName.equals("__swift_async_resume_project_context")) {
1735 ResumeNameSuffix = "TQ";
1736 UseSwiftMangling = true;
1737 } else if (ProjectionFunctionName.equals(
1738 "__swift_async_resume_get_context")) {
1739 ResumeNameSuffix = "TY";
1740 UseSwiftMangling = true;
1741 }
1742 auto *Continuation = createCloneDeclaration(
1743 F, Shape,
1744 UseSwiftMangling ? ResumeNameSuffix + Twine(Idx) + "_"
1745 : ResumeNameSuffix + Twine(Idx),
1746 NextF, Suspend);
1747 Clones.push_back(Continuation);
1748
1749 // Insert a branch to a new return block immediately before the suspend
1750 // point.
1751 auto *SuspendBB = Suspend->getParent();
1752 auto *NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);
1753 auto *Branch = cast<BranchInst>(SuspendBB->getTerminator());
1754
1755 // Place it before the first suspend.
1756 auto *ReturnBB =
1757 BasicBlock::Create(F.getContext(), "coro.return", &F, NewSuspendBB);
1758 Branch->setSuccessor(0, ReturnBB);
1759
1760 IRBuilder<> Builder(ReturnBB);
1761
1762 // Insert the call to the tail call function and inline it.
1763 auto *Fn = Suspend->getMustTailCallFunction();
1764 SmallVector<Value *, 8> Args(Suspend->args());
1765 auto FnArgs = ArrayRef<Value *>(Args).drop_front(
1767 auto *TailCall =
1768 coro::createMustTailCall(Suspend->getDebugLoc(), Fn, FnArgs, Builder);
1769 Builder.CreateRetVoid();
1770 InlineFunctionInfo FnInfo;
1771 auto InlineRes = InlineFunction(*TailCall, FnInfo);
1772 assert(InlineRes.isSuccess() && "Expected inlining to succeed");
1773 (void)InlineRes;
1774
1775 // Replace the lvm.coro.async.resume intrisic call.
1776 replaceAsyncResumeFunction(Suspend, Continuation);
1777 }
1778
1779 assert(Clones.size() == Shape.CoroSuspends.size());
1780 for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) {
1781 auto *Suspend = Shape.CoroSuspends[Idx];
1782 auto *Clone = Clones[Idx];
1783
1784 CoroCloner(F, "resume." + Twine(Idx), Shape, Clone, Suspend).create();
1785 }
1786}
1787
1790 assert(Shape.ABI == coro::ABI::Retcon ||
1791 Shape.ABI == coro::ABI::RetconOnce);
1792 assert(Clones.empty());
1793
1794 // Reset various things that the optimizer might have decided it
1795 // "knows" about the coroutine function due to not seeing a return.
1796 F.removeFnAttr(Attribute::NoReturn);
1797 F.removeRetAttr(Attribute::NoAlias);
1798 F.removeRetAttr(Attribute::NonNull);
1799
1800 // Allocate the frame.
1801 auto *Id = cast<AnyCoroIdRetconInst>(Shape.CoroBegin->getId());
1802 Value *RawFramePtr;
1804 RawFramePtr = Id->getStorage();
1805 } else {
1806 IRBuilder<> Builder(Id);
1807
1808 // Determine the size of the frame.
1809 const DataLayout &DL = F.getParent()->getDataLayout();
1810 auto Size = DL.getTypeAllocSize(Shape.FrameTy);
1811
1812 // Allocate. We don't need to update the call graph node because we're
1813 // going to recompute it from scratch after splitting.
1814 // FIXME: pass the required alignment
1815 RawFramePtr = Shape.emitAlloc(Builder, Builder.getInt64(Size), nullptr);
1816 RawFramePtr =
1817 Builder.CreateBitCast(RawFramePtr, Shape.CoroBegin->getType());
1818
1819 // Stash the allocated frame pointer in the continuation storage.
1820 auto Dest = Builder.CreateBitCast(Id->getStorage(),
1821 RawFramePtr->getType()->getPointerTo());
1822 Builder.CreateStore(RawFramePtr, Dest);
1823 }
1824
1825 // Map all uses of llvm.coro.begin to the allocated frame pointer.
1826 {
1827 // Make sure we don't invalidate Shape.FramePtr.
1828 TrackingVH<Value> Handle(Shape.FramePtr);
1829 Shape.CoroBegin->replaceAllUsesWith(RawFramePtr);
1830 Shape.FramePtr = Handle.getValPtr();
1831 }
1832
1833 // Create a unique return block.
1834 BasicBlock *ReturnBB = nullptr;
1835 SmallVector<PHINode *, 4> ReturnPHIs;
1836
1837 // Create all the functions in order after the main function.
1838 auto NextF = std::next(F.getIterator());
1839
1840 // Create a continuation function for each of the suspend points.
1841 Clones.reserve(Shape.CoroSuspends.size());
1842 for (size_t i = 0, e = Shape.CoroSuspends.size(); i != e; ++i) {
1843 auto Suspend = cast<CoroSuspendRetconInst>(Shape.CoroSuspends[i]);
1844
1845 // Create the clone declaration.
1846 auto Continuation =
1847 createCloneDeclaration(F, Shape, ".resume." + Twine(i), NextF, nullptr);
1848 Clones.push_back(Continuation);
1849
1850 // Insert a branch to the unified return block immediately before
1851 // the suspend point.
1852 auto SuspendBB = Suspend->getParent();
1853 auto NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);
1854 auto Branch = cast<BranchInst>(SuspendBB->getTerminator());
1855
1856 // Create the unified return block.
1857 if (!ReturnBB) {
1858 // Place it before the first suspend.
1859 ReturnBB = BasicBlock::Create(F.getContext(), "coro.return", &F,
1860 NewSuspendBB);
1861 Shape.RetconLowering.ReturnBlock = ReturnBB;
1862
1863 IRBuilder<> Builder(ReturnBB);
1864
1865 // Create PHIs for all the return values.
1866 assert(ReturnPHIs.empty());
1867
1868 // First, the continuation.
1869 ReturnPHIs.push_back(Builder.CreatePHI(Continuation->getType(),
1870 Shape.CoroSuspends.size()));
1871
1872 // Next, all the directly-yielded values.
1873 for (auto *ResultTy : Shape.getRetconResultTypes())
1874 ReturnPHIs.push_back(Builder.CreatePHI(ResultTy,
1875 Shape.CoroSuspends.size()));
1876
1877 // Build the return value.
1878 auto RetTy = F.getReturnType();
1879
1880 // Cast the continuation value if necessary.
1881 // We can't rely on the types matching up because that type would
1882 // have to be infinite.
1883 auto CastedContinuationTy =
1884 (ReturnPHIs.size() == 1 ? RetTy : RetTy->getStructElementType(0));
1885 auto *CastedContinuation =
1886 Builder.CreateBitCast(ReturnPHIs[0], CastedContinuationTy);
1887
1888 Value *RetV;
1889 if (ReturnPHIs.size() == 1) {
1890 RetV = CastedContinuation;
1891 } else {
1892 RetV = UndefValue::get(RetTy);
1893 RetV = Builder.CreateInsertValue(RetV, CastedContinuation, 0);
1894 for (size_t I = 1, E = ReturnPHIs.size(); I != E; ++I)
1895 RetV = Builder.CreateInsertValue(RetV, ReturnPHIs[I], I);
1896 }
1897
1898 Builder.CreateRet(RetV);
1899 }
1900
1901 // Branch to the return block.
1902 Branch->setSuccessor(0, ReturnBB);
1903 ReturnPHIs[0]->addIncoming(Continuation, SuspendBB);
1904 size_t NextPHIIndex = 1;
1905 for (auto &VUse : Suspend->value_operands())
1906 ReturnPHIs[NextPHIIndex++]->addIncoming(&*VUse, SuspendBB);
1907 assert(NextPHIIndex == ReturnPHIs.size());
1908 }
1909
1910 assert(Clones.size() == Shape.CoroSuspends.size());
1911 for (size_t i = 0, e = Shape.CoroSuspends.size(); i != e; ++i) {
1912 auto Suspend = Shape.CoroSuspends[i];
1913 auto Clone = Clones[i];
1914
1915 CoroCloner(F, "resume." + Twine(i), Shape, Clone, Suspend).create();
1916 }
1917}
1918
1919namespace {
1920 class PrettyStackTraceFunction : public PrettyStackTraceEntry {
1921 Function &F;
1922 public:
1923 PrettyStackTraceFunction(Function &F) : F(F) {}
1924 void print(raw_ostream &OS) const override {
1925 OS << "While splitting coroutine ";
1926 F.printAsOperand(OS, /*print type*/ false, F.getParent());
1927 OS << "\n";
1928 }
1929 };
1930}
1931
1935 bool OptimizeFrame) {
1936 PrettyStackTraceFunction prettyStackTrace(F);
1937
1938 // The suspend-crossing algorithm in buildCoroutineFrame get tripped
1939 // up by uses in unreachable blocks, so remove them as a first pass.
1941
1942 coro::Shape Shape(F, OptimizeFrame);
1943 if (!Shape.CoroBegin)
1944 return Shape;
1945
1946 simplifySuspendPoints(Shape);
1947 buildCoroutineFrame(F, Shape);
1949
1950 // If there are no suspend points, no split required, just remove
1951 // the allocation and deallocation blocks, they are not needed.
1952 if (Shape.CoroSuspends.empty()) {
1954 } else {
1955 switch (Shape.ABI) {
1956 case coro::ABI::Switch:
1957 splitSwitchCoroutine(F, Shape, Clones, TTI);
1958 break;
1959 case coro::ABI::Async:
1960 splitAsyncCoroutine(F, Shape, Clones);
1961 break;
1962 case coro::ABI::Retcon:
1963 case coro::ABI::RetconOnce:
1964 splitRetconCoroutine(F, Shape, Clones);
1965 break;
1966 }
1967 }
1968
1969 // Replace all the swifterror operations in the original function.
1970 // This invalidates SwiftErrorOps in the Shape.
1971 replaceSwiftErrorOps(F, Shape, nullptr);
1972
1973 // Finally, salvage the llvm.dbg.{declare,addr} in our original function that
1974 // point into the coroutine frame. We only do this for the current function
1975 // since the Cloner salvaged debug info for us in the new coroutine funclets.
1978 for (auto &BB : F) {
1979 for (auto &I : BB) {
1980 if (auto *DDI = dyn_cast<DbgDeclareInst>(&I)) {
1981 Worklist.push_back(DDI);
1982 continue;
1983 }
1984 if (auto *DDI = dyn_cast<DbgAddrIntrinsic>(&I)) {
1985 Worklist.push_back(DDI);
1986 continue;
1987 }
1988 }
1989 }
1990 for (auto *DDI : Worklist)
1991 coro::salvageDebugInfo(DbgPtrAllocaCache, DDI, Shape.OptimizeFrame);
1992
1993 return Shape;
1994}
1995
1996/// Remove calls to llvm.coro.end in the original function.
1997static void removeCoroEnds(const coro::Shape &Shape) {
1998 for (auto *End : Shape.CoroEnds) {
1999 replaceCoroEnd(End, Shape, Shape.FramePtr, /*in resume*/ false, nullptr);
2000 }
2001}
2002
2004 LazyCallGraph::Node &N, const coro::Shape &Shape,
2008 if (!Shape.CoroBegin)
2009 return;
2010
2011 if (Shape.ABI != coro::ABI::Switch)
2012 removeCoroEnds(Shape);
2013 else {
2014 for (llvm::AnyCoroEndInst *End : Shape.CoroEnds) {
2015 auto &Context = End->getContext();
2016 End->replaceAllUsesWith(ConstantInt::getFalse(Context));
2017 End->eraseFromParent();
2018 }
2019 }
2020
2021 if (!Clones.empty()) {
2022 switch (Shape.ABI) {
2023 case coro::ABI::Switch:
2024 // Each clone in the Switch lowering is independent of the other clones.
2025 // Let the LazyCallGraph know about each one separately.
2026 for (Function *Clone : Clones)
2027 CG.addSplitFunction(N.getFunction(), *Clone);
2028 break;
2029 case coro::ABI::Async:
2030 case coro::ABI::Retcon:
2031 case coro::ABI::RetconOnce:
2032 // Each clone in the Async/Retcon lowering references of the other clones.
2033 // Let the LazyCallGraph know about all of them at once.
2034 if (!Clones.empty())
2035 CG.addSplitRefRecursiveFunctions(N.getFunction(), Clones);
2036 break;
2037 }
2038
2039 // Let the CGSCC infra handle the changes to the original function.
2041 }
2042
2043 // Do some cleanup and let the CGSCC infra see if we've cleaned up any edges
2044 // to the split functions.
2045 postSplitCleanup(N.getFunction());
2047}
2048
2049/// Replace a call to llvm.coro.prepare.retcon.
2050static void replacePrepare(CallInst *Prepare, LazyCallGraph &CG,
2052 auto CastFn = Prepare->getArgOperand(0); // as an i8*
2053 auto Fn = CastFn->stripPointerCasts(); // as its original type
2054
2055 // Attempt to peephole this pattern:
2056 // %0 = bitcast [[TYPE]] @some_function to i8*
2057 // %1 = call @llvm.coro.prepare.retcon(i8* %0)
2058 // %2 = bitcast %1 to [[TYPE]]
2059 // ==>
2060 // %2 = @some_function
2061 for (Use &U : llvm::make_early_inc_range(Prepare->uses())) {
2062 // Look for bitcasts back to the original function type.
2063 auto *Cast = dyn_cast<BitCastInst>(U.getUser());
2064 if (!Cast || Cast->getType() != Fn->getType())
2065 continue;
2066
2067 // Replace and remove the cast.
2068 Cast->replaceAllUsesWith(Fn);
2069 Cast->eraseFromParent();
2070 }
2071
2072 // Replace any remaining uses with the function as an i8*.
2073 // This can never directly be a callee, so we don't need to update CG.
2074 Prepare->replaceAllUsesWith(CastFn);
2075 Prepare->eraseFromParent();
2076
2077 // Kill dead bitcasts.
2078 while (auto *Cast = dyn_cast<BitCastInst>(CastFn)) {
2079 if (!Cast->use_empty())
2080 break;
2081 CastFn = Cast->getOperand(0);
2082 Cast->eraseFromParent();
2083 }
2084}
2085
2086static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG,
2088 bool Changed = false;
2089 for (Use &P : llvm::make_early_inc_range(PrepareFn->uses())) {
2090 // Intrinsics can only be used in calls.
2091 auto *Prepare = cast<CallInst>(P.getUser());
2092 replacePrepare(Prepare, CG, C);
2093 Changed = true;
2094 }
2095
2096 return Changed;
2097}
2098
2099static void addPrepareFunction(const Module &M,
2101 StringRef Name) {
2102 auto *PrepareFn = M.getFunction(Name);
2103 if (PrepareFn && !PrepareFn->use_empty())
2104 Fns.push_back(PrepareFn);
2105}
2106
2110 // NB: One invariant of a valid LazyCallGraph::SCC is that it must contain a
2111 // non-zero number of nodes, so we assume that here and grab the first
2112 // node's function's module.
2113 Module &M = *C.begin()->getFunction().getParent();
2114 auto &FAM =
2115 AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager();
2116
2117 // Check for uses of llvm.coro.prepare.retcon/async.
2118 SmallVector<Function *, 2> PrepareFns;
2119 addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.retcon");
2120 addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.async");
2121
2122 // Find coroutines for processing.
2124 for (LazyCallGraph::Node &N : C)
2125 if (N.getFunction().isPresplitCoroutine())
2126 Coroutines.push_back(&N);
2127
2128 if (Coroutines.empty() && PrepareFns.empty())
2129 return PreservedAnalyses::all();
2130
2131 if (Coroutines.empty()) {
2132 for (auto *PrepareFn : PrepareFns) {
2133 replaceAllPrepares(PrepareFn, CG, C);
2134 }
2135 }
2136
2137 // Split all the coroutines.
2138 for (LazyCallGraph::Node *N : Coroutines) {
2139 Function &F = N->getFunction();
2140 LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F.getName()
2141 << "\n");
2142 F.setSplittedCoroutine();
2143
2145 const coro::Shape Shape = splitCoroutine(
2147 updateCallGraphAfterCoroutineSplit(*N, Shape, Clones, C, CG, AM, UR, FAM);
2148
2149 if (!Shape.CoroSuspends.empty()) {
2150 // Run the CGSCC pipeline on the original and newly split functions.
2151 UR.CWorklist.insert(&C);
2152 for (Function *Clone : Clones)
2153 UR.CWorklist.insert(CG.lookupSCC(CG.get(*Clone)));
2154 }
2155 }
2156
2157 if (!PrepareFns.empty()) {
2158 for (auto *PrepareFn : PrepareFns) {
2159 replaceAllPrepares(PrepareFn, CG, C);
2160 }
2161 }
2162
2163 return PreservedAnalyses::none();
2164}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
AMDGPU Lower Kernel Arguments
SmallPtrSet< MachineInstr *, 2 > Uses
assume Assume Builder
This file contains the simple types necessary to represent the attributes associated with functions a...
SmallVector< MachineOperand, 4 > Cond
BlockVerifier::State From
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file provides interfaces used to manipulate a call graph, regardless if it is a "old style" Call...
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Remove calls to llvm coro end in the original static function void removeCoroEnds(const coro::Shape &Shape)
Definition: CoroSplit.cpp:1997
static void addSwiftSelfAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)
Definition: CoroSplit.cpp:866
static void splitSwitchCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI)
Definition: CoroSplit.cpp:1606
static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy)
Definition: CoroSplit.cpp:1480
static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape, ValueToValueMapTy *VMap)
Definition: CoroSplit.cpp:620
static void addAsyncContextAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)
Definition: CoroSplit.cpp:859
static void addMustTailToCoroResumes(Function &F, TargetTransformInfo &TTI)
Definition: CoroSplit.cpp:1379
static void maybeFreeRetconStorage(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr, CallGraph *CG)
Definition: CoroSplit.cpp:169
static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB)
Definition: CoroSplit.cpp:1451
static Function * createCloneDeclaration(Function &OrigF, coro::Shape &Shape, const Twine &Suffix, Module::iterator InsertBefore, AnyCoroSuspendInst *ActiveSuspend)
Definition: CoroSplit.cpp:507
static FunctionType * getFunctionTypeFromAsyncSuspend(AnyCoroSuspendInst *Suspend)
Definition: CoroSplit.cpp:499
static void addPrepareFunction(const Module &M, SmallVectorImpl< Function * > &Fns, StringRef Name)
Definition: CoroSplit.cpp:2099
static void updateCallGraphAfterCoroutineSplit(LazyCallGraph::Node &N, const coro::Shape &Shape, const SmallVectorImpl< Function * > &Clones, LazyCallGraph::SCC &C, LazyCallGraph &CG, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Definition: CoroSplit.cpp:2003
static void simplifySuspendPoints(coro::Shape &Shape)
Definition: CoroSplit.cpp:1565
static void addFramePointerAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex, uint64_t Size, Align Alignment, bool NoAlias)
Definition: CoroSplit.cpp:844
static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG, LazyCallGraph::SCC &C)
Definition: CoroSplit.cpp:2086
static void replaceFallthroughCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Replace a non-unwind call to llvm.coro.end.
Definition: CoroSplit.cpp:226
static void replaceFrameSizeAndAlignment(coro::Shape &Shape)
Definition: CoroSplit.cpp:1114
static coro::Shape splitCoroutine(Function &F, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI, bool OptimizeFrame)
Definition: CoroSplit.cpp:1932
static bool replaceCoroEndAsync(AnyCoroEndInst *End)
Replace an llvm.coro.end.async.
Definition: CoroSplit.cpp:183
Replace a call to llvm coro prepare static retcon void replacePrepare(CallInst *Prepare, LazyCallGraph &CG, LazyCallGraph::SCC &C)
Definition: CoroSplit.cpp:2050
static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Replace an unwind call to llvm.coro.end.
Definition: CoroSplit.cpp:305
static bool simplifySuspendPoint(CoroSuspendInst *Suspend, CoroBeginInst *CoroBegin)
Definition: CoroSplit.cpp:1505
static bool hasCallsInBlockBetween(Instruction *From, Instruction *To)
Definition: CoroSplit.cpp:1439
static void markCoroutineAsDone(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr)
Definition: CoroSplit.cpp:291
static void splitAsyncCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones)
Definition: CoroSplit.cpp:1691
static void updateAsyncFuncPointerContextSize(coro::Shape &Shape)
Definition: CoroSplit.cpp:1099
static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Definition: CoroSplit.cpp:343
static void setCoroInfo(Function &F, coro::Shape &Shape, ArrayRef< Function * > Fns)
Definition: CoroSplit.cpp:1151
static void handleNoSuspendCoroutine(coro::Shape &Shape)
Definition: CoroSplit.cpp:1404
static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn, Function *DestroyFn, Function *CleanupFn)
Definition: CoroSplit.cpp:1175
static void createResumeEntryBlock(Function &F, coro::Shape &Shape)
Definition: CoroSplit.cpp:358
static void postSplitCleanup(Function &F)
Definition: CoroSplit.cpp:1201
static bool simplifyTerminatorLeadingToRet(Instruction *InitialInst)
Definition: CoroSplit.cpp:1233
static void splitRetconCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones)
Definition: CoroSplit.cpp:1788
static void scanPHIsAndUpdateValueMap(Instruction *Prev, BasicBlock *NewBlock, DenseMap< Value *, Value * > &ResolvedValues)
Definition: CoroSplit.cpp:1216
Coerce the arguments in p FnArgs according to p FnTy in p static CallArgs void coerceArguments(IRBuilder<> &Builder, FunctionType *FnTy, ArrayRef< Value * > FnArgs, SmallVectorImpl< Value * > &CallArgs)
Definition: CoroSplit.cpp:1660
static void replaceAsyncResumeFunction(CoroSuspendAsyncInst *Suspend, Value *Continuation)
Definition: CoroSplit.cpp:1645
static bool shouldBeMustTail(const CallInst &CI, const Function &F)
Definition: CoroSplit.cpp:1344
static Function * createClone(Function &F, const Twine &Suffix, coro::Shape &Shape, CoroCloner::Kind FKind)
Definition: CoroSplit.cpp:1092
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseMap class.
This file contains constants used for implementing Dwarf debug support.
std::string Name
uint64_t Size
static Function * getFunction(Constant *C)
Definition: Evaluator.cpp:236
@ InlineInfo
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Implements a lazy call graph analysis and related passes for the new pass manager.
static M68kRelType getType(unsigned Kind, MCSymbolRefExpr::VariantKind &Modifier, bool &IsPCRel)
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Module.h This file contains the declarations for the Module class.
print must be executed print the must be executed context for all instructions
LLVMContext & Context
#define P(N)
FunctionAnalysisManager FAM
This file provides a priority worklist.
@ SI
@ VI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This pass exposes codegen information to IR-level passes.
static const unsigned FramePtr
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:620
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:774
CoroAllocInst * getCoroAlloc()
Definition: CoroInstr.h:84
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
Definition: ArrayRef.h:202
iterator end() const
Definition: ArrayRef.h:152
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:163
iterator begin() const
Definition: ArrayRef.h:151
AttrBuilder & addAlignmentAttr(MaybeAlign Align)
This turns an alignment into the form used internally in Attribute.
AttrBuilder & addAttribute(Attribute::AttrKind Val)
Add an attribute to the builder.
AttrBuilder & addDereferenceableAttr(uint64_t Bytes)
This turns the number of dereferenceable bytes into the form used internally in Attribute.
AttrKind
This enumeration lists the attributes that can be associated with parameters, function results,...
Definition: Attributes.h:86
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:372
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:105
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
Definition: BasicBlock.cpp:401
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:284
const Instruction * getFirstNonPHIOrDbgOrLifetime(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode, a debug intrinsic,...
Definition: BasicBlock.cpp:229
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:112
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:127
void removePredecessor(BasicBlock *Pred, bool KeepOneInputPHIs=false)
Update PHI nodes in this BasicBlock before removal of predecessor Pred.
Definition: BasicBlock.cpp:341
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1184
bool isInlineAsm() const
Check if this call is an inline asm statement.
Definition: InstrTypes.h:1474
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1465
Value * getCalledOperand() const
Definition: InstrTypes.h:1399
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1351
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1264
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1484
The basic data container for the call graph of a Module of IR.
Definition: CallGraph.h:72
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1249
static Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition: Constants.cpp:2041
This is the shared class of boolean and integer constants.
Definition: Constants.h:78
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:835
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:887
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:842
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1707
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1314
static ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition: Constants.cpp:1429
This represents the llvm.coro.align instruction.
Definition: CoroInstr.h:603
This represents the llvm.coro.alloc instruction.
Definition: CoroInstr.h:70
This class represents the llvm.coro.begin instruction.
Definition: CoroInstr.h:420
AnyCoroIdInst * getId() const
Definition: CoroInstr.h:424
This represents the llvm.coro.id instruction.
Definition: CoroInstr.h:113
void setInfo(Constant *C)
Definition: CoroInstr.h:180
This represents the llvm.coro.size instruction.
Definition: CoroInstr.h:591
This represents the llvm.coro.suspend.async instruction.
Definition: CoroInstr.h:525
CoroAsyncResumeInst * getResumeFunction() const
Definition: CoroInstr.h:546
This represents the llvm.coro.suspend instruction.
Definition: CoroInstr.h:493
CoroSaveInst * getCoroSave() const
Definition: CoroInstr.h:497
DISubprogram * getSubprogram() const
Get the subprogram for this scope.
Subprogram description.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:114
This is the common base class for debug info intrinsics for variables.
A debug info location.
Definition: DebugLoc.h:33
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:150
iterator end()
Definition: DenseMap.h:84
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:166
This class represents a freeze function that returns random concrete value if an operand is either a ...
A proxy from a FunctionAnalysisManager to an SCC.
Type * getReturnType() const
Definition: DerivedTypes.h:124
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:136
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:174
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:237
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:315
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
PointerType * getType() const
Global values are always pointers.
Definition: GlobalValue.h:290
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
void setInitializer(Constant *InitVal)
setInitializer - Sets the initializer for this global variable, removing any existing initializer if ...
Definition: Globals.cpp:472
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2550
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
Definition: Cloning.h:203
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:70
const BasicBlock * getParent() const
Definition: Instruction.h:90
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:82
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
A node in the call graph.
An SCC of the call graph.
A lazily constructed view of the call graph of a module.
void addSplitFunction(Function &OriginalFunction, Function &NewFunction)
Add a new function split/outlined from an existing function.
void addSplitRefRecursiveFunctions(Function &OriginalFunction, ArrayRef< Function * > NewFunctions)
Add new ref-recursive functions split/outlined from an existing function.
Node & get(Function &F)
Get a graph node for a given function, scanning it to populate the graph data as necessary.
SCC * lookupSCC(Node &N) const
Lookup a function's SCC in the graph.
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:497
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:398
FunctionListType::iterator iterator
The Function iterators.
Definition: Module.h:90
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:155
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
PrettyStackTraceEntry - This class is used to represent a frame of the "pretty" stack trace that is d...
virtual void print(raw_ostream &OS) const =0
print - Emit information about this stack frame to OS.
Return a value (possibly void), from a function.
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
Definition: SmallPtrSet.h:379
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:365
bool contains(ConstPtrType Ptr) const
Definition: SmallPtrSet.h:389
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void reserve(size_type N)
Definition: SmallVector.h:667
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Type * getTypeAtIndex(const Value *V) const
Given an index value into the type, return the type of the element.
Definition: Type.cpp:605
Analysis pass providing the TargetTransformInfo.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
bool supportsTailCallFor(const CallBase *CB) const
If target supports tail call on CB.
bool supportsTailCalls() const
If the target supports tail calls.
Value handle that tracks a Value across RAUW.
Definition: ValueHandle.h:331
ValueTy * getValPtr() const
Definition: ValueHandle.h:335
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:249
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt8Ty(LLVMContext &C)
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1740
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
void setOperand(unsigned i, Value *Val)
Definition: User.h:174
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:532
iterator_range< user_iterator > users()
Definition: Value.h:421
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:685
iterator_range< use_iterator > uses()
Definition: Value.h:376
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:308
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:381
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:289
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
void salvageDebugInfo(SmallDenseMap< llvm::Value *, llvm::AllocaInst *, 4 > &DbgPtrAllocaCache, DbgVariableIntrinsic *DVI, bool OptimizeFrame)
Recover a dbg.declare prepared by the frontend and emit an alloca holding a pointer to the coroutine ...
Definition: CoroFrame.cpp:2637
void replaceCoroFree(CoroIdInst *CoroId, bool Elide)
Definition: Coroutines.cpp:130
CallInst * createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, ArrayRef< Value * > Arguments, IRBuilder<> &)
Definition: CoroSplit.cpp:1675
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
bool ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions=false, const TargetLibraryInfo *TLI=nullptr, DomTreeUpdater *DTU=nullptr)
If a terminator instruction is predicated on a constant value, convert it into an unconditional branc...
Definition: Local.cpp:126
bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
Definition: Verifier.cpp:6428
void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
Definition: Utils.cpp:1367
LazyCallGraph::SCC & updateCGAndAnalysisManagerForFunctionPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a function pass.
LazyCallGraph::SCC & updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a CGSCC pass.
Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Instruction *I=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:721
bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction is not used, and the instruction will return.
Definition: Local.cpp:398
@ Async
"Asynchronous" unwind tables (instr precise)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
Definition: Local.cpp:2244
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)
This function inlines the called function into the basic block of the caller.
void CloneFunctionInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, CloneFunctionChangeType Changes, SmallVectorImpl< ReturnInst * > &Returns, const char *NameSuffix="", ClonedCodeInfo *CodeInfo=nullptr, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Clone OldFunc into NewFunc, transforming the old arguments into references to VMap values.
auto predecessors(const MachineBasicBlock *BB)
bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Remove all blocks that can not be reached from the function's entry.
Definition: Local.cpp:2608
bool isPotentiallyReachable(const Instruction *From, const Instruction *To, const SmallPtrSetImpl< BasicBlock * > *ExclusionSet=nullptr, const DominatorTree *DT=nullptr, const LoopInfo *LI=nullptr)
Determine whether instruction 'To' is reachable from 'From', without passing through any blocks in Ex...
Definition: CFG.cpp:231
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:853
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
SmallPriorityWorklist< LazyCallGraph::SCC *, 1 > & CWorklist
Worklist of the SCCs queued for processing.
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)
Definition: CoroSplit.cpp:2107
AsyncLoweringStorage AsyncLowering
Definition: CoroInternal.h:143
FunctionType * getResumeFunctionType() const
Definition: CoroInternal.h:182
IntegerType * getIndexType() const
Definition: CoroInternal.h:167
StructType * FrameTy
Definition: CoroInternal.h:98
CoroIdInst * getSwitchCoroId() const
Definition: CoroInternal.h:146
Instruction * getInsertPtAfterFramePtr() const
Definition: CoroInternal.h:240
SmallVector< CoroSizeInst *, 2 > CoroSizes
Definition: CoroInternal.h:76
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
Definition: CoroInternal.h:78
Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const
Allocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:453
SmallVector< CallInst *, 2 > SwiftErrorOps
Definition: CoroInternal.h:79
ConstantInt * getIndex(uint64_t Value) const
Definition: CoroInternal.h:172
bool OptimizeFrame
This would only be true if optimization are enabled.
Definition: CoroInternal.h:105
SwitchLoweringStorage SwitchLowering
Definition: CoroInternal.h:141
CoroBeginInst * CoroBegin
Definition: CoroInternal.h:74
ArrayRef< Type * > getRetconResultTypes() const
Definition: CoroInternal.h:198
void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const
Deallocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:476
RetconLoweringStorage RetconLowering
Definition: CoroInternal.h:142
SmallVector< CoroAlignInst *, 2 > CoroAligns
Definition: CoroInternal.h:77
SmallVector< AnyCoroEndInst *, 4 > CoroEnds
Definition: CoroInternal.h:75
unsigned getSwitchIndexField() const
Definition: CoroInternal.h:162