LLVM 17.0.0git
CoroSplit.cpp
Go to the documentation of this file.
1//===- CoroSplit.cpp - Converts a coroutine into a state machine ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// This pass builds the coroutine frame and outlines resume and destroy parts
9// of the coroutine into separate functions.
10//
11// We present a coroutine to an LLVM as an ordinary function with suspension
12// points marked up with intrinsics. We let the optimizer party on the coroutine
13// as a single function for as long as possible. Shortly before the coroutine is
14// eligible to be inlined into its callers, we split up the coroutine into parts
15// corresponding to an initial, resume and destroy invocations of the coroutine,
16// add them to the current SCC and restart the IPO pipeline to optimize the
17// coroutine subfunctions we extracted before proceeding to the caller of the
18// coroutine.
19//===----------------------------------------------------------------------===//
20
22#include "CoroInstr.h"
23#include "CoroInternal.h"
24#include "llvm/ADT/DenseMap.h"
28#include "llvm/ADT/StringRef.h"
29#include "llvm/ADT/Twine.h"
30#include "llvm/Analysis/CFG.h"
37#include "llvm/IR/Argument.h"
38#include "llvm/IR/Attributes.h"
39#include "llvm/IR/BasicBlock.h"
40#include "llvm/IR/CFG.h"
41#include "llvm/IR/CallingConv.h"
42#include "llvm/IR/Constants.h"
43#include "llvm/IR/DataLayout.h"
45#include "llvm/IR/Dominators.h"
46#include "llvm/IR/Function.h"
47#include "llvm/IR/GlobalValue.h"
49#include "llvm/IR/IRBuilder.h"
51#include "llvm/IR/InstrTypes.h"
52#include "llvm/IR/Instruction.h"
55#include "llvm/IR/LLVMContext.h"
56#include "llvm/IR/Module.h"
57#include "llvm/IR/Type.h"
58#include "llvm/IR/Value.h"
59#include "llvm/IR/Verifier.h"
61#include "llvm/Support/Debug.h"
70#include <cassert>
71#include <cstddef>
72#include <cstdint>
73#include <initializer_list>
74#include <iterator>
75
76using namespace llvm;
77
78#define DEBUG_TYPE "coro-split"
79
80namespace {
81
82/// A little helper class for building
83class CoroCloner {
84public:
85 enum class Kind {
86 /// The shared resume function for a switch lowering.
87 SwitchResume,
88
89 /// The shared unwind function for a switch lowering.
90 SwitchUnwind,
91
92 /// The shared cleanup function for a switch lowering.
93 SwitchCleanup,
94
95 /// An individual continuation function.
96 Continuation,
97
98 /// An async resume function.
99 Async,
100 };
101
102private:
103 Function &OrigF;
104 Function *NewF;
105 const Twine &Suffix;
106 coro::Shape &Shape;
107 Kind FKind;
110 Value *NewFramePtr = nullptr;
111
112 /// The active suspend instruction; meaningful only for continuation and async
113 /// ABIs.
114 AnyCoroSuspendInst *ActiveSuspend = nullptr;
115
116public:
117 /// Create a cloner for a switch lowering.
118 CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
119 Kind FKind)
120 : OrigF(OrigF), NewF(nullptr), Suffix(Suffix), Shape(Shape),
121 FKind(FKind), Builder(OrigF.getContext()) {
122 assert(Shape.ABI == coro::ABI::Switch);
123 }
124
125 /// Create a cloner for a continuation lowering.
126 CoroCloner(Function &OrigF, const Twine &Suffix, coro::Shape &Shape,
127 Function *NewF, AnyCoroSuspendInst *ActiveSuspend)
128 : OrigF(OrigF), NewF(NewF), Suffix(Suffix), Shape(Shape),
129 FKind(Shape.ABI == coro::ABI::Async ? Kind::Async : Kind::Continuation),
130 Builder(OrigF.getContext()), ActiveSuspend(ActiveSuspend) {
131 assert(Shape.ABI == coro::ABI::Retcon ||
132 Shape.ABI == coro::ABI::RetconOnce || Shape.ABI == coro::ABI::Async);
133 assert(NewF && "need existing function for continuation");
134 assert(ActiveSuspend && "need active suspend point for continuation");
135 }
136
137 Function *getFunction() const {
138 assert(NewF != nullptr && "declaration not yet set");
139 return NewF;
140 }
141
142 void create();
143
144private:
145 bool isSwitchDestroyFunction() {
146 switch (FKind) {
147 case Kind::Async:
148 case Kind::Continuation:
149 case Kind::SwitchResume:
150 return false;
151 case Kind::SwitchUnwind:
152 case Kind::SwitchCleanup:
153 return true;
154 }
155 llvm_unreachable("Unknown CoroCloner::Kind enum");
156 }
157
158 void replaceEntryBlock();
159 Value *deriveNewFramePointer();
160 void replaceRetconOrAsyncSuspendUses();
161 void replaceCoroSuspends();
162 void replaceCoroEnds();
164 void salvageDebugInfo();
165 void handleFinalSuspend();
166};
167
168} // end anonymous namespace
169
171 const coro::Shape &Shape, Value *FramePtr,
172 CallGraph *CG) {
173 assert(Shape.ABI == coro::ABI::Retcon ||
174 Shape.ABI == coro::ABI::RetconOnce);
176 return;
177
178 Shape.emitDealloc(Builder, FramePtr, CG);
179}
180
181/// Replace an llvm.coro.end.async.
182/// Will inline the must tail call function call if there is one.
183/// \returns true if cleanup of the coro.end block is needed, false otherwise.
186
187 auto *EndAsync = dyn_cast<CoroAsyncEndInst>(End);
188 if (!EndAsync) {
189 Builder.CreateRetVoid();
190 return true /*needs cleanup of coro.end block*/;
191 }
192
193 auto *MustTailCallFunc = EndAsync->getMustTailCallFunction();
194 if (!MustTailCallFunc) {
195 Builder.CreateRetVoid();
196 return true /*needs cleanup of coro.end block*/;
197 }
198
199 // Move the must tail call from the predecessor block into the end block.
200 auto *CoroEndBlock = End->getParent();
201 auto *MustTailCallFuncBlock = CoroEndBlock->getSinglePredecessor();
202 assert(MustTailCallFuncBlock && "Must have a single predecessor block");
203 auto It = MustTailCallFuncBlock->getTerminator()->getIterator();
204 auto *MustTailCall = cast<CallInst>(&*std::prev(It));
205 CoroEndBlock->splice(End->getIterator(), MustTailCallFuncBlock,
206 MustTailCall->getIterator());
207
208 // Insert the return instruction.
209 Builder.SetInsertPoint(End);
210 Builder.CreateRetVoid();
211 InlineFunctionInfo FnInfo;
212
213 // Remove the rest of the block, by splitting it into an unreachable block.
214 auto *BB = End->getParent();
215 BB->splitBasicBlock(End);
216 BB->getTerminator()->eraseFromParent();
217
218 auto InlineRes = InlineFunction(*MustTailCall, FnInfo);
219 assert(InlineRes.isSuccess() && "Expected inlining to succeed");
220 (void)InlineRes;
221
222 // We have cleaned up the coro.end block above.
223 return false;
224}
225
226/// Replace a non-unwind call to llvm.coro.end.
228 const coro::Shape &Shape, Value *FramePtr,
229 bool InResume, CallGraph *CG) {
230 // Start inserting right before the coro.end.
232
233 // Create the return instruction.
234 switch (Shape.ABI) {
235 // The cloned functions in switch-lowering always return void.
236 case coro::ABI::Switch:
237 // coro.end doesn't immediately end the coroutine in the main function
238 // in this lowering, because we need to deallocate the coroutine.
239 if (!InResume)
240 return;
241 Builder.CreateRetVoid();
242 break;
243
244 // In async lowering this returns.
245 case coro::ABI::Async: {
246 bool CoroEndBlockNeedsCleanup = replaceCoroEndAsync(End);
247 if (!CoroEndBlockNeedsCleanup)
248 return;
249 break;
250 }
251
252 // In unique continuation lowering, the continuations always return void.
253 // But we may have implicitly allocated storage.
254 case coro::ABI::RetconOnce:
256 Builder.CreateRetVoid();
257 break;
258
259 // In non-unique continuation lowering, we signal completion by returning
260 // a null continuation.
261 case coro::ABI::Retcon: {
263 auto RetTy = Shape.getResumeFunctionType()->getReturnType();
264 auto RetStructTy = dyn_cast<StructType>(RetTy);
265 PointerType *ContinuationTy =
266 cast<PointerType>(RetStructTy ? RetStructTy->getElementType(0) : RetTy);
267
268 Value *ReturnValue = ConstantPointerNull::get(ContinuationTy);
269 if (RetStructTy) {
270 ReturnValue = Builder.CreateInsertValue(UndefValue::get(RetStructTy),
271 ReturnValue, 0);
272 }
273 Builder.CreateRet(ReturnValue);
274 break;
275 }
276 }
277
278 // Remove the rest of the block, by splitting it into an unreachable block.
279 auto *BB = End->getParent();
280 BB->splitBasicBlock(End);
281 BB->getTerminator()->eraseFromParent();
282}
283
284// Mark a coroutine as done, which implies that the coroutine is finished and
285// never get resumed.
286//
287// In resume-switched ABI, the done state is represented by storing zero in
288// ResumeFnAddr.
289//
290// NOTE: We couldn't omit the argument `FramePtr`. It is necessary because the
291// pointer to the frame in splitted function is not stored in `Shape`.
292static void markCoroutineAsDone(IRBuilder<> &Builder, const coro::Shape &Shape,
293 Value *FramePtr) {
294 assert(
295 Shape.ABI == coro::ABI::Switch &&
296 "markCoroutineAsDone is only supported for Switch-Resumed ABI for now.");
297 auto *GepIndex = Builder.CreateStructGEP(
299 "ResumeFn.addr");
300 auto *NullPtr = ConstantPointerNull::get(cast<PointerType>(
302 Builder.CreateStore(NullPtr, GepIndex);
303}
304
305/// Replace an unwind call to llvm.coro.end.
307 Value *FramePtr, bool InResume,
308 CallGraph *CG) {
310
311 switch (Shape.ABI) {
312 // In switch-lowering, this does nothing in the main function.
313 case coro::ABI::Switch: {
314 // In C++'s specification, the coroutine should be marked as done
315 // if promise.unhandled_exception() throws. The frontend will
316 // call coro.end(true) along this path.
317 //
318 // FIXME: We should refactor this once there is other language
319 // which uses Switch-Resumed style other than C++.
321 if (!InResume)
322 return;
323 break;
324 }
325 // In async lowering this does nothing.
326 case coro::ABI::Async:
327 break;
328 // In continuation-lowering, this frees the continuation storage.
329 case coro::ABI::Retcon:
330 case coro::ABI::RetconOnce:
332 break;
333 }
334
335 // If coro.end has an associated bundle, add cleanupret instruction.
336 if (auto Bundle = End->getOperandBundle(LLVMContext::OB_funclet)) {
337 auto *FromPad = cast<CleanupPadInst>(Bundle->Inputs[0]);
338 auto *CleanupRet = Builder.CreateCleanupRet(FromPad, nullptr);
339 End->getParent()->splitBasicBlock(End);
340 CleanupRet->getParent()->getTerminator()->eraseFromParent();
341 }
342}
343
344static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape,
345 Value *FramePtr, bool InResume, CallGraph *CG) {
346 if (End->isUnwind())
347 replaceUnwindCoroEnd(End, Shape, FramePtr, InResume, CG);
348 else
349 replaceFallthroughCoroEnd(End, Shape, FramePtr, InResume, CG);
350
351 auto &Context = End->getContext();
352 End->replaceAllUsesWith(InResume ? ConstantInt::getTrue(Context)
354 End->eraseFromParent();
355}
356
357// Create an entry block for a resume function with a switch that will jump to
358// suspend points.
360 assert(Shape.ABI == coro::ABI::Switch);
361 LLVMContext &C = F.getContext();
362
363 // resume.entry:
364 // %index.addr = getelementptr inbounds %f.Frame, %f.Frame* %FramePtr, i32 0,
365 // i32 2
366 // % index = load i32, i32* %index.addr
367 // switch i32 %index, label %unreachable [
368 // i32 0, label %resume.0
369 // i32 1, label %resume.1
370 // ...
371 // ]
372
373 auto *NewEntry = BasicBlock::Create(C, "resume.entry", &F);
374 auto *UnreachBB = BasicBlock::Create(C, "unreachable", &F);
375
376 IRBuilder<> Builder(NewEntry);
377 auto *FramePtr = Shape.FramePtr;
378 auto *FrameTy = Shape.FrameTy;
379 auto *GepIndex = Builder.CreateStructGEP(
380 FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
381 auto *Index = Builder.CreateLoad(Shape.getIndexType(), GepIndex, "index");
382 auto *Switch =
383 Builder.CreateSwitch(Index, UnreachBB, Shape.CoroSuspends.size());
384 Shape.SwitchLowering.ResumeSwitch = Switch;
385
386 size_t SuspendIndex = 0;
387 for (auto *AnyS : Shape.CoroSuspends) {
388 auto *S = cast<CoroSuspendInst>(AnyS);
389 ConstantInt *IndexVal = Shape.getIndex(SuspendIndex);
390
391 // Replace CoroSave with a store to Index:
392 // %index.addr = getelementptr %f.frame... (index field number)
393 // store i32 %IndexVal, i32* %index.addr1
394 auto *Save = S->getCoroSave();
395 Builder.SetInsertPoint(Save);
396 if (S->isFinal()) {
397 // The coroutine should be marked done if it reaches the final suspend
398 // point.
400 }
401
402 // If the coroutine don't have unwind coro end, we could omit the store to
403 // the final suspend point since we could infer the coroutine is suspended
404 // at the final suspend point by the nullness of ResumeFnAddr.
405 // However, we can't skip it if the coroutine have unwind coro end. Since
406 // the coroutine reaches unwind coro end is considered suspended at the
407 // final suspend point (the ResumeFnAddr is null) but in fact the coroutine
408 // didn't complete yet. We need the IndexVal for the final suspend point
409 // to make the states clear.
410 if (!S->isFinal() || Shape.SwitchLowering.HasUnwindCoroEnd) {
411 auto *GepIndex = Builder.CreateStructGEP(
412 FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
413 Builder.CreateStore(IndexVal, GepIndex);
414 }
415
416 Save->replaceAllUsesWith(ConstantTokenNone::get(C));
417 Save->eraseFromParent();
418
419 // Split block before and after coro.suspend and add a jump from an entry
420 // switch:
421 //
422 // whateverBB:
423 // whatever
424 // %0 = call i8 @llvm.coro.suspend(token none, i1 false)
425 // switch i8 %0, label %suspend[i8 0, label %resume
426 // i8 1, label %cleanup]
427 // becomes:
428 //
429 // whateverBB:
430 // whatever
431 // br label %resume.0.landing
432 //
433 // resume.0: ; <--- jump from the switch in the resume.entry
434 // %0 = tail call i8 @llvm.coro.suspend(token none, i1 false)
435 // br label %resume.0.landing
436 //
437 // resume.0.landing:
438 // %1 = phi i8[-1, %whateverBB], [%0, %resume.0]
439 // switch i8 % 1, label %suspend [i8 0, label %resume
440 // i8 1, label %cleanup]
441
442 auto *SuspendBB = S->getParent();
443 auto *ResumeBB =
444 SuspendBB->splitBasicBlock(S, "resume." + Twine(SuspendIndex));
445 auto *LandingBB = ResumeBB->splitBasicBlock(
446 S->getNextNode(), ResumeBB->getName() + Twine(".landing"));
447 Switch->addCase(IndexVal, ResumeBB);
448
449 cast<BranchInst>(SuspendBB->getTerminator())->setSuccessor(0, LandingBB);
450 auto *PN = PHINode::Create(Builder.getInt8Ty(), 2, "", &LandingBB->front());
451 S->replaceAllUsesWith(PN);
452 PN->addIncoming(Builder.getInt8(-1), SuspendBB);
453 PN->addIncoming(S, ResumeBB);
454
455 ++SuspendIndex;
456 }
457
458 Builder.SetInsertPoint(UnreachBB);
459 Builder.CreateUnreachable();
460
461 Shape.SwitchLowering.ResumeEntryBlock = NewEntry;
462}
463
464// In the resume function, we remove the last case (when coro::Shape is built,
465// the final suspend point (if present) is always the last element of
466// CoroSuspends array) since it is an undefined behavior to resume a coroutine
467// suspended at the final suspend point.
468// In the destroy function, if it isn't possible that the ResumeFnAddr is NULL
469// and the coroutine doesn't suspend at the final suspend point actually (this
470// is possible since the coroutine is considered suspended at the final suspend
471// point if promise.unhandled_exception() exits via an exception), we can
472// remove the last case.
473void CoroCloner::handleFinalSuspend() {
474 assert(Shape.ABI == coro::ABI::Switch &&
475 Shape.SwitchLowering.HasFinalSuspend);
476
477 if (isSwitchDestroyFunction() && Shape.SwitchLowering.HasUnwindCoroEnd)
478 return;
479
480 auto *Switch = cast<SwitchInst>(VMap[Shape.SwitchLowering.ResumeSwitch]);
481 auto FinalCaseIt = std::prev(Switch->case_end());
482 BasicBlock *ResumeBB = FinalCaseIt->getCaseSuccessor();
483 Switch->removeCase(FinalCaseIt);
484 if (isSwitchDestroyFunction()) {
485 BasicBlock *OldSwitchBB = Switch->getParent();
486 auto *NewSwitchBB = OldSwitchBB->splitBasicBlock(Switch, "Switch");
487 Builder.SetInsertPoint(OldSwitchBB->getTerminator());
488 auto *GepIndex = Builder.CreateStructGEP(Shape.FrameTy, NewFramePtr,
490 "ResumeFn.addr");
491 auto *Load = Builder.CreateLoad(Shape.getSwitchResumePointerType(),
492 GepIndex);
493 auto *Cond = Builder.CreateIsNull(Load);
494 Builder.CreateCondBr(Cond, ResumeBB, NewSwitchBB);
495 OldSwitchBB->getTerminator()->eraseFromParent();
496 }
497}
498
499static FunctionType *
501 auto *AsyncSuspend = cast<CoroSuspendAsyncInst>(Suspend);
502 auto *StructTy = cast<StructType>(AsyncSuspend->getType());
503 auto &Context = Suspend->getParent()->getParent()->getContext();
504 auto *VoidTy = Type::getVoidTy(Context);
505 return FunctionType::get(VoidTy, StructTy->elements(), false);
506}
507
509 const Twine &Suffix,
510 Module::iterator InsertBefore,
511 AnyCoroSuspendInst *ActiveSuspend) {
512 Module *M = OrigF.getParent();
513 auto *FnTy = (Shape.ABI != coro::ABI::Async)
514 ? Shape.getResumeFunctionType()
515 : getFunctionTypeFromAsyncSuspend(ActiveSuspend);
516
517 Function *NewF =
518 Function::Create(FnTy, GlobalValue::LinkageTypes::InternalLinkage,
519 OrigF.getName() + Suffix);
520
521 M->getFunctionList().insert(InsertBefore, NewF);
522
523 return NewF;
524}
525
526/// Replace uses of the active llvm.coro.suspend.retcon/async call with the
527/// arguments to the continuation function.
528///
529/// This assumes that the builder has a meaningful insertion point.
530void CoroCloner::replaceRetconOrAsyncSuspendUses() {
531 assert(Shape.ABI == coro::ABI::Retcon || Shape.ABI == coro::ABI::RetconOnce ||
532 Shape.ABI == coro::ABI::Async);
533
534 auto NewS = VMap[ActiveSuspend];
535 if (NewS->use_empty()) return;
536
537 // Copy out all the continuation arguments after the buffer pointer into
538 // an easily-indexed data structure for convenience.
540 // The async ABI includes all arguments -- including the first argument.
541 bool IsAsyncABI = Shape.ABI == coro::ABI::Async;
542 for (auto I = IsAsyncABI ? NewF->arg_begin() : std::next(NewF->arg_begin()),
543 E = NewF->arg_end();
544 I != E; ++I)
545 Args.push_back(&*I);
546
547 // If the suspend returns a single scalar value, we can just do a simple
548 // replacement.
549 if (!isa<StructType>(NewS->getType())) {
550 assert(Args.size() == 1);
551 NewS->replaceAllUsesWith(Args.front());
552 return;
553 }
554
555 // Try to peephole extracts of an aggregate return.
556 for (Use &U : llvm::make_early_inc_range(NewS->uses())) {
557 auto *EVI = dyn_cast<ExtractValueInst>(U.getUser());
558 if (!EVI || EVI->getNumIndices() != 1)
559 continue;
560
561 EVI->replaceAllUsesWith(Args[EVI->getIndices().front()]);
562 EVI->eraseFromParent();
563 }
564
565 // If we have no remaining uses, we're done.
566 if (NewS->use_empty()) return;
567
568 // Otherwise, we need to create an aggregate.
569 Value *Agg = UndefValue::get(NewS->getType());
570 for (size_t I = 0, E = Args.size(); I != E; ++I)
571 Agg = Builder.CreateInsertValue(Agg, Args[I], I);
572
573 NewS->replaceAllUsesWith(Agg);
574}
575
576void CoroCloner::replaceCoroSuspends() {
577 Value *SuspendResult;
578
579 switch (Shape.ABI) {
580 // In switch lowering, replace coro.suspend with the appropriate value
581 // for the type of function we're extracting.
582 // Replacing coro.suspend with (0) will result in control flow proceeding to
583 // a resume label associated with a suspend point, replacing it with (1) will
584 // result in control flow proceeding to a cleanup label associated with this
585 // suspend point.
586 case coro::ABI::Switch:
587 SuspendResult = Builder.getInt8(isSwitchDestroyFunction() ? 1 : 0);
588 break;
589
590 // In async lowering there are no uses of the result.
591 case coro::ABI::Async:
592 return;
593
594 // In returned-continuation lowering, the arguments from earlier
595 // continuations are theoretically arbitrary, and they should have been
596 // spilled.
597 case coro::ABI::RetconOnce:
598 case coro::ABI::Retcon:
599 return;
600 }
601
602 for (AnyCoroSuspendInst *CS : Shape.CoroSuspends) {
603 // The active suspend was handled earlier.
604 if (CS == ActiveSuspend) continue;
605
606 auto *MappedCS = cast<AnyCoroSuspendInst>(VMap[CS]);
607 MappedCS->replaceAllUsesWith(SuspendResult);
608 MappedCS->eraseFromParent();
609 }
610}
611
612void CoroCloner::replaceCoroEnds() {
613 for (AnyCoroEndInst *CE : Shape.CoroEnds) {
614 // We use a null call graph because there's no call graph node for
615 // the cloned function yet. We'll just be rebuilding that later.
616 auto *NewCE = cast<AnyCoroEndInst>(VMap[CE]);
617 replaceCoroEnd(NewCE, Shape, NewFramePtr, /*in resume*/ true, nullptr);
618 }
619}
620
622 ValueToValueMapTy *VMap) {
623 if (Shape.ABI == coro::ABI::Async && Shape.CoroSuspends.empty())
624 return;
625 Value *CachedSlot = nullptr;
626 auto getSwiftErrorSlot = [&](Type *ValueTy) -> Value * {
627 if (CachedSlot) {
628 assert(cast<PointerType>(CachedSlot->getType())
629 ->isOpaqueOrPointeeTypeMatches(ValueTy) &&
630 "multiple swifterror slots in function with different types");
631 return CachedSlot;
632 }
633
634 // Check if the function has a swifterror argument.
635 for (auto &Arg : F.args()) {
636 if (Arg.isSwiftError()) {
637 CachedSlot = &Arg;
638 assert(cast<PointerType>(Arg.getType())
639 ->isOpaqueOrPointeeTypeMatches(ValueTy) &&
640 "swifterror argument does not have expected type");
641 return &Arg;
642 }
643 }
644
645 // Create a swifterror alloca.
646 IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHIOrDbg());
647 auto Alloca = Builder.CreateAlloca(ValueTy);
648 Alloca->setSwiftError(true);
649
650 CachedSlot = Alloca;
651 return Alloca;
652 };
653
654 for (CallInst *Op : Shape.SwiftErrorOps) {
655 auto MappedOp = VMap ? cast<CallInst>((*VMap)[Op]) : Op;
656 IRBuilder<> Builder(MappedOp);
657
658 // If there are no arguments, this is a 'get' operation.
659 Value *MappedResult;
660 if (Op->arg_empty()) {
661 auto ValueTy = Op->getType();
662 auto Slot = getSwiftErrorSlot(ValueTy);
663 MappedResult = Builder.CreateLoad(ValueTy, Slot);
664 } else {
665 assert(Op->arg_size() == 1);
666 auto Value = MappedOp->getArgOperand(0);
667 auto ValueTy = Value->getType();
668 auto Slot = getSwiftErrorSlot(ValueTy);
669 Builder.CreateStore(Value, Slot);
670 MappedResult = Slot;
671 }
672
673 MappedOp->replaceAllUsesWith(MappedResult);
674 MappedOp->eraseFromParent();
675 }
676
677 // If we're updating the original function, we've invalidated SwiftErrorOps.
678 if (VMap == nullptr) {
679 Shape.SwiftErrorOps.clear();
680 }
681}
682
683/// Returns all DbgVariableIntrinsic in F.
687 for (auto &I : instructions(F))
688 if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I))
689 Intrinsics.push_back(DVI);
690 return Intrinsics;
691}
692
693void CoroCloner::replaceSwiftErrorOps() {
694 ::replaceSwiftErrorOps(*NewF, Shape, &VMap);
695}
696
697void CoroCloner::salvageDebugInfo() {
701 for (DbgVariableIntrinsic *DVI : Worklist)
702 coro::salvageDebugInfo(ArgToAllocaMap, DVI, Shape.OptimizeFrame);
703
704 // Remove all salvaged dbg.declare intrinsics that became
705 // either unreachable or stale due to the CoroSplit transformation.
706 DominatorTree DomTree(*NewF);
707 auto IsUnreachableBlock = [&](BasicBlock *BB) {
708 return !isPotentiallyReachable(&NewF->getEntryBlock(), BB, nullptr,
709 &DomTree);
710 };
711 for (DbgVariableIntrinsic *DVI : Worklist) {
712 if (IsUnreachableBlock(DVI->getParent()))
713 DVI->eraseFromParent();
714 else if (isa_and_nonnull<AllocaInst>(DVI->getVariableLocationOp(0))) {
715 // Count all non-debuginfo uses in reachable blocks.
716 unsigned Uses = 0;
717 for (auto *User : DVI->getVariableLocationOp(0)->users())
718 if (auto *I = dyn_cast<Instruction>(User))
719 if (!isa<AllocaInst>(I) && !IsUnreachableBlock(I->getParent()))
720 ++Uses;
721 if (!Uses)
722 DVI->eraseFromParent();
723 }
724 }
725}
726
727void CoroCloner::replaceEntryBlock() {
728 // In the original function, the AllocaSpillBlock is a block immediately
729 // following the allocation of the frame object which defines GEPs for
730 // all the allocas that have been moved into the frame, and it ends by
731 // branching to the original beginning of the coroutine. Make this
732 // the entry block of the cloned function.
733 auto *Entry = cast<BasicBlock>(VMap[Shape.AllocaSpillBlock]);
734 auto *OldEntry = &NewF->getEntryBlock();
735 Entry->setName("entry" + Suffix);
736 Entry->moveBefore(OldEntry);
737 Entry->getTerminator()->eraseFromParent();
738
739 // Clear all predecessors of the new entry block. There should be
740 // exactly one predecessor, which we created when splitting out
741 // AllocaSpillBlock to begin with.
742 assert(Entry->hasOneUse());
743 auto BranchToEntry = cast<BranchInst>(Entry->user_back());
744 assert(BranchToEntry->isUnconditional());
745 Builder.SetInsertPoint(BranchToEntry);
746 Builder.CreateUnreachable();
747 BranchToEntry->eraseFromParent();
748
749 // Branch from the entry to the appropriate place.
750 Builder.SetInsertPoint(Entry);
751 switch (Shape.ABI) {
752 case coro::ABI::Switch: {
753 // In switch-lowering, we built a resume-entry block in the original
754 // function. Make the entry block branch to this.
755 auto *SwitchBB =
756 cast<BasicBlock>(VMap[Shape.SwitchLowering.ResumeEntryBlock]);
757 Builder.CreateBr(SwitchBB);
758 break;
759 }
760 case coro::ABI::Async:
761 case coro::ABI::Retcon:
762 case coro::ABI::RetconOnce: {
763 // In continuation ABIs, we want to branch to immediately after the
764 // active suspend point. Earlier phases will have put the suspend in its
765 // own basic block, so just thread our jump directly to its successor.
766 assert((Shape.ABI == coro::ABI::Async &&
767 isa<CoroSuspendAsyncInst>(ActiveSuspend)) ||
768 ((Shape.ABI == coro::ABI::Retcon ||
769 Shape.ABI == coro::ABI::RetconOnce) &&
770 isa<CoroSuspendRetconInst>(ActiveSuspend)));
771 auto *MappedCS = cast<AnyCoroSuspendInst>(VMap[ActiveSuspend]);
772 auto Branch = cast<BranchInst>(MappedCS->getNextNode());
773 assert(Branch->isUnconditional());
774 Builder.CreateBr(Branch->getSuccessor(0));
775 break;
776 }
777 }
778
779 // Any static alloca that's still being used but not reachable from the new
780 // entry needs to be moved to the new entry.
781 Function *F = OldEntry->getParent();
782 DominatorTree DT{*F};
784 auto *Alloca = dyn_cast<AllocaInst>(&I);
785 if (!Alloca || I.use_empty())
786 continue;
787 if (DT.isReachableFromEntry(I.getParent()) ||
788 !isa<ConstantInt>(Alloca->getArraySize()))
789 continue;
790 I.moveBefore(*Entry, Entry->getFirstInsertionPt());
791 }
792}
793
794/// Derive the value of the new frame pointer.
795Value *CoroCloner::deriveNewFramePointer() {
796 // Builder should be inserting to the front of the new entry block.
797
798 switch (Shape.ABI) {
799 // In switch-lowering, the argument is the frame pointer.
800 case coro::ABI::Switch:
801 return &*NewF->arg_begin();
802 // In async-lowering, one of the arguments is an async context as determined
803 // by the `llvm.coro.id.async` intrinsic. We can retrieve the async context of
804 // the resume function from the async context projection function associated
805 // with the active suspend. The frame is located as a tail to the async
806 // context header.
807 case coro::ABI::Async: {
808 auto *ActiveAsyncSuspend = cast<CoroSuspendAsyncInst>(ActiveSuspend);
809 auto ContextIdx = ActiveAsyncSuspend->getStorageArgumentIndex() & 0xff;
810 auto *CalleeContext = NewF->getArg(ContextIdx);
811 auto *FramePtrTy = Shape.FrameTy->getPointerTo();
812 auto *ProjectionFunc =
813 ActiveAsyncSuspend->getAsyncContextProjectionFunction();
814 auto DbgLoc =
815 cast<CoroSuspendAsyncInst>(VMap[ActiveSuspend])->getDebugLoc();
816 // Calling i8* (i8*)
817 auto *CallerContext = Builder.CreateCall(ProjectionFunc->getFunctionType(),
818 ProjectionFunc, CalleeContext);
819 CallerContext->setCallingConv(ProjectionFunc->getCallingConv());
820 CallerContext->setDebugLoc(DbgLoc);
821 // The frame is located after the async_context header.
822 auto &Context = Builder.getContext();
823 auto *FramePtrAddr = Builder.CreateConstInBoundsGEP1_32(
824 Type::getInt8Ty(Context), CallerContext,
825 Shape.AsyncLowering.FrameOffset, "async.ctx.frameptr");
826 // Inline the projection function.
828 auto InlineRes = InlineFunction(*CallerContext, InlineInfo);
829 assert(InlineRes.isSuccess());
830 (void)InlineRes;
831 return Builder.CreateBitCast(FramePtrAddr, FramePtrTy);
832 }
833 // In continuation-lowering, the argument is the opaque storage.
834 case coro::ABI::Retcon:
835 case coro::ABI::RetconOnce: {
836 Argument *NewStorage = &*NewF->arg_begin();
837 auto FramePtrTy = Shape.FrameTy->getPointerTo();
838
839 // If the storage is inline, just bitcast to the storage to the frame type.
840 if (Shape.RetconLowering.IsFrameInlineInStorage)
841 return Builder.CreateBitCast(NewStorage, FramePtrTy);
842
843 // Otherwise, load the real frame from the opaque storage.
844 auto FramePtrPtr =
845 Builder.CreateBitCast(NewStorage, FramePtrTy->getPointerTo());
846 return Builder.CreateLoad(FramePtrTy, FramePtrPtr);
847 }
848 }
849 llvm_unreachable("bad ABI");
850}
851
852static void addFramePointerAttrs(AttributeList &Attrs, LLVMContext &Context,
853 unsigned ParamIndex, uint64_t Size,
854 Align Alignment, bool NoAlias) {
855 AttrBuilder ParamAttrs(Context);
856 ParamAttrs.addAttribute(Attribute::NonNull);
857 ParamAttrs.addAttribute(Attribute::NoUndef);
858
859 if (NoAlias)
860 ParamAttrs.addAttribute(Attribute::NoAlias);
861
862 ParamAttrs.addAlignmentAttr(Alignment);
863 ParamAttrs.addDereferenceableAttr(Size);
864 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
865}
866
867static void addAsyncContextAttrs(AttributeList &Attrs, LLVMContext &Context,
868 unsigned ParamIndex) {
869 AttrBuilder ParamAttrs(Context);
870 ParamAttrs.addAttribute(Attribute::SwiftAsync);
871 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
872}
873
874static void addSwiftSelfAttrs(AttributeList &Attrs, LLVMContext &Context,
875 unsigned ParamIndex) {
876 AttrBuilder ParamAttrs(Context);
877 ParamAttrs.addAttribute(Attribute::SwiftSelf);
878 Attrs = Attrs.addParamAttributes(Context, ParamIndex, ParamAttrs);
879}
880
881/// Clone the body of the original function into a resume function of
882/// some sort.
883void CoroCloner::create() {
884 // Create the new function if we don't already have one.
885 if (!NewF) {
886 NewF = createCloneDeclaration(OrigF, Shape, Suffix,
887 OrigF.getParent()->end(), ActiveSuspend);
888 }
889
890 // Replace all args with dummy instructions. If an argument is the old frame
891 // pointer, the dummy will be replaced by the new frame pointer once it is
892 // computed below. Uses of all other arguments should have already been
893 // rewritten by buildCoroutineFrame() to use loads/stores on the coroutine
894 // frame.
896 for (Argument &A : OrigF.args()) {
897 DummyArgs.push_back(new FreezeInst(UndefValue::get(A.getType())));
898 VMap[&A] = DummyArgs.back();
899 }
900
902
903 // Ignore attempts to change certain attributes of the function.
904 // TODO: maybe there should be a way to suppress this during cloning?
905 auto savedVisibility = NewF->getVisibility();
906 auto savedUnnamedAddr = NewF->getUnnamedAddr();
907 auto savedDLLStorageClass = NewF->getDLLStorageClass();
908
909 // NewF's linkage (which CloneFunctionInto does *not* change) might not
910 // be compatible with the visibility of OrigF (which it *does* change),
911 // so protect against that.
912 auto savedLinkage = NewF->getLinkage();
913 NewF->setLinkage(llvm::GlobalValue::ExternalLinkage);
914
915 CloneFunctionInto(NewF, &OrigF, VMap,
916 CloneFunctionChangeType::LocalChangesOnly, Returns);
917
918 auto &Context = NewF->getContext();
919
920 // For async functions / continuations, adjust the scope line of the
921 // clone to the line number of the suspend point. However, only
922 // adjust the scope line when the files are the same. This ensures
923 // line number and file name belong together. The scope line is
924 // associated with all pre-prologue instructions. This avoids a jump
925 // in the linetable from the function declaration to the suspend point.
926 if (DISubprogram *SP = NewF->getSubprogram()) {
927 assert(SP != OrigF.getSubprogram() && SP->isDistinct());
928 if (ActiveSuspend)
929 if (auto DL = ActiveSuspend->getDebugLoc())
930 if (SP->getFile() == DL->getFile())
931 SP->setScopeLine(DL->getLine());
932 // Update the linkage name to reflect the modified symbol name. It
933 // is necessary to update the linkage name in Swift, since the
934 // mangling changes for resume functions. It might also be the
935 // right thing to do in C++, but due to a limitation in LLVM's
936 // AsmPrinter we can only do this if the function doesn't have an
937 // abstract specification, since the DWARF backend expects the
938 // abstract specification to contain the linkage name and asserts
939 // that they are identical.
940 if (!SP->getDeclaration() && SP->getUnit() &&
941 SP->getUnit()->getSourceLanguage() == dwarf::DW_LANG_Swift)
942 SP->replaceLinkageName(MDString::get(Context, NewF->getName()));
943 }
944
945 NewF->setLinkage(savedLinkage);
946 NewF->setVisibility(savedVisibility);
947 NewF->setUnnamedAddr(savedUnnamedAddr);
948 NewF->setDLLStorageClass(savedDLLStorageClass);
949 // The function sanitizer metadata needs to match the signature of the
950 // function it is being attached to. However this does not hold for split
951 // functions here. Thus remove the metadata for split functions.
952 if (Shape.ABI == coro::ABI::Switch &&
953 NewF->hasMetadata(LLVMContext::MD_func_sanitize))
954 NewF->eraseMetadata(LLVMContext::MD_func_sanitize);
955
956 // Replace the attributes of the new function:
957 auto OrigAttrs = NewF->getAttributes();
958 auto NewAttrs = AttributeList();
959
960 switch (Shape.ABI) {
961 case coro::ABI::Switch:
962 // Bootstrap attributes by copying function attributes from the
963 // original function. This should include optimization settings and so on.
964 NewAttrs = NewAttrs.addFnAttributes(
965 Context, AttrBuilder(Context, OrigAttrs.getFnAttrs()));
966
967 addFramePointerAttrs(NewAttrs, Context, 0, Shape.FrameSize,
968 Shape.FrameAlign, /*NoAlias=*/false);
969 break;
970 case coro::ABI::Async: {
971 auto *ActiveAsyncSuspend = cast<CoroSuspendAsyncInst>(ActiveSuspend);
972 if (OrigF.hasParamAttribute(Shape.AsyncLowering.ContextArgNo,
973 Attribute::SwiftAsync)) {
974 uint32_t ArgAttributeIndices =
975 ActiveAsyncSuspend->getStorageArgumentIndex();
976 auto ContextArgIndex = ArgAttributeIndices & 0xff;
977 addAsyncContextAttrs(NewAttrs, Context, ContextArgIndex);
978
979 // `swiftasync` must preceed `swiftself` so 0 is not a valid index for
980 // `swiftself`.
981 auto SwiftSelfIndex = ArgAttributeIndices >> 8;
982 if (SwiftSelfIndex)
983 addSwiftSelfAttrs(NewAttrs, Context, SwiftSelfIndex);
984 }
985
986 // Transfer the original function's attributes.
987 auto FnAttrs = OrigF.getAttributes().getFnAttrs();
988 NewAttrs = NewAttrs.addFnAttributes(Context, AttrBuilder(Context, FnAttrs));
989 break;
990 }
991 case coro::ABI::Retcon:
992 case coro::ABI::RetconOnce:
993 // If we have a continuation prototype, just use its attributes,
994 // full-stop.
995 NewAttrs = Shape.RetconLowering.ResumePrototype->getAttributes();
996
997 /// FIXME: Is it really good to add the NoAlias attribute?
998 addFramePointerAttrs(NewAttrs, Context, 0,
999 Shape.getRetconCoroId()->getStorageSize(),
1000 Shape.getRetconCoroId()->getStorageAlignment(),
1001 /*NoAlias=*/true);
1002
1003 break;
1004 }
1005
1006 switch (Shape.ABI) {
1007 // In these ABIs, the cloned functions always return 'void', and the
1008 // existing return sites are meaningless. Note that for unique
1009 // continuations, this includes the returns associated with suspends;
1010 // this is fine because we can't suspend twice.
1011 case coro::ABI::Switch:
1012 case coro::ABI::RetconOnce:
1013 // Remove old returns.
1014 for (ReturnInst *Return : Returns)
1015 changeToUnreachable(Return);
1016 break;
1017
1018 // With multi-suspend continuations, we'll already have eliminated the
1019 // original returns and inserted returns before all the suspend points,
1020 // so we want to leave any returns in place.
1021 case coro::ABI::Retcon:
1022 break;
1023 // Async lowering will insert musttail call functions at all suspend points
1024 // followed by a return.
1025 // Don't change returns to unreachable because that will trip up the verifier.
1026 // These returns should be unreachable from the clone.
1027 case coro::ABI::Async:
1028 break;
1029 }
1030
1031 NewF->setAttributes(NewAttrs);
1032 NewF->setCallingConv(Shape.getResumeFunctionCC());
1033
1034 // Set up the new entry block.
1035 replaceEntryBlock();
1036
1037 Builder.SetInsertPoint(&NewF->getEntryBlock().front());
1038 NewFramePtr = deriveNewFramePointer();
1039
1040 // Remap frame pointer.
1041 Value *OldFramePtr = VMap[Shape.FramePtr];
1042 NewFramePtr->takeName(OldFramePtr);
1043 OldFramePtr->replaceAllUsesWith(NewFramePtr);
1044
1045 // Remap vFrame pointer.
1046 auto *NewVFrame = Builder.CreateBitCast(
1047 NewFramePtr, Type::getInt8PtrTy(Builder.getContext()), "vFrame");
1048 Value *OldVFrame = cast<Value>(VMap[Shape.CoroBegin]);
1049 if (OldVFrame != NewVFrame)
1050 OldVFrame->replaceAllUsesWith(NewVFrame);
1051
1052 // All uses of the arguments should have been resolved by this point,
1053 // so we can safely remove the dummy values.
1054 for (Instruction *DummyArg : DummyArgs) {
1055 DummyArg->replaceAllUsesWith(UndefValue::get(DummyArg->getType()));
1056 DummyArg->deleteValue();
1057 }
1058
1059 switch (Shape.ABI) {
1060 case coro::ABI::Switch:
1061 // Rewrite final suspend handling as it is not done via switch (allows to
1062 // remove final case from the switch, since it is undefined behavior to
1063 // resume the coroutine suspended at the final suspend point.
1064 if (Shape.SwitchLowering.HasFinalSuspend)
1065 handleFinalSuspend();
1066 break;
1067 case coro::ABI::Async:
1068 case coro::ABI::Retcon:
1069 case coro::ABI::RetconOnce:
1070 // Replace uses of the active suspend with the corresponding
1071 // continuation-function arguments.
1072 assert(ActiveSuspend != nullptr &&
1073 "no active suspend when lowering a continuation-style coroutine");
1074 replaceRetconOrAsyncSuspendUses();
1075 break;
1076 }
1077
1078 // Handle suspends.
1079 replaceCoroSuspends();
1080
1081 // Handle swifterror.
1083
1084 // Remove coro.end intrinsics.
1085 replaceCoroEnds();
1086
1087 // Salvage debug info that points into the coroutine frame.
1089
1090 // Eliminate coro.free from the clones, replacing it with 'null' in cleanup,
1091 // to suppress deallocation code.
1092 if (Shape.ABI == coro::ABI::Switch)
1093 coro::replaceCoroFree(cast<CoroIdInst>(VMap[Shape.CoroBegin->getId()]),
1094 /*Elide=*/ FKind == CoroCloner::Kind::SwitchCleanup);
1095}
1096
1097// Create a resume clone by cloning the body of the original function, setting
1098// new entry block and replacing coro.suspend an appropriate value to force
1099// resume or cleanup pass for every suspend point.
1100static Function *createClone(Function &F, const Twine &Suffix,
1101 coro::Shape &Shape, CoroCloner::Kind FKind) {
1102 CoroCloner Cloner(F, Suffix, Shape, FKind);
1103 Cloner.create();
1104 return Cloner.getFunction();
1105}
1106
1108 assert(Shape.ABI == coro::ABI::Async);
1109
1110 auto *FuncPtrStruct = cast<ConstantStruct>(
1112 auto *OrigRelativeFunOffset = FuncPtrStruct->getOperand(0);
1113 auto *OrigContextSize = FuncPtrStruct->getOperand(1);
1114 auto *NewContextSize = ConstantInt::get(OrigContextSize->getType(),
1116 auto *NewFuncPtrStruct = ConstantStruct::get(
1117 FuncPtrStruct->getType(), OrigRelativeFunOffset, NewContextSize);
1118
1119 Shape.AsyncLowering.AsyncFuncPointer->setInitializer(NewFuncPtrStruct);
1120}
1121
1123 if (Shape.ABI == coro::ABI::Async)
1125
1126 for (CoroAlignInst *CA : Shape.CoroAligns) {
1128 ConstantInt::get(CA->getType(), Shape.FrameAlign.value()));
1129 CA->eraseFromParent();
1130 }
1131
1132 if (Shape.CoroSizes.empty())
1133 return;
1134
1135 // In the same function all coro.sizes should have the same result type.
1136 auto *SizeIntrin = Shape.CoroSizes.back();
1137 Module *M = SizeIntrin->getModule();
1138 const DataLayout &DL = M->getDataLayout();
1139 auto Size = DL.getTypeAllocSize(Shape.FrameTy);
1140 auto *SizeConstant = ConstantInt::get(SizeIntrin->getType(), Size);
1141
1142 for (CoroSizeInst *CS : Shape.CoroSizes) {
1143 CS->replaceAllUsesWith(SizeConstant);
1144 CS->eraseFromParent();
1145 }
1146}
1147
1148// Create a global constant array containing pointers to functions provided and
1149// set Info parameter of CoroBegin to point at this constant. Example:
1150//
1151// @f.resumers = internal constant [2 x void(%f.frame*)*]
1152// [void(%f.frame*)* @f.resume, void(%f.frame*)* @f.destroy]
1153// define void @f() {
1154// ...
1155// call i8* @llvm.coro.begin(i8* null, i32 0, i8* null,
1156// i8* bitcast([2 x void(%f.frame*)*] * @f.resumers to i8*))
1157//
1158// Assumes that all the functions have the same signature.
1159static void setCoroInfo(Function &F, coro::Shape &Shape,
1161 // This only works under the switch-lowering ABI because coro elision
1162 // only works on the switch-lowering ABI.
1163 assert(Shape.ABI == coro::ABI::Switch);
1164
1165 SmallVector<Constant *, 4> Args(Fns.begin(), Fns.end());
1166 assert(!Args.empty());
1167 Function *Part = *Fns.begin();
1168 Module *M = Part->getParent();
1169 auto *ArrTy = ArrayType::get(Part->getType(), Args.size());
1170
1171 auto *ConstVal = ConstantArray::get(ArrTy, Args);
1172 auto *GV = new GlobalVariable(*M, ConstVal->getType(), /*isConstant=*/true,
1173 GlobalVariable::PrivateLinkage, ConstVal,
1174 F.getName() + Twine(".resumers"));
1175
1176 // Update coro.begin instruction to refer to this constant.
1177 LLVMContext &C = F.getContext();
1179 Shape.getSwitchCoroId()->setInfo(BC);
1180}
1181
1182// Store addresses of Resume/Destroy/Cleanup functions in the coroutine frame.
1183static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn,
1184 Function *DestroyFn, Function *CleanupFn) {
1185 assert(Shape.ABI == coro::ABI::Switch);
1186
1188
1189 auto *ResumeAddr = Builder.CreateStructGEP(
1191 "resume.addr");
1192 Builder.CreateStore(ResumeFn, ResumeAddr);
1193
1194 Value *DestroyOrCleanupFn = DestroyFn;
1195
1196 CoroIdInst *CoroId = Shape.getSwitchCoroId();
1197 if (CoroAllocInst *CA = CoroId->getCoroAlloc()) {
1198 // If there is a CoroAlloc and it returns false (meaning we elide the
1199 // allocation, use CleanupFn instead of DestroyFn).
1200 DestroyOrCleanupFn = Builder.CreateSelect(CA, DestroyFn, CleanupFn);
1201 }
1202
1203 auto *DestroyAddr = Builder.CreateStructGEP(
1205 "destroy.addr");
1206 Builder.CreateStore(DestroyOrCleanupFn, DestroyAddr);
1207}
1208
1211
1212#ifndef NDEBUG
1213 // For now, we do a mandatory verification step because we don't
1214 // entirely trust this pass. Note that we don't want to add a verifier
1215 // pass to FPM below because it will also verify all the global data.
1216 if (verifyFunction(F, &errs()))
1217 report_fatal_error("Broken function");
1218#endif
1219}
1220
1221// Assuming we arrived at the block NewBlock from Prev instruction, store
1222// PHI's incoming values in the ResolvedValues map.
1223static void
1225 DenseMap<Value *, Value *> &ResolvedValues) {
1226 auto *PrevBB = Prev->getParent();
1227 for (PHINode &PN : NewBlock->phis()) {
1228 auto V = PN.getIncomingValueForBlock(PrevBB);
1229 // See if we already resolved it.
1230 auto VI = ResolvedValues.find(V);
1231 if (VI != ResolvedValues.end())
1232 V = VI->second;
1233 // Remember the value.
1234 ResolvedValues[&PN] = V;
1235 }
1236}
1237
1238// Replace a sequence of branches leading to a ret, with a clone of a ret
1239// instruction. Suspend instruction represented by a switch, track the PHI
1240// values and select the correct case successor when possible.
1242 DenseMap<Value *, Value *> ResolvedValues;
1243 BasicBlock *UnconditionalSucc = nullptr;
1244 assert(InitialInst->getModule());
1245 const DataLayout &DL = InitialInst->getModule()->getDataLayout();
1246
1247 auto GetFirstValidInstruction = [](Instruction *I) {
1248 while (I) {
1249 // BitCastInst wouldn't generate actual code so that we could skip it.
1250 if (isa<BitCastInst>(I) || I->isDebugOrPseudoInst() ||
1251 I->isLifetimeStartOrEnd())
1252 I = I->getNextNode();
1253 else if (isInstructionTriviallyDead(I))
1254 // Duing we are in the middle of the transformation, we need to erase
1255 // the dead instruction manually.
1256 I = &*I->eraseFromParent();
1257 else
1258 break;
1259 }
1260 return I;
1261 };
1262
1263 auto TryResolveConstant = [&ResolvedValues](Value *V) {
1264 auto It = ResolvedValues.find(V);
1265 if (It != ResolvedValues.end())
1266 V = It->second;
1267 return dyn_cast<ConstantInt>(V);
1268 };
1269
1270 Instruction *I = InitialInst;
1271 while (I->isTerminator() || isa<CmpInst>(I)) {
1272 if (isa<ReturnInst>(I)) {
1273 if (I != InitialInst) {
1274 // If InitialInst is an unconditional branch,
1275 // remove PHI values that come from basic block of InitialInst
1276 if (UnconditionalSucc)
1277 UnconditionalSucc->removePredecessor(InitialInst->getParent(), true);
1278 ReplaceInstWithInst(InitialInst, I->clone());
1279 }
1280 return true;
1281 }
1282 if (auto *BR = dyn_cast<BranchInst>(I)) {
1283 if (BR->isUnconditional()) {
1284 BasicBlock *Succ = BR->getSuccessor(0);
1285 if (I == InitialInst)
1286 UnconditionalSucc = Succ;
1287 scanPHIsAndUpdateValueMap(I, Succ, ResolvedValues);
1288 I = GetFirstValidInstruction(Succ->getFirstNonPHIOrDbgOrLifetime());
1289 continue;
1290 }
1291
1292 BasicBlock *BB = BR->getParent();
1293 // Handle the case the condition of the conditional branch is constant.
1294 // e.g.,
1295 //
1296 // br i1 false, label %cleanup, label %CoroEnd
1297 //
1298 // It is possible during the transformation. We could continue the
1299 // simplifying in this case.
1300 if (ConstantFoldTerminator(BB, /*DeleteDeadConditions=*/true)) {
1301 // Handle this branch in next iteration.
1302 I = BB->getTerminator();
1303 continue;
1304 }
1305 } else if (auto *CondCmp = dyn_cast<CmpInst>(I)) {
1306 // If the case number of suspended switch instruction is reduced to
1307 // 1, then it is simplified to CmpInst in llvm::ConstantFoldTerminator.
1308 auto *BR = dyn_cast<BranchInst>(
1309 GetFirstValidInstruction(CondCmp->getNextNode()));
1310 if (!BR || !BR->isConditional() || CondCmp != BR->getCondition())
1311 return false;
1312
1313 // And the comparsion looks like : %cond = icmp eq i8 %V, constant.
1314 // So we try to resolve constant for the first operand only since the
1315 // second operand should be literal constant by design.
1316 ConstantInt *Cond0 = TryResolveConstant(CondCmp->getOperand(0));
1317 auto *Cond1 = dyn_cast<ConstantInt>(CondCmp->getOperand(1));
1318 if (!Cond0 || !Cond1)
1319 return false;
1320
1321 // Both operands of the CmpInst are Constant. So that we could evaluate
1322 // it immediately to get the destination.
1323 auto *ConstResult =
1324 dyn_cast_or_null<ConstantInt>(ConstantFoldCompareInstOperands(
1325 CondCmp->getPredicate(), Cond0, Cond1, DL));
1326 if (!ConstResult)
1327 return false;
1328
1329 CondCmp->replaceAllUsesWith(ConstResult);
1330 CondCmp->eraseFromParent();
1331
1332 // Handle this branch in next iteration.
1333 I = BR;
1334 continue;
1335 } else if (auto *SI = dyn_cast<SwitchInst>(I)) {
1336 ConstantInt *Cond = TryResolveConstant(SI->getCondition());
1337 if (!Cond)
1338 return false;
1339
1340 BasicBlock *BB = SI->findCaseValue(Cond)->getCaseSuccessor();
1341 scanPHIsAndUpdateValueMap(I, BB, ResolvedValues);
1342 I = GetFirstValidInstruction(BB->getFirstNonPHIOrDbgOrLifetime());
1343 continue;
1344 }
1345
1346 return false;
1347 }
1348 return false;
1349}
1350
1351// Check whether CI obeys the rules of musttail attribute.
1352static bool shouldBeMustTail(const CallInst &CI, const Function &F) {
1353 if (CI.isInlineAsm())
1354 return false;
1355
1356 // Match prototypes and calling conventions of resume function.
1357 FunctionType *CalleeTy = CI.getFunctionType();
1358 if (!CalleeTy->getReturnType()->isVoidTy() || (CalleeTy->getNumParams() != 1))
1359 return false;
1360
1361 Type *CalleeParmTy = CalleeTy->getParamType(0);
1362 if (!CalleeParmTy->isPointerTy() ||
1363 (CalleeParmTy->getPointerAddressSpace() != 0))
1364 return false;
1365
1366 if (CI.getCallingConv() != F.getCallingConv())
1367 return false;
1368
1369 // CI should not has any ABI-impacting function attributes.
1370 static const Attribute::AttrKind ABIAttrs[] = {
1371 Attribute::StructRet, Attribute::ByVal, Attribute::InAlloca,
1372 Attribute::Preallocated, Attribute::InReg, Attribute::Returned,
1373 Attribute::SwiftSelf, Attribute::SwiftError};
1374 AttributeList Attrs = CI.getAttributes();
1375 for (auto AK : ABIAttrs)
1376 if (Attrs.hasParamAttr(0, AK))
1377 return false;
1378
1379 return true;
1380}
1381
1382// Add musttail to any resume instructions that is immediately followed by a
1383// suspend (i.e. ret). We do this even in -O0 to support guaranteed tail call
1384// for symmetrical coroutine control transfer (C++ Coroutines TS extension).
1385// This transformation is done only in the resume part of the coroutine that has
1386// identical signature and calling convention as the coro.resume call.
1388 bool changed = false;
1389
1390 // Collect potential resume instructions.
1392 for (auto &I : instructions(F))
1393 if (auto *Call = dyn_cast<CallInst>(&I))
1394 if (shouldBeMustTail(*Call, F))
1395 Resumes.push_back(Call);
1396
1397 // Set musttail on those that are followed by a ret instruction.
1398 for (CallInst *Call : Resumes)
1399 // Skip targets which don't support tail call on the specific case.
1400 if (TTI.supportsTailCallFor(Call) &&
1401 simplifyTerminatorLeadingToRet(Call->getNextNode())) {
1402 Call->setTailCallKind(CallInst::TCK_MustTail);
1403 changed = true;
1404 }
1405
1406 if (changed)
1408}
1409
1410// Coroutine has no suspend points. Remove heap allocation for the coroutine
1411// frame if possible.
1413 auto *CoroBegin = Shape.CoroBegin;
1414 auto *CoroId = CoroBegin->getId();
1415 auto *AllocInst = CoroId->getCoroAlloc();
1416 switch (Shape.ABI) {
1417 case coro::ABI::Switch: {
1418 auto SwitchId = cast<CoroIdInst>(CoroId);
1419 coro::replaceCoroFree(SwitchId, /*Elide=*/AllocInst != nullptr);
1420 if (AllocInst) {
1421 IRBuilder<> Builder(AllocInst);
1422 auto *Frame = Builder.CreateAlloca(Shape.FrameTy);
1423 Frame->setAlignment(Shape.FrameAlign);
1424 auto *VFrame = Builder.CreateBitCast(Frame, Builder.getInt8PtrTy());
1425 AllocInst->replaceAllUsesWith(Builder.getFalse());
1426 AllocInst->eraseFromParent();
1427 CoroBegin->replaceAllUsesWith(VFrame);
1428 } else {
1429 CoroBegin->replaceAllUsesWith(CoroBegin->getMem());
1430 }
1431
1432 break;
1433 }
1434 case coro::ABI::Async:
1435 case coro::ABI::Retcon:
1436 case coro::ABI::RetconOnce:
1437 CoroBegin->replaceAllUsesWith(UndefValue::get(CoroBegin->getType()));
1438 break;
1439 }
1440
1441 CoroBegin->eraseFromParent();
1442}
1443
1444// SimplifySuspendPoint needs to check that there is no calls between
1445// coro_save and coro_suspend, since any of the calls may potentially resume
1446// the coroutine and if that is the case we cannot eliminate the suspend point.
1448 for (Instruction *I = From; I != To; I = I->getNextNode()) {
1449 // Assume that no intrinsic can resume the coroutine.
1450 if (isa<IntrinsicInst>(I))
1451 continue;
1452
1453 if (isa<CallBase>(I))
1454 return true;
1455 }
1456 return false;
1457}
1458
1459static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB) {
1462
1463 Set.insert(SaveBB);
1464 Worklist.push_back(ResDesBB);
1465
1466 // Accumulate all blocks between SaveBB and ResDesBB. Because CoroSaveIntr
1467 // returns a token consumed by suspend instruction, all blocks in between
1468 // will have to eventually hit SaveBB when going backwards from ResDesBB.
1469 while (!Worklist.empty()) {
1470 auto *BB = Worklist.pop_back_val();
1471 Set.insert(BB);
1472 for (auto *Pred : predecessors(BB))
1473 if (!Set.contains(Pred))
1474 Worklist.push_back(Pred);
1475 }
1476
1477 // SaveBB and ResDesBB are checked separately in hasCallsBetween.
1478 Set.erase(SaveBB);
1479 Set.erase(ResDesBB);
1480
1481 for (auto *BB : Set)
1482 if (hasCallsInBlockBetween(BB->getFirstNonPHI(), nullptr))
1483 return true;
1484
1485 return false;
1486}
1487
1488static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy) {
1489 auto *SaveBB = Save->getParent();
1490 auto *ResumeOrDestroyBB = ResumeOrDestroy->getParent();
1491
1492 if (SaveBB == ResumeOrDestroyBB)
1493 return hasCallsInBlockBetween(Save->getNextNode(), ResumeOrDestroy);
1494
1495 // Any calls from Save to the end of the block?
1496 if (hasCallsInBlockBetween(Save->getNextNode(), nullptr))
1497 return true;
1498
1499 // Any calls from begging of the block up to ResumeOrDestroy?
1500 if (hasCallsInBlockBetween(ResumeOrDestroyBB->getFirstNonPHI(),
1501 ResumeOrDestroy))
1502 return true;
1503
1504 // Any calls in all of the blocks between SaveBB and ResumeOrDestroyBB?
1505 if (hasCallsInBlocksBetween(SaveBB, ResumeOrDestroyBB))
1506 return true;
1507
1508 return false;
1509}
1510
1511// If a SuspendIntrin is preceded by Resume or Destroy, we can eliminate the
1512// suspend point and replace it with nornal control flow.
1514 CoroBeginInst *CoroBegin) {
1515 Instruction *Prev = Suspend->getPrevNode();
1516 if (!Prev) {
1517 auto *Pred = Suspend->getParent()->getSinglePredecessor();
1518 if (!Pred)
1519 return false;
1520 Prev = Pred->getTerminator();
1521 }
1522
1523 CallBase *CB = dyn_cast<CallBase>(Prev);
1524 if (!CB)
1525 return false;
1526
1527 auto *Callee = CB->getCalledOperand()->stripPointerCasts();
1528
1529 // See if the callsite is for resumption or destruction of the coroutine.
1530 auto *SubFn = dyn_cast<CoroSubFnInst>(Callee);
1531 if (!SubFn)
1532 return false;
1533
1534 // Does not refer to the current coroutine, we cannot do anything with it.
1535 if (SubFn->getFrame() != CoroBegin)
1536 return false;
1537
1538 // See if the transformation is safe. Specifically, see if there are any
1539 // calls in between Save and CallInstr. They can potenitally resume the
1540 // coroutine rendering this optimization unsafe.
1541 auto *Save = Suspend->getCoroSave();
1542 if (hasCallsBetween(Save, CB))
1543 return false;
1544
1545 // Replace llvm.coro.suspend with the value that results in resumption over
1546 // the resume or cleanup path.
1547 Suspend->replaceAllUsesWith(SubFn->getRawIndex());
1548 Suspend->eraseFromParent();
1549 Save->eraseFromParent();
1550
1551 // No longer need a call to coro.resume or coro.destroy.
1552 if (auto *Invoke = dyn_cast<InvokeInst>(CB)) {
1553 BranchInst::Create(Invoke->getNormalDest(), Invoke);
1554 }
1555
1556 // Grab the CalledValue from CB before erasing the CallInstr.
1557 auto *CalledValue = CB->getCalledOperand();
1558 CB->eraseFromParent();
1559
1560 // If no more users remove it. Usually it is a bitcast of SubFn.
1561 if (CalledValue != SubFn && CalledValue->user_empty())
1562 if (auto *I = dyn_cast<Instruction>(CalledValue))
1563 I->eraseFromParent();
1564
1565 // Now we are good to remove SubFn.
1566 if (SubFn->user_empty())
1567 SubFn->eraseFromParent();
1568
1569 return true;
1570}
1571
1572// Remove suspend points that are simplified.
1574 // Currently, the only simplification we do is switch-lowering-specific.
1575 if (Shape.ABI != coro::ABI::Switch)
1576 return;
1577
1578 auto &S = Shape.CoroSuspends;
1579 size_t I = 0, N = S.size();
1580 if (N == 0)
1581 return;
1582
1583 size_t ChangedFinalIndex = std::numeric_limits<size_t>::max();
1584 while (true) {
1585 auto SI = cast<CoroSuspendInst>(S[I]);
1586 // Leave final.suspend to handleFinalSuspend since it is undefined behavior
1587 // to resume a coroutine suspended at the final suspend point.
1588 if (!SI->isFinal() && simplifySuspendPoint(SI, Shape.CoroBegin)) {
1589 if (--N == I)
1590 break;
1591
1592 std::swap(S[I], S[N]);
1593
1594 if (cast<CoroSuspendInst>(S[I])->isFinal()) {
1596 ChangedFinalIndex = I;
1597 }
1598
1599 continue;
1600 }
1601 if (++I == N)
1602 break;
1603 }
1604 S.resize(N);
1605
1606 // Maintain final.suspend in case final suspend was swapped.
1607 // Due to we requrie the final suspend to be the last element of CoroSuspends.
1608 if (ChangedFinalIndex < N) {
1609 assert(cast<CoroSuspendInst>(S[ChangedFinalIndex])->isFinal());
1610 std::swap(S[ChangedFinalIndex], S.back());
1611 }
1612}
1613
1617 assert(Shape.ABI == coro::ABI::Switch);
1618
1619 createResumeEntryBlock(F, Shape);
1620 auto ResumeClone = createClone(F, ".resume", Shape,
1621 CoroCloner::Kind::SwitchResume);
1622 auto DestroyClone = createClone(F, ".destroy", Shape,
1623 CoroCloner::Kind::SwitchUnwind);
1624 auto CleanupClone = createClone(F, ".cleanup", Shape,
1625 CoroCloner::Kind::SwitchCleanup);
1626
1627 postSplitCleanup(*ResumeClone);
1628 postSplitCleanup(*DestroyClone);
1629 postSplitCleanup(*CleanupClone);
1630
1631 // Adding musttail call to support symmetric transfer.
1632 // Skip targets which don't support tail call.
1633 //
1634 // FIXME: Could we support symmetric transfer effectively without musttail
1635 // call?
1636 if (TTI.supportsTailCalls())
1637 addMustTailToCoroResumes(*ResumeClone, TTI);
1638
1639 // Store addresses resume/destroy/cleanup functions in the coroutine frame.
1640 updateCoroFrame(Shape, ResumeClone, DestroyClone, CleanupClone);
1641
1642 assert(Clones.empty());
1643 Clones.push_back(ResumeClone);
1644 Clones.push_back(DestroyClone);
1645 Clones.push_back(CleanupClone);
1646
1647 // Create a constant array referring to resume/destroy/clone functions pointed
1648 // by the last argument of @llvm.coro.info, so that CoroElide pass can
1649 // determined correct function to call.
1650 setCoroInfo(F, Shape, Clones);
1651}
1652
1654 Value *Continuation) {
1655 auto *ResumeIntrinsic = Suspend->getResumeFunction();
1656 auto &Context = Suspend->getParent()->getParent()->getContext();
1657 auto *Int8PtrTy = Type::getInt8PtrTy(Context);
1658
1659 IRBuilder<> Builder(ResumeIntrinsic);
1660 auto *Val = Builder.CreateBitOrPointerCast(Continuation, Int8PtrTy);
1661 ResumeIntrinsic->replaceAllUsesWith(Val);
1662 ResumeIntrinsic->eraseFromParent();
1664 UndefValue::get(Int8PtrTy));
1665}
1666
1667/// Coerce the arguments in \p FnArgs according to \p FnTy in \p CallArgs.
1668static void coerceArguments(IRBuilder<> &Builder, FunctionType *FnTy,
1669 ArrayRef<Value *> FnArgs,
1670 SmallVectorImpl<Value *> &CallArgs) {
1671 size_t ArgIdx = 0;
1672 for (auto *paramTy : FnTy->params()) {
1673 assert(ArgIdx < FnArgs.size());
1674 if (paramTy != FnArgs[ArgIdx]->getType())
1675 CallArgs.push_back(
1676 Builder.CreateBitOrPointerCast(FnArgs[ArgIdx], paramTy));
1677 else
1678 CallArgs.push_back(FnArgs[ArgIdx]);
1679 ++ArgIdx;
1680 }
1681}
1682
1685 IRBuilder<> &Builder) {
1686 auto *FnTy = MustTailCallFn->getFunctionType();
1687 // Coerce the arguments, llvm optimizations seem to ignore the types in
1688 // vaarg functions and throws away casts in optimized mode.
1689 SmallVector<Value *, 8> CallArgs;
1690 coerceArguments(Builder, FnTy, Arguments, CallArgs);
1691
1692 auto *TailCall = Builder.CreateCall(FnTy, MustTailCallFn, CallArgs);
1693 TailCall->setTailCallKind(CallInst::TCK_MustTail);
1694 TailCall->setDebugLoc(Loc);
1695 TailCall->setCallingConv(MustTailCallFn->getCallingConv());
1696 return TailCall;
1697}
1698
1701 assert(Shape.ABI == coro::ABI::Async);
1702 assert(Clones.empty());
1703 // Reset various things that the optimizer might have decided it
1704 // "knows" about the coroutine function due to not seeing a return.
1705 F.removeFnAttr(Attribute::NoReturn);
1706 F.removeRetAttr(Attribute::NoAlias);
1707 F.removeRetAttr(Attribute::NonNull);
1708
1709 auto &Context = F.getContext();
1710 auto *Int8PtrTy = Type::getInt8PtrTy(Context);
1711
1712 auto *Id = cast<CoroIdAsyncInst>(Shape.CoroBegin->getId());
1713 IRBuilder<> Builder(Id);
1714
1715 auto *FramePtr = Id->getStorage();
1716 FramePtr = Builder.CreateBitOrPointerCast(FramePtr, Int8PtrTy);
1717 FramePtr = Builder.CreateConstInBoundsGEP1_32(
1719 "async.ctx.frameptr");
1720
1721 // Map all uses of llvm.coro.begin to the allocated frame pointer.
1722 {
1723 // Make sure we don't invalidate Shape.FramePtr.
1724 TrackingVH<Value> Handle(Shape.FramePtr);
1726 Shape.FramePtr = Handle.getValPtr();
1727 }
1728
1729 // Create all the functions in order after the main function.
1730 auto NextF = std::next(F.getIterator());
1731
1732 // Create a continuation function for each of the suspend points.
1733 Clones.reserve(Shape.CoroSuspends.size());
1734 for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) {
1735 auto *Suspend = cast<CoroSuspendAsyncInst>(Shape.CoroSuspends[Idx]);
1736
1737 // Create the clone declaration.
1738 auto ResumeNameSuffix = ".resume.";
1739 auto ProjectionFunctionName =
1740 Suspend->getAsyncContextProjectionFunction()->getName();
1741 bool UseSwiftMangling = false;
1742 if (ProjectionFunctionName.equals("__swift_async_resume_project_context")) {
1743 ResumeNameSuffix = "TQ";
1744 UseSwiftMangling = true;
1745 } else if (ProjectionFunctionName.equals(
1746 "__swift_async_resume_get_context")) {
1747 ResumeNameSuffix = "TY";
1748 UseSwiftMangling = true;
1749 }
1750 auto *Continuation = createCloneDeclaration(
1751 F, Shape,
1752 UseSwiftMangling ? ResumeNameSuffix + Twine(Idx) + "_"
1753 : ResumeNameSuffix + Twine(Idx),
1754 NextF, Suspend);
1755 Clones.push_back(Continuation);
1756
1757 // Insert a branch to a new return block immediately before the suspend
1758 // point.
1759 auto *SuspendBB = Suspend->getParent();
1760 auto *NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);
1761 auto *Branch = cast<BranchInst>(SuspendBB->getTerminator());
1762
1763 // Place it before the first suspend.
1764 auto *ReturnBB =
1765 BasicBlock::Create(F.getContext(), "coro.return", &F, NewSuspendBB);
1766 Branch->setSuccessor(0, ReturnBB);
1767
1768 IRBuilder<> Builder(ReturnBB);
1769
1770 // Insert the call to the tail call function and inline it.
1771 auto *Fn = Suspend->getMustTailCallFunction();
1772 SmallVector<Value *, 8> Args(Suspend->args());
1773 auto FnArgs = ArrayRef<Value *>(Args).drop_front(
1775 auto *TailCall =
1776 coro::createMustTailCall(Suspend->getDebugLoc(), Fn, FnArgs, Builder);
1777 Builder.CreateRetVoid();
1778 InlineFunctionInfo FnInfo;
1779 auto InlineRes = InlineFunction(*TailCall, FnInfo);
1780 assert(InlineRes.isSuccess() && "Expected inlining to succeed");
1781 (void)InlineRes;
1782
1783 // Replace the lvm.coro.async.resume intrisic call.
1784 replaceAsyncResumeFunction(Suspend, Continuation);
1785 }
1786
1787 assert(Clones.size() == Shape.CoroSuspends.size());
1788 for (size_t Idx = 0, End = Shape.CoroSuspends.size(); Idx != End; ++Idx) {
1789 auto *Suspend = Shape.CoroSuspends[Idx];
1790 auto *Clone = Clones[Idx];
1791
1792 CoroCloner(F, "resume." + Twine(Idx), Shape, Clone, Suspend).create();
1793 }
1794}
1795
1798 assert(Shape.ABI == coro::ABI::Retcon ||
1799 Shape.ABI == coro::ABI::RetconOnce);
1800 assert(Clones.empty());
1801
1802 // Reset various things that the optimizer might have decided it
1803 // "knows" about the coroutine function due to not seeing a return.
1804 F.removeFnAttr(Attribute::NoReturn);
1805 F.removeRetAttr(Attribute::NoAlias);
1806 F.removeRetAttr(Attribute::NonNull);
1807
1808 // Allocate the frame.
1809 auto *Id = cast<AnyCoroIdRetconInst>(Shape.CoroBegin->getId());
1810 Value *RawFramePtr;
1812 RawFramePtr = Id->getStorage();
1813 } else {
1814 IRBuilder<> Builder(Id);
1815
1816 // Determine the size of the frame.
1817 const DataLayout &DL = F.getParent()->getDataLayout();
1818 auto Size = DL.getTypeAllocSize(Shape.FrameTy);
1819
1820 // Allocate. We don't need to update the call graph node because we're
1821 // going to recompute it from scratch after splitting.
1822 // FIXME: pass the required alignment
1823 RawFramePtr = Shape.emitAlloc(Builder, Builder.getInt64(Size), nullptr);
1824 RawFramePtr =
1825 Builder.CreateBitCast(RawFramePtr, Shape.CoroBegin->getType());
1826
1827 // Stash the allocated frame pointer in the continuation storage.
1828 auto Dest = Builder.CreateBitCast(Id->getStorage(),
1829 RawFramePtr->getType()->getPointerTo());
1830 Builder.CreateStore(RawFramePtr, Dest);
1831 }
1832
1833 // Map all uses of llvm.coro.begin to the allocated frame pointer.
1834 {
1835 // Make sure we don't invalidate Shape.FramePtr.
1836 TrackingVH<Value> Handle(Shape.FramePtr);
1837 Shape.CoroBegin->replaceAllUsesWith(RawFramePtr);
1838 Shape.FramePtr = Handle.getValPtr();
1839 }
1840
1841 // Create a unique return block.
1842 BasicBlock *ReturnBB = nullptr;
1843 SmallVector<PHINode *, 4> ReturnPHIs;
1844
1845 // Create all the functions in order after the main function.
1846 auto NextF = std::next(F.getIterator());
1847
1848 // Create a continuation function for each of the suspend points.
1849 Clones.reserve(Shape.CoroSuspends.size());
1850 for (size_t i = 0, e = Shape.CoroSuspends.size(); i != e; ++i) {
1851 auto Suspend = cast<CoroSuspendRetconInst>(Shape.CoroSuspends[i]);
1852
1853 // Create the clone declaration.
1854 auto Continuation =
1855 createCloneDeclaration(F, Shape, ".resume." + Twine(i), NextF, nullptr);
1856 Clones.push_back(Continuation);
1857
1858 // Insert a branch to the unified return block immediately before
1859 // the suspend point.
1860 auto SuspendBB = Suspend->getParent();
1861 auto NewSuspendBB = SuspendBB->splitBasicBlock(Suspend);
1862 auto Branch = cast<BranchInst>(SuspendBB->getTerminator());
1863
1864 // Create the unified return block.
1865 if (!ReturnBB) {
1866 // Place it before the first suspend.
1867 ReturnBB = BasicBlock::Create(F.getContext(), "coro.return", &F,
1868 NewSuspendBB);
1869 Shape.RetconLowering.ReturnBlock = ReturnBB;
1870
1871 IRBuilder<> Builder(ReturnBB);
1872
1873 // Create PHIs for all the return values.
1874 assert(ReturnPHIs.empty());
1875
1876 // First, the continuation.
1877 ReturnPHIs.push_back(Builder.CreatePHI(Continuation->getType(),
1878 Shape.CoroSuspends.size()));
1879
1880 // Next, all the directly-yielded values.
1881 for (auto *ResultTy : Shape.getRetconResultTypes())
1882 ReturnPHIs.push_back(Builder.CreatePHI(ResultTy,
1883 Shape.CoroSuspends.size()));
1884
1885 // Build the return value.
1886 auto RetTy = F.getReturnType();
1887
1888 // Cast the continuation value if necessary.
1889 // We can't rely on the types matching up because that type would
1890 // have to be infinite.
1891 auto CastedContinuationTy =
1892 (ReturnPHIs.size() == 1 ? RetTy : RetTy->getStructElementType(0));
1893 auto *CastedContinuation =
1894 Builder.CreateBitCast(ReturnPHIs[0], CastedContinuationTy);
1895
1896 Value *RetV;
1897 if (ReturnPHIs.size() == 1) {
1898 RetV = CastedContinuation;
1899 } else {
1900 RetV = UndefValue::get(RetTy);
1901 RetV = Builder.CreateInsertValue(RetV, CastedContinuation, 0);
1902 for (size_t I = 1, E = ReturnPHIs.size(); I != E; ++I)
1903 RetV = Builder.CreateInsertValue(RetV, ReturnPHIs[I], I);
1904 }
1905
1906 Builder.CreateRet(RetV);
1907 }
1908
1909 // Branch to the return block.
1910 Branch->setSuccessor(0, ReturnBB);
1911 ReturnPHIs[0]->addIncoming(Continuation, SuspendBB);
1912 size_t NextPHIIndex = 1;
1913 for (auto &VUse : Suspend->value_operands())
1914 ReturnPHIs[NextPHIIndex++]->addIncoming(&*VUse, SuspendBB);
1915 assert(NextPHIIndex == ReturnPHIs.size());
1916 }
1917
1918 assert(Clones.size() == Shape.CoroSuspends.size());
1919 for (size_t i = 0, e = Shape.CoroSuspends.size(); i != e; ++i) {
1920 auto Suspend = Shape.CoroSuspends[i];
1921 auto Clone = Clones[i];
1922
1923 CoroCloner(F, "resume." + Twine(i), Shape, Clone, Suspend).create();
1924 }
1925}
1926
1927namespace {
1928 class PrettyStackTraceFunction : public PrettyStackTraceEntry {
1929 Function &F;
1930 public:
1931 PrettyStackTraceFunction(Function &F) : F(F) {}
1932 void print(raw_ostream &OS) const override {
1933 OS << "While splitting coroutine ";
1934 F.printAsOperand(OS, /*print type*/ false, F.getParent());
1935 OS << "\n";
1936 }
1937 };
1938}
1939
1940static coro::Shape
1942 TargetTransformInfo &TTI, bool OptimizeFrame,
1943 std::function<bool(Instruction &)> MaterializableCallback) {
1944 PrettyStackTraceFunction prettyStackTrace(F);
1945
1946 // The suspend-crossing algorithm in buildCoroutineFrame get tripped
1947 // up by uses in unreachable blocks, so remove them as a first pass.
1949
1950 coro::Shape Shape(F, OptimizeFrame);
1951 if (!Shape.CoroBegin)
1952 return Shape;
1953
1954 simplifySuspendPoints(Shape);
1955 buildCoroutineFrame(F, Shape, MaterializableCallback);
1957
1958 // If there are no suspend points, no split required, just remove
1959 // the allocation and deallocation blocks, they are not needed.
1960 if (Shape.CoroSuspends.empty()) {
1962 } else {
1963 switch (Shape.ABI) {
1964 case coro::ABI::Switch:
1965 splitSwitchCoroutine(F, Shape, Clones, TTI);
1966 break;
1967 case coro::ABI::Async:
1968 splitAsyncCoroutine(F, Shape, Clones);
1969 break;
1970 case coro::ABI::Retcon:
1971 case coro::ABI::RetconOnce:
1972 splitRetconCoroutine(F, Shape, Clones);
1973 break;
1974 }
1975 }
1976
1977 // Replace all the swifterror operations in the original function.
1978 // This invalidates SwiftErrorOps in the Shape.
1979 replaceSwiftErrorOps(F, Shape, nullptr);
1980
1981 // Salvage debug intrinsics that point into the coroutine frame in the
1982 // original function. The Cloner has already salvaged debug info in the new
1983 // coroutine funclets.
1985 for (auto *DDI : collectDbgVariableIntrinsics(F))
1986 coro::salvageDebugInfo(ArgToAllocaMap, DDI, Shape.OptimizeFrame);
1987
1988 return Shape;
1989}
1990
1991/// Remove calls to llvm.coro.end in the original function.
1992static void removeCoroEnds(const coro::Shape &Shape) {
1993 for (auto *End : Shape.CoroEnds) {
1994 replaceCoroEnd(End, Shape, Shape.FramePtr, /*in resume*/ false, nullptr);
1995 }
1996}
1997
1999 LazyCallGraph::Node &N, const coro::Shape &Shape,
2003 if (!Shape.CoroBegin)
2004 return;
2005
2006 if (Shape.ABI != coro::ABI::Switch)
2007 removeCoroEnds(Shape);
2008 else {
2009 for (llvm::AnyCoroEndInst *End : Shape.CoroEnds) {
2010 auto &Context = End->getContext();
2011 End->replaceAllUsesWith(ConstantInt::getFalse(Context));
2012 End->eraseFromParent();
2013 }
2014 }
2015
2016 if (!Clones.empty()) {
2017 switch (Shape.ABI) {
2018 case coro::ABI::Switch:
2019 // Each clone in the Switch lowering is independent of the other clones.
2020 // Let the LazyCallGraph know about each one separately.
2021 for (Function *Clone : Clones)
2022 CG.addSplitFunction(N.getFunction(), *Clone);
2023 break;
2024 case coro::ABI::Async:
2025 case coro::ABI::Retcon:
2026 case coro::ABI::RetconOnce:
2027 // Each clone in the Async/Retcon lowering references of the other clones.
2028 // Let the LazyCallGraph know about all of them at once.
2029 if (!Clones.empty())
2030 CG.addSplitRefRecursiveFunctions(N.getFunction(), Clones);
2031 break;
2032 }
2033
2034 // Let the CGSCC infra handle the changes to the original function.
2036 }
2037
2038 // Do some cleanup and let the CGSCC infra see if we've cleaned up any edges
2039 // to the split functions.
2040 postSplitCleanup(N.getFunction());
2042}
2043
2044/// Replace a call to llvm.coro.prepare.retcon.
2045static void replacePrepare(CallInst *Prepare, LazyCallGraph &CG,
2047 auto CastFn = Prepare->getArgOperand(0); // as an i8*
2048 auto Fn = CastFn->stripPointerCasts(); // as its original type
2049
2050 // Attempt to peephole this pattern:
2051 // %0 = bitcast [[TYPE]] @some_function to i8*
2052 // %1 = call @llvm.coro.prepare.retcon(i8* %0)
2053 // %2 = bitcast %1 to [[TYPE]]
2054 // ==>
2055 // %2 = @some_function
2056 for (Use &U : llvm::make_early_inc_range(Prepare->uses())) {
2057 // Look for bitcasts back to the original function type.
2058 auto *Cast = dyn_cast<BitCastInst>(U.getUser());
2059 if (!Cast || Cast->getType() != Fn->getType())
2060 continue;
2061
2062 // Replace and remove the cast.
2063 Cast->replaceAllUsesWith(Fn);
2064 Cast->eraseFromParent();
2065 }
2066
2067 // Replace any remaining uses with the function as an i8*.
2068 // This can never directly be a callee, so we don't need to update CG.
2069 Prepare->replaceAllUsesWith(CastFn);
2070 Prepare->eraseFromParent();
2071
2072 // Kill dead bitcasts.
2073 while (auto *Cast = dyn_cast<BitCastInst>(CastFn)) {
2074 if (!Cast->use_empty())
2075 break;
2076 CastFn = Cast->getOperand(0);
2077 Cast->eraseFromParent();
2078 }
2079}
2080
2081static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG,
2083 bool Changed = false;
2084 for (Use &P : llvm::make_early_inc_range(PrepareFn->uses())) {
2085 // Intrinsics can only be used in calls.
2086 auto *Prepare = cast<CallInst>(P.getUser());
2087 replacePrepare(Prepare, CG, C);
2088 Changed = true;
2089 }
2090
2091 return Changed;
2092}
2093
2094static void addPrepareFunction(const Module &M,
2096 StringRef Name) {
2097 auto *PrepareFn = M.getFunction(Name);
2098 if (PrepareFn && !PrepareFn->use_empty())
2099 Fns.push_back(PrepareFn);
2100}
2101
2103 : MaterializableCallback(coro::defaultMaterializable),
2104 OptimizeFrame(OptimizeFrame) {}
2105
2109 // NB: One invariant of a valid LazyCallGraph::SCC is that it must contain a
2110 // non-zero number of nodes, so we assume that here and grab the first
2111 // node's function's module.
2112 Module &M = *C.begin()->getFunction().getParent();
2113 auto &FAM =
2114 AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager();
2115
2116 // Check for uses of llvm.coro.prepare.retcon/async.
2117 SmallVector<Function *, 2> PrepareFns;
2118 addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.retcon");
2119 addPrepareFunction(M, PrepareFns, "llvm.coro.prepare.async");
2120
2121 // Find coroutines for processing.
2123 for (LazyCallGraph::Node &N : C)
2124 if (N.getFunction().isPresplitCoroutine())
2125 Coroutines.push_back(&N);
2126
2127 if (Coroutines.empty() && PrepareFns.empty())
2128 return PreservedAnalyses::all();
2129
2130 if (Coroutines.empty()) {
2131 for (auto *PrepareFn : PrepareFns) {
2132 replaceAllPrepares(PrepareFn, CG, C);
2133 }
2134 }
2135
2136 // Split all the coroutines.
2137 for (LazyCallGraph::Node *N : Coroutines) {
2138 Function &F = N->getFunction();
2139 LLVM_DEBUG(dbgs() << "CoroSplit: Processing coroutine '" << F.getName()
2140 << "\n");
2141 F.setSplittedCoroutine();
2142
2145 const coro::Shape Shape =
2148 updateCallGraphAfterCoroutineSplit(*N, Shape, Clones, C, CG, AM, UR, FAM);
2149
2150 ORE.emit([&]() {
2151 return OptimizationRemark(DEBUG_TYPE, "CoroSplit", &F)
2152 << "Split '" << ore::NV("function", F.getName())
2153 << "' (frame_size=" << ore::NV("frame_size", Shape.FrameSize)
2154 << ", align=" << ore::NV("align", Shape.FrameAlign.value()) << ")";
2155 });
2156
2157 if (!Shape.CoroSuspends.empty()) {
2158 // Run the CGSCC pipeline on the original and newly split functions.
2159 UR.CWorklist.insert(&C);
2160 for (Function *Clone : Clones)
2161 UR.CWorklist.insert(CG.lookupSCC(CG.get(*Clone)));
2162 }
2163 }
2164
2165 if (!PrepareFns.empty()) {
2166 for (auto *PrepareFn : PrepareFns) {
2167 replaceAllPrepares(PrepareFn, CG, C);
2168 }
2169 }
2170
2171 return PreservedAnalyses::none();
2172}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
AMDGPU Lower Kernel Arguments
assume Assume Builder
This file contains the simple types necessary to represent the attributes associated with functions a...
SmallVector< MachineOperand, 4 > Cond
BlockVerifier::State From
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file provides interfaces used to manipulate a call graph, regardless if it is a "old style" Call...
This file provides interfaces used to build and manipulate a call graph, which is a very useful tool ...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Remove calls to llvm coro end in the original static function void removeCoroEnds(const coro::Shape &Shape)
Definition: CoroSplit.cpp:1992
static void addSwiftSelfAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)
Definition: CoroSplit.cpp:874
static void splitSwitchCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI)
Definition: CoroSplit.cpp:1614
static bool hasCallsBetween(Instruction *Save, Instruction *ResumeOrDestroy)
Definition: CoroSplit.cpp:1488
static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape, ValueToValueMapTy *VMap)
Definition: CoroSplit.cpp:621
static void addAsyncContextAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex)
Definition: CoroSplit.cpp:867
static void addMustTailToCoroResumes(Function &F, TargetTransformInfo &TTI)
Definition: CoroSplit.cpp:1387
static void maybeFreeRetconStorage(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr, CallGraph *CG)
Definition: CoroSplit.cpp:170
static bool hasCallsInBlocksBetween(BasicBlock *SaveBB, BasicBlock *ResDesBB)
Definition: CoroSplit.cpp:1459
static Function * createCloneDeclaration(Function &OrigF, coro::Shape &Shape, const Twine &Suffix, Module::iterator InsertBefore, AnyCoroSuspendInst *ActiveSuspend)
Definition: CoroSplit.cpp:508
static FunctionType * getFunctionTypeFromAsyncSuspend(AnyCoroSuspendInst *Suspend)
Definition: CoroSplit.cpp:500
static void addPrepareFunction(const Module &M, SmallVectorImpl< Function * > &Fns, StringRef Name)
Definition: CoroSplit.cpp:2094
static void updateCallGraphAfterCoroutineSplit(LazyCallGraph::Node &N, const coro::Shape &Shape, const SmallVectorImpl< Function * > &Clones, LazyCallGraph::SCC &C, LazyCallGraph &CG, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Definition: CoroSplit.cpp:1998
static void simplifySuspendPoints(coro::Shape &Shape)
Definition: CoroSplit.cpp:1573
static void addFramePointerAttrs(AttributeList &Attrs, LLVMContext &Context, unsigned ParamIndex, uint64_t Size, Align Alignment, bool NoAlias)
Definition: CoroSplit.cpp:852
static bool replaceAllPrepares(Function *PrepareFn, LazyCallGraph &CG, LazyCallGraph::SCC &C)
Definition: CoroSplit.cpp:2081
static void replaceFallthroughCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Replace a non-unwind call to llvm.coro.end.
Definition: CoroSplit.cpp:227
static void replaceFrameSizeAndAlignment(coro::Shape &Shape)
Definition: CoroSplit.cpp:1122
static SmallVector< DbgVariableIntrinsic *, 8 > collectDbgVariableIntrinsics(Function &F)
Returns all DbgVariableIntrinsic in F.
Definition: CoroSplit.cpp:685
static bool replaceCoroEndAsync(AnyCoroEndInst *End)
Replace an llvm.coro.end.async.
Definition: CoroSplit.cpp:184
Replace a call to llvm coro prepare static retcon void replacePrepare(CallInst *Prepare, LazyCallGraph &CG, LazyCallGraph::SCC &C)
Definition: CoroSplit.cpp:2045
static void replaceUnwindCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Replace an unwind call to llvm.coro.end.
Definition: CoroSplit.cpp:306
static bool simplifySuspendPoint(CoroSuspendInst *Suspend, CoroBeginInst *CoroBegin)
Definition: CoroSplit.cpp:1513
static bool hasCallsInBlockBetween(Instruction *From, Instruction *To)
Definition: CoroSplit.cpp:1447
static void markCoroutineAsDone(IRBuilder<> &Builder, const coro::Shape &Shape, Value *FramePtr)
Definition: CoroSplit.cpp:292
static void splitAsyncCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones)
Definition: CoroSplit.cpp:1699
static void updateAsyncFuncPointerContextSize(coro::Shape &Shape)
Definition: CoroSplit.cpp:1107
static void replaceCoroEnd(AnyCoroEndInst *End, const coro::Shape &Shape, Value *FramePtr, bool InResume, CallGraph *CG)
Definition: CoroSplit.cpp:344
static void setCoroInfo(Function &F, coro::Shape &Shape, ArrayRef< Function * > Fns)
Definition: CoroSplit.cpp:1159
static void handleNoSuspendCoroutine(coro::Shape &Shape)
Definition: CoroSplit.cpp:1412
static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn, Function *DestroyFn, Function *CleanupFn)
Definition: CoroSplit.cpp:1183
static void createResumeEntryBlock(Function &F, coro::Shape &Shape)
Definition: CoroSplit.cpp:359
static coro::Shape splitCoroutine(Function &F, SmallVectorImpl< Function * > &Clones, TargetTransformInfo &TTI, bool OptimizeFrame, std::function< bool(Instruction &)> MaterializableCallback)
Definition: CoroSplit.cpp:1941
static void postSplitCleanup(Function &F)
Definition: CoroSplit.cpp:1209
static bool simplifyTerminatorLeadingToRet(Instruction *InitialInst)
Definition: CoroSplit.cpp:1241
static void splitRetconCoroutine(Function &F, coro::Shape &Shape, SmallVectorImpl< Function * > &Clones)
Definition: CoroSplit.cpp:1796
static void scanPHIsAndUpdateValueMap(Instruction *Prev, BasicBlock *NewBlock, DenseMap< Value *, Value * > &ResolvedValues)
Definition: CoroSplit.cpp:1224
Coerce the arguments in p FnArgs according to p FnTy in p static CallArgs void coerceArguments(IRBuilder<> &Builder, FunctionType *FnTy, ArrayRef< Value * > FnArgs, SmallVectorImpl< Value * > &CallArgs)
Definition: CoroSplit.cpp:1668
static void replaceAsyncResumeFunction(CoroSuspendAsyncInst *Suspend, Value *Continuation)
Definition: CoroSplit.cpp:1653
static bool shouldBeMustTail(const CallInst &CI, const Function &F)
Definition: CoroSplit.cpp:1352
static Function * createClone(Function &F, const Twine &Suffix, coro::Shape &Shape, CoroCloner::Kind FKind)
Definition: CoroSplit.cpp:1100
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseMap class.
This file contains constants used for implementing Dwarf debug support.
std::string Name
uint64_t Size
bool End
Definition: ELF_riscv.cpp:464
static Function * getFunction(Constant *C)
Definition: Evaluator.cpp:236
@ InlineInfo
Rewrite Partial Register Uses
#define DEBUG_TYPE
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
Implements a lazy call graph analysis and related passes for the new pass manager.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Module.h This file contains the declarations for the Module class.
print must be executed print the must be executed context for all instructions
LLVMContext & Context
#define P(N)
FunctionAnalysisManager FAM
This file provides a priority worklist.
@ SI
@ VI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
raw_pwrite_stream & OS
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
Definition: TapiFile.cpp:40
This pass exposes codegen information to IR-level passes.
static const unsigned FramePtr
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:620
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:774
CoroAllocInst * getCoroAlloc()
Definition: CoroInstr.h:84
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
Definition: ArrayRef.h:202
iterator end() const
Definition: ArrayRef.h:152
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:163
iterator begin() const
Definition: ArrayRef.h:151
AttrBuilder & addAlignmentAttr(MaybeAlign Align)
This turns an alignment into the form used internally in Attribute.
AttrBuilder & addAttribute(Attribute::AttrKind Val)
Add an attribute to the builder.
AttrBuilder & addDereferenceableAttr(uint64_t Bytes)
This turns the number of dereferenceable bytes into the form used internally in Attribute.
AttrKind
This enumeration lists the attributes that can be associated with parameters, function results,...
Definition: Attributes.h:87
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:384
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:105
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
Definition: BasicBlock.cpp:409
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:292
const Instruction * getFirstNonPHIOrDbgOrLifetime(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode, a debug intrinsic,...
Definition: BasicBlock.cpp:237
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:112
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:127
void removePredecessor(BasicBlock *Pred, bool KeepOneInputPHIs=false)
Update PHI nodes in this BasicBlock before removal of predecessor Pred.
Definition: BasicBlock.cpp:349
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1190
bool isInlineAsm() const
Check if this call is an inline asm statement.
Definition: InstrTypes.h:1480
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1471
Value * getCalledOperand() const
Definition: InstrTypes.h:1405
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1357
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1270
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1490
The basic data container for the call graph of a Module of IR.
Definition: CallGraph.h:72
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1235
static Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition: Constants.cpp:2025
This is the shared class of boolean and integer constants.
Definition: Constants.h:78
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:833
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:888
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:840
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1691
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1300
static ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
Definition: Constants.cpp:1415
This represents the llvm.coro.align instruction.
Definition: CoroInstr.h:603
This represents the llvm.coro.alloc instruction.
Definition: CoroInstr.h:70
This class represents the llvm.coro.begin instruction.
Definition: CoroInstr.h:420
AnyCoroIdInst * getId() const
Definition: CoroInstr.h:424
This represents the llvm.coro.id instruction.
Definition: CoroInstr.h:113
void setInfo(Constant *C)
Definition: CoroInstr.h:180
This represents the llvm.coro.size instruction.
Definition: CoroInstr.h:591
This represents the llvm.coro.suspend.async instruction.
Definition: CoroInstr.h:525
CoroAsyncResumeInst * getResumeFunction() const
Definition: CoroInstr.h:546
This represents the llvm.coro.suspend instruction.
Definition: CoroInstr.h:493
CoroSaveInst * getCoroSave() const
Definition: CoroInstr.h:497
DISubprogram * getSubprogram() const
Get the subprogram for this scope.
Subprogram description.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
This is the common base class for debug info intrinsics for variables.
A debug info location.
Definition: DebugLoc.h:33
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
iterator end()
Definition: DenseMap.h:84
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:166
This class represents a freeze function that returns random concrete value if an operand is either a ...
A proxy from a FunctionAnalysisManager to an SCC.
Type * getReturnType() const
Definition: DerivedTypes.h:124
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:136
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:174
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:237
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:319
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
PointerType * getType() const
Global values are always pointers.
Definition: GlobalValue.h:290
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
void setInitializer(Constant *InitVal)
setInitializer - Sets the initializer for this global variable, removing any existing initializer if ...
Definition: Globals.cpp:458
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2570
This class captures the data input to the InlineFunction call, and records the auxiliary results prod...
Definition: Cloning.h:203
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:70
const BasicBlock * getParent() const
Definition: Instruction.h:90
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:82
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
A node in the call graph.
An SCC of the call graph.
A lazily constructed view of the call graph of a module.
void addSplitFunction(Function &OriginalFunction, Function &NewFunction)
Add a new function split/outlined from an existing function.
void addSplitRefRecursiveFunctions(Function &OriginalFunction, ArrayRef< Function * > NewFunctions)
Add new ref-recursive functions split/outlined from an existing function.
Node & get(Function &F)
Get a graph node for a given function, scanning it to populate the graph data as necessary.
SCC * lookupSCC(Node &N) const
Lookup a function's SCC in the graph.
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:499
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:398
FunctionListType::iterator iterator
The Function iterators.
Definition: Module.h:90
Diagnostic information for applied optimization remarks.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:155
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
PrettyStackTraceEntry - This class is used to represent a frame of the "pretty" stack trace that is d...
virtual void print(raw_ostream &OS) const =0
print - Emit information about this stack frame to OS.
Return a value (possibly void), from a function.
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
Definition: SmallPtrSet.h:379
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:365
bool contains(ConstPtrType Ptr) const
Definition: SmallPtrSet.h:389
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void reserve(size_type N)
Definition: SmallVector.h:667
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Type * getTypeAtIndex(const Value *V) const
Given an index value into the type, return the type of the element.
Definition: Type.cpp:673
Analysis pass providing the TargetTransformInfo.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
bool supportsTailCallFor(const CallBase *CB) const
If target supports tail call on CB.
bool supportsTailCalls() const
If the target supports tail calls.
Value handle that tracks a Value across RAUW.
Definition: ValueHandle.h:331
ValueTy * getValPtr() const
Definition: ValueHandle.h:335
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:256
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt8Ty(LLVMContext &C)
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1724
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
void setOperand(unsigned i, Value *Val)
Definition: User.h:174
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:535
iterator_range< user_iterator > users()
Definition: Value.h:421
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:688
iterator_range< use_iterator > uses()
Definition: Value.h:376
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:384
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:289
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
void salvageDebugInfo(SmallDenseMap< Argument *, AllocaInst *, 4 > &ArgToAllocaMap, DbgVariableIntrinsic *DVI, bool OptimizeFrame)
Attempts to rewrite the location operand of debug intrinsics in terms of the coroutine frame pointer,...
Definition: CoroFrame.cpp:2832
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
void replaceCoroFree(CoroIdInst *CoroId, bool Elide)
Definition: Coroutines.cpp:130
CallInst * createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, ArrayRef< Value * > Arguments, IRBuilder<> &)
Definition: CoroSplit.cpp:1683
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
bool ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions=false, const TargetLibraryInfo *TLI=nullptr, DomTreeUpdater *DTU=nullptr)
If a terminator instruction is predicated on a constant value, convert it into an unconditional branc...
Definition: Local.cpp:126
bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
Definition: Verifier.cpp:6511
void salvageDebugInfo(const MachineRegisterInfo &MRI, MachineInstr &MI)
Assuming the instruction MI is going to be deleted, attempt to salvage debug users of MI by writing t...
Definition: Utils.cpp:1364
LazyCallGraph::SCC & updateCGAndAnalysisManagerForFunctionPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a function pass.
LazyCallGraph::SCC & updateCGAndAnalysisManagerForCGSCCPass(LazyCallGraph &G, LazyCallGraph::SCC &C, LazyCallGraph::Node &N, CGSCCAnalysisManager &AM, CGSCCUpdateResult &UR, FunctionAnalysisManager &FAM)
Helper to update the call graph after running a CGSCC pass.
Constant * ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr, const Instruction *I=nullptr)
Attempt to constant fold a compare instruction (icmp/fcmp) with the specified operands.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:748
bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction is not used, and the instruction will return.
Definition: Local.cpp:398
@ Async
"Asynchronous" unwind tables (instr precise)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
unsigned changeToUnreachable(Instruction *I, bool PreserveLCSSA=false, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
Definition: Local.cpp:2296
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
InlineResult InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, bool MergeAttributes=false, AAResults *CalleeAAR=nullptr, bool InsertLifetime=true, Function *ForwardVarArgsTo=nullptr)
This function inlines the called function into the basic block of the caller.
void CloneFunctionInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, CloneFunctionChangeType Changes, SmallVectorImpl< ReturnInst * > &Returns, const char *NameSuffix="", ClonedCodeInfo *CodeInfo=nullptr, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Clone OldFunc into NewFunc, transforming the old arguments into references to VMap values.
auto predecessors(const MachineBasicBlock *BB)
bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Remove all blocks that can not be reached from the function's entry.
Definition: Local.cpp:2660
bool isPotentiallyReachable(const Instruction *From, const Instruction *To, const SmallPtrSetImpl< BasicBlock * > *ExclusionSet=nullptr, const DominatorTree *DT=nullptr, const LoopInfo *LI=nullptr)
Determine whether instruction 'To' is reachable from 'From', without passing through any blocks in Ex...
Definition: CFG.cpp:231
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
SmallPriorityWorklist< LazyCallGraph::SCC *, 1 > & CWorklist
Worklist of the SCCs queued for processing.
const std::function< bool(Instruction &)> MaterializableCallback
Definition: CoroSplit.h:25
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)
Definition: CoroSplit.cpp:2106
CoroSplitPass(bool OptimizeFrame=false)
Definition: CoroSplit.cpp:2102
AsyncLoweringStorage AsyncLowering
Definition: CoroInternal.h:145
FunctionType * getResumeFunctionType() const
Definition: CoroInternal.h:184
IntegerType * getIndexType() const
Definition: CoroInternal.h:169
StructType * FrameTy
Definition: CoroInternal.h:101
CoroIdInst * getSwitchCoroId() const
Definition: CoroInternal.h:148
Instruction * getInsertPtAfterFramePtr() const
Definition: CoroInternal.h:242
SmallVector< CoroSizeInst *, 2 > CoroSizes
Definition: CoroInternal.h:79
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
Definition: CoroInternal.h:81
Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const
Allocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:453
SmallVector< CallInst *, 2 > SwiftErrorOps
Definition: CoroInternal.h:82
ConstantInt * getIndex(uint64_t Value) const
Definition: CoroInternal.h:174
bool OptimizeFrame
This would only be true if optimization are enabled.
Definition: CoroInternal.h:108
SwitchLoweringStorage SwitchLowering
Definition: CoroInternal.h:143
CoroBeginInst * CoroBegin
Definition: CoroInternal.h:77
ArrayRef< Type * > getRetconResultTypes() const
Definition: CoroInternal.h:200
void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const
Deallocate memory according to the rules of the active lowering.
Definition: Coroutines.cpp:476
RetconLoweringStorage RetconLowering
Definition: CoroInternal.h:144
SmallVector< CoroAlignInst *, 2 > CoroAligns
Definition: CoroInternal.h:80
SmallVector< AnyCoroEndInst *, 4 > CoroEnds
Definition: CoroInternal.h:78
unsigned getSwitchIndexField() const
Definition: CoroInternal.h:164