35 #define DEBUG_TYPE "coro-split"
60 auto *Index = Builder.
CreateLoad(GepIndex,
"index");
65 size_t SuspendIndex = 0;
79 cast<PointerType>(GepIndex->getType())->getElementType()));
87 Save->eraseFromParent();
117 Switch->addCase(IndexVal, ResumeBB);
119 cast<BranchInst>(SuspendBB->getTerminator())->setSuccessor(0, LandingBB);
122 PN->addIncoming(Builder.
getInt8(-1), SuspendBB);
123 PN->addIncoming(S, ResumeBB);
138 auto *NewE = cast<IntrinsicInst>(VMap[
End]);
160 auto FinalCase = --Switch->
case_end();
161 BasicBlock *ResumeBB = FinalCase.getCaseSuccessor();
168 0, 0,
"ResumeFn.addr");
185 auto *FnPtrTy = cast<PointerType>(FrameTy->getElementType(0));
186 auto *FnTy = cast<FunctionType>(FnPtrTy->getElementType());
206 auto &MD = VMap.
MD();
207 MD[SP->getUnit()].reset(SP->getUnit());
208 MD[SP->getType()].reset(SP->getType());
209 MD[SP->getFile()].reset(SP->getFile());
218 NewF->removeAttributes(
225 auto *SwitchBB = cast<BasicBlock>(VMap[ResumeEntry]);
227 Entry->moveBefore(&NewF->getEntryBlock());
228 Entry->getTerminator()->eraseFromParent();
230 Entry->setName(
"entry" + Suffix);
234 Entry->replaceAllUsesWith(
Switch->getDefaultDest());
236 IRBuilder<> Builder(&NewF->getEntryBlock().front());
239 Argument *NewFramePtr = &NewF->getArgumentList().front();
241 NewFramePtr->takeName(OldFramePtr);
245 auto *NewVFrame = Builder.CreateBitCast(
255 bool IsDestroy = FnIndex != 0;
264 auto *NewValue = Builder.getInt8(FnIndex ? 1 : 0);
266 auto *MappedCS = cast<CoroSuspendInst>(VMap[CS]);
267 MappedCS->replaceAllUsesWith(NewValue);
268 MappedCS->eraseFromParent();
296 auto *SizeIntrin = Shape.
CoroSizes.back();
297 Module *M = SizeIntrin->getModule();
320 std::initializer_list<Function *> Fns) {
325 Module *M = Part->getParent();
347 Builder.CreateStore(ResumeFn, ResumeAddr);
349 Value *DestroyOrCleanupFn = DestroyFn;
355 DestroyOrCleanupFn = Builder.CreateSelect(CA, DestroyFn, CleanupFn);
358 auto *DestroyAddr = Builder.CreateConstInBoundsGEP2_32(
361 Builder.CreateStore(DestroyOrCleanupFn, DestroyAddr);
374 FPM.doInitialization();
376 FPM.doFinalization();
382 auto *CoroId = CoroBegin->
getId();
391 AllocInst->eraseFromParent();
412 if (BB != Save->getParent())
419 I =
I->getNextNode()) {
420 if (isa<CoroFrameInst>(
I))
422 if (isa<CoroSubFnInst>(
I))
443 if (SubFn->getFrame() != CoroBegin)
450 Save->eraseFromParent();
453 CallInstr->eraseFromParent();
455 if (SubFn->user_empty())
456 SubFn->eraseFromParent();
464 size_t I = 0,
N = S.size();
500 auto ResumeClone =
createClone(F,
".resume", Shape, ResumeEntry, 0);
501 auto DestroyClone =
createClone(F,
".destroy", Shape, ResumeEntry, 1);
502 auto CleanupClone =
createClone(F,
".cleanup", Shape, ResumeEntry, 2);
531 assert(DevirtFn &&
"coro.devirt.trigger function not found");
543 auto *
Null = ConstantPointerNull::get(Type::getInt8PtrTy(F.
getContext()));
546 auto *IndirectCall = CallInst::Create(DevirtFnAddr,
Null,
"", InsertPt);
561 auto *FnTy = FunctionType::get(Type::getVoidTy(C), Type::getInt8PtrTy(C),
564 Function::Create(FnTy, GlobalValue::LinkageTypes::PrivateLinkage,
566 DevirtFn->
addFnAttr(Attribute::AlwaysInline);
567 auto *Entry = BasicBlock::Create(C,
"entry", DevirtFn);
568 ReturnInst::Create(C, Entry);
603 if (
auto *
F = CGN->getFunction())
607 if (Coroutines.
empty())
610 CallGraph &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph();
616 DEBUG(
dbgs() <<
"CoroSplit: Processing coroutine '" <<
F->getName()
617 <<
"' state: " << Value <<
"\n");
636 CoroSplit,
"coro-split",
637 "Split coroutine into a set of functions driving its state machine",
false,
Pass interface - Implemented by all 'passes'.
Return a value (possibly void), from a function.
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void push_back(const T &Elt)
A parsed version of the target data layout string in and methods for querying it. ...
static void createDevirtTriggerFunc(CallGraph &CG, CallGraphSCC &SCC)
BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
CaseIt case_end()
Returns a read/write iterator that points one past the last in the SwitchInst.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
LLVM Argument representation.
CoroBeginInst * CoroBegin
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
This represents the llvm.coro.alloc instruction.
A Module instance is used to store all the information related to an LLVM module. ...
static void splitCoroutine(Function &F, CallGraph &CG, CallGraphSCC &SCC)
static void setCoroInfo(Function &F, CoroBeginInst *CoroBegin, std::initializer_list< Function * > Fns)
Pass * createCoroSplitPass()
Split up coroutines into multiple functions driving their state machines.
INITIALIZE_PASS(CoroSplit,"coro-split","Split coroutine into a set of functions driving its state machine", false, false) Pass *llvm
virtual bool doInitialization(CallGraph &CG)
doInitialization - This method is called before the SCC's of the program has been processed...
FunctionPass * createVerifierPass(bool FatalErrors=true)
Like Internal, but omit from symbol table.
The two locations do not alias at all.
static void replaceFrameSize(coro::Shape &Shape)
void initialize(ArrayRef< CallGraphNode * > NewNodes)
unsigned changeToUnreachable(Instruction *I, bool UseLLVMTrap, bool PreserveLCSSA=false)
Insert an unreachable instruction before the specified instruction, making it and the rest of the cod...
A node in the call graph for a module.
void getAnalysisUsage(AnalysisUsage &Info) const override
getAnalysisUsage - For this class, we declare that we require and preserve the call graph...
Module & getModule() const
Returns the module the call graph corresponds to.
ConstantInt * getIndex(uint64_t Value) const
StringRef getName() const
Return a constant reference to the value's name.
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, Instruction *InsertBefore=nullptr)
AllocaInst * CreateAlloca(Type *Ty, Value *ArraySize=nullptr, const Twine &Name="")
static bool simplifySuspendPoint(CoroSuspendInst *Suspend, CoroBeginInst *CoroBegin)
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static void handleNoSuspendCoroutine(CoroBeginInst *CoroBegin, Type *FrameTy)
ValTy * getCalledValue() const
getCalledValue - Return the pointer to function that is being called.
void CloneFunctionInto(Function *NewFunc, const Function *OldFunc, ValueToValueMapTy &VMap, bool ModuleLevelChanges, SmallVectorImpl< ReturnInst * > &Returns, const char *NameSuffix="", ClonedCodeInfo *CodeInfo=nullptr, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Clone OldFunc into NewFunc, transforming the old arguments into references to VMap values...
This represents the llvm.coro.suspend instruction.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
FunctionPass * createSCCPPass()
CoroIdInst * getId() const
This class represents the llvm.coro.subfn.addr instruction.
#define PREPARED_FOR_SPLIT
LLVM_NODISCARD bool empty() const
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
void updateCallGraph(Function &Caller, ArrayRef< Function * > Funcs, CallGraph &CG, CallGraphSCC &SCC)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
static void replaceFallthroughCoroEnd(IntrinsicInst *End, ValueToValueMapTy &VMap)
This represents the llvm.coro.alloc instruction.
SmallVector< CoroSizeInst *, 2 > CoroSizes
void add(Pass *P) override
Add a pass to the queue of passes to run.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
CoroSaveInst * getCoroSave() const
static void simplifySuspendPoints(coro::Shape &Shape)
SmallVector< CoroSuspendInst *, 4 > CoroSuspends
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block...
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
LoadInst * CreateLoad(Value *Ptr, const char *Name)
This represents the llvm.coro.size instruction.
static void postSplitCleanup(Function &F)
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
LLVM Basic Block Representation.
#define CORO_PRESPLIT_ATTR
The instances of the Type class are immutable: once they are created, they are never changed...
This is an important class for using LLVM in a threaded context.
UnreachableInst * CreateUnreachable()
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
FunctionPass * createCFGSimplificationPass(int Threshold=-1, std::function< bool(const Function &)> Ftor=nullptr)
static void prepareForSplit(Function &F, CallGraph &CG)
Represent the analysis usage information of a pass.
void addAttribute(unsigned i, Attribute::AttrKind Kind)
adds the attribute to the list of attributes.
This represents the llvm.coro.end instruction.
static const unsigned End
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
FunctionPassManager manages FunctionPasses and BasicBlockPassManagers.
static Function * createClone(Function &F, Twine Suffix, coro::Shape &Shape, BasicBlock *ResumeEntry, int8_t FnIndex)
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
PointerType * getInt8PtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer to an 8-bit integer value.
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
static Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
This is the shared class of boolean and integer constants.
InstrTy * getInstruction() const
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
bool removeUnreachableBlocks(Function &F, LazyValueInfo *LVI=nullptr)
Remove all blocks that can not be reached from the function's entry.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
void buildCoroutineFrame(Function &F, Shape &Shape)
Value * stripPointerCasts()
Strip off pointer casts, all-zero GEPs, and aliases.
#define UNPREPARED_FOR_SPLIT
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
const BasicBlock & getEntryBlock() const
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This class represents the llvm.coro.begin instruction.
Value * makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt)
DISubprogram * getSubprogram() const
Get the attached subprogram.
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn, Function *DestroyFn, Function *CleanupFn)
static BasicBlock * createResumeEntryBlock(Function &F, coro::Shape &Shape)
ConstantInt * getFalse()
Get the constant value for i1 false.
SwitchInst * CreateSwitch(Value *V, BasicBlock *Dest, unsigned NumCases=10, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a switch instruction with the specified value, default dest, and with a hint for the number of...
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
The basic data container for the call graph of a Module of IR.
SwitchInst * ResumeSwitch
static void handleFinalSuspend(IRBuilder<> &Builder, Value *FramePtr, coro::Shape &Shape, SwitchInst *Switch, bool IsDestroy)
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Value * CreateConstInBoundsGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0, unsigned Idx1, const Twine &Name="")
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
#define CORO_DEVIRT_TRIGGER_FN
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
static ConstantTokenNone * get(LLVMContext &Context)
Return the ConstantTokenNone.
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="")
Split the basic block into two basic blocks at the specified instruction.
void removeCase(CaseIt i)
This method removes the specified case and its successor from the switch instruction.
void replaceCoroFree(CoroIdInst *CoroId, bool Elide)
StringRef getValueAsString() const
Return the attribute's value as a string.
virtual bool runOnSCC(CallGraphSCC &SCC)=0
runOnSCC - This method should be implemented by the subclass to perform whatever action is necessary ...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SmallVector< CoroEndInst *, 4 > CoroEnds
Module * getParent()
Get the module that this global value is contained inside of...
LLVM Value Representation.
void setInfo(Constant *C)
CallGraphSCC - This is a single SCC that a CallGraphSCCPass is run on.
const ArgumentListType & getArgumentList() const
Get the underlying elements of the Function...
CallGraphNode * getOrInsertFunction(const Function *F)
Similar to operator[], but this will insert a new CallGraphNode for F if one does not already exist...
AttrBuilder typeIncompatible(Type *Ty)
Which attributes cannot be applied to a type.
static const unsigned FramePtr
bool declaresIntrinsics(Module &M, std::initializer_list< StringRef >)
void addFnAttr(Attribute::AttrKind Kind)
Add function attributes to this function.
FunctionPass * createEarlyCSEPass(bool UseMemorySSA=false)
ConstantInt * getInt8(uint8_t C)
Get a constant 8-bit value.
static void removeCoroEnds(coro::Shape &Shape)
StringRef - Represent a constant reference to a string, i.e.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, const Twine &N="", Module *M=nullptr)
CoroAllocInst * getCoroAlloc()
BasicBlock * AllocaSpillBlock
static volatile int * Null
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
Fast - This calling convention attempts to make calls as fast as possible (e.g.
const BasicBlock * getParent() const
A wrapper class for inspecting calls to intrinsic functions.
LLVMContext & getContext() const
Get the global data context.
CallGraphNode * getCallsExternalNode() const