LLVM  14.0.0git
CoroInternal.h
Go to the documentation of this file.
1 //===- CoroInternal.h - Internal Coroutine interfaces ---------*- C++ -*---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 // Common definitions/declarations used internally by coroutine lowering passes.
9 //===----------------------------------------------------------------------===//
10 
11 #ifndef LLVM_LIB_TRANSFORMS_COROUTINES_COROINTERNAL_H
12 #define LLVM_LIB_TRANSFORMS_COROUTINES_COROINTERNAL_H
13 
14 #include "CoroInstr.h"
15 #include "llvm/IR/IRBuilder.h"
17 
18 namespace llvm {
19 
20 class CallGraph;
21 class CallGraphSCC;
22 class PassRegistry;
23 
24 void initializeCoroEarlyLegacyPass(PassRegistry &);
25 void initializeCoroSplitLegacyPass(PassRegistry &);
26 void initializeCoroElideLegacyPass(PassRegistry &);
27 void initializeCoroCleanupLegacyPass(PassRegistry &);
28 
29 // CoroEarly pass marks every function that has coro.begin with a string
30 // attribute "coroutine.presplit"="0". CoroSplit pass processes the coroutine
31 // twice. First, it lets it go through complete IPO optimization pipeline as a
32 // single function. It forces restart of the pipeline by inserting an indirect
33 // call to an empty function "coro.devirt.trigger" which is devirtualized by
34 // CoroElide pass that triggers a restart of the pipeline by CGPassManager.
35 // When CoroSplit pass sees the same coroutine the second time, it splits it up,
36 // adds coroutine subfunctions to the SCC to be processed by IPO pipeline.
37 // Async lowering similarily triggers a restart of the pipeline after it has
38 // split the coroutine.
39 #define CORO_PRESPLIT_ATTR "coroutine.presplit"
40 #define UNPREPARED_FOR_SPLIT "0"
41 #define PREPARED_FOR_SPLIT "1"
42 #define ASYNC_RESTART_AFTER_SPLIT "2"
43 
44 #define CORO_DEVIRT_TRIGGER_FN "coro.devirt.trigger"
45 
46 namespace coro {
47 
48 bool declaresIntrinsics(const Module &M,
49  const std::initializer_list<StringRef>);
50 void replaceCoroFree(CoroIdInst *CoroId, bool Elide);
52  CallGraph &CG, CallGraphSCC &SCC);
53 /// Recover a dbg.declare prepared by the frontend and emit an alloca
54 /// holding a pointer to the coroutine frame.
55 void salvageDebugInfo(
57  DbgVariableIntrinsic *DVI, bool ReuseFrameSlot);
58 
59 // Keeps data and helper functions for lowering coroutine intrinsics.
60 struct LowererBase {
66 
67  LowererBase(Module &M);
68  Value *makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt);
69 };
70 
71 enum class ABI {
72  /// The "resume-switch" lowering, where there are separate resume and
73  /// destroy functions that are shared between all suspend points. The
74  /// coroutine frame implicitly stores the resume and destroy functions,
75  /// the current index, and any promise value.
76  Switch,
77 
78  /// The "returned-continuation" lowering, where each suspend point creates a
79  /// single continuation function that is used for both resuming and
80  /// destroying. Does not support promises.
81  Retcon,
82 
83  /// The "unique returned-continuation" lowering, where each suspend point
84  /// creates a single continuation function that is used for both resuming
85  /// and destroying. Does not support promises. The function is known to
86  /// suspend at most once during its execution, and the return value of
87  /// the continuation is void.
88  RetconOnce,
89 
90  /// The "async continuation" lowering, where each suspend point creates a
91  /// single continuation function. The continuation function is available as an
92  /// intrinsic.
93  Async,
94 };
95 
96 // Holds structural Coroutine Intrinsics for a particular function and other
97 // values used during CoroSplit pass.
104 
105  // Field indexes for special fields in the switch lowering.
107  enum {
109  Destroy
110 
111  // The promise field is always at a fixed offset from the start of
112  // frame given its type, but the index isn't a constant for all
113  // possible frames.
114 
115  // The switch-index field isn't at a fixed offset or index, either;
116  // we just work it in where it fits best.
117  };
118  };
119 
121 
127 
128  /// This would only be true if optimization are enabled.
130 
135  unsigned IndexField;
136  unsigned IndexAlign;
137  unsigned IndexOffset;
139  };
140 
147  };
148 
153  unsigned ContextArgNo;
156  uint64_t FrameOffset; // Start of the frame.
157  uint64_t ContextSize; // Includes frame size.
159 
160  Align getContextAlignment() const { return Align(ContextAlignment); }
161  };
162 
163  union {
167  };
168 
171  return cast<CoroIdInst>(CoroBegin->getId());
172  }
173 
177  return cast<AnyCoroIdRetconInst>(CoroBegin->getId());
178  }
179 
182  return cast<CoroIdAsyncInst>(CoroBegin->getId());
183  }
184 
185  unsigned getSwitchIndexField() const {
187  assert(FrameTy && "frame type not assigned");
188  return SwitchLowering.IndexField;
189  }
192  assert(FrameTy && "frame type not assigned");
193  return cast<IntegerType>(FrameTy->getElementType(getSwitchIndexField()));
194  }
196  return ConstantInt::get(getIndexType(), Value);
197  }
198 
201  assert(FrameTy && "frame type not assigned");
202  return cast<PointerType>(FrameTy->getElementType(SwitchFieldIndex::Resume));
203  }
204 
206  switch (ABI) {
207  case coro::ABI::Switch: {
208  auto *FnPtrTy = getSwitchResumePointerType();
209  return cast<FunctionType>(FnPtrTy->getPointerElementType());
210  }
211  case coro::ABI::Retcon:
213  return RetconLowering.ResumePrototype->getFunctionType();
214  case coro::ABI::Async:
215  // Not used. The function type depends on the active suspend.
216  return nullptr;
217  }
218 
219  llvm_unreachable("Unknown coro::ABI enum");
220  }
221 
225  auto FTy = CoroBegin->getFunction()->getFunctionType();
226 
227  // The safety of all this is checked by checkWFRetconPrototype.
228  if (auto STy = dyn_cast<StructType>(FTy->getReturnType())) {
229  return STy->elements().slice(1);
230  } else {
231  return ArrayRef<Type*>();
232  }
233  }
234 
238 
239  // The safety of all this is checked by checkWFRetconPrototype.
240  auto FTy = RetconLowering.ResumePrototype->getFunctionType();
241  return FTy->params().slice(1);
242  }
243 
245  switch (ABI) {
246  case coro::ABI::Switch:
247  return CallingConv::Fast;
248 
249  case coro::ABI::Retcon:
251  return RetconLowering.ResumePrototype->getCallingConv();
252  case coro::ABI::Async:
253  return AsyncLowering.AsyncCC;
254  }
255  llvm_unreachable("Unknown coro::ABI enum");
256  }
257 
259  if (ABI == coro::ABI::Switch)
260  return SwitchLowering.PromiseAlloca;
261  return nullptr;
262  }
263 
264  /// Allocate memory according to the rules of the active lowering.
265  ///
266  /// \param CG - if non-null, will be updated for the new call
267  Value *emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const;
268 
269  /// Deallocate memory according to the rules of the active lowering.
270  ///
271  /// \param CG - if non-null, will be updated for the new call
272  void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const;
273 
274  Shape() = default;
275  explicit Shape(Function &F, bool ReuseFrameSlot = false)
276  : ReuseFrameSlot(ReuseFrameSlot) {
277  buildFrom(F);
278  }
279  void buildFrom(Function &F);
280 };
281 
282 void buildCoroutineFrame(Function &F, Shape &Shape);
283 CallInst *createMustTailCall(DebugLoc Loc, Function *MustTailCallFn,
285 } // End namespace coro.
286 } // End namespace llvm
287 
288 #endif
llvm::coro::Shape::CoroSizes
SmallVector< CoroSizeInst *, 2 > CoroSizes
Definition: CoroInternal.h:101
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
llvm::coro::createMustTailCall
CallInst * createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, ArrayRef< Value * > Arguments, IRBuilder<> &)
Definition: CoroSplit.cpp:1545
llvm::coro::Shape::AsyncLoweringStorage::FrameOffset
uint64_t FrameOffset
Definition: CoroInternal.h:156
llvm::coro::Shape::RetconLoweringStorage::ResumePrototype
Function * ResumePrototype
Definition: CoroInternal.h:142
llvm::coro::LowererBase::NullPtr
ConstantPointerNull *const NullPtr
Definition: CoroInternal.h:65
llvm
---------------------— PointerInfo ------------------------------------—
Definition: AllocatorList.h:23
llvm::coro::salvageDebugInfo
void salvageDebugInfo(SmallDenseMap< llvm::Value *, llvm::AllocaInst *, 4 > &DbgPtrAllocaCache, DbgVariableIntrinsic *DVI, bool ReuseFrameSlot)
Recover a dbg.declare prepared by the frontend and emit an alloca holding a pointer to the coroutine ...
Definition: CoroFrame.cpp:2502
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::coro::Shape::SwitchLowering
SwitchLoweringStorage SwitchLowering
Definition: CoroInternal.h:164
llvm::initializeCoroSplitLegacyPass
void initializeCoroSplitLegacyPass(PassRegistry &)
llvm::coro::Shape::FrameTy
StructType * FrameTy
Definition: CoroInternal.h:122
llvm::Function
Definition: Function.h:61
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
llvm::IRBuilder<>
llvm::coro::ABI::Retcon
@ Retcon
The "returned-continuation" lowering, where each suspend point creates a single continuation function...
llvm::GlobalVariable
Definition: GlobalVariable.h:40
llvm::coro::LowererBase::Context
LLVMContext & Context
Definition: CoroInternal.h:62
llvm::SmallDenseMap
Definition: DenseMap.h:880
llvm::coro::Shape::AsyncLoweringStorage::AsyncCC
CallingConv::ID AsyncCC
Definition: CoroInternal.h:152
llvm::CallGraph
The basic data container for the call graph of a Module of IR.
Definition: CallGraph.h:73
llvm::coro::Shape::AsyncLoweringStorage::AsyncFuncTy
FunctionType * AsyncFuncTy
Definition: CoroInternal.h:150
llvm::coro::ABI::Switch
@ Switch
The "resume-switch" lowering, where there are separate resume and destroy functions that are shared b...
llvm::coro::LowererBase::makeSubFnCall
Value * makeSubFnCall(Value *Arg, int Index, Instruction *InsertPt)
Definition: Coroutines.cpp:107
llvm::coro::Shape::ABI
coro::ABI ABI
Definition: CoroInternal.h:120
llvm::CoroBeginInst::getId
AnyCoroIdInst * getId() const
Definition: CoroInstr.h:424
llvm::coro::declaresIntrinsics
bool declaresIntrinsics(const Module &M, const std::initializer_list< StringRef >)
Definition: Coroutines.cpp:161
llvm::coro::Shape::getPromiseAlloca
AllocaInst * getPromiseAlloca() const
Definition: CoroInternal.h:258
llvm::coro::Shape::SwitchLoweringStorage::ResumeSwitch
SwitchInst * ResumeSwitch
Definition: CoroInternal.h:132
llvm::coro::buildCoroutineFrame
void buildCoroutineFrame(Function &F, Shape &Shape)
Definition: CoroFrame.cpp:2587
llvm::coro::Shape::getAsyncCoroId
CoroIdAsyncInst * getAsyncCoroId() const
Definition: CoroInternal.h:180
llvm::coro::Shape::SwitchLoweringStorage::PromiseAlloca
AllocaInst * PromiseAlloca
Definition: CoroInternal.h:133
llvm::coro::Shape::getResumeFunctionCC
CallingConv::ID getResumeFunctionCC() const
Definition: CoroInternal.h:244
llvm::initializeCoroEarlyLegacyPass
void initializeCoroEarlyLegacyPass(PassRegistry &)
llvm::coro::Shape::AsyncLoweringStorage::ContextHeaderSize
uint64_t ContextHeaderSize
Definition: CoroInternal.h:154
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:206
llvm::ConstantPointerNull
A constant pointer value that points to null.
Definition: Constants.h:534
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
llvm::coro::Shape::AllocaSpillBlock
BasicBlock * AllocaSpillBlock
Definition: CoroInternal.h:126
llvm::initializeCoroCleanupLegacyPass
void initializeCoroCleanupLegacyPass(PassRegistry &)
llvm::coro::Shape::RetconLoweringStorage::IsFrameInlineInStorage
bool IsFrameInlineInStorage
Definition: CoroInternal.h:146
llvm::coro::Shape::RetconLoweringStorage
Definition: CoroInternal.h:141
llvm::coro::Shape::getRetconResumeTypes
ArrayRef< Type * > getRetconResumeTypes() const
Definition: CoroInternal.h:235
llvm::CallGraphSCC
CallGraphSCC - This is a single SCC that a CallGraphSCCPass is run on.
Definition: CallGraphSCCPass.h:87
llvm::coro::Shape::AsyncLoweringStorage::AsyncFuncPointer
GlobalVariable * AsyncFuncPointer
Definition: CoroInternal.h:158
llvm::coro::Shape::getRetconResultTypes
ArrayRef< Type * > getRetconResultTypes() const
Definition: CoroInternal.h:222
llvm::coro::Shape::CoroSuspends
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
Definition: CoroInternal.h:102
llvm::IntegerType
Class to represent integer types.
Definition: DerivedTypes.h:40
llvm::coro::Shape::SwitchLoweringStorage
Definition: CoroInternal.h:131
llvm::coro::Shape::CoroEnds
SmallVector< AnyCoroEndInst *, 4 > CoroEnds
Definition: CoroInternal.h:100
llvm::coro::Shape::SwitchFieldIndex
Definition: CoroInternal.h:106
llvm::coro::Shape::RetconLoweringStorage::Alloc
Function * Alloc
Definition: CoroInternal.h:143
llvm::coro::replaceCoroFree
void replaceCoroFree(CoroIdInst *CoroId, bool Elide)
Definition: Coroutines.cpp:174
llvm::Instruction
Definition: Instruction.h:45
llvm::coro::Shape::SwitchLoweringStorage::HasFinalSuspend
bool HasFinalSuspend
Definition: CoroInternal.h:138
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:900
llvm::coro::Shape::Shape
Shape(Function &F, bool ReuseFrameSlot=false)
Definition: CoroInternal.h:275
Align
uint64_t Align
Definition: ELFObjHandler.cpp:83
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm::coro::Shape::FrameAlign
Align FrameAlign
Definition: CoroInternal.h:123
llvm::coro::LowererBase::LowererBase
LowererBase(Module &M)
Definition: Coroutines.cpp:94
llvm::coro::Shape::getIndexType
IntegerType * getIndexType() const
Definition: CoroInternal.h:190
llvm::coro::Shape
Definition: CoroInternal.h:98
llvm::coro::Shape::AsyncLowering
AsyncLoweringStorage AsyncLowering
Definition: CoroInternal.h:166
Coroutines.h
llvm::coro::Shape::AsyncLoweringStorage::ContextAlignment
uint64_t ContextAlignment
Definition: CoroInternal.h:155
llvm::DbgVariableIntrinsic
This is the common base class for debug info intrinsics for variables.
Definition: IntrinsicInst.h:148
llvm::coro::Shape::SwiftErrorOps
SmallVector< CallInst *, 2 > SwiftErrorOps
Definition: CoroInternal.h:103
Index
uint32_t Index
Definition: ELFObjHandler.cpp:84
uint64_t
llvm::coro::Shape::SwitchLoweringStorage::IndexOffset
unsigned IndexOffset
Definition: CoroInternal.h:137
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
llvm::coro::Shape::SwitchLoweringStorage::IndexAlign
unsigned IndexAlign
Definition: CoroInternal.h:136
llvm::coro::Shape::SwitchFieldIndex::Resume
@ Resume
Definition: CoroInternal.h:108
llvm::coro::Shape::ReuseFrameSlot
bool ReuseFrameSlot
This would only be true if optimization are enabled.
Definition: CoroInternal.h:129
llvm::CoroIdAsyncInst
This represents the llvm.coro.id.async instruction.
Definition: CoroInstr.h:277
llvm::PointerType
Class to represent pointers.
Definition: DerivedTypes.h:632
llvm::coro::Shape::AsyncLoweringStorage::Context
Value * Context
Definition: CoroInternal.h:151
llvm::coro::Shape::FramePtr
Instruction * FramePtr
Definition: CoroInternal.h:125
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::coro::ABI::Async
@ Async
The "async continuation" lowering, where each suspend point creates a single continuation function.
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:67
llvm::initializeCoroElideLegacyPass
void initializeCoroElideLegacyPass(PassRegistry &)
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:650
llvm::coro::Shape::AsyncLoweringStorage::ContextSize
uint64_t ContextSize
Definition: CoroInternal.h:157
llvm::AMDGPU::CPol::SCC
@ SCC
Definition: SIDefines.h:292
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::StructType
Class to represent struct types.
Definition: DerivedTypes.h:213
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:136
llvm::CoroIdInst
This represents the llvm.coro.id instruction.
Definition: CoroInstr.h:113
llvm::Instruction::getFunction
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:70
llvm::coro::ABI
ABI
Definition: CoroInternal.h:71
LLVM_LIBRARY_VISIBILITY
#define LLVM_LIBRARY_VISIBILITY
LLVM_LIBRARY_VISIBILITY - If a class marked with this attribute is linked into a shared library,...
Definition: Compiler.h:131
llvm::coro::Shape::CoroBegin
CoroBeginInst * CoroBegin
Definition: CoroInternal.h:99
llvm::coro::Shape::getResumeFunctionType
FunctionType * getResumeFunctionType() const
Definition: CoroInternal.h:205
llvm::coro::Shape::SwitchLoweringStorage::ResumeEntryBlock
BasicBlock * ResumeEntryBlock
Definition: CoroInternal.h:134
llvm::coro::Shape::RetconLowering
RetconLoweringStorage RetconLowering
Definition: CoroInternal.h:165
llvm::coro::ABI::RetconOnce
@ RetconOnce
The "unique returned-continuation" lowering, where each suspend point creates a single continuation f...
llvm::coro::Shape::getSwitchResumePointerType
PointerType * getSwitchResumePointerType() const
Definition: CoroInternal.h:199
llvm::coro::Shape::getIndex
ConstantInt * getIndex(uint64_t Value) const
Definition: CoroInternal.h:195
llvm::coro::LowererBase::TheModule
Module & TheModule
Definition: CoroInternal.h:61
Arguments
AMDGPU Lower Kernel Arguments
Definition: AMDGPULowerKernelArguments.cpp:243
llvm::Function::getFunctionType
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:176
llvm::coro::Shape::getSwitchCoroId
CoroIdInst * getSwitchCoroId() const
Definition: CoroInternal.h:169
llvm::coro::Shape::getSwitchIndexField
unsigned getSwitchIndexField() const
Definition: CoroInternal.h:185
llvm::CallingConv::Fast
@ Fast
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:42
llvm::AnyCoroIdRetconInst
This represents either the llvm.coro.id.retcon or llvm.coro.id.retcon.once instruction.
Definition: CoroInstr.h:204
llvm::coro::Shape::getRetconCoroId
AnyCoroIdRetconInst * getRetconCoroId() const
Definition: CoroInternal.h:174
llvm::coro::Shape::RetconLoweringStorage::ReturnBlock
BasicBlock * ReturnBlock
Definition: CoroInternal.h:145
llvm::coro::updateCallGraph
void updateCallGraph(Function &Caller, ArrayRef< Function * > Funcs, CallGraph &CG, CallGraphSCC &SCC)
Definition: Coroutines.cpp:215
llvm::coro::Shape::FrameSize
uint64_t FrameSize
Definition: CoroInternal.h:124
llvm::coro::Shape::AsyncLoweringStorage::ContextArgNo
unsigned ContextArgNo
Definition: CoroInternal.h:153
llvm::coro::LowererBase
Definition: CoroInternal.h:60
llvm::coro::LowererBase::ResumeFnType
FunctionType *const ResumeFnType
Definition: CoroInternal.h:64
llvm::CoroBeginInst
This class represents the llvm.coro.begin instruction.
Definition: CoroInstr.h:420
CoroInstr.h
llvm::StructType::getElementType
Type * getElementType(unsigned N) const
Definition: DerivedTypes.h:328
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1475
llvm::coro::LowererBase::Int8Ptr
PointerType *const Int8Ptr
Definition: CoroInternal.h:63
llvm::SwitchInst
Multiway switch.
Definition: Instructions.h:3204
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::AllocaInst
an instruction to allocate memory on the stack
Definition: Instructions.h:62
llvm::coro::Shape::AsyncLoweringStorage::getContextAlignment
Align getContextAlignment() const
Definition: CoroInternal.h:160
llvm::coro::Shape::RetconLoweringStorage::Dealloc
Function * Dealloc
Definition: CoroInternal.h:144
llvm::Value
LLVM Value Representation.
Definition: Value.h:75
llvm::coro::Shape::SwitchLoweringStorage::IndexField
unsigned IndexField
Definition: CoroInternal.h:135
llvm::FunctionType
Class to represent function types.
Definition: DerivedTypes.h:103
llvm::coro::Shape::AsyncLoweringStorage
Definition: CoroInternal.h:149