36 #define DEBUG_TYPE "coro-suspend-crossing"
42 class BlockToIndexMapping {
46 size_t size()
const {
return V.
size(); }
51 std::sort(V.begin(), V.end());
55 auto *
I = std::lower_bound(V.begin(), V.end(), BB);
56 assert(
I != V.end() && *
I == BB &&
"BasicBlockNumberng: Unknown block");
60 BasicBlock *indexToBlock(
unsigned Index)
const {
return V[Index]; }
77 struct SuspendCrossingInfo {
78 BlockToIndexMapping Mapping;
89 BasicBlock *BB = Mapping.indexToBlock(&BD - &Block[0]);
94 return Block[Mapping.blockToIndex(BB)];
103 size_t const DefIndex = Mapping.blockToIndex(DefBB);
104 size_t const UseIndex = Mapping.blockToIndex(UseBB);
106 assert(Block[UseIndex].Consumes[DefIndex] &&
"use must consume def");
107 bool const Result =
Block[UseIndex].Kills[DefIndex];
109 <<
" answer is " << Result <<
"\n");
113 bool isDefinitionAcrossSuspend(
BasicBlock *DefBB,
User *U)
const {
114 auto *
I = cast<Instruction>(U);
118 if (
auto *PN = dyn_cast<PHINode>(
I))
119 if (PN->getNumIncomingValues() > 1)
123 return hasPathCrossingSuspendPoint(DefBB, UseBB);
131 return isDefinitionAcrossSuspend(I.
getParent(), U);
138 dbgs() << Label <<
":";
139 for (
size_t I = 0,
N = BV.
size();
I <
N; ++
I)
141 dbgs() <<
" " << Mapping.indexToBlock(I)->getName();
146 for (
size_t I = 0, N =
Block.size(); I <
N; ++
I) {
149 dump(
" Consumes", Block[I].Consumes);
150 dump(
" Kills", Block[I].Kills);
157 const size_t N = Mapping.size();
161 for (
size_t I = 0; I <
N; ++
I) {
163 B.Consumes.resize(N);
172 getBlockData(
CE->getParent()).
End =
true;
180 auto &B = getBlockData(SuspendBlock);
182 B.Kills |= B.Consumes;
185 markSuspendBlock(CSI);
195 DEBUG(
dbgs() <<
"iteration " << ++Iteration);
199 for (
size_t I = 0; I <
N; ++
I) {
203 auto SuccNo = Mapping.blockToIndex(
SI);
207 auto &S =
Block[SuccNo];
208 auto SavedConsumes = S.Consumes;
209 auto SavedKills = S.Kills;
212 S.Consumes |= B.Consumes;
218 S.Kills |= B.Consumes;
223 S.Kills |= S.Consumes;
233 S.Kills.reset(SuccNo);
237 Changed |= (S.Kills != SavedKills) || (S.Consumes != SavedConsumes);
239 if (S.Kills != SavedKills) {
240 DEBUG(
dbgs() <<
"\nblock " << I <<
" follower " <<
SI->getName()
245 if (S.Consumes != SavedConsumes) {
246 DEBUG(
dbgs() <<
"\nblock " << I <<
" follower " <<
SI <<
"\n");
256 #undef DEBUG_TYPE // "coro-suspend-crossing"
257 #define DEBUG_TYPE "coro-frame"
262 struct Spill : std::pair<Value *, Instruction *> {
263 using base = std::pair<Value *, Instruction *>;
271 std::pair<Value *, BasicBlock *>
getKey()
const {
284 dbgs() <<
"------------- " << Title <<
"--------------\n";
285 Value *CurrentValue =
nullptr;
286 for (
auto const &
E : Spills) {
287 if (CurrentValue !=
E.def()) {
288 CurrentValue =
E.def();
289 CurrentValue->
dump();
309 Name.append(
".Frame");
312 auto *FnTy = FunctionType::get(Type::getVoidTy(C), FramePtrTy,
314 auto *FnPtrTy = FnTy->getPointerTo();
320 : Type::getInt1Ty(C);
322 Type::getIntNTy(C, IndexBits)};
323 Value *CurrentDef =
nullptr;
326 for (
auto const &S : Spills) {
327 if (CurrentDef == S.def())
330 CurrentDef = S.def();
336 if (
auto *AI = dyn_cast<AllocaInst>(CurrentDef))
337 Ty = AI->getAllocatedType();
375 cast<Instruction>(Builder.CreateBitCast(CB, FramePtrTy,
"FramePtr"));
376 Type *FrameTy = FramePtrTy->getElementType();
378 Value *CurrentValue =
nullptr;
380 Value *CurrentReload =
nullptr;
381 unsigned Index = coro::Shape::LastKnownField;
395 auto CreateReload = [&](
Instruction *InsertBefore) {
396 Builder.SetInsertPoint(InsertBefore);
397 auto *
G = Builder.CreateConstInBoundsGEP2_32(FrameTy,
FramePtr, 0, Index,
399 Twine(
".reload.addr"));
400 return isa<AllocaInst>(CurrentValue)
402 : Builder.CreateLoad(
G,
406 for (
auto const &
E : Spills) {
408 if (CurrentValue !=
E.def()) {
409 CurrentValue =
E.def();
410 CurrentBlock =
nullptr;
411 CurrentReload =
nullptr;
415 if (
auto *AI = dyn_cast<AllocaInst>(CurrentValue)) {
419 if (!AI->isStaticAlloca())
427 Builder.SetInsertPoint(
428 isa<Argument>(CurrentValue)
432 auto *
G = Builder.CreateConstInBoundsGEP2_32(
435 Builder.CreateStore(CurrentValue,
G);
440 if (CurrentBlock !=
E.userBlock()) {
441 CurrentBlock =
E.userBlock();
448 if (
auto *PN = dyn_cast<PHINode>(
E.user())) {
449 assert(PN->getNumIncomingValues() == 1 &&
"unexpected number of incoming "
450 "values in the PHINode");
451 PN->replaceAllUsesWith(CurrentReload);
452 PN->eraseFromParent();
457 E.user()->replaceUsesOfWith(CurrentValue, CurrentReload);
468 for (
auto &
P : Allocas) {
470 Builder.CreateConstInBoundsGEP2_32(FrameTy,
FramePtr, 0,
P.second);
473 G->takeName(
P.first);
474 P.first->replaceAllUsesWith(
G);
475 P.first->eraseFromParent();
505 IncomingBB->setName(BB.
getName() +
Twine(
".from.") + Pred->getName());
506 auto *PN = cast<PHINode>(&BB.
front());
508 int Index = PN->getBasicBlockIndex(IncomingBB);
509 Value *V = PN->getIncomingValue(Index);
510 PHINode *InputV = PHINode::Create(
512 &IncomingBB->
front());
514 PN->setIncomingValue(Index, InputV);
524 if (
auto *PN = dyn_cast<PHINode>(&BB.
front()))
525 if (PN->getNumIncomingValues() > 1)
535 return isa<CastInst>(&V) || isa<GetElementPtrInst>(&V) ||
536 isa<BinaryOperator>(&V) || isa<CmpInst>(&V) || isa<SelectInst>(&V);
542 return isa<CoroIdInst>(&
I) || isa<CoroBeginInst>(&I) ||
543 isa<CoroSaveInst>(&
I) || isa<CoroSuspendInst>(&I);
554 for (
auto const &
E : Spills) {
556 if (CurrentDef !=
E.def()) {
557 CurrentDef = cast<Instruction>(
E.def());
558 CurrentBlock =
nullptr;
559 CurrentMaterialization =
nullptr;
563 if (CurrentBlock !=
E.userBlock()) {
564 CurrentBlock =
E.userBlock();
565 CurrentMaterialization = cast<Instruction>(CurrentDef)->clone();
571 if (
auto *PN = dyn_cast<PHINode>(
E.user())) {
572 assert(PN->getNumIncomingValues() == 1 &&
"unexpected number of incoming "
573 "values in the PHINode");
574 PN->replaceAllUsesWith(CurrentMaterialization);
575 PN->eraseFromParent();
581 E.user()->replaceUsesOfWith(CurrentDef, CurrentMaterialization);
599 Value *CurrentValue =
nullptr;
601 for (
auto const &
E : Spills) {
602 if (CurrentValue ==
E.def())
605 CurrentValue =
E.def();
614 if (!DT.
dominates(CoroBegin, cast<Instruction>(UI)))
616 " dominated by CoroBegin");
619 DEBUG(
dbgs() <<
"will move: " << *I <<
"\n");
677 SuspendCrossingInfo Checker(F, Shape);
686 if (Checker.isDefinitionAcrossSuspend(I, U))
687 Spills.emplace_back(&I, U);
690 std::sort(Spills.begin(), Spills.end());
698 if (Checker.isDefinitionAcrossSuspend(A, U))
699 Spills.emplace_back(&A, U);
712 if (Checker.isDefinitionAcrossSuspend(I, U)) {
714 if (I.getType()->isTokenTy())
716 "token definition is separated from the use by a suspend point");
718 "rewriteMaterializable did not do its job");
719 Spills.emplace_back(&I, U);
722 std::sort(Spills.begin(), Spills.end());
void push_back(const T &Elt)
size_type size() const
size - Returns the number of bits in this bitvector.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
LLVM Argument representation.
CoroBeginInst * CoroBegin
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds...
static void rewritePHIs(BasicBlock &BB)
BasicBlock * userBlock() const
static void dump(StringRef Title, SpillInfo const &Spills)
const Function * getParent() const
Return the enclosing method, or null if none.
const Instruction & front() const
Type * getElementType() const
StringRef getName() const
Return a constant reference to the value's name.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Class to represent struct types.
static bool materializable(Instruction &V)
This represents the llvm.coro.suspend instruction.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Windows NT (Windows on ARM)
void setName(const Twine &Name)
Change the name of the value.
CoroIdInst * getId() const
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
CoroSaveInst * getCoroSave() const
LLVM_NODISCARD char front() const
front - Get the first character in the string.
SmallVector< CoroSuspendInst *, 4 > CoroSuspends
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Spill(Value *Def, User *U)
std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type cast(const Y &Val)
Class to represent pointers.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
Instruction * user() const
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
static void moveSpillUsesAfterCoroBegin(Function &F, SpillInfo const &Spills, CoroBeginInst *CoroBegin)
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction...
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
This is an important class for using LLVM in a threaded context.
const Function * getParent() const
PointerType * getType() const
Overload to return most specific pointer type.
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Select target instructions out of generic instructions
bool operator<(Spill const &rhs) const
static const unsigned End
Interval::pred_iterator pred_end(Interval *I)
std::pair< Value *, BasicBlock * > getKey() const
bool LowerDbgDeclare(Function &F)
Lowers llvm.dbg.declare intrinsics into appropriate set of llvm.dbg.value intrinsics.
void setBody(ArrayRef< Type * > Elements, bool isPacked=false)
Specify a body for an opaque identified type.
void dump() const
Support for debugging, callable in GDB: V->dump()
static bool isCoroutineStructureIntrinsic(Instruction &I)
bool dominates(const Instruction *Def, const Use &U) const
Return true if Def dominates a use in User.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Type * getType() const
All values are typed, get the type of this value.
void buildCoroutineFrame(Function &F, Shape &Shape)
const BasicBlock & getEntryBlock() const
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
unsigned Log2_64_Ceil(uint64_t Value)
Log2_64_Ceil - This function returns the ceil log base 2 of the specified value, 64 if the value is z...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
This class represents the llvm.coro.begin instruction.
A range adaptor for a pair of iterators.
iterator_range< user_iterator > users()
BasicBlock * getSinglePredecessor()
Return the predecessor of this block if it has a single predecessor block.
static BasicBlock * splitBlockIfNotFirst(Instruction *I, const Twine &Name)
static void rewriteMaterializableInstructions(IRBuilder<> &IRB, SpillInfo const &Spills)
void emplace_back(ArgTypes &&...Args)
static Instruction * insertSpills(SpillInfo &Spills, coro::Shape &Shape)
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="")
Split the basic block into two basic blocks at the specified instruction.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SmallVector< CoroEndInst *, 4 > CoroEnds
AllocaInst * PromiseAlloca
static StructType * buildFrameType(Function &F, coro::Shape &Shape, SpillInfo &Spills)
LLVM Value Representation.
succ_range successors(BasicBlock *BB)
const ArgumentListType & getArgumentList() const
Get the underlying elements of the Function...
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
AllocaInst * getPromise() const
static const unsigned FramePtr
BasicBlock * SplitEdge(BasicBlock *From, BasicBlock *To, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr)
Split the edge connecting specified block.
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
StringRef - Represent a constant reference to a string, i.e.
BasicBlock * AllocaSpillBlock
iterator getFirstInsertionPt()
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
std::pair< Value *, Instruction * > base
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
static void splitAround(Instruction *I, const Twine &Name)
const BasicBlock * getParent() const
A wrapper class for inspecting calls to intrinsic functions.