23#include "llvm/Config/llvm-config.h"
46#define DEBUG_TYPE "coro-suspend-crossing"
52class BlockToIndexMapping {
56 size_t size()
const {
return V.
size(); }
66 assert(
I != V.
end() && *
I == BB &&
"BasicBlockNumberng: Unknown block");
90struct SuspendCrossingInfo {
91 BlockToIndexMapping Mapping;
98 bool KillLoop =
false;
103 BasicBlock *BB = Mapping.indexToBlock(&BD - &Block[0]);
108 return Block[Mapping.blockToIndex(BB)];
119 size_t const FromIndex = Mapping.blockToIndex(
From);
120 size_t const ToIndex = Mapping.blockToIndex(To);
121 bool const Result =
Block[ToIndex].Kills[FromIndex];
123 <<
" answer is " << Result <<
"\n");
132 size_t const FromIndex = Mapping.blockToIndex(
From);
133 size_t const ToIndex = Mapping.blockToIndex(To);
137 <<
" answer is " << Result <<
" (path or loop)\n");
141 bool isDefinitionAcrossSuspend(
BasicBlock *DefBB,
User *U)
const {
142 auto *
I = cast<Instruction>(U);
146 if (
auto *PN = dyn_cast<PHINode>(
I))
147 if (PN->getNumIncomingValues() > 1)
155 if (isa<CoroSuspendRetconInst>(
I) || isa<CoroSuspendAsyncInst>(
I)) {
157 assert(UseBB &&
"should have split coro.suspend into its own block");
160 return hasPathCrossingSuspendPoint(DefBB, UseBB);
164 return isDefinitionAcrossSuspend(&
A.getParent()->getEntryBlock(), U);
168 auto *DefBB =
I.getParent();
173 if (isa<AnyCoroSuspendInst>(
I)) {
175 assert(DefBB &&
"should have split coro.suspend into its own block");
178 return isDefinitionAcrossSuspend(DefBB, U);
181 bool isDefinitionAcrossSuspend(
Value &V,
User *U)
const {
182 if (
auto *
Arg = dyn_cast<Argument>(&V))
183 return isDefinitionAcrossSuspend(*
Arg, U);
184 if (
auto *Inst = dyn_cast<Instruction>(&V))
185 return isDefinitionAcrossSuspend(*Inst, U);
188 "Coroutine could only collect Argument and Instruction now.");
193#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
197 for (
size_t I = 0,
N = BV.
size();
I <
N; ++
I)
199 dbgs() <<
" " << Mapping.indexToBlock(
I)->getName();
204 for (
size_t I = 0,
N =
Block.size();
I <
N; ++
I) {
206 dbgs() <<
B->getName() <<
":\n";
207 dump(
" Consumes", Block[
I].Consumes);
208 dump(
" Kills", Block[
I].Kills);
216 const size_t N = Mapping.size();
220 for (
size_t I = 0;
I <
N; ++
I) {
222 B.Consumes.resize(
N);
231 getBlockData(
CE->getParent()).End =
true;
239 auto &
B = getBlockData(SuspendBlock);
241 B.Kills |=
B.Consumes;
244 markSuspendBlock(CSI);
245 if (
auto *Save = CSI->getCoroSave())
246 markSuspendBlock(Save);
259 for (
size_t I = 0;
I <
N; ++
I) {
263 auto SuccNo = Mapping.blockToIndex(SI);
267 auto &S =
Block[SuccNo];
268 auto SavedConsumes = S.Consumes;
269 auto SavedKills = S.Kills;
272 S.Consumes |=
B.Consumes;
278 S.Kills |=
B.Consumes;
283 S.Kills |= S.Consumes;
293 S.KillLoop |= S.Kills[SuccNo];
294 S.Kills.reset(SuccNo);
298 Changed |= (S.Kills != SavedKills) || (S.Consumes != SavedConsumes);
300 if (S.Kills != SavedKills) {
306 if (S.Consumes != SavedConsumes) {
318#define DEBUG_TYPE "coro-frame"
321class FrameTypeBuilder;
327 bool MayWriteBeforeCoroBegin;
330 bool MayWriteBeforeCoroBegin)
331 : Alloca(Alloca), Aliases(std::move(Aliases)),
332 MayWriteBeforeCoroBegin(MayWriteBeforeCoroBegin) {}
334struct FrameDataInfo {
344 for (
const auto &
P : Spills)
346 for (
const auto &
A : Allocas)
352 auto Itr = FieldIndexMap.find(V);
353 assert(Itr != FieldIndexMap.end() &&
354 "Value does not have a frame field index");
359 assert((LayoutIndexUpdateStarted || FieldIndexMap.count(V) == 0) &&
360 "Cannot set the index for the same field twice.");
361 FieldIndexMap[V] =
Index;
365 auto Iter = FieldAlignMap.find(V);
366 assert(Iter != FieldAlignMap.end());
371 assert(FieldAlignMap.count(V) == 0);
372 FieldAlignMap.insert({V, AL});
376 auto Iter = FieldDynamicAlignMap.find(V);
377 assert(Iter != FieldDynamicAlignMap.end());
382 assert(FieldDynamicAlignMap.count(V) == 0);
383 FieldDynamicAlignMap.insert({V,
Align});
387 auto Iter = FieldOffsetMap.find(V);
388 assert(Iter != FieldOffsetMap.end());
393 assert(FieldOffsetMap.count(V) == 0);
394 FieldOffsetMap.insert({V,
Offset});
398 void updateLayoutIndex(FrameTypeBuilder &
B);
403 bool LayoutIndexUpdateStarted =
false;
420 dbgs() <<
"------------- " << Title <<
"--------------\n";
421 for (
const auto &
E : Spills) {
424 for (
auto *
I :
E.second)
430 dbgs() <<
"------------- Allocas --------------\n";
431 for (
const auto &
A : Allocas) {
438using FieldIDType = size_t;
443class FrameTypeBuilder {
449 FieldIDType LayoutFieldIndex;
459 bool IsFinished =
false;
461 std::optional<Align> MaxFrameAlignment;
468 std::optional<Align> MaxFrameAlignment)
473 [[nodiscard]] FieldIDType addFieldForAlloca(
AllocaInst *AI,
474 bool IsHeader =
false) {
479 if (
auto *CI = dyn_cast<ConstantInt>(AI->
getArraySize()))
480 Ty = ArrayType::get(Ty, CI->getValue().getZExtValue());
485 return addField(Ty, AI->
getAlign(), IsHeader);
515 void addFieldForAllocas(
const Function &
F, FrameDataInfo &FrameData,
519 [[nodiscard]] FieldIDType addField(
Type *Ty,
MaybeAlign MaybeFieldAlignment,
520 bool IsHeader =
false,
521 bool IsSpillOfValue =
false) {
522 assert(!IsFinished &&
"adding fields to a finished builder");
523 assert(Ty &&
"must provide a type for a field");
530 if (FieldSize == 0) {
538 Align ABIAlign =
DL.getABITypeAlign(Ty);
539 Align TyAlignment = ABIAlign;
540 if (IsSpillOfValue && MaxFrameAlignment && *MaxFrameAlignment < ABIAlign)
541 TyAlignment = *MaxFrameAlignment;
542 Align FieldAlignment = MaybeFieldAlignment.value_or(TyAlignment);
548 if (MaxFrameAlignment && (FieldAlignment > *MaxFrameAlignment)) {
551 FieldAlignment = *MaxFrameAlignment;
552 FieldSize = FieldSize + DynamicAlignBuffer;
559 StructSize =
Offset + FieldSize;
566 Fields.
push_back({FieldSize,
Offset, Ty, 0, FieldAlignment, TyAlignment,
567 DynamicAlignBuffer});
568 return Fields.
size() - 1;
575 assert(IsFinished &&
"not yet finished!");
579 Align getStructAlign()
const {
580 assert(IsFinished &&
"not yet finished!");
584 FieldIDType getLayoutFieldIndex(FieldIDType Id)
const {
585 assert(IsFinished &&
"not yet finished!");
586 return Fields[
Id].LayoutFieldIndex;
589 Field getLayoutField(FieldIDType Id)
const {
590 assert(IsFinished &&
"not yet finished!");
596void FrameDataInfo::updateLayoutIndex(FrameTypeBuilder &
B) {
597 auto Updater = [&](
Value *
I) {
598 auto Field =
B.getLayoutField(getFieldIndex(
I));
599 setFieldIndex(
I,
Field.LayoutFieldIndex);
602 Field.DynamicAlignBuffer
605 setDynamicAlign(
I, dynamicAlign);
608 LayoutIndexUpdateStarted =
true;
609 for (
auto &S : Spills)
611 for (
const auto &
A : Allocas)
613 LayoutIndexUpdateStarted =
false;
616void FrameTypeBuilder::addFieldForAllocas(
const Function &
F,
617 FrameDataInfo &FrameData,
624 for (
auto AllocaList : NonOverlapedAllocas) {
625 auto *LargestAI = *AllocaList.begin();
626 FieldIDType
Id = addFieldForAlloca(LargestAI);
627 for (
auto *Alloca : AllocaList)
635 NonOverlapedAllocas.emplace_back(AllocaSetType(1, Alloca));
655 if (
auto *ConstSWI = dyn_cast<SwitchInst>(U)) {
656 auto *SWI =
const_cast<SwitchInst *
>(ConstSWI);
657 DefaultSuspendDest[SWI] = SWI->getDefaultDest();
658 SWI->setDefaultDest(SWI->getSuccessor(1));
663 auto ExtractAllocas = [&]() {
664 AllocaSetType Allocas;
667 Allocas.push_back(
A.Alloca);
671 StackLifetime::LivenessType::May);
672 StackLifetimeAnalyzer.run();
674 return StackLifetimeAnalyzer.getLiveRange(AI1).overlaps(
675 StackLifetimeAnalyzer.getLiveRange(AI2));
677 auto GetAllocaSize = [&](
const AllocaInfo &
A) {
678 std::optional<TypeSize> RetSize =
A.Alloca->getAllocationSize(
DL);
679 assert(RetSize &&
"Variable Length Arrays (VLA) are not supported.\n");
680 assert(!RetSize->isScalable() &&
"Scalable vectors are not yet supported");
681 return RetSize->getFixedValue();
687 sort(
FrameData.Allocas, [&](
const auto &Iter1,
const auto &Iter2) {
688 return GetAllocaSize(Iter1) > GetAllocaSize(Iter2);
696 for (
auto &AllocaSet : NonOverlapedAllocas) {
697 assert(!AllocaSet.empty() &&
"Processing Alloca Set is not empty.\n");
698 bool NoInference =
none_of(AllocaSet, [&](
auto Iter) {
699 return IsAllocaInferenre(Alloca, Iter);
707 bool Alignable = [&]() ->
bool {
708 auto *LargestAlloca = *AllocaSet.begin();
709 return LargestAlloca->getAlign().value() % Alloca->
getAlign().
value() ==
712 bool CouldMerge = NoInference && Alignable;
715 AllocaSet.push_back(Alloca);
720 NonOverlapedAllocas.emplace_back(AllocaSetType(1, Alloca));
725 for (
auto SwitchAndDefaultDest : DefaultSuspendDest) {
727 BasicBlock *DestBB = SwitchAndDefaultDest.second;
732 : NonOverlapedAllocas) {
733 if (AllocaSet.size() > 1) {
734 dbgs() <<
"In Function:" << F.getName() <<
"\n";
735 dbgs() <<
"Find Union Set "
737 dbgs() <<
"\tAllocas are \n";
738 for (auto Alloca : AllocaSet)
739 dbgs() <<
"\t\t" << *Alloca <<
"\n";
744void FrameTypeBuilder::finish(
StructType *Ty) {
745 assert(!IsFinished &&
"already finished!");
751 for (
auto &
Field : Fields) {
758 StructSize = SizeAndAlign.first;
759 StructAlign = SizeAndAlign.second;
762 return *
static_cast<Field *
>(
const_cast<void*
>(LayoutField.Id));
768 for (
auto &LayoutField : LayoutFields) {
769 auto &
F = getField(LayoutField);
770 if (!
isAligned(
F.TyAlignment, LayoutField.Offset))
778 FieldTypes.
reserve(LayoutFields.size() * 3 / 2);
780 for (
auto &LayoutField : LayoutFields) {
781 auto &
F = getField(LayoutField);
783 auto Offset = LayoutField.Offset;
789 if (
Offset != LastOffset) {
796 F.LayoutFieldIndex = FieldTypes.
size();
799 if (
F.DynamicAlignBuffer) {
806 Ty->
setBody(FieldTypes, Packed);
810 auto Layout =
DL.getStructLayout(Ty);
811 for (
auto &
F : Fields) {
813 assert(Layout->getElementOffset(
F.LayoutFieldIndex) ==
F.Offset);
822 for (
auto *V : FrameData.getAllDefs()) {
823 if (DIVarCache.
find(V) != DIVarCache.
end())
831 DIVarCache.
insert({V, (*I)->getVariable()});
842 OS <<
"__int_" << cast<IntegerType>(Ty)->getBitWidth();
844 return MDName->getString();
852 return "__floating_type_";
855 if (
auto *PtrTy = dyn_cast<PointerType>(Ty)) {
856 if (PtrTy->isOpaque())
857 return "PointerType";
860 if (
Name ==
"UnknownType")
861 return "PointerType";
865 return MDName->getString();
869 if (!cast<StructType>(Ty)->hasName())
870 return "__LiteralStructType_";
875 for (
auto &Iter : Buffer)
876 if (Iter ==
'.' || Iter ==
':')
879 return MDName->getString();
882 return "UnknownType";
894 DIType *RetType =
nullptr;
897 auto BitWidth = cast<IntegerType>(Ty)->getBitWidth();
899 llvm::DINode::FlagArtificial);
903 llvm::DINode::FlagArtificial);
917 auto *DIStruct =
Builder.createStructType(
920 llvm::DINode::FlagArtificial,
nullptr, llvm::DINodeArray());
922 auto *StructTy = cast<StructType>(Ty);
924 for (
unsigned I = 0;
I < StructTy->getNumElements();
I++) {
926 Scope, LineNum, DITypeCache);
928 Elements.push_back(
Builder.createMemberType(
929 Scope, DITy->
getName(), Scope->getFile(), LineNum,
932 llvm::DINode::FlagArtificial, DITy));
935 Builder.replaceArrays(DIStruct,
Builder.getOrCreateArray(Elements));
941 auto *CharSizeType =
Builder.createBasicType(
942 Name, 8, dwarf::DW_ATE_unsigned_char, llvm::DINode::FlagArtificial);
945 RetType = CharSizeType;
950 RetType =
Builder.createArrayType(
956 DITypeCache.
insert({Ty, RetType});
973 FrameDataInfo &FrameData) {
978 if (!DIS || !DIS->getUnit() ||
984 "We could only build debug infomation for C++ coroutine now.\n");
990 "Coroutine with switch ABI should own Promise alloca");
1001 unsigned LineNum = PromiseDIVariable->
getLine();
1004 DIS->getUnit(),
Twine(
F.getName() +
".coro_frame_ty").
str(),
1007 llvm::DINodeArray());
1010 DataLayout Layout =
F.getParent()->getDataLayout();
1020 NameCache.
insert({ResumeIndex,
"__resume_fn"});
1021 NameCache.
insert({DestroyIndex,
"__destroy_fn"});
1022 NameCache.
insert({IndexIndex,
"__coro_index"});
1043 dwarf::DW_ATE_unsigned_char)});
1045 for (
auto *V : FrameData.getAllDefs()) {
1046 if (DIVarCache.
find(V) == DIVarCache.
end())
1049 auto Index = FrameData.getFieldIndex(V);
1051 NameCache.
insert({
Index, DIVarCache[V]->getName()});
1052 TyCache.
insert({
Index, DIVarCache[V]->getType()});
1058 OffsetCache.
insert({ResumeIndex, {8, 0}});
1059 OffsetCache.
insert({DestroyIndex, {8, 8}});
1064 for (
auto *V : FrameData.getAllDefs()) {
1065 auto Index = FrameData.getFieldIndex(V);
1068 {
Index, {FrameData.getAlign(V).
value(), FrameData.getOffset(V)}});
1076 unsigned UnknownTypeNum = 0;
1088 assert(Ty->
isSized() &&
"We can't handle type which is not sized.\n");
1090 AlignInBits = OffsetCache[
Index].first * 8;
1091 OffsetInBits = OffsetCache[
Index].second * 8;
1095 DITy = TyCache[
Index];
1097 DITy =
solveDIType(DBuilder, Ty, Layout, FrameDITy, LineNum, DITypeCache);
1098 assert(DITy &&
"SolveDIType shouldn't return nullptr.\n");
1100 Name +=
"_" + std::to_string(UnknownTypeNum);
1105 FrameDITy,
Name, DFile, LineNum, SizeInBits, AlignInBits, OffsetInBits,
1106 llvm::DINode::FlagArtificial, DITy));
1112 DFile, LineNum, FrameDITy,
1113 true, DINode::FlagArtificial);
1123 if (
auto *SubProgram = dyn_cast<DISubprogram>(PromiseDIScope)) {
1124 auto RetainedNodes = SubProgram->getRetainedNodes();
1126 RetainedNodes.end());
1128 SubProgram->replaceOperandWith(
1132 DBuilder.insertDeclare(Shape.
FramePtr, FrameDIVar,
1146 FrameDataInfo &FrameData) {
1151 Name.append(
".Frame");
1156 std::optional<Align> MaxFrameAlignment;
1157 if (Shape.
ABI == coro::ABI::Async)
1159 FrameTypeBuilder
B(
C,
DL, MaxFrameAlignment);
1162 std::optional<FieldIDType> SwitchIndexFieldId;
1164 if (Shape.
ABI == coro::ABI::Switch) {
1168 auto *FnPtrTy = FnTy->getPointerTo();
1172 (void)
B.addField(FnPtrTy, std::nullopt,
true);
1173 (void)
B.addField(FnPtrTy, std::nullopt,
true);
1179 FrameData.setFieldIndex(
1180 PromiseAlloca,
B.addFieldForAlloca(PromiseAlloca,
true));
1187 SwitchIndexFieldId =
B.addField(IndexType, std::nullopt);
1189 assert(PromiseAlloca ==
nullptr &&
"lowering doesn't support promises");
1194 B.addFieldForAllocas(
F, FrameData, Shape);
1199 if (Shape.
ABI == coro::ABI::Switch && PromiseAlloca)
1202 FrameData.Allocas.emplace_back(
1205 for (
auto &S : FrameData.Spills) {
1206 Type *FieldType = S.first->getType();
1209 if (
const Argument *
A = dyn_cast<Argument>(S.first))
1210 if (
A->hasByValAttr())
1211 FieldType =
A->getParamByValType();
1212 FieldIDType Id =
B.addField(FieldType, std::nullopt,
false ,
1214 FrameData.setFieldIndex(S.first, Id);
1218 FrameData.updateLayoutIndex(
B);
1222 switch (Shape.
ABI) {
1223 case coro::ABI::Switch: {
1225 auto IndexField =
B.getLayoutField(*SwitchIndexFieldId);
1237 case coro::ABI::Retcon:
1238 case coro::ABI::RetconOnce: {
1241 = (
B.getStructSize() <= Id->getStorageSize() &&
1242 B.getStructAlign() <= Id->getStorageAlignment());
1245 case coro::ABI::Async: {
1255 "The alignment requirment of frame variables cannot be higher than "
1256 "the alignment of the async function context");
1294 const CoroBeginInst &CB,
const SuspendCrossingInfo &Checker,
1295 bool ShouldUseLifetimeStartInfo)
1297 ShouldUseLifetimeStartInfo(ShouldUseLifetimeStartInfo) {}
1304 if (PI.isEscaped() && !DT.dominates(&CoroBegin, PI.getEscapingInst())) {
1305 MayWriteBeforeCoroBegin =
true;
1327 if (
SI.getValueOperand() != U->get())
1340 auto IsSimpleStoreThenLoad = [&]() {
1341 auto *AI = dyn_cast<AllocaInst>(
SI.getPointerOperand());
1349 while (!StoreAliases.
empty()) {
1351 for (
User *U :
I->users()) {
1354 if (
auto *LI = dyn_cast<LoadInst>(U)) {
1361 if (
auto *S = dyn_cast<StoreInst>(U))
1362 if (S->getPointerOperand() ==
I)
1364 if (
auto *II = dyn_cast<IntrinsicInst>(U))
1365 if (II->isLifetimeStartOrEnd())
1369 if (
auto *BI = dyn_cast<BitCastInst>(U)) {
1380 if (!IsSimpleStoreThenLoad())
1407 if (II.
getIntrinsicID() != Intrinsic::lifetime_start || !IsOffsetKnown ||
1410 LifetimeStarts.insert(&II);
1414 for (
unsigned Op = 0, OpCount = CB.
arg_size(); Op < OpCount; ++Op)
1420 bool getShouldLiveOnFrame()
const {
1421 if (!ShouldLiveOnFrame)
1422 ShouldLiveOnFrame = computeShouldLiveOnFrame();
1423 return *ShouldLiveOnFrame;
1426 bool getMayWriteBeforeCoroBegin()
const {
return MayWriteBeforeCoroBegin; }
1429 assert(getShouldLiveOnFrame() &&
"This method should only be called if the "
1430 "alloca needs to live on the frame.");
1431 for (
const auto &
P : AliasOffetMap)
1434 "created before CoroBegin.");
1435 return AliasOffetMap;
1441 const SuspendCrossingInfo &Checker;
1448 bool MayWriteBeforeCoroBegin{
false};
1449 bool ShouldUseLifetimeStartInfo{
true};
1451 mutable std::optional<bool> ShouldLiveOnFrame{};
1453 bool computeShouldLiveOnFrame()
const {
1458 if (ShouldUseLifetimeStartInfo && !LifetimeStarts.empty()) {
1459 for (
auto *
I : Users)
1460 for (
auto *S : LifetimeStarts)
1461 if (Checker.isDefinitionAcrossSuspend(*S,
I))
1467 if (PI.isEscaped()) {
1468 for (
auto *
A : LifetimeStarts) {
1469 for (
auto *
B : LifetimeStarts) {
1470 if (Checker.hasPathOrLoopCrossingSuspendPoint(
A->getParent(),
1491 for (
auto *U1 : Users)
1492 for (
auto *U2 : Users)
1493 if (Checker.isDefinitionAcrossSuspend(*U1, U2))
1501 MayWriteBeforeCoroBegin =
true;
1505 for (
auto &U :
I.uses())
1515 if (DT.
dominates(&CoroBegin, &
I) || !usedAfterCoroBegin(
I))
1518 if (!IsOffsetKnown) {
1519 AliasOffetMap[&
I].reset();
1521 auto Itr = AliasOffetMap.
find(&
I);
1522 if (Itr == AliasOffetMap.end()) {
1524 }
else if (Itr->second && *Itr->second !=
Offset) {
1527 AliasOffetMap[&
I].reset();
1561 cast<Instruction>(
Builder.CreateBitCast(CB, FramePtrTy,
"FramePtr"));
1598 auto GetFramePointer = [&](
Value *Orig) ->
Value * {
1599 FieldIDType
Index = FrameData.getFieldIndex(Orig);
1605 if (
auto *AI = dyn_cast<AllocaInst>(Orig)) {
1606 if (
auto *CI = dyn_cast<ConstantInt>(AI->
getArraySize())) {
1607 auto Count = CI->getValue().getZExtValue();
1616 auto GEP = cast<GetElementPtrInst>(
1618 if (
auto *AI = dyn_cast<AllocaInst>(Orig)) {
1619 if (FrameData.getDynamicAlign(Orig) != 0) {
1622 auto *IntPtrTy = M->getDataLayout().getIntPtrType(AI->
getType());
1623 auto *PtrValue =
Builder.CreatePtrToInt(
GEP, IntPtrTy);
1626 PtrValue =
Builder.CreateAdd(PtrValue, AlignMask);
1627 PtrValue =
Builder.CreateAnd(PtrValue,
Builder.CreateNot(AlignMask));
1637 if (
GEP->getType() != Orig->getType())
1638 return Builder.CreateBitCast(
GEP, Orig->getType(),
1639 Orig->getName() +
Twine(
".cast"));
1644 for (
auto const &
E : FrameData.Spills) {
1646 auto SpillAlignment =
Align(FrameData.getAlign(Def));
1650 Type *ByValTy =
nullptr;
1651 if (
auto *
Arg = dyn_cast<Argument>(Def)) {
1659 Arg->getParent()->removeParamAttr(
Arg->getArgNo(), Attribute::NoCapture);
1661 if (
Arg->hasByValAttr())
1662 ByValTy =
Arg->getParamByValType();
1663 }
else if (
auto *CSI = dyn_cast<AnyCoroSuspendInst>(Def)) {
1668 auto *
I = cast<Instruction>(Def);
1673 }
else if (
auto *II = dyn_cast<InvokeInst>(
I)) {
1677 InsertPt = NewBB->getTerminator();
1678 }
else if (isa<PHINode>(
I)) {
1681 if (
auto *CSI = dyn_cast<CatchSwitchInst>(DefBlock->
getTerminator()))
1686 assert(!
I->isTerminator() &&
"unexpected terminator");
1689 InsertPt =
I->getNextNode();
1693 auto Index = FrameData.getFieldIndex(Def);
1694 Builder.SetInsertPoint(InsertPt);
1695 auto *
G =
Builder.CreateConstInBoundsGEP2_32(
1703 Builder.CreateAlignedStore(Def,
G, SpillAlignment);
1707 Value *CurrentReload =
nullptr;
1708 for (
auto *U :
E.second) {
1712 if (CurrentBlock != U->getParent()) {
1713 CurrentBlock = U->getParent();
1716 auto *
GEP = GetFramePointer(
E.first);
1717 GEP->setName(
E.first->getName() +
Twine(
".reload.addr"));
1719 CurrentReload =
GEP;
1721 CurrentReload =
Builder.CreateAlignedLoad(
1723 SpillAlignment,
E.first->getName() +
Twine(
".reload"));
1727 bool AllowUnresolved =
false;
1732 .insertDeclare(CurrentReload, DDI->getVariable(),
1733 DDI->getExpression(), DDI->getDebugLoc(),
1743 if (
auto *DI = dyn_cast<DbgAddrIntrinsic>(U)) {
1750 if (
auto *PN = dyn_cast<PHINode>(U)) {
1751 assert(PN->getNumIncomingValues() == 1 &&
1752 "unexpected number of incoming "
1753 "values in the PHINode");
1754 PN->replaceAllUsesWith(CurrentReload);
1755 PN->eraseFromParent();
1773 if (Shape.
ABI == coro::ABI::Retcon || Shape.
ABI == coro::ABI::RetconOnce ||
1774 Shape.
ABI == coro::ABI::Async) {
1776 Builder.SetInsertPoint(&SpillBlock->front());
1777 for (
const auto &
P : FrameData.Allocas) {
1779 auto *
G = GetFramePointer(Alloca);
1783 G->takeName(Alloca);
1797 for (
const auto &
A : FrameData.Allocas) {
1799 UsersToUpdate.
clear();
1801 auto *
I = cast<Instruction>(U);
1805 if (UsersToUpdate.
empty())
1807 auto *
G = GetFramePointer(Alloca);
1812 for (
auto *DVI : DIs)
1813 DVI->replaceUsesOfWith(Alloca,
G);
1819 if (
I->isLifetimeStartOrEnd())
1822 I->replaceUsesOfWith(Alloca,
G);
1826 for (
const auto &
A : FrameData.Allocas) {
1828 if (
A.MayWriteBeforeCoroBegin) {
1832 "Coroutines cannot handle copying of array allocas yet");
1834 auto *
G = GetFramePointer(Alloca);
1841 for (
const auto &Alias :
A.Aliases) {
1842 auto *
FramePtr = GetFramePointer(Alloca);
1845 auto &
Value = *Alias.second;
1849 auto *AliasPtrTyped =
1850 Builder.CreateBitCast(AliasPtr, Alias.first->getType());
1851 Alias.first->replaceUsesWithIf(
1852 AliasPtrTyped, [&](
Use &U) {
return DT.
dominates(CB, U); });
1864 auto *Inst = dyn_cast<Instruction>(U.getUser());
1865 if (!Inst || DT.dominates(CB, Inst))
1868 if (auto *CI = dyn_cast<CallInst>(Inst)) {
1873 if (CI->onlyReadsMemory() ||
1874 CI->onlyReadsMemory(CI->getArgOperandNo(&U)))
1879 return isa<StoreInst>(Inst) ||
1882 isa<GetElementPtrInst>(Inst) ||
1887 isa<BitCastInst>(Inst);
1889 if (HasAccessingPromiseBeforeCB) {
1891 auto *
G = GetFramePointer(PA);
1892 auto *
Value =
Builder.CreateLoad(PA->getAllocatedType(), PA);
1903 PHINode *UntilPHI =
nullptr) {
1904 auto *PN = cast<PHINode>(&SuccBB->
front());
1906 int Index = PN->getBasicBlockIndex(InsertedBB);
1910 &InsertedBB->
front());
1912 PN->setIncomingValue(
Index, InputV);
1913 PN = dyn_cast<PHINode>(PN->getNextNode());
1914 }
while (PN != UntilPHI);
1954 auto *NewCleanupPadBB =
1957 CleanupPadBB->
getParent(), CleanupPadBB);
1958 Builder.SetInsertPoint(NewCleanupPadBB);
1959 auto *SwitchType =
Builder.getInt8Ty();
1960 auto *SetDispatchValuePN =
1964 auto *SwitchOnDispatch =
Builder.CreateSwitch(SetDispatchValuePN, UnreachBB,
1967 int SwitchIndex = 0;
1973 Twine(
".from.") + Pred->getName(),
1974 CleanupPadBB->
getParent(), CleanupPadBB);
1976 CaseBB->setName(CleanupPadBB->
getName() +
Twine(
".from.") +
1978 Builder.SetInsertPoint(CaseBB);
1979 Builder.CreateBr(CleanupPadBB);
1987 SetDispatchValuePN->addIncoming(SwitchConstant, Pred);
1988 SwitchOnDispatch->addCase(SwitchConstant, CaseBB);
1995 for (
auto &BB :
F) {
1996 for (
auto &Phi : BB.
phis()) {
1997 if (Phi.getNumIncomingValues() == 1) {
2003 while (!Worklist.
empty()) {
2005 auto *OriginalValue = Phi->getIncomingValue(0);
2006 Phi->replaceAllUsesWith(OriginalValue);
2034 if (
auto *CleanupPad =
2039 dyn_cast<CatchSwitchInst>(Pred->getTerminator())) {
2042 assert(CS->getUnwindDest() == &BB);
2052 if ((LandingPad = dyn_cast_or_null<LandingPadInst>(BB.
getFirstNonPHI()))) {
2066 IncomingBB->setName(BB.
getName() +
Twine(
".from.") + Pred->getName());
2084 if (
auto *PN = dyn_cast<PHINode>(&BB.
front()))
2085 if (PN->getNumIncomingValues() > 1)
2095 return isa<CastInst>(&V) || isa<GetElementPtrInst>(&V) ||
2096 isa<BinaryOperator>(&V) || isa<CmpInst>(&V) || isa<SelectInst>(&V);
2102 return isa<CoroIdInst>(&
I) || isa<CoroSaveInst>(&
I) ||
2103 isa<CoroSuspendInst>(&
I);
2109 const SpillInfo &Spills) {
2110 for (
const auto &
E : Spills) {
2118 bool IsInCoroSuspendBlock = isa<AnyCoroSuspendInst>(U);
2120 auto *InsertBlock = IsInCoroSuspendBlock
2123 CurrentMaterialization = cast<Instruction>(Def)->
clone();
2124 CurrentMaterialization->
setName(Def->getName());
2126 IsInCoroSuspendBlock ? InsertBlock->getTerminator()
2127 : &*InsertBlock->getFirstInsertionPt());
2129 if (
auto *PN = dyn_cast<PHINode>(U)) {
2130 assert(PN->getNumIncomingValues() == 1 &&
2131 "unexpected number of incoming "
2132 "values in the PHINode");
2133 PN->replaceAllUsesWith(CurrentMaterialization);
2134 PN->eraseFromParent();
2147 auto *BB =
I->getParent();
2165 return isa<AnyCoroSuspendInst>(BB->
front());
2200 if (
auto FI = dyn_cast<CoroAllocaFreeInst>(
User))
2201 VisitedOrFreeBBs.
insert(FI->getParent());
2210 unsigned depth = 3) {
2213 if (depth == 0)
return false;
2232 for (
auto *U : AI->
users()) {
2233 auto FI = dyn_cast<CoroAllocaFreeInst>(U);
2248 for (
auto *AI : LocalAllocas) {
2254 Value *StackSave =
nullptr;
2256 StackSave =
Builder.CreateCall(
2260 auto Alloca =
Builder.CreateAlloca(
Builder.getInt8Ty(), AI->getSize());
2263 for (
auto *U : AI->
users()) {
2265 if (isa<CoroAllocaGetInst>(U)) {
2272 auto FI = cast<CoroAllocaFreeInst>(U);
2280 DeadInsts.
push_back(cast<Instruction>(U));
2297 if (isa<CoroAllocaGetInst>(U)) {
2300 auto FI = cast<CoroAllocaFreeInst>(U);
2304 DeadInsts.
push_back(cast<Instruction>(U));
2311 return cast<Instruction>(Alloc);
2318 auto FnTy = FunctionType::get(ValueTy, {},
false);
2321 auto Call =
Builder.CreateCall(FnTy, Fn, {});
2334 {V->getType()},
false);
2337 auto Call =
Builder.CreateCall(FnTy, Fn, { V });
2356 auto ValueBeforeCall =
Builder.CreateLoad(ValueTy, Alloca);
2362 if (isa<CallInst>(Call)) {
2363 Builder.SetInsertPoint(Call->getNextNode());
2365 auto Invoke = cast<InvokeInst>(Call);
2366 Builder.SetInsertPoint(Invoke->getNormalDest()->getFirstNonPHIOrDbg());
2371 Builder.CreateStore(ValueAfterCall, Alloca);
2384 if (isa<LoadInst>(
User) || isa<StoreInst>(
User))
2388 auto Call = cast<Instruction>(
User);
2409 auto ArgTy = cast<PointerType>(
Arg.getType());
2412 auto ValueTy = ArgTy->isOpaque() ? PointerType::getUnqual(
F.getContext())
2413 : ArgTy->getNonOpaquePointerElementType();
2418 auto Alloca =
Builder.CreateAlloca(ValueTy, ArgTy->getAddressSpace());
2419 Arg.replaceAllUsesWith(Alloca);
2423 Builder.CreateStore(InitialValue, Alloca);
2433 auto FinalValue =
Builder.CreateLoad(ValueTy, Alloca);
2448 for (
auto &
Arg :
F.args()) {
2449 if (!
Arg.hasSwiftErrorAttr())
continue;
2456 for (
auto &Inst :
F.getEntryBlock()) {
2457 auto Alloca = dyn_cast<AllocaInst>(&Inst);
2469 if (!AllocasToPromote.
empty()) {
2478 const FrameDataInfo &FrameData,
2486 for (
auto *Def : FrameData.getAllDefs()) {
2487 for (
User *U : Def->users()) {
2488 auto Inst = cast<Instruction>(U);
2489 if (Inst->getParent() != CoroBegin->
getParent() ||
2497 while (!Worklist.
empty()) {
2499 for (
User *U : Def->users()) {
2500 auto Inst = cast<Instruction>(U);
2525 SuspendCrossingInfo &Checker) {
2530 DomSet.
insert(&
F.getEntryBlock());
2534 "should have split coro.suspend into its own block");
2548 if (
auto* II = dyn_cast<IntrinsicInst>(
I))
2574 Checker.isDefinitionAcrossSuspend(DomBB, UI)) {
2577 if (collectLifetimeStart(UI, AI))
2585 if (Valid && Lifetimes.
size() != 0) {
2588 if (isa<AllocaInst>(Lifetimes[0]->getOperand(1)))
2592 DomBB->getTerminator());
2595 auto *NewLifetime = Lifetimes[0]->clone();
2596 NewLifetime->replaceUsesOfWith(NewLifetime->getOperand(1), NewBitCast);
2597 NewLifetime->insertBefore(DomBB->getTerminator());
2601 S->eraseFromParent();
2610 const SuspendCrossingInfo &Checker,
2624 bool ShouldUseLifetimeStartInfo =
2625 (Shape.
ABI != coro::ABI::Async && Shape.
ABI != coro::ABI::Retcon &&
2626 Shape.
ABI != coro::ABI::RetconOnce);
2629 ShouldUseLifetimeStartInfo};
2630 Visitor.visitPtr(*AI);
2631 if (!Visitor.getShouldLiveOnFrame())
2634 Visitor.getMayWriteBeforeCoroBegin());
2642 auto InsertPt =
F->getEntryBlock().getFirstInsertionPt();
2643 while (isa<IntrinsicInst>(InsertPt))
2645 Builder.SetInsertPoint(&
F->getEntryBlock(), InsertPt);
2649 bool SkipOutermostLoad = !isa<DbgValueInst>(DVI);
2651 Value *OriginalStorage = Storage;
2653 while (
auto *Inst = dyn_cast_or_null<Instruction>(Storage)) {
2654 if (
auto *LdInst = dyn_cast<LoadInst>(Inst)) {
2655 Storage = LdInst->getOperand(0);
2662 if (!SkipOutermostLoad)
2664 }
else if (
auto *StInst = dyn_cast<StoreInst>(Inst)) {
2665 Storage = StInst->getOperand(0);
2672 if (!Op || !AdditionalValues.
empty()) {
2680 SkipOutermostLoad =
false;
2693 if (
auto *
Arg = dyn_cast<llvm::Argument>(Storage)) {
2694 auto &Cached = DbgPtrAllocaCache[Storage];
2697 Arg->getName() +
".debug");
2698 Builder.CreateStore(Storage, Cached);
2716 if (!isa<DbgValueInst>(DVI) && !isa<DbgAddrIntrinsic>(DVI)) {
2718 if (
auto *
I = dyn_cast<Instruction>(Storage))
2719 InsertPt =
I->getInsertionPointAfterDef();
2720 else if (isa<Argument>(Storage))
2721 InsertPt = &*
F->getEntryBlock().begin();
2732 if (
Shape.
ABI == coro::ABI::Switch &&
2741 if (
auto *Save = CSI->getCoroSave())
2754 if (
auto *AsyncEnd = dyn_cast<CoroAsyncEndInst>(CE)) {
2755 auto *MustTailCallFn = AsyncEnd->getMustTailCallFunction();
2756 if (!MustTailCallFn)
2776 SuspendCrossingInfo Checker(
F,
Shape);
2779 FrameDataInfo FrameData;
2785 for (
int Repeat = 0; Repeat < 4; ++Repeat) {
2791 for (
User *U :
I.users())
2792 if (Checker.isDefinitionAcrossSuspend(
I, U))
2808 Shape.
ABI != coro::ABI::RetconOnce)
2813 for (
User *U :
A.users())
2814 if (Checker.isDefinitionAcrossSuspend(
A, U))
2815 FrameData.Spills[&
A].push_back(cast<Instruction>(U));
2825 if (
auto AI = dyn_cast<CoroAllocaAllocInst>(&
I)) {
2839 for (
User *U : Alloc->users()) {
2840 if (Checker.isDefinitionAcrossSuspend(*Alloc, U))
2841 FrameData.Spills[Alloc].push_back(cast<Instruction>(U));
2847 if (isa<CoroAllocaGetInst>(
I))
2850 if (
auto *AI = dyn_cast<AllocaInst>(&
I)) {
2855 for (
User *U :
I.users())
2856 if (Checker.isDefinitionAcrossSuspend(
I, U)) {
2858 if (
I.getType()->isTokenTy())
2860 "token definition is separated from the use by a suspend point");
2861 FrameData.Spills[&
I].push_back(cast<Instruction>(U));
2871 for (
auto &Iter : FrameData.Spills) {
2872 auto *V = Iter.first;
2876 if (Checker.isDefinitionAcrossSuspend(*V, DVI))
2877 FrameData.Spills[V].push_back(DVI);
2881 if (
Shape.
ABI == coro::ABI::Retcon ||
Shape.
ABI == coro::ABI::RetconOnce ||
2891 for (
auto *
I : DeadInstructions)
2892 I->eraseFromParent();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
AMDGPU Lower Kernel Arguments
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
static void cleanupSinglePredPHIs(Function &F)
static bool isSuspendReachableFrom(BasicBlock *From, VisitedBlocksSet &VisitedOrFreeBBs)
Does control flow starting at the given block ever reach a suspend instruction before reaching a bloc...
static bool isCoroutineStructureIntrinsic(Instruction &I)
SmallPtrSet< BasicBlock *, 8 > VisitedBlocksSet
static void createFramePtr(coro::Shape &Shape)
static Instruction * lowerNonLocalAlloca(CoroAllocaAllocInst *AI, coro::Shape &Shape, SmallVectorImpl< Instruction * > &DeadInsts)
Turn the given coro.alloca.alloc call into a dynamic allocation.
static Instruction * splitBeforeCatchSwitch(CatchSwitchInst *CatchSwitch)
static void eliminateSwiftError(Function &F, coro::Shape &Shape)
Eliminate all problematic uses of swifterror arguments and allocas from the function.
static void lowerLocalAllocas(ArrayRef< CoroAllocaAllocInst * > LocalAllocas, SmallVectorImpl< Instruction * > &DeadInsts)
Turn each of the given local allocas into a normal (dynamic) alloca instruction.
static bool isLocalAlloca(CoroAllocaAllocInst *AI)
Is the given alloca "local", i.e.
static Value * emitSetSwiftErrorValue(IRBuilder<> &Builder, Value *V, coro::Shape &Shape)
Set the given value as the current swifterror value.
static Value * emitSetAndGetSwiftErrorValueAround(Instruction *Call, AllocaInst *Alloca, coro::Shape &Shape)
Set the swifterror value from the given alloca before a call, then put in back in the alloca afterwar...
static void cacheDIVar(FrameDataInfo &FrameData, DenseMap< Value *, DILocalVariable * > &DIVarCache)
static void collectFrameAlloca(AllocaInst *AI, coro::Shape &Shape, const SuspendCrossingInfo &Checker, SmallVectorImpl< AllocaInfo > &Allocas, const DominatorTree &DT)
static bool localAllocaNeedsStackSave(CoroAllocaAllocInst *AI)
static void splitAround(Instruction *I, const Twine &Name)
static void eliminateSwiftErrorAlloca(Function &F, AllocaInst *Alloca, coro::Shape &Shape)
Eliminate a formerly-swifterror alloca by inserting the get/set intrinsics and attempting to MemToReg...
static void rewritePHIs(BasicBlock &BB)
static void movePHIValuesToInsertedBlock(BasicBlock *SuccBB, BasicBlock *InsertedBB, BasicBlock *PredBB, PHINode *UntilPHI=nullptr)
static DIType * solveDIType(DIBuilder &Builder, Type *Ty, const DataLayout &Layout, DIScope *Scope, unsigned LineNum, DenseMap< Type *, DIType * > &DITypeCache)
static bool willLeaveFunctionImmediatelyAfter(BasicBlock *BB, unsigned depth=3)
After we split the coroutine, will the given basic block be along an obvious exit path for the resump...
static void rewriteMaterializableInstructions(IRBuilder<> &IRB, const SpillInfo &Spills)
static void eliminateSwiftErrorArgument(Function &F, Argument &Arg, coro::Shape &Shape, SmallVectorImpl< AllocaInst * > &AllocasToPromote)
"Eliminate" a swifterror argument by reducing it to the alloca case and then loading and storing in t...
static void buildFrameDebugInfo(Function &F, coro::Shape &Shape, FrameDataInfo &FrameData)
Build artificial debug info for C++ coroutine frames to allow users to inspect the contents of the fr...
static StructType * buildFrameType(Function &F, coro::Shape &Shape, FrameDataInfo &FrameData)
static BasicBlock * splitBlockIfNotFirst(Instruction *I, const Twine &Name)
static void sinkSpillUsesAfterCoroBegin(Function &F, const FrameDataInfo &FrameData, CoroBeginInst *CoroBegin)
retcon and retcon.once conventions assume that all spill uses can be sunk after the coro....
static void sinkLifetimeStartMarkers(Function &F, coro::Shape &Shape, SuspendCrossingInfo &Checker)
For each local variable that all of its user are only used inside one of suspended region,...
static bool materializable(Instruction &V)
static bool isSuspendBlock(BasicBlock *BB)
static void rewritePHIsForCleanupPad(BasicBlock *CleanupPadBB, CleanupPadInst *CleanupPad)
static void dumpAllocas(const SmallVectorImpl< AllocaInfo > &Allocas)
static StringRef solveTypeName(Type *Ty)
Create name for Type.
static void dumpSpills(StringRef Title, const SpillInfo &Spills)
static Value * emitGetSwiftErrorValue(IRBuilder<> &Builder, Type *ValueTy, coro::Shape &Shape)
Get the current swifterror value.
static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape)
Given that RA is a live value
static bool isLifetimeStart(const Instruction *Inst)
static MaybeAlign getAlign(Value *Ptr)
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
iv Induction Variable Users
print must be executed print the must be executed context for all instructions
return ToRemove size() > 0
This file provides an interface for laying out a sequence of fields as a struct in a way that attempt...
This file provides a collection of visitors which walk the (instruction) uses of a pointer.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallString class.
static const unsigned FramePtr
This class represents a conversion between pointers from one address space to another.
an instruction to allocate memory on the stack
bool isSwiftError() const
Return true if this alloca is used as a swifterror argument to a call.
void setSwiftError(bool V)
Specify whether this alloca is used to represent a swifterror.
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
PointerType * getType() const
Overload to return most specific pointer type.
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
bool isArrayAllocation() const
Return true if there is an allocation size parameter to the allocation instruction that is not 1.
void setAlignment(Align Align)
const Value * getArraySize() const
Get the number of elements allocated.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
LLVM Basic Block Representation.
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const Instruction & front() const
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
const Function * getParent() const
Return the enclosing method, or null if none.
LLVMContext & getContext() const
Get the context in which this basic block lives.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
This class represents a no-op cast from one type to another.
size_type size() const
size - Returns the number of bits in this bitvector.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
bool doesNotCapture(unsigned OpNo) const
Determine whether this data operand is not captured.
Value * getArgOperand(unsigned i) const
unsigned arg_size() const
static CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", Instruction *InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Value * getParentPad() const
static CleanupPadInst * Create(Value *ParentPad, ArrayRef< Value * > Args=std::nullopt, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
static CleanupReturnInst * Create(Value *CleanupPad, BasicBlock *UnwindBB=nullptr, Instruction *InsertBefore=nullptr)
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
This represents the llvm.coro.alloca.alloc instruction.
This class represents the llvm.coro.begin instruction.
This represents the llvm.coro.suspend instruction.
DICompositeType * createStructType(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNumber, uint64_t SizeInBits, uint32_t AlignInBits, DINode::DIFlags Flags, DIType *DerivedFrom, DINodeArray Elements, unsigned RunTimeLang=0, DIType *VTableHolder=nullptr, StringRef UniqueIdentifier="")
Create debugging information entry for a struct.
DIDerivedType * createPointerType(DIType *PointeeTy, uint64_t SizeInBits, uint32_t AlignInBits=0, std::optional< unsigned > DWARFAddressSpace=std::nullopt, StringRef Name="", DINodeArray Annotations=nullptr)
Create debugging information entry for a pointer.
DIExpression * createExpression(ArrayRef< uint64_t > Addr=std::nullopt)
Create a new descriptor for the specified variable which has a complex address expression for its add...
DIBasicType * createBasicType(StringRef Name, uint64_t SizeInBits, unsigned Encoding, DINode::DIFlags Flags=DINode::FlagZero)
Create debugging information entry for a basic type.
DINodeArray getOrCreateArray(ArrayRef< Metadata * > Elements)
Get a DINodeArray, create one if required.
DIDerivedType * createMemberType(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo, uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits, DINode::DIFlags Flags, DIType *Ty, DINodeArray Annotations=nullptr)
Create debugging information entry for a member.
DILocalVariable * createAutoVariable(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo, DIType *Ty, bool AlwaysPreserve=false, DINode::DIFlags Flags=DINode::FlagZero, uint32_t AlignInBits=0)
Create a new descriptor for an auto variable.
void replaceArrays(DICompositeType *&T, DINodeArray Elements, DINodeArray TParams=DINodeArray())
Replace arrays on a composite type.
static DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
uint64_t getNumLocationOperands() const
Return the number of unique location operands referred to (via DW_OP_LLVM_arg) in this expression; th...
static DIExpression * prepend(const DIExpression *Expr, uint8_t Flags, int64_t Offset=0)
Prepend DIExpr with a deref and offset operation and optionally turn it into a stack value or/and an ...
DILocalScope * getScope() const
Get the local scope for this variable.
Base class for scope-like contexts.
StringRef getName() const
uint64_t getSizeInBits() const
uint32_t getAlignInBits() const
A parsed version of the target data layout string in and methods for querying it.
const StructLayout * getStructLayout(StructType *Ty) const
Returns a StructLayout object, indicating the alignment of the struct, its size, and the offsets of i...
Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
This represents the llvm.dbg.declare instruction.
This represents the llvm.dbg.value instruction.
This is the common base class for debug info intrinsics for variables.
void replaceVariableLocationOp(Value *OldValue, Value *NewValue)
Value * getVariableLocationOp(unsigned OpIdx) const
void setExpression(DIExpression *NewExpr)
DILocalVariable * getVariable() const
DIExpression * getExpression() const
DILocation * get() const
Get the underlying DILocation.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
void reserve(size_type NumEntries)
Grow the densemap so that it can contain at least NumEntries items before resizing again.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Module * getParent()
Get the module that this global value is contained inside of...
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
void visitIntrinsicInst(IntrinsicInst &I)
void visitBitCastInst(BitCastInst &I)
void visit(Iterator Start, Iterator End)
void visitPHINode(PHINode &I)
void visitAddrSpaceCastInst(AddrSpaceCastInst &I)
void visitSelectInst(SelectInst &I)
void visitGetElementPtrInst(GetElementPtrInst &I)
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
const BasicBlock * getParent() const
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
const Function * getFunction() const
Return the function this instruction belongs to.
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void insertAfter(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately after the specified instruction.
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
static MDString * get(LLVMContext &Context, StringRef Str)
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
This is the common base class for memset/memcpy/memmove.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
A base class for visitors over the uses of a pointer value.
void visitCallBase(CallBase &CB)
void visitGetElementPtrInst(GetElementPtrInst &GEPI)
void visitAddrSpaceCastInst(AddrSpaceCastInst &ASC)
void visitBitCastInst(BitCastInst &BC)
void visitStoreInst(StoreInst &SI)
void visitIntrinsicInst(IntrinsicInst &II)
void visitMemIntrinsic(MemIntrinsic &I)
This class represents the LLVM 'select' instruction.
bool insert(const value_type &X)
Insert a new element into the SetVector.
iterator begin()
Get an iterator to the beginning of the SetVector.
iterator end()
Get an iterator to the end of the SetVector.
iterator find(ConstPtrType Ptr) const
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
StringRef str() const
Explicit conversion to StringRef.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Compute live ranges of allocas.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
std::string str() const
str - Get the contents as an std::string.
uint64_t getElementOffsetInBits(unsigned Idx) const
Class to represent struct types.
void setBody(ArrayRef< Type * > Elements, bool isPacked=false)
Specify a body for an opaque identified type.
static StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
unsigned getNumElements() const
Random access to the elements.
Type * getElementType(unsigned N) const
void setDefaultDest(BasicBlock *DefaultCase)
TinyPtrVector - This class is specialized for cases where there are normally 0 or 1 element in a vect...
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
std::string str() const
Return the twine contents as a std::string.
StringRef toStringRef(SmallVectorImpl< char > &Out) const
This returns the twine as a single StringRef if it can be represented as such.
static constexpr TypeSize Fixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
bool isPointerTy() const
True if this is an instance of PointerType.
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Type * getNonOpaquePointerElementType() const
Only use this method in code that is not reachable with opaque pointers, or part of deprecated method...
StringRef getStructName() const
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
static Type * getVoidTy(LLVMContext &C)
bool isStructTy() const
True if this is an instance of StructType.
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
static IntegerType * getInt32Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
LLVMContext & getContext() const
All values hold a context through their type.
iterator_range< use_iterator > uses()
StringRef getName() const
Return a constant reference to the value's name.
void takeName(Value *V)
Transfer the name from V to this value.
APInt Offset
The constant offset of the use if that is known.
void enqueueUsers(Instruction &I)
Enqueue the users of this instruction in the visit worklist.
constexpr ScalarTy getFixedValue() const
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
A range adaptor for a pair of iterators.
A raw_ostream that writes to an SmallVector or SmallString.
StringRef str() const
Return a StringRef for the vector contents.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ CE
Windows NT (Windows on ARM)
void salvageDebugInfo(SmallDenseMap< llvm::Value *, llvm::AllocaInst *, 4 > &DbgPtrAllocaCache, DbgVariableIntrinsic *DVI, bool OptimizeFrame)
Recover a dbg.declare prepared by the frontend and emit an alloca holding a pointer to the coroutine ...
void buildCoroutineFrame(Function &F, Shape &Shape)
CallInst * createMustTailCall(DebugLoc Loc, Function *MustTailCallFn, ArrayRef< Value * > Arguments, IRBuilder<> &)
bool isCPlusPlus(SourceLanguage S)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
TinyPtrVector< DbgDeclareInst * > FindDbgDeclareUses(Value *V)
Like FindDbgAddrUses, but only returns dbg.declare intrinsics, not dbg.addr.
void PromoteMemToReg(ArrayRef< AllocaInst * > Allocas, DominatorTree &DT, AssumptionCache *AC=nullptr)
Promote the specified list of alloca instructions into scalar registers, inserting PHI nodes as appro...
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
unsigned Log2_64_Ceil(uint64_t Value)
Return the ceil log base 2 of the specified value, 64 if the value is zero.
bool isAligned(Align Lhs, uint64_t SizeInBytes)
Checks that SizeInBytes is a multiple of the alignment.
auto successors(const MachineBasicBlock *BB)
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool isAllocaPromotable(const AllocaInst *AI)
Return true if this alloca is legal for promotion.
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
void findDbgUsers(SmallVectorImpl< DbgVariableIntrinsic * > &DbgInsts, Value *V)
Finds the debug info intrinsics describing a value.
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
BasicBlock * ehAwareSplitEdge(BasicBlock *BB, BasicBlock *Succ, LandingPadInst *OriginalPad=nullptr, PHINode *LandingPadReplacement=nullptr, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions(), const Twine &BBName="")
Split the edge connect the specficed blocks in the case that Succ is an Exception Handling Block.
void findDbgValues(SmallVectorImpl< DbgValueInst * > &DbgValues, Value *V)
Finds the llvm.dbg.value intrinsics describing a value.
Value * salvageDebugInfoImpl(Instruction &I, uint64_t CurrentLocOps, SmallVectorImpl< uint64_t > &Ops, SmallVectorImpl< Value * > &AdditionalValues)
uint64_t offsetToAlignment(uint64_t Value, Align Alignment)
Returns the offset to the next integer (mod 2**64) that is greater than or equal to Value and is a mu...
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
std::pair< uint64_t, Align > performOptimizedStructLayout(MutableArrayRef< OptimizedStructLayoutField > Fields)
Compute a layout for a struct containing the given fields, making a best-effort attempt to minimize t...
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr unsigned BitWidth
void updatePhiNodes(BasicBlock *DestBB, BasicBlock *OldPred, BasicBlock *NewPred, PHINode *Until=nullptr)
Replaces all uses of OldPred with the NewPred block in all PHINodes in a block.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
auto predecessors(const MachineBasicBlock *BB)
BasicBlock * SplitEdge(BasicBlock *From, BasicBlock *To, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")
Split the edge connecting the specified blocks, and return the newly created basic block between From...
void setUnwindEdgeTo(Instruction *TI, BasicBlock *Succ)
Sets the unwind edge of an instruction to a particular successor.
unsigned pred_size(const MachineBasicBlock *BB)
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Align Alignment
The required alignment of this field.
uint64_t Offset
The offset of this field in the final layout.
uint64_t Size
The required size of this field in bytes.
static constexpr uint64_t FlexibleOffset
A special value for Offset indicating that the field can be moved anywhere.
A MapVector that performs no allocations if smaller than a certain size.
Align getContextAlignment() const
uint64_t ContextHeaderSize
bool IsFrameInlineInStorage
AllocaInst * PromiseAlloca
AsyncLoweringStorage AsyncLowering
AnyCoroIdRetconInst * getRetconCoroId() const
CoroIdInst * getSwitchCoroId() const
Instruction * getInsertPtAfterFramePtr() const
SmallVector< AnyCoroSuspendInst *, 4 > CoroSuspends
Value * emitAlloc(IRBuilder<> &Builder, Value *Size, CallGraph *CG) const
Allocate memory according to the rules of the active lowering.
SmallVector< CallInst *, 2 > SwiftErrorOps
AllocaInst * getPromiseAlloca() const
bool OptimizeFrame
This would only be true if optimization are enabled.
SwitchLoweringStorage SwitchLowering
CoroBeginInst * CoroBegin
void emitDealloc(IRBuilder<> &Builder, Value *Ptr, CallGraph *CG) const
Deallocate memory according to the rules of the active lowering.
RetconLoweringStorage RetconLowering
SmallVector< AnyCoroEndInst *, 4 > CoroEnds
BasicBlock * AllocaSpillBlock