74#define DEBUG_TYPE "code-extractor"
82 cl::desc(
"Aggregate arguments to code-extracted functions"));
87 bool AllowVarArgs,
bool AllowAlloca) {
100 while (!ToVisit.
empty()) {
102 if (!Visited.
insert(Curr).second)
104 if (isa<BlockAddress const>(Curr))
107 if (isa<Instruction>(Curr) && cast<Instruction>(Curr)->
getParent() != &BB)
110 for (
auto const &U : Curr->
operands()) {
111 if (
auto *UU = dyn_cast<User>(U))
119 if (isa<AllocaInst>(
I)) {
125 if (
const auto *II = dyn_cast<InvokeInst>(
I)) {
128 if (
auto *UBB = II->getUnwindDest())
129 if (!Result.count(UBB))
136 if (
const auto *CSI = dyn_cast<CatchSwitchInst>(
I)) {
137 if (
auto *UBB = CSI->getUnwindDest())
138 if (!Result.count(UBB))
140 for (
const auto *HBB : CSI->handlers())
141 if (!Result.count(
const_cast<BasicBlock*
>(HBB)))
148 if (
const auto *CPI = dyn_cast<CatchPadInst>(
I)) {
149 for (
const auto *U : CPI->users())
150 if (
const auto *CRI = dyn_cast<CatchReturnInst>(U))
151 if (!Result.count(
const_cast<BasicBlock*
>(CRI->getParent())))
159 if (
const auto *CPI = dyn_cast<CleanupPadInst>(
I)) {
160 for (
const auto *U : CPI->users())
161 if (
const auto *CRI = dyn_cast<CleanupReturnInst>(U))
162 if (!Result.count(
const_cast<BasicBlock*
>(CRI->getParent())))
166 if (
const auto *CRI = dyn_cast<CleanupReturnInst>(
I)) {
167 if (
auto *UBB = CRI->getUnwindDest())
168 if (!Result.count(UBB))
173 if (
const CallInst *CI = dyn_cast<CallInst>(
I)) {
174 if (
const Function *
F = CI->getCalledFunction()) {
175 auto IID =
F->getIntrinsicID();
176 if (IID == Intrinsic::vastart) {
185 if (IID == Intrinsic::eh_typeid_for)
197 bool AllowVarArgs,
bool AllowAlloca) {
198 assert(!BBs.
empty() &&
"The set of blocks to extract must be non-empty");
208 if (!Result.insert(BB))
212 LLVM_DEBUG(
dbgs() <<
"Region front block: " << Result.front()->getName()
215 for (
auto *BB : Result) {
220 if (BB == Result.front()) {
222 LLVM_DEBUG(
dbgs() <<
"The first block cannot be an unwind block\n");
231 if (!Result.count(PBB)) {
232 LLVM_DEBUG(
dbgs() <<
"No blocks in this region may have entries from "
233 "outside the region except for the first block!\n"
234 <<
"Problematic source BB: " << BB->getName() <<
"\n"
235 <<
"Problematic destination BB: " << PBB->getName()
247 bool AllowVarArgs,
bool AllowAlloca,
248 BasicBlock *AllocationBlock, std::string Suffix,
249 bool ArgsInZeroAddressSpace)
251 BPI(BPI), AC(AC), AllocationBlock(AllocationBlock),
252 AllowVarArgs(AllowVarArgs),
254 Suffix(Suffix), ArgsInZeroAddressSpace(ArgsInZeroAddressSpace) {}
261 BPI(BPI), AC(AC), AllocationBlock(nullptr), AllowVarArgs(
false),
271 if (
Blocks.count(
I->getParent()))
280 if (isa<Argument>(V))
return true;
282 if (!
Blocks.count(
I->getParent()))
294 if (!CommonExitBlock) {
295 CommonExitBlock = Succ;
298 if (CommonExitBlock != Succ)
307 return CommonExitBlock;
312 for (
Instruction &II : BB.instructionsWithoutDebug())
313 if (
auto *AI = dyn_cast<AllocaInst>(&II))
314 Allocas.push_back(AI);
316 findSideEffectInfoForBlock(BB);
320void CodeExtractorAnalysisCache::findSideEffectInfoForBlock(
BasicBlock &BB) {
322 unsigned Opcode = II.getOpcode();
323 Value *MemAddr =
nullptr;
325 case Instruction::Store:
326 case Instruction::Load: {
327 if (Opcode == Instruction::Store) {
329 MemAddr = SI->getPointerOperand();
335 if (isa<Constant>(MemAddr))
338 if (!isa<AllocaInst>(
Base)) {
339 SideEffectingBlocks.insert(&BB);
342 BaseMemAddrs[&BB].insert(
Base);
350 SideEffectingBlocks.insert(&BB);
354 if (II.mayHaveSideEffects()) {
355 SideEffectingBlocks.insert(&BB);
365 if (SideEffectingBlocks.count(&BB))
367 auto It = BaseMemAddrs.find(&BB);
368 if (It != BaseMemAddrs.end())
369 return It->second.count(
Addr);
375 AllocaInst *AI = cast<AllocaInst>(
Addr->stripInBoundsConstantOffsets());
378 if (Blocks.count(&BB))
388 BasicBlock *SinglePredFromOutlineRegion =
nullptr;
389 assert(!Blocks.count(CommonExitBlock) &&
390 "Expect a block outside the region!");
392 if (!Blocks.count(Pred))
394 if (!SinglePredFromOutlineRegion) {
395 SinglePredFromOutlineRegion = Pred;
396 }
else if (SinglePredFromOutlineRegion != Pred) {
397 SinglePredFromOutlineRegion =
nullptr;
402 if (SinglePredFromOutlineRegion)
403 return SinglePredFromOutlineRegion;
409 while (
I != BB->
end()) {
410 PHINode *Phi = dyn_cast<PHINode>(
I);
422 assert(!getFirstPHI(CommonExitBlock) &&
"Phi not expected");
430 if (Blocks.count(Pred))
435 Blocks.insert(CommonExitBlock);
436 OldTargets.push_back(NewExitBlock);
437 return CommonExitBlock;
444CodeExtractor::LifetimeMarkerInfo
448 LifetimeMarkerInfo
Info;
458 Info.LifeStart = IntrInst;
464 Info.LifeEnd = IntrInst;
469 if (isa<DbgInfoIntrinsic>(IntrInst))
477 if (!
Info.LifeStart || !
Info.LifeEnd)
483 if ((
Info.SinkLifeStart ||
Info.HoistLifeEnd) &&
488 if (
Info.HoistLifeEnd && !ExitBlock)
500 auto moveOrIgnoreLifetimeMarkers =
501 [&](
const LifetimeMarkerInfo &LMI) ->
bool {
504 if (LMI.SinkLifeStart) {
507 SinkCands.
insert(LMI.LifeStart);
509 if (LMI.HoistLifeEnd) {
510 LLVM_DEBUG(
dbgs() <<
"Hoisting lifetime.end: " << *LMI.LifeEnd <<
"\n");
511 HoistCands.
insert(LMI.LifeEnd);
520 if (Blocks.count(BB))
529 LifetimeMarkerInfo MarkerInfo = getLifetimeMarkers(CEAC, AI, ExitBlock);
530 bool Moved = moveOrIgnoreLifetimeMarkers(MarkerInfo);
542 for (
User *U : AI->users()) {
546 if (U->stripInBoundsConstantOffsets() != AI)
550 for (
User *BU : Bitcast->users()) {
562 << *Bitcast <<
" in out-of-region lifetime marker "
563 << *IntrInst <<
"\n");
564 LifetimeBitcastUsers.
push_back(IntrInst);
574 I->replaceUsesOfWith(
I->getOperand(1), CastI);
580 for (
User *U : AI->users()) {
581 if (U->stripInBoundsConstantOffsets() == AI) {
583 LifetimeMarkerInfo LMI = getLifetimeMarkers(CEAC, Bitcast, ExitBlock);
599 if (Bitcasts.
empty())
602 LLVM_DEBUG(
dbgs() <<
"Sinking alloca (via bitcast): " << *AI <<
"\n");
604 for (
unsigned I = 0, E = Bitcasts.
size();
I != E; ++
I) {
606 const LifetimeMarkerInfo &LMI = BitcastLifetimeInfo[
I];
608 "Unsafe to sink bitcast without lifetime markers");
609 moveOrIgnoreLifetimeMarkers(LMI);
611 LLVM_DEBUG(
dbgs() <<
"Sinking bitcast-of-alloca: " << *BitcastAddr
613 SinkCands.
insert(BitcastAddr);
627 if (AllowVarArgs &&
F->getFunctionType()->isVarArg()) {
628 auto containsVarArgIntrinsic = [](
const Instruction &
I) {
629 if (
const CallInst *CI = dyn_cast<CallInst>(&
I))
630 if (
const Function *Callee = CI->getCalledFunction())
631 return Callee->getIntrinsicID() == Intrinsic::vastart ||
632 Callee->getIntrinsicID() == Intrinsic::vaend;
636 for (
auto &BB : *
F) {
637 if (Blocks.count(&BB))
652 for (
auto &OI : II.operands()) {
658 for (
User *U : II.users())
670void CodeExtractor::severSplitPHINodesOfEntry(
BasicBlock *&Header) {
671 unsigned NumPredsFromRegion = 0;
672 unsigned NumPredsOutsideRegion = 0;
674 if (Header != &Header->getParent()->getEntryBlock()) {
675 PHINode *PN = dyn_cast<PHINode>(Header->begin());
683 ++NumPredsFromRegion;
685 ++NumPredsOutsideRegion;
689 if (NumPredsOutsideRegion <= 1)
return;
701 Blocks.remove(OldPred);
702 Blocks.insert(NewBB);
707 if (NumPredsFromRegion) {
720 for (AfterPHIs = OldPred->
begin(); isa<PHINode>(AfterPHIs); ++AfterPHIs) {
721 PHINode *PN = cast<PHINode>(AfterPHIs);
747void CodeExtractor::severSplitPHINodesOfExits(
752 for (
PHINode &PN : ExitBB->phis()) {
762 if (IncomingVals.
size() <= 1)
769 ExitBB->getName() +
".split",
770 ExitBB->getParent(), ExitBB);
774 if (Blocks.count(PredBB))
775 PredBB->getTerminator()->replaceUsesOfWith(ExitBB, NewBB);
777 Blocks.insert(NewBB);
784 for (
unsigned i : IncomingVals)
786 for (
unsigned i :
reverse(IncomingVals))
793void CodeExtractor::splitReturnBlocks() {
797 Block->splitBasicBlock(RI->getIterator(),
Block->getName() +
".ret");
815Function *CodeExtractor::constructFunction(
const ValueSet &inputs,
816 const ValueSet &outputs,
826 switch (NumExitBlocks) {
833 std::vector<Type *> ParamTy;
834 std::vector<Type *> AggParamTy;
835 ValueSet StructValues;
841 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(
value)) {
842 AggParamTy.push_back(
value->getType());
843 StructValues.insert(
value);
845 ParamTy.push_back(
value->getType());
849 for (
Value *output : outputs) {
851 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(output)) {
852 AggParamTy.push_back(output->getType());
853 StructValues.insert(output);
860 (ParamTy.size() + AggParamTy.size()) ==
861 (inputs.size() + outputs.size()) &&
862 "Number of scalar and aggregate params does not match inputs, outputs");
863 assert((StructValues.empty() || AggregateArgs) &&
864 "Expeced StructValues only with AggregateArgs set");
867 size_t NumScalarParams = ParamTy.size();
869 if (AggregateArgs && !AggParamTy.empty()) {
872 StructTy, ArgsInZeroAddressSpace ? 0 :
DL.getAllocaAddrSpace()));
876 dbgs() <<
"Function type: " << *RetTy <<
" f(";
877 for (
Type *i : ParamTy)
878 dbgs() << *i <<
", ";
883 RetTy, ParamTy, AllowVarArgs && oldFunction->
isVarArg());
885 std::string SuffixToUse =
892 oldFunction->
getName() +
"." + SuffixToUse, M);
903 if (Attr.isStringAttribute()) {
904 if (Attr.getKindAsString() ==
"thunk")
907 switch (Attr.getKindAsEnum()) {
910 case Attribute::AllocSize:
911 case Attribute::Builtin:
912 case Attribute::Convergent:
913 case Attribute::JumpTable:
914 case Attribute::Naked:
915 case Attribute::NoBuiltin:
916 case Attribute::NoMerge:
917 case Attribute::NoReturn:
918 case Attribute::NoSync:
919 case Attribute::ReturnsTwice:
920 case Attribute::Speculatable:
921 case Attribute::StackAlignment:
922 case Attribute::WillReturn:
923 case Attribute::AllocKind:
924 case Attribute::PresplitCoroutine:
925 case Attribute::Memory:
926 case Attribute::NoFPClass:
927 case Attribute::CoroDestroyOnlyWhenComplete:
930 case Attribute::AlwaysInline:
931 case Attribute::Cold:
932 case Attribute::DisableSanitizerInstrumentation:
933 case Attribute::FnRetThunkExtern:
935 case Attribute::NoRecurse:
936 case Attribute::InlineHint:
937 case Attribute::MinSize:
938 case Attribute::NoCallback:
939 case Attribute::NoDuplicate:
940 case Attribute::NoFree:
941 case Attribute::NoImplicitFloat:
942 case Attribute::NoInline:
943 case Attribute::NonLazyBind:
944 case Attribute::NoRedZone:
945 case Attribute::NoUnwind:
946 case Attribute::NoSanitizeBounds:
947 case Attribute::NoSanitizeCoverage:
948 case Attribute::NullPointerIsValid:
949 case Attribute::OptimizeForDebugging:
950 case Attribute::OptForFuzzing:
951 case Attribute::OptimizeNone:
952 case Attribute::OptimizeForSize:
953 case Attribute::SafeStack:
954 case Attribute::ShadowCallStack:
955 case Attribute::SanitizeAddress:
956 case Attribute::SanitizeMemory:
957 case Attribute::SanitizeThread:
958 case Attribute::SanitizeHWAddress:
959 case Attribute::SanitizeMemTag:
960 case Attribute::SpeculativeLoadHardening:
961 case Attribute::StackProtect:
962 case Attribute::StackProtectReq:
963 case Attribute::StackProtectStrong:
964 case Attribute::StrictFP:
965 case Attribute::UWTable:
966 case Attribute::VScaleRange:
967 case Attribute::NoCfCheck:
968 case Attribute::MustProgress:
969 case Attribute::NoProfile:
970 case Attribute::SkipProfile:
973 case Attribute::Alignment:
974 case Attribute::AllocatedPointer:
975 case Attribute::AllocAlign:
976 case Attribute::ByVal:
977 case Attribute::Dereferenceable:
978 case Attribute::DereferenceableOrNull:
979 case Attribute::ElementType:
980 case Attribute::InAlloca:
981 case Attribute::InReg:
982 case Attribute::Nest:
983 case Attribute::NoAlias:
984 case Attribute::NoCapture:
985 case Attribute::NoUndef:
986 case Attribute::NonNull:
987 case Attribute::Preallocated:
988 case Attribute::ReadNone:
989 case Attribute::ReadOnly:
990 case Attribute::Returned:
991 case Attribute::SExt:
992 case Attribute::StructRet:
993 case Attribute::SwiftError:
994 case Attribute::SwiftSelf:
995 case Attribute::SwiftAsync:
996 case Attribute::ZExt:
997 case Attribute::ImmArg:
998 case Attribute::ByRef:
999 case Attribute::WriteOnly:
1000 case Attribute::Writable:
1001 case Attribute::DeadOnUnwind:
1002 case Attribute::Range:
1014 if (NumExitBlocks == 0) {
1020 return isa<ReturnInst>(Term) || isa<ResumeInst>(Term);
1025 newFunction->
insert(newFunction->
end(), newRootNode);
1034 for (
unsigned i = 0, e = inputs.size(), aggIdx = 0; i != e; ++i) {
1036 if (AggregateArgs && StructValues.contains(inputs[i])) {
1042 StructTy, &*AggAI,
Idx,
"gep_" + inputs[i]->
getName(), TI);
1044 "loadgep_" + inputs[i]->getName(), TI);
1047 RewriteVal = &*ScalarAI++;
1049 std::vector<User *>
Users(inputs[i]->user_begin(), inputs[i]->user_end());
1052 if (
Blocks.count(inst->getParent()))
1053 inst->replaceUsesOfWith(inputs[i], RewriteVal);
1057 if (NumScalarParams) {
1059 for (
unsigned i = 0, e = inputs.size(); i != e; ++i, ++ScalarAI)
1060 if (!StructValues.contains(inputs[i]))
1062 for (
unsigned i = 0, e = outputs.size(); i != e; ++i, ++ScalarAI)
1063 if (!StructValues.contains(outputs[i]))
1071 for (
auto &U :
Users)
1075 if (
I->isTerminator() &&
I->getFunction() == oldFunction &&
1076 !
Blocks.count(
I->getParent()))
1077 I->replaceUsesOfWith(header, newHeader);
1093 auto *II = dyn_cast<IntrinsicInst>(&
I);
1094 if (!II || !II->isLifetimeStartOrEnd())
1104 if (II->getIntrinsicID() == Intrinsic::lifetime_start)
1105 LifetimesStart.
insert(Mem);
1106 II->eraseFromParent();
1123 bool InsertBefore) {
1124 for (
Value *Mem : Objects) {
1127 "Input memory not defined in original function");
1134 Marker->insertBefore(Term);
1138 if (!LifetimesStart.
empty()) {
1139 insertMarkers(Intrinsic::lifetime_start, LifetimesStart,
1143 if (!LifetimesEnd.
empty()) {
1144 insertMarkers(Intrinsic::lifetime_end, LifetimesEnd,
1155 ValueSet &outputs) {
1158 std::vector<Value *> params, ReloadOutputs, Reloads;
1159 ValueSet StructValues;
1167 unsigned ScalarInputArgNo = 0;
1169 for (
Value *input : inputs) {
1170 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(input))
1171 StructValues.
insert(input);
1173 params.push_back(input);
1174 if (input->isSwiftError())
1175 SwiftErrorArgs.
push_back(ScalarInputArgNo);
1181 unsigned ScalarOutputArgNo = 0;
1182 for (
Value *output : outputs) {
1183 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(output)) {
1184 StructValues.insert(output);
1187 new AllocaInst(output->getType(),
DL.getAllocaAddrSpace(),
1188 nullptr, output->
getName() +
".loc",
1190 ReloadOutputs.push_back(alloca);
1191 params.push_back(alloca);
1192 ++ScalarOutputArgNo;
1198 unsigned NumAggregatedInputs = 0;
1199 if (AggregateArgs && !StructValues.empty()) {
1200 std::vector<Type *> ArgTypes;
1201 for (
Value *V : StructValues)
1202 ArgTypes.push_back(
V->getType());
1207 StructArgTy,
DL.getAllocaAddrSpace(),
nullptr,
"structArg",
1211 if (ArgsInZeroAddressSpace &&
DL.getAllocaAddrSpace() != 0) {
1213 Struct, PointerType ::get(Context, 0),
"structArg.ascast");
1214 StructSpaceCast->insertAfter(
Struct);
1215 params.push_back(StructSpaceCast);
1217 params.push_back(
Struct);
1220 for (
unsigned i = 0, e = StructValues.size(); i != e; ++i) {
1221 if (inputs.contains(StructValues[i])) {
1227 GEP->insertInto(codeReplacer, codeReplacer->
end());
1229 NumAggregatedInputs++;
1236 NumExitBlocks > 1 ?
"targetBlock" :
"");
1248 for (
unsigned SwiftErrArgNo : SwiftErrorArgs) {
1249 call->
addParamAttr(SwiftErrArgNo, Attribute::SwiftError);
1250 newFunction->
addParamAttr(SwiftErrArgNo, Attribute::SwiftError);
1255 for (
unsigned i = 0, e = outputs.size(), scalarIdx = 0,
1256 aggIdx = NumAggregatedInputs;
1258 Value *Output =
nullptr;
1259 if (AggregateArgs && StructValues.contains(outputs[i])) {
1265 GEP->insertInto(codeReplacer, codeReplacer->
end());
1269 Output = ReloadOutputs[scalarIdx];
1273 outputs[i]->
getName() +
".reload",
1275 Reloads.push_back(
load);
1276 std::vector<User *>
Users(outputs[i]->user_begin(), outputs[i]->user_end());
1287 codeReplacer, 0, codeReplacer);
1294 std::map<BasicBlock *, BasicBlock *> ExitBlockMap;
1298 unsigned switchVal = 0;
1300 if (
Blocks.count(OldTarget))
1302 BasicBlock *&NewTarget = ExitBlockMap[OldTarget];
1309 OldTarget->getName() +
".exitStub",
1311 unsigned SuccNum = switchVal++;
1313 Value *brVal =
nullptr;
1314 assert(NumExitBlocks < 0xffff &&
"too many exit blocks for switch");
1315 switch (NumExitBlocks) {
1341 BasicBlock *NewTarget = ExitBlockMap[OldTarget];
1342 assert(NewTarget &&
"Unknown target block!");
1353 std::advance(ScalarOutputArgBegin, ScalarInputArgNo);
1355 std::advance(AggOutputArgBegin, ScalarInputArgNo + ScalarOutputArgNo);
1357 for (
unsigned i = 0, e = outputs.size(), aggIdx = NumAggregatedInputs; i != e;
1359 auto *OutI = dyn_cast<Instruction>(outputs[i]);
1367 if (
auto *InvokeI = dyn_cast<InvokeInst>(OutI))
1368 InsertPt = InvokeI->getNormalDest()->getFirstInsertionPt();
1369 else if (
auto *Phi = dyn_cast<PHINode>(OutI))
1370 InsertPt =
Phi->getParent()->getFirstInsertionPt();
1372 InsertPt = std::next(OutI->getIterator());
1374 assert((InsertPt->getFunction() == newFunction ||
1375 Blocks.count(InsertPt->getParent())) &&
1376 "InsertPt should be in new function");
1377 if (AggregateArgs && StructValues.contains(outputs[i])) {
1379 "Number of aggregate output arguments should match "
1380 "the number of defined values");
1385 StructArgTy, &*AggOutputArgBegin,
Idx,
"gep_" + outputs[i]->
getName(),
1394 "Number of scalar output arguments should match "
1395 "the number of defined values");
1396 new StoreInst(outputs[i], &*ScalarOutputArgBegin, InsertPt);
1397 ++ScalarOutputArgBegin;
1403 switch (NumExitBlocks) {
1411 }
else if (OldFnRetTy->
isVoidTy()) {
1456void CodeExtractor::moveCodeToFunction(
Function *newFunction) {
1460 Block->removeFromParent();
1467 newFuncIt = newFunction->
insert(std::next(newFuncIt),
Block);
1471void CodeExtractor::calculateNewCallTerminatorWeights(
1483 Distribution BranchDist;
1490 BlockNode ExitNode(i);
1493 BranchDist.addExit(ExitNode, ExitFreq);
1499 if (BranchDist.Total == 0) {
1505 BranchDist.normalize();
1508 for (
unsigned I = 0, E = BranchDist.Weights.size();
I < E; ++
I) {
1509 const auto &Weight = BranchDist.Weights[
I];
1512 BranchWeights[Weight.TargetNode.Index] = Weight.Amount;
1514 EdgeProbabilities[Weight.TargetNode.Index] = BP;
1518 LLVMContext::MD_prof,
1530 if (DVI->getFunction() != &
F)
1531 DVI->eraseFromParent();
1533 if (DVR->getFunction() != &
F)
1534 DVR->eraseFromParent();
1557 assert(OldSP->getUnit() &&
"Missing compile unit for subprogram");
1563 DISubprogram::SPFlagOptimized |
1564 DISubprogram::SPFlagLocalToUnit;
1567 0, SPType, 0, DINode::FlagZero, SPFlags);
1570 auto IsInvalidLocation = [&NewFunc](
Value *Location) {
1574 (!isa<Constant>(Location) && !isa<Instruction>(Location)))
1576 Instruction *LocationInst = dyn_cast<Instruction>(Location);
1577 return LocationInst && LocationInst->
getFunction() != &NewFunc;
1592 DINode *&NewVar = RemappedMetadata[OldVar];
1595 *OldVar->getScope(), *NewSP, Ctx, Cache);
1597 NewScope, OldVar->
getName(), OldVar->getFile(), OldVar->getLine(),
1598 OldVar->getType(),
false, DINode::FlagZero,
1599 OldVar->getAlignInBits());
1601 return cast<DILocalVariable>(NewVar);
1604 auto UpdateDbgLabel = [&](
auto *LabelRecord) {
1607 if (LabelRecord->getDebugLoc().getInlinedAt())
1609 DILabel *OldLabel = LabelRecord->getLabel();
1610 DINode *&NewLabel = RemappedMetadata[OldLabel];
1613 *OldLabel->
getScope(), *NewSP, Ctx, Cache);
1617 LabelRecord->setLabel(cast<DILabel>(NewLabel));
1620 auto UpdateDbgRecordsOnInst = [&](
Instruction &
I) ->
void {
1621 for (
DbgRecord &DR :
I.getDbgRecordRange()) {
1623 UpdateDbgLabel(DLR);
1644 UpdateDbgRecordsOnInst(
I);
1646 auto *DII = dyn_cast<DbgInfoIntrinsic>(&
I);
1652 if (
auto *DLI = dyn_cast<DbgLabelInst>(&
I)) {
1653 UpdateDbgLabel(DLI);
1657 auto *DVI = cast<DbgVariableIntrinsic>(DII);
1659 if (
any_of(DVI->location_ops(), IsInvalidLocation)) {
1664 if (
auto *DAI = dyn_cast<DbgAssignIntrinsic>(DVI);
1665 DAI && IsInvalidLocation(DAI->getAddress())) {
1671 if (!DVI->getDebugLoc().getInlinedAt())
1672 DVI->setVariable(GetUpdatedDIVariable(DVI->getVariable()));
1675 for (
auto *DII : DebugIntrinsicsToDelete)
1676 DII->eraseFromParent();
1677 for (
auto *DVR : DVRsToDelete)
1678 DVR->getMarker()->MarkedInstr->dropOneDbgRecord(DVR);
1689 *NewSP, Ctx, Cache));
1692 auto updateLoopInfoLoc = [&Ctx, &Cache, NewSP](
Metadata *MD) ->
Metadata * {
1693 if (
auto *Loc = dyn_cast_or_null<DILocation>(MD))
1726 assert(BPI &&
"Both BPI and BFI are required to preserve profile info");
1739 if (
auto *AI = dyn_cast<AssumeInst>(&
I)) {
1742 AI->eraseFromParent();
1749 splitReturnBlocks();
1757 if (!
Blocks.count(Succ)) {
1767 NumExitBlocks = ExitBlocks.
size();
1771 if (!
Blocks.contains(OldTarget))
1772 OldTargets.push_back(OldTarget);
1776 severSplitPHINodesOfEntry(header);
1777 severSplitPHINodesOfExits(ExitBlocks);
1781 "codeRepl", oldFunction,
1799 if (!
I.getDebugLoc())
1803 if (isa<DbgInfoIntrinsic>(
I))
1805 BranchI->setDebugLoc(
I.getDebugLoc());
1810 BranchI->insertInto(newFuncRoot, newFuncRoot->
end());
1812 ValueSet SinkingCands, HoistingCands;
1814 findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);
1824 for (
auto *II : SinkingCands) {
1825 if (
auto *AI = dyn_cast<AllocaInst>(II)) {
1827 if (!FirstSunkAlloca)
1828 FirstSunkAlloca = AI;
1831 assert((SinkingCands.
empty() || FirstSunkAlloca) &&
1832 "Did not expect a sink candidate without any allocas");
1833 for (
auto *II : SinkingCands) {
1834 if (!isa<AllocaInst>(II)) {
1835 cast<Instruction>(II)->moveAfter(FirstSunkAlloca);
1839 if (!HoistingCands.
empty()) {
1842 for (
auto *II : HoistingCands)
1855 constructFunction(inputs, outputs, header, newFuncRoot, codeReplacer,
1868 emitCallAndSwitchStatement(newFunction, codeReplacer, inputs, outputs);
1870 moveCodeToFunction(newFunction);
1882 if (BFI && NumExitBlocks > 1)
1883 calculateNewCallTerminatorWeights(codeReplacer, ExitWeights, BPI);
1895 for (
PHINode &PN : ExitBB->phis()) {
1896 Value *IncomingCodeReplacerVal =
nullptr;
1903 if (!IncomingCodeReplacerVal) {
1908 "PHI has two incompatbile incoming values from codeRepl");
1915 newFunction->
dump();
1929 auto *
I = dyn_cast_or_null<CallInst>(AssumeVH);
1934 if (
I->getFunction() != &OldFunc)
1941 auto *AffectedCI = dyn_cast_or_null<CallInst>(AffectedValVH);
1944 if (AffectedCI->getFunction() != &OldFunc)
1946 auto *AssumedInst = cast<Instruction>(AffectedCI->getOperand(0));
1947 if (AssumedInst->getFunction() != &OldFunc)
1955 ExcludeArgsFromAggregate.
insert(Arg);
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
AMDGPU Mark last scratch load
Expand Atomic instructions
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
Analysis containing CSE Info
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Given that RA is a live value
This file defines the DenseMap class.
DenseMap< Block *, BlockRelaxAux > Blocks
static Function * getFunction(Constant *C)
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
iv Induction Variable Users
Move duplicate certain instructions close to their use
Module.h This file contains the declarations for the Module class.
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
This class represents a conversion between pointers from one address space to another.
an instruction to allocate memory on the stack
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
bool empty() const
empty - Check if the array is empty.
A cache of @llvm.assume calls within a function.
MutableArrayRef< ResultElem > assumptions()
Access the list of assumption handles currently tracked for this function.
void unregisterAssumption(AssumeInst *CI)
Remove an @llvm.assume intrinsic from this function's cache if it has been added to the cache earlier...
MutableArrayRef< ResultElem > assumptionsFor(const Value *V)
Access the list of assumptions which affect this value.
AttributeSet getFnAttrs() const
The function attributes are returned.
@ TombstoneKey
Use as Tombstone key for DenseMap of AttrKind.
@ None
No attributes have been set.
@ EmptyKey
Use as Empty key for DenseMap of AttrKind.
@ EndAttrKinds
Sentinel value useful for loops.
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
iterator_range< filter_iterator< BasicBlock::const_iterator, std::function< bool(const Instruction &)> > > instructionsWithoutDebug(bool SkipPseudoOp=true) const
Return a const iterator range over the instructions in the block, skipping any debug instructions.
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches,...
InstListType::const_iterator getFirstNonPHIIt() const
Iterator returning form of getFirstNonPHI.
InstListType::const_iterator const_iterator
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
const Function * getParent() const
Return the enclosing method, or null if none.
InstListType::iterator iterator
Instruction iterators...
LLVMContext & getContext() const
Get the context in which this basic block lives.
bool IsNewDbgInfoFormat
Flag recording whether or not this block stores debug-info in the form of intrinsic instructions (fal...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
std::optional< uint64_t > getProfileCountFromFreq(BlockFrequency Freq) const
Returns the estimated profile count of Freq.
void setBlockFreq(const BasicBlock *BB, BlockFrequency Freq)
BlockFrequency getBlockFreq(const BasicBlock *BB) const
getblockFreq - Return block frequency.
static BranchInst * Create(BasicBlock *IfTrue, BasicBlock::iterator InsertBefore)
Analysis providing branch probability information.
void setEdgeProbability(const BasicBlock *Src, const SmallVectorImpl< BranchProbability > &Probs)
Set the raw probabilities for all edges from the given block.
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
static BranchProbability getUnknown()
static BranchProbability getZero()
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr, BasicBlock::iterator InsertBefore)
This is the base class for all instructions that perform data casts.
static CastInst * CreatePointerCast(Value *S, Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd)
Create a BitCast AddrSpaceCast, or a PtrToInt cast instruction.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
DISubroutineType * createSubroutineType(DITypeRefArray ParameterTypes, DINode::DIFlags Flags=DINode::FlagZero, unsigned CC=0)
Create subroutine type.
void finalizeSubprogram(DISubprogram *SP)
Finalize a specific subprogram - no new variables may be added to this subprogram afterwards.
DISubprogram * createFunction(DIScope *Scope, StringRef Name, StringRef LinkageName, DIFile *File, unsigned LineNo, DISubroutineType *Ty, unsigned ScopeLine, DINode::DIFlags Flags=DINode::FlagZero, DISubprogram::DISPFlags SPFlags=DISubprogram::SPFlagZero, DITemplateParameterArray TParams=nullptr, DISubprogram *Decl=nullptr, DITypeArray ThrownTypes=nullptr, DINodeArray Annotations=nullptr, StringRef TargetFuncName="")
Create a new descriptor for the specified subprogram.
DITypeRefArray getOrCreateTypeArray(ArrayRef< Metadata * > Elements)
Get a DITypeRefArray, create one if required.
DILocalVariable * createAutoVariable(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo, DIType *Ty, bool AlwaysPreserve=false, DINode::DIFlags Flags=DINode::FlagZero, uint32_t AlignInBits=0)
Create a new descriptor for an auto variable.
StringRef getName() const
DILocalScope * getScope() const
Get the local scope for this label.
static DILocalScope * cloneScopeForSubprogram(DILocalScope &RootScope, DISubprogram &NewSP, LLVMContext &Ctx, DenseMap< const MDNode *, MDNode * > &Cache)
Traverses the scope chain rooted at RootScope until it hits a Subprogram, recreating the chain with "...
Tagged DWARF-like metadata node.
StringRef getName() const
DISPFlags
Debug info subprogram flags.
A parsed version of the target data layout string in and methods for querying it.
Records a position in IR for a source label (DILabel).
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
This is the common base class for debug info intrinsics for variables.
Record of a variable value-assignment, aka a non instruction representation of the dbg....
void setVariable(DILocalVariable *NewVar)
Value * getAddress() const
DILocalVariable * getVariable() const
iterator_range< location_op_iterator > location_ops() const
Get the locations corresponding to the variable referenced by the debug info intrinsic.
static DebugLoc replaceInlinedAtSubprogram(const DebugLoc &DL, DISubprogram &NewSP, LLVMContext &Ctx, DenseMap< const MDNode *, MDNode * > &Cache)
Rebuild the entire inline-at chain by replacing the subprogram at the end of the chain with NewSP.
DILocation * getInlinedAt() const
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
DomTreeNodeBase< NodeT > * addNewBlock(NodeT *BB, NodeT *DomBB)
Add a new node to the dominator tree information.
DomTreeNodeBase< NodeT > * getNode(const NodeT *BB) const
getNode - return the (Post)DominatorTree node for the specified basic block.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Class to represent profile counts.
void addFnAttr(Attribute::AttrKind Kind)
Add function attributes to this function.
void setSubprogram(DISubprogram *SP)
Set the attached subprogram.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
const BasicBlock & getEntryBlock() const
const BasicBlock & front() const
DISubprogram * getSubprogram() const
Get the attached subprogram.
bool IsNewDbgInfoFormat
Is this function using intrinsics to record the position of debugging information,...
bool hasPersonalityFn() const
Check whether this function has a personality function.
Constant * getPersonalityFn() const
Get the personality function associated with this function.
void setPersonalityFn(Constant *Fn)
AttributeList getAttributes() const
Return the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
adds the attribute to the list of attributes for the given arg.
Function::iterator insert(Function::iterator Position, BasicBlock *BB)
Insert BB in the basic block list at Position.
bool doesNotReturn() const
Determine if the function cannot return.
Type * getReturnType() const
Returns the type of the ret val.
void setEntryCount(ProfileCount Count, const DenseSet< GlobalValue::GUID > *Imports=nullptr)
Set the entry count for this function.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr, BasicBlock::iterator InsertBefore)
unsigned getAddressSpace() const
Module * getParent()
Get the module that this global value is contained inside of...
@ InternalLinkage
Rename collisions when linking (static functions).
bool isLifetimeStartOrEnd() const LLVM_READONLY
Return true if the instruction is a llvm.lifetime.start or llvm.lifetime.end marker.
unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const BasicBlock * getParent() const
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
const Function * getFunction() const
Return the function this instruction belongs to.
BasicBlock * getSuccessor(unsigned Idx) const LLVM_READONLY
Return the specified successor. This instruction must be a terminator.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
void setSuccessor(unsigned Idx, BasicBlock *BB)
Update the specified successor to point at the provided block.
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
InstListType::iterator insertInto(BasicBlock *ParentBB, InstListType::iterator It)
Inserts an unlinked instruction into ParentBB at position It and returns the iterator of the inserted...
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
Value * getPointerOperand()
Represents a single loop in the control flow graph.
MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight)
Return metadata containing two branch weights.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
A Module instance is used to store all the information related to an LLVM module.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
void setIncomingBlock(unsigned i, BasicBlock *BB)
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr, BasicBlock::iterator InsertBefore)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
Value * removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty=true)
Remove an incoming value.
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Return a value (possibly void), from a function.
static ReturnInst * Create(LLVMContext &C, Value *retVal, BasicBlock::iterator InsertBefore)
A vector that has set insertion semantics.
ArrayRef< value_type > getArrayRef() const
size_type size() const
Determine the number of elements in the SetVector.
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
std::string str() const
str - Get the contents as an std::string.
constexpr bool empty() const
empty - Check if the string is empty.
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Type * getElementType(unsigned N) const
static SwitchInst * Create(Value *Value, BasicBlock *Default, unsigned NumCases, BasicBlock::iterator InsertBefore)
BasicBlock * getSuccessor(unsigned idx) const
void setCondition(Value *V)
void addCase(ConstantInt *OnVal, BasicBlock *Dest)
Add an entry to the switch instruction.
void setDefaultDest(BasicBlock *DefaultCase)
Value * getCondition() const
CaseIt removeCase(CaseIt I)
This method removes the specified case and its successor from the switch instruction.
The instances of the Type class are immutable: once they are created, they are never changed.
static IntegerType * getInt1Ty(LLVMContext &C)
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isVoidTy() const
Return true if this is 'void'.
This function has undefined behavior.
bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
user_iterator user_begin()
void setName(const Twine &Name)
Change the name of the value.
const Value * stripInBoundsConstantOffsets() const
Strip off pointer casts and all-constant inbounds GEPs.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
const Value * stripInBoundsOffsets(function_ref< void(const Value *)> Func=[](const Value *) {}) const
Strip off pointer casts and inbounds GEPs.
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
void dump() const
Support for debugging, callable in GDB: V->dump()
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
NodeAddr< PhiNode * > Phi
This is an optimization pass for GlobalISel generic memory operations.
bool stripDebugInfo(Function &F)
Function::ProfileCount ProfileCount
bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
void findDbgUsers(SmallVectorImpl< DbgVariableIntrinsic * > &DbgInsts, Value *V, SmallVectorImpl< DbgVariableRecord * > *DbgVariableRecords=nullptr)
Finds the debug info intrinsics describing a value.
auto successors(const MachineBasicBlock *BB)
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
auto predecessors(const MachineBasicBlock *BB)
void updateLoopMetadataDebugLocations(Instruction &I, function_ref< Metadata *(Metadata *)> Updater)
Update the debug locations contained within the MD_loop metadata attached to the instruction I,...
Representative of a block.
Distribution of unscaled probability weight.