74#define DEBUG_TYPE "code-extractor"
82 cl::desc(
"Aggregate arguments to code-extracted functions"));
87 bool AllowVarArgs,
bool AllowAlloca) {
100 while (!ToVisit.
empty()) {
102 if (!Visited.
insert(Curr).second)
104 if (isa<BlockAddress const>(Curr))
107 if (isa<Instruction>(Curr) && cast<Instruction>(Curr)->
getParent() != &BB)
110 for (
auto const &U : Curr->
operands()) {
111 if (
auto *UU = dyn_cast<User>(U))
119 if (isa<AllocaInst>(
I)) {
125 if (
const auto *II = dyn_cast<InvokeInst>(
I)) {
128 if (
auto *UBB = II->getUnwindDest())
129 if (!Result.count(UBB))
136 if (
const auto *CSI = dyn_cast<CatchSwitchInst>(
I)) {
137 if (
auto *UBB = CSI->getUnwindDest())
138 if (!Result.count(UBB))
140 for (
const auto *HBB : CSI->handlers())
141 if (!Result.count(
const_cast<BasicBlock*
>(HBB)))
148 if (
const auto *CPI = dyn_cast<CatchPadInst>(
I)) {
149 for (
const auto *U : CPI->users())
150 if (
const auto *CRI = dyn_cast<CatchReturnInst>(U))
151 if (!Result.count(
const_cast<BasicBlock*
>(CRI->getParent())))
159 if (
const auto *CPI = dyn_cast<CleanupPadInst>(
I)) {
160 for (
const auto *U : CPI->users())
161 if (
const auto *CRI = dyn_cast<CleanupReturnInst>(U))
162 if (!Result.count(
const_cast<BasicBlock*
>(CRI->getParent())))
166 if (
const auto *CRI = dyn_cast<CleanupReturnInst>(
I)) {
167 if (
auto *UBB = CRI->getUnwindDest())
168 if (!Result.count(UBB))
173 if (
const CallInst *CI = dyn_cast<CallInst>(
I)) {
174 if (
const Function *
F = CI->getCalledFunction()) {
175 auto IID =
F->getIntrinsicID();
176 if (IID == Intrinsic::vastart) {
185 if (IID == Intrinsic::eh_typeid_for)
197 bool AllowVarArgs,
bool AllowAlloca) {
198 assert(!BBs.
empty() &&
"The set of blocks to extract must be non-empty");
208 if (!Result.insert(BB))
212 LLVM_DEBUG(
dbgs() <<
"Region front block: " << Result.front()->getName()
215 for (
auto *BB : Result) {
220 if (BB == Result.front()) {
222 LLVM_DEBUG(
dbgs() <<
"The first block cannot be an unwind block\n");
231 if (!Result.count(PBB)) {
232 LLVM_DEBUG(
dbgs() <<
"No blocks in this region may have entries from "
233 "outside the region except for the first block!\n"
234 <<
"Problematic source BB: " << BB->getName() <<
"\n"
235 <<
"Problematic destination BB: " << PBB->getName()
247 bool AllowVarArgs,
bool AllowAlloca,
248 BasicBlock *AllocationBlock, std::string Suffix,
249 bool ArgsInZeroAddressSpace)
251 BPI(BPI), AC(AC), AllocationBlock(AllocationBlock),
252 AllowVarArgs(AllowVarArgs),
254 Suffix(Suffix), ArgsInZeroAddressSpace(ArgsInZeroAddressSpace) {}
261 BPI(BPI), AC(AC), AllocationBlock(nullptr), AllowVarArgs(
false),
271 if (
Blocks.count(
I->getParent()))
280 if (isa<Argument>(V))
return true;
282 if (!
Blocks.count(
I->getParent()))
294 if (!CommonExitBlock) {
295 CommonExitBlock = Succ;
298 if (CommonExitBlock != Succ)
307 return CommonExitBlock;
312 for (
Instruction &II : BB.instructionsWithoutDebug())
313 if (
auto *AI = dyn_cast<AllocaInst>(&II))
314 Allocas.push_back(AI);
316 findSideEffectInfoForBlock(BB);
320void CodeExtractorAnalysisCache::findSideEffectInfoForBlock(
BasicBlock &BB) {
322 unsigned Opcode = II.getOpcode();
323 Value *MemAddr =
nullptr;
325 case Instruction::Store:
326 case Instruction::Load: {
327 if (
Opcode == Instruction::Store) {
329 MemAddr = SI->getPointerOperand();
335 if (isa<Constant>(MemAddr))
338 if (!isa<AllocaInst>(
Base)) {
339 SideEffectingBlocks.insert(&BB);
342 BaseMemAddrs[&BB].insert(
Base);
350 SideEffectingBlocks.insert(&BB);
354 if (II.mayHaveSideEffects()) {
355 SideEffectingBlocks.insert(&BB);
365 if (SideEffectingBlocks.count(&BB))
367 auto It = BaseMemAddrs.find(&BB);
368 if (It != BaseMemAddrs.end())
369 return It->second.count(
Addr);
375 AllocaInst *AI = cast<AllocaInst>(
Addr->stripInBoundsConstantOffsets());
378 if (Blocks.count(&BB))
388 BasicBlock *SinglePredFromOutlineRegion =
nullptr;
389 assert(!Blocks.count(CommonExitBlock) &&
390 "Expect a block outside the region!");
392 if (!Blocks.count(Pred))
394 if (!SinglePredFromOutlineRegion) {
395 SinglePredFromOutlineRegion = Pred;
396 }
else if (SinglePredFromOutlineRegion != Pred) {
397 SinglePredFromOutlineRegion =
nullptr;
402 if (SinglePredFromOutlineRegion)
403 return SinglePredFromOutlineRegion;
409 while (
I != BB->
end()) {
410 PHINode *Phi = dyn_cast<PHINode>(
I);
422 assert(!getFirstPHI(CommonExitBlock) &&
"Phi not expected");
430 if (Blocks.count(Pred))
435 Blocks.insert(CommonExitBlock);
436 OldTargets.push_back(NewExitBlock);
437 return CommonExitBlock;
444CodeExtractor::LifetimeMarkerInfo
448 LifetimeMarkerInfo
Info;
458 Info.LifeStart = IntrInst;
464 Info.LifeEnd = IntrInst;
469 if (isa<DbgInfoIntrinsic>(IntrInst))
477 if (!
Info.LifeStart || !
Info.LifeEnd)
483 if ((
Info.SinkLifeStart ||
Info.HoistLifeEnd) &&
488 if (
Info.HoistLifeEnd && !ExitBlock)
500 auto moveOrIgnoreLifetimeMarkers =
501 [&](
const LifetimeMarkerInfo &LMI) ->
bool {
504 if (LMI.SinkLifeStart) {
507 SinkCands.
insert(LMI.LifeStart);
509 if (LMI.HoistLifeEnd) {
510 LLVM_DEBUG(
dbgs() <<
"Hoisting lifetime.end: " << *LMI.LifeEnd <<
"\n");
511 HoistCands.
insert(LMI.LifeEnd);
520 if (Blocks.count(BB))
529 LifetimeMarkerInfo MarkerInfo = getLifetimeMarkers(CEAC, AI, ExitBlock);
530 bool Moved = moveOrIgnoreLifetimeMarkers(MarkerInfo);
542 for (
User *U : AI->users()) {
546 if (U->stripInBoundsConstantOffsets() != AI)
550 for (
User *BU : Bitcast->users()) {
562 << *Bitcast <<
" in out-of-region lifetime marker "
563 << *IntrInst <<
"\n");
564 LifetimeBitcastUsers.
push_back(IntrInst);
574 I->replaceUsesOfWith(
I->getOperand(1), CastI);
580 for (
User *U : AI->users()) {
581 if (U->stripInBoundsConstantOffsets() == AI) {
583 LifetimeMarkerInfo LMI = getLifetimeMarkers(CEAC, Bitcast, ExitBlock);
599 if (Bitcasts.
empty())
602 LLVM_DEBUG(
dbgs() <<
"Sinking alloca (via bitcast): " << *AI <<
"\n");
604 for (
unsigned I = 0,
E = Bitcasts.
size();
I !=
E; ++
I) {
606 const LifetimeMarkerInfo &LMI = BitcastLifetimeInfo[
I];
608 "Unsafe to sink bitcast without lifetime markers");
609 moveOrIgnoreLifetimeMarkers(LMI);
611 LLVM_DEBUG(
dbgs() <<
"Sinking bitcast-of-alloca: " << *BitcastAddr
613 SinkCands.
insert(BitcastAddr);
627 if (AllowVarArgs &&
F->getFunctionType()->isVarArg()) {
628 auto containsVarArgIntrinsic = [](
const Instruction &
I) {
629 if (
const CallInst *CI = dyn_cast<CallInst>(&
I))
630 if (
const Function *Callee = CI->getCalledFunction())
631 return Callee->getIntrinsicID() == Intrinsic::vastart ||
632 Callee->getIntrinsicID() == Intrinsic::vaend;
636 for (
auto &BB : *
F) {
637 if (Blocks.count(&BB))
652 for (
auto &OI : II.operands()) {
658 for (
User *U : II.users())
670void CodeExtractor::severSplitPHINodesOfEntry(
BasicBlock *&Header) {
671 unsigned NumPredsFromRegion = 0;
672 unsigned NumPredsOutsideRegion = 0;
674 if (Header != &Header->getParent()->getEntryBlock()) {
675 PHINode *PN = dyn_cast<PHINode>(Header->begin());
683 ++NumPredsFromRegion;
685 ++NumPredsOutsideRegion;
689 if (NumPredsOutsideRegion <= 1)
return;
701 Blocks.remove(OldPred);
702 Blocks.insert(NewBB);
707 if (NumPredsFromRegion) {
720 for (AfterPHIs = OldPred->
begin(); isa<PHINode>(AfterPHIs); ++AfterPHIs) {
721 PHINode *PN = cast<PHINode>(AfterPHIs);
747void CodeExtractor::severSplitPHINodesOfExits(
752 for (
PHINode &PN : ExitBB->phis()) {
762 if (IncomingVals.
size() <= 1)
769 ExitBB->getName() +
".split",
770 ExitBB->getParent(), ExitBB);
774 if (Blocks.count(PredBB))
775 PredBB->getTerminator()->replaceUsesOfWith(ExitBB, NewBB);
777 Blocks.insert(NewBB);
784 for (
unsigned i : IncomingVals)
786 for (
unsigned i :
reverse(IncomingVals))
793void CodeExtractor::splitReturnBlocks() {
797 Block->splitBasicBlock(RI->getIterator(),
Block->getName() +
".ret");
815Function *CodeExtractor::constructFunction(
const ValueSet &inputs,
816 const ValueSet &outputs,
826 switch (NumExitBlocks) {
833 std::vector<Type *> ParamTy;
834 std::vector<Type *> AggParamTy;
835 ValueSet StructValues;
841 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(
value)) {
842 AggParamTy.push_back(
value->getType());
843 StructValues.insert(
value);
845 ParamTy.push_back(
value->getType());
849 for (
Value *output : outputs) {
851 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(output)) {
852 AggParamTy.push_back(output->getType());
853 StructValues.insert(output);
860 (ParamTy.size() + AggParamTy.size()) ==
861 (inputs.size() + outputs.size()) &&
862 "Number of scalar and aggregate params does not match inputs, outputs");
863 assert((StructValues.empty() || AggregateArgs) &&
864 "Expeced StructValues only with AggregateArgs set");
867 size_t NumScalarParams = ParamTy.size();
869 if (AggregateArgs && !AggParamTy.empty()) {
872 StructTy, ArgsInZeroAddressSpace ? 0 :
DL.getAllocaAddrSpace()));
876 dbgs() <<
"Function type: " << *RetTy <<
" f(";
877 for (
Type *i : ParamTy)
878 dbgs() << *i <<
", ";
883 RetTy, ParamTy, AllowVarArgs && oldFunction->
isVarArg());
885 std::string SuffixToUse =
892 oldFunction->
getName() +
"." + SuffixToUse, M);
903 if (Attr.isStringAttribute()) {
904 if (Attr.getKindAsString() ==
"thunk")
907 switch (Attr.getKindAsEnum()) {
910 case Attribute::AllocSize:
911 case Attribute::Builtin:
912 case Attribute::Convergent:
913 case Attribute::JumpTable:
914 case Attribute::Naked:
915 case Attribute::NoBuiltin:
916 case Attribute::NoMerge:
917 case Attribute::NoReturn:
918 case Attribute::NoSync:
919 case Attribute::ReturnsTwice:
920 case Attribute::Speculatable:
921 case Attribute::StackAlignment:
922 case Attribute::WillReturn:
923 case Attribute::AllocKind:
924 case Attribute::PresplitCoroutine:
925 case Attribute::Memory:
926 case Attribute::NoFPClass:
927 case Attribute::CoroDestroyOnlyWhenComplete:
930 case Attribute::AlwaysInline:
931 case Attribute::Cold:
932 case Attribute::DisableSanitizerInstrumentation:
933 case Attribute::FnRetThunkExtern:
935 case Attribute::NoRecurse:
936 case Attribute::InlineHint:
937 case Attribute::MinSize:
938 case Attribute::NoCallback:
939 case Attribute::NoDuplicate:
940 case Attribute::NoFree:
941 case Attribute::NoImplicitFloat:
942 case Attribute::NoInline:
943 case Attribute::NonLazyBind:
944 case Attribute::NoRedZone:
945 case Attribute::NoUnwind:
946 case Attribute::NoSanitizeBounds:
947 case Attribute::NoSanitizeCoverage:
948 case Attribute::NullPointerIsValid:
949 case Attribute::OptimizeForDebugging:
950 case Attribute::OptForFuzzing:
951 case Attribute::OptimizeNone:
952 case Attribute::OptimizeForSize:
953 case Attribute::SafeStack:
954 case Attribute::ShadowCallStack:
955 case Attribute::SanitizeAddress:
956 case Attribute::SanitizeMemory:
957 case Attribute::SanitizeThread:
958 case Attribute::SanitizeHWAddress:
959 case Attribute::SanitizeMemTag:
960 case Attribute::SpeculativeLoadHardening:
961 case Attribute::StackProtect:
962 case Attribute::StackProtectReq:
963 case Attribute::StackProtectStrong:
964 case Attribute::StrictFP:
965 case Attribute::UWTable:
966 case Attribute::VScaleRange:
967 case Attribute::NoCfCheck:
968 case Attribute::MustProgress:
969 case Attribute::NoProfile:
970 case Attribute::SkipProfile:
973 case Attribute::Alignment:
974 case Attribute::AllocatedPointer:
975 case Attribute::AllocAlign:
976 case Attribute::ByVal:
977 case Attribute::Dereferenceable:
978 case Attribute::DereferenceableOrNull:
979 case Attribute::ElementType:
980 case Attribute::InAlloca:
981 case Attribute::InReg:
982 case Attribute::Nest:
983 case Attribute::NoAlias:
984 case Attribute::NoCapture:
985 case Attribute::NoUndef:
986 case Attribute::NonNull:
987 case Attribute::Preallocated:
988 case Attribute::ReadNone:
989 case Attribute::ReadOnly:
990 case Attribute::Returned:
991 case Attribute::SExt:
992 case Attribute::StructRet:
993 case Attribute::SwiftError:
994 case Attribute::SwiftSelf:
995 case Attribute::SwiftAsync:
996 case Attribute::ZExt:
997 case Attribute::ImmArg:
998 case Attribute::ByRef:
999 case Attribute::WriteOnly:
1000 case Attribute::Writable:
1011 newFunction->
insert(newFunction->
end(), newRootNode);
1020 for (
unsigned i = 0, e = inputs.size(), aggIdx = 0; i != e; ++i) {
1022 if (AggregateArgs && StructValues.contains(inputs[i])) {
1028 StructTy, &*AggAI,
Idx,
"gep_" + inputs[i]->
getName(), TI);
1030 "loadgep_" + inputs[i]->getName(), TI);
1033 RewriteVal = &*ScalarAI++;
1035 std::vector<User *>
Users(inputs[i]->user_begin(), inputs[i]->user_end());
1038 if (
Blocks.count(inst->getParent()))
1039 inst->replaceUsesOfWith(inputs[i], RewriteVal);
1043 if (NumScalarParams) {
1045 for (
unsigned i = 0, e = inputs.size(); i != e; ++i, ++ScalarAI)
1046 if (!StructValues.contains(inputs[i]))
1048 for (
unsigned i = 0, e = outputs.size(); i != e; ++i, ++ScalarAI)
1049 if (!StructValues.contains(outputs[i]))
1057 for (
auto &U :
Users)
1061 if (
I->isTerminator() &&
I->getFunction() == oldFunction &&
1062 !
Blocks.count(
I->getParent()))
1063 I->replaceUsesOfWith(header, newHeader);
1079 auto *II = dyn_cast<IntrinsicInst>(&
I);
1080 if (!II || !II->isLifetimeStartOrEnd())
1090 if (II->getIntrinsicID() == Intrinsic::lifetime_start)
1091 LifetimesStart.
insert(Mem);
1092 II->eraseFromParent();
1109 bool InsertBefore) {
1110 for (
Value *Mem : Objects) {
1113 "Input memory not defined in original function");
1120 Marker->insertBefore(Term);
1124 if (!LifetimesStart.
empty()) {
1125 insertMarkers(Intrinsic::lifetime_start, LifetimesStart,
1129 if (!LifetimesEnd.
empty()) {
1130 insertMarkers(Intrinsic::lifetime_end, LifetimesEnd,
1141 ValueSet &outputs) {
1144 std::vector<Value *> params, ReloadOutputs, Reloads;
1145 ValueSet StructValues;
1153 unsigned ScalarInputArgNo = 0;
1155 for (
Value *input : inputs) {
1156 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(input))
1157 StructValues.
insert(input);
1159 params.push_back(input);
1160 if (input->isSwiftError())
1161 SwiftErrorArgs.
push_back(ScalarInputArgNo);
1167 unsigned ScalarOutputArgNo = 0;
1168 for (
Value *output : outputs) {
1169 if (AggregateArgs && !ExcludeArgsFromAggregate.
contains(output)) {
1170 StructValues.insert(output);
1173 new AllocaInst(output->getType(),
DL.getAllocaAddrSpace(),
1174 nullptr, output->
getName() +
".loc",
1176 ReloadOutputs.push_back(alloca);
1177 params.push_back(alloca);
1178 ++ScalarOutputArgNo;
1184 unsigned NumAggregatedInputs = 0;
1185 if (AggregateArgs && !StructValues.empty()) {
1186 std::vector<Type *> ArgTypes;
1187 for (
Value *V : StructValues)
1188 ArgTypes.push_back(
V->getType());
1193 StructArgTy,
DL.getAllocaAddrSpace(),
nullptr,
"structArg",
1197 if (ArgsInZeroAddressSpace &&
DL.getAllocaAddrSpace() != 0) {
1199 Struct, PointerType ::get(Context, 0),
"structArg.ascast");
1200 StructSpaceCast->insertAfter(
Struct);
1201 params.push_back(StructSpaceCast);
1203 params.push_back(
Struct);
1206 for (
unsigned i = 0, e = StructValues.size(); i != e; ++i) {
1207 if (inputs.contains(StructValues[i])) {
1213 GEP->insertInto(codeReplacer, codeReplacer->
end());
1215 NumAggregatedInputs++;
1222 NumExitBlocks > 1 ?
"targetBlock" :
"");
1234 for (
unsigned SwiftErrArgNo : SwiftErrorArgs) {
1235 call->
addParamAttr(SwiftErrArgNo, Attribute::SwiftError);
1236 newFunction->
addParamAttr(SwiftErrArgNo, Attribute::SwiftError);
1241 for (
unsigned i = 0, e = outputs.size(), scalarIdx = 0,
1242 aggIdx = NumAggregatedInputs;
1244 Value *Output =
nullptr;
1245 if (AggregateArgs && StructValues.contains(outputs[i])) {
1251 GEP->insertInto(codeReplacer, codeReplacer->
end());
1255 Output = ReloadOutputs[scalarIdx];
1259 outputs[i]->
getName() +
".reload",
1261 Reloads.push_back(load);
1262 std::vector<User *>
Users(outputs[i]->user_begin(), outputs[i]->user_end());
1273 codeReplacer, 0, codeReplacer);
1280 std::map<BasicBlock *, BasicBlock *> ExitBlockMap;
1284 unsigned switchVal = 0;
1286 if (
Blocks.count(OldTarget))
1288 BasicBlock *&NewTarget = ExitBlockMap[OldTarget];
1295 OldTarget->getName() +
".exitStub",
1297 unsigned SuccNum = switchVal++;
1299 Value *brVal =
nullptr;
1300 assert(NumExitBlocks < 0xffff &&
"too many exit blocks for switch");
1301 switch (NumExitBlocks) {
1327 BasicBlock *NewTarget = ExitBlockMap[OldTarget];
1328 assert(NewTarget &&
"Unknown target block!");
1339 std::advance(ScalarOutputArgBegin, ScalarInputArgNo);
1341 std::advance(AggOutputArgBegin, ScalarInputArgNo + ScalarOutputArgNo);
1343 for (
unsigned i = 0, e = outputs.size(), aggIdx = NumAggregatedInputs; i != e;
1345 auto *OutI = dyn_cast<Instruction>(outputs[i]);
1353 if (
auto *InvokeI = dyn_cast<InvokeInst>(OutI))
1354 InsertPt = InvokeI->getNormalDest()->getFirstInsertionPt();
1355 else if (
auto *Phi = dyn_cast<PHINode>(OutI))
1356 InsertPt =
Phi->getParent()->getFirstInsertionPt();
1358 InsertPt = std::next(OutI->getIterator());
1363 "InsertPt should be in new function");
1364 if (AggregateArgs && StructValues.contains(outputs[i])) {
1366 "Number of aggregate output arguments should match "
1367 "the number of defined values");
1372 StructArgTy, &*AggOutputArgBegin,
Idx,
"gep_" + outputs[i]->
getName(),
1381 "Number of scalar output arguments should match "
1382 "the number of defined values");
1383 new StoreInst(outputs[i], &*ScalarOutputArgBegin, InsertBefore);
1384 ++ScalarOutputArgBegin;
1390 switch (NumExitBlocks) {
1439void CodeExtractor::moveCodeToFunction(
Function *newFunction) {
1443 Block->removeFromParent();
1450 newFuncIt = newFunction->
insert(std::next(newFuncIt),
Block);
1454void CodeExtractor::calculateNewCallTerminatorWeights(
1466 Distribution BranchDist;
1473 BlockNode ExitNode(i);
1476 BranchDist.addExit(ExitNode, ExitFreq);
1482 if (BranchDist.Total == 0) {
1488 BranchDist.normalize();
1491 for (
unsigned I = 0,
E = BranchDist.Weights.size();
I <
E; ++
I) {
1492 const auto &Weight = BranchDist.Weights[
I];
1495 BranchWeights[Weight.TargetNode.Index] = Weight.Amount;
1497 EdgeProbabilities[Weight.TargetNode.Index] = BP;
1501 LLVMContext::MD_prof,
1513 if (DVI->getFunction() != &
F)
1514 DVI->eraseFromParent();
1516 if (DPV->getFunction() != &
F)
1517 DPV->eraseFromParent();
1540 assert(OldSP->getUnit() &&
"Missing compile unit for subprogram");
1546 DISubprogram::SPFlagOptimized |
1547 DISubprogram::SPFlagLocalToUnit;
1550 0, SPType, 0, DINode::FlagZero, SPFlags);
1553 auto IsInvalidLocation = [&NewFunc](
Value *Location) {
1557 (!isa<Constant>(Location) && !isa<Instruction>(Location)))
1559 Instruction *LocationInst = dyn_cast<Instruction>(Location);
1560 return LocationInst && LocationInst->
getFunction() != &NewFunc;
1575 DINode *&NewVar = RemappedMetadata[OldVar];
1578 *OldVar->getScope(), *NewSP, Ctx, Cache);
1580 NewScope, OldVar->
getName(), OldVar->getFile(), OldVar->getLine(),
1581 OldVar->getType(),
false, DINode::FlagZero,
1582 OldVar->getAlignInBits());
1584 return cast<DILocalVariable>(NewVar);
1587 auto UpdateDPValuesOnInst = [&](
Instruction &
I) ->
void {
1588 for (
auto &DPV :
I.getDbgValueRange()) {
1592 if (
any_of(DPV.location_ops(), IsInvalidLocation)) {
1596 if (!DPV.getDebugLoc().getInlinedAt())
1597 DPV.setVariable(GetUpdatedDIVariable(DPV.getVariable()));
1599 *NewSP, Ctx, Cache));
1604 UpdateDPValuesOnInst(
I);
1606 auto *DII = dyn_cast<DbgInfoIntrinsic>(&
I);
1612 if (
auto *DLI = dyn_cast<DbgLabelInst>(&
I)) {
1613 if (DLI->getDebugLoc().getInlinedAt())
1615 DILabel *OldLabel = DLI->getLabel();
1616 DINode *&NewLabel = RemappedMetadata[OldLabel];
1619 *OldLabel->
getScope(), *NewSP, Ctx, Cache);
1627 auto *DVI = cast<DbgVariableIntrinsic>(DII);
1629 if (
any_of(DVI->location_ops(), IsInvalidLocation)) {
1634 if (
auto *DAI = dyn_cast<DbgAssignIntrinsic>(DVI);
1635 DAI && IsInvalidLocation(DAI->getAddress())) {
1641 if (!DVI->getDebugLoc().getInlinedAt())
1642 DVI->setVariable(GetUpdatedDIVariable(DVI->getVariable()));
1645 for (
auto *DII : DebugIntrinsicsToDelete)
1646 DII->eraseFromParent();
1647 for (
auto *DPV : DPVsToDelete)
1648 DPV->getMarker()->MarkedInstr->dropOneDbgValue(DPV);
1659 auto updateLoopInfoLoc = [&Ctx, &Cache, NewSP](
Metadata *MD) ->
Metadata * {
1660 if (
auto *Loc = dyn_cast_or_null<DILocation>(MD))
1693 assert(BPI &&
"Both BPI and BFI are required to preserve profile info");
1706 if (
auto *AI = dyn_cast<AssumeInst>(&
I)) {
1709 AI->eraseFromParent();
1716 splitReturnBlocks();
1724 if (!
Blocks.count(Succ)) {
1734 NumExitBlocks = ExitBlocks.
size();
1742 OldTargets.push_back(OldTarget);
1747 severSplitPHINodesOfEntry(header);
1748 severSplitPHINodesOfExits(ExitBlocks);
1752 "codeRepl", oldFunction,
1770 if (!
I.getDebugLoc())
1772 BranchI->setDebugLoc(
I.getDebugLoc());
1777 BranchI->insertInto(newFuncRoot, newFuncRoot->
end());
1779 ValueSet SinkingCands, HoistingCands;
1781 findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);
1791 for (
auto *II : SinkingCands) {
1792 if (
auto *AI = dyn_cast<AllocaInst>(II)) {
1794 if (!FirstSunkAlloca)
1795 FirstSunkAlloca = AI;
1798 assert((SinkingCands.
empty() || FirstSunkAlloca) &&
1799 "Did not expect a sink candidate without any allocas");
1800 for (
auto *II : SinkingCands) {
1801 if (!isa<AllocaInst>(II)) {
1802 cast<Instruction>(II)->moveAfter(FirstSunkAlloca);
1806 if (!HoistingCands.
empty()) {
1809 for (
auto *II : HoistingCands)
1822 constructFunction(inputs, outputs, header, newFuncRoot, codeReplacer,
1835 emitCallAndSwitchStatement(newFunction, codeReplacer, inputs, outputs);
1837 moveCodeToFunction(newFunction);
1849 if (BFI && NumExitBlocks > 1)
1850 calculateNewCallTerminatorWeights(codeReplacer, ExitWeights, BPI);
1862 for (
PHINode &PN : ExitBB->phis()) {
1863 Value *IncomingCodeReplacerVal =
nullptr;
1870 if (!IncomingCodeReplacerVal) {
1875 "PHI has two incompatbile incoming values from codeRepl");
1886 return isa<ReturnInst>(Term) || isa<ResumeInst>(Term);
1892 newFunction->
dump();
1906 auto *
I = dyn_cast_or_null<CallInst>(AssumeVH);
1911 if (
I->getFunction() != &OldFunc)
1918 auto *AffectedCI = dyn_cast_or_null<CallInst>(AffectedValVH);
1921 if (AffectedCI->getFunction() != &OldFunc)
1923 auto *AssumedInst = cast<Instruction>(AffectedCI->getOperand(0));
1924 if (AssumedInst->getFunction() != &OldFunc)
1932 ExcludeArgsFromAggregate.
insert(Arg);
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Given that RA is a live value
This file defines the DenseMap class.
DenseMap< Block *, BlockRelaxAux > Blocks
static Function * getFunction(Constant *C)
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
iv Induction Variable Users
Select target instructions out of generic instructions
Move duplicate certain instructions close to their use
Module.h This file contains the declarations for the Module class.
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
static constexpr uint32_t Opcode
This class represents a conversion between pointers from one address space to another.
an instruction to allocate memory on the stack
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
bool empty() const
empty - Check if the array is empty.
A cache of @llvm.assume calls within a function.
MutableArrayRef< ResultElem > assumptions()
Access the list of assumption handles currently tracked for this function.
void unregisterAssumption(AssumeInst *CI)
Remove an @llvm.assume intrinsic from this function's cache if it has been added to the cache earlier...
MutableArrayRef< ResultElem > assumptionsFor(const Value *V)
Access the list of assumptions which affect this value.
AttributeSet getFnAttrs() const
The function attributes are returned.
@ TombstoneKey
Use as Tombstone key for DenseMap of AttrKind.
@ None
No attributes have been set.
@ EmptyKey
Use as Empty key for DenseMap of AttrKind.
@ EndAttrKinds
Sentinal value useful for loops.
LLVM Basic Block Representation.
iterator begin()
Instruction iterator methods.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
iterator_range< filter_iterator< BasicBlock::const_iterator, std::function< bool(const Instruction &)> > > instructionsWithoutDebug(bool SkipPseudoOp=true) const
Return a const iterator range over the instructions in the block, skipping any debug instructions.
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches,...
InstListType::const_iterator getFirstNonPHIIt() const
Iterator returning form of getFirstNonPHI.
InstListType::const_iterator const_iterator
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const Instruction & front() const
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
const Function * getParent() const
Return the enclosing method, or null if none.
InstListType::iterator iterator
Instruction iterators...
LLVMContext & getContext() const
Get the context in which this basic block lives.
bool IsNewDbgInfoFormat
Flag recording whether or not this block stores debug-info in the form of intrinsic instructions (fal...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
std::optional< uint64_t > getProfileCountFromFreq(BlockFrequency Freq) const
Returns the estimated profile count of Freq.
void setBlockFreq(const BasicBlock *BB, BlockFrequency Freq)
BlockFrequency getBlockFreq(const BasicBlock *BB) const
getblockFreq - Return block frequency.
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
Analysis providing branch probability information.
void setEdgeProbability(const BasicBlock *Src, const SmallVectorImpl< BranchProbability > &Probs)
Set the raw probabilities for all edges from the given block.
BranchProbability getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const
Get an edge's probability, relative to other out-edges of the Src.
static BranchProbability getUnknown()
static BranchProbability getZero()
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
This is the base class for all instructions that perform data casts.
static CastInst * CreatePointerCast(Value *S, Type *Ty, const Twine &Name, BasicBlock *InsertAtEnd)
Create a BitCast AddrSpaceCast, or a PtrToInt cast instruction.
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
DISubroutineType * createSubroutineType(DITypeRefArray ParameterTypes, DINode::DIFlags Flags=DINode::FlagZero, unsigned CC=0)
Create subroutine type.
void finalizeSubprogram(DISubprogram *SP)
Finalize a specific subprogram - no new variables may be added to this subprogram afterwards.
DISubprogram * createFunction(DIScope *Scope, StringRef Name, StringRef LinkageName, DIFile *File, unsigned LineNo, DISubroutineType *Ty, unsigned ScopeLine, DINode::DIFlags Flags=DINode::FlagZero, DISubprogram::DISPFlags SPFlags=DISubprogram::SPFlagZero, DITemplateParameterArray TParams=nullptr, DISubprogram *Decl=nullptr, DITypeArray ThrownTypes=nullptr, DINodeArray Annotations=nullptr, StringRef TargetFuncName="")
Create a new descriptor for the specified subprogram.
DITypeRefArray getOrCreateTypeArray(ArrayRef< Metadata * > Elements)
Get a DITypeRefArray, create one if required.
DILocalVariable * createAutoVariable(DIScope *Scope, StringRef Name, DIFile *File, unsigned LineNo, DIType *Ty, bool AlwaysPreserve=false, DINode::DIFlags Flags=DINode::FlagZero, uint32_t AlignInBits=0)
Create a new descriptor for an auto variable.
StringRef getName() const
DILocalScope * getScope() const
Get the local scope for this label.
static DILocalScope * cloneScopeForSubprogram(DILocalScope &RootScope, DISubprogram &NewSP, LLVMContext &Ctx, DenseMap< const MDNode *, MDNode * > &Cache)
Traverses the scope chain rooted at RootScope until it hits a Subprogram, recreating the chain with "...
Tagged DWARF-like metadata node.
StringRef getName() const
DISPFlags
Debug info subprogram flags.
Record of a variable value-assignment, aka a non instruction representation of the dbg....
A parsed version of the target data layout string in and methods for querying it.
This is the common base class for debug info intrinsics for variables.
static DebugLoc replaceInlinedAtSubprogram(const DebugLoc &DL, DISubprogram &NewSP, LLVMContext &Ctx, DenseMap< const MDNode *, MDNode * > &Cache)
Rebuild the entire inline-at chain by replacing the subprogram at the end of the chain with NewSP.
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
DomTreeNodeBase< NodeT > * addNewBlock(NodeT *BB, NodeT *DomBB)
Add a new node to the dominator tree information.
DomTreeNodeBase< NodeT > * getNode(const NodeT *BB) const
getNode - return the (Post)DominatorTree node for the specified basic block.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Class to represent profile counts.
void addFnAttr(Attribute::AttrKind Kind)
Add function attributes to this function.
void setSubprogram(DISubprogram *SP)
Set the attached subprogram.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
const BasicBlock & getEntryBlock() const
const BasicBlock & front() const
DISubprogram * getSubprogram() const
Get the attached subprogram.
bool IsNewDbgInfoFormat
Is this function using intrinsics to record the position of debugging information,...
bool hasPersonalityFn() const
Check whether this function has a personality function.
Constant * getPersonalityFn() const
Get the personality function associated with this function.
void setPersonalityFn(Constant *Fn)
AttributeList getAttributes() const
Return the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
adds the attribute to the list of attributes for the given arg.
Function::iterator insert(Function::iterator Position, BasicBlock *BB)
Insert BB in the basic block list at Position.
Type * getReturnType() const
Returns the type of the ret val.
void setEntryCount(ProfileCount Count, const DenseSet< GlobalValue::GUID > *Imports=nullptr)
Set the entry count for this function.
bool isVarArg() const
isVarArg - Return true if this function takes a variable number of arguments.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
unsigned getAddressSpace() const
Module * getParent()
Get the module that this global value is contained inside of...
@ InternalLinkage
Rename collisions when linking (static functions).
bool isLifetimeStartOrEnd() const LLVM_READONLY
Return true if the instruction is a llvm.lifetime.start or llvm.lifetime.end marker.
unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const BasicBlock * getParent() const
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
const Function * getFunction() const
Return the function this instruction belongs to.
BasicBlock * getSuccessor(unsigned Idx) const LLVM_READONLY
Return the specified successor. This instruction must be a terminator.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
void setSuccessor(unsigned Idx, BasicBlock *BB)
Update the specified successor to point at the provided block.
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
InstListType::iterator insertInto(BasicBlock *ParentBB, InstListType::iterator It)
Inserts an unlinked instruction into ParentBB at position It and returns the iterator of the inserted...
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
Value * getPointerOperand()
Represents a single loop in the control flow graph.
MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight)
Return metadata containing two branch weights.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
A Module instance is used to store all the information related to an LLVM module.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
void setIncomingBlock(unsigned i, BasicBlock *BB)
Value * removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty=true)
Remove an incoming value.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Return a value (possibly void), from a function.
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, Instruction *InsertBefore=nullptr)
A vector that has set insertion semantics.
ArrayRef< value_type > getArrayRef() const
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
std::string str() const
str - Get the contents as an std::string.
constexpr bool empty() const
empty - Check if the string is empty.
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Type * getElementType(unsigned N) const
BasicBlock * getSuccessor(unsigned idx) const
static SwitchInst * Create(Value *Value, BasicBlock *Default, unsigned NumCases, Instruction *InsertBefore=nullptr)
void setCondition(Value *V)
void addCase(ConstantInt *OnVal, BasicBlock *Dest)
Add an entry to the switch instruction.
void setDefaultDest(BasicBlock *DefaultCase)
Value * getCondition() const
CaseIt removeCase(CaseIt I)
This method removes the specified case and its successor from the switch instruction.
The instances of the Type class are immutable: once they are created, they are never changed.
static IntegerType * getInt1Ty(LLVMContext &C)
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isVoidTy() const
Return true if this is 'void'.
bool replaceUsesOfWith(Value *From, Value *To)
Replace uses of one Value with another.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
user_iterator user_begin()
void setName(const Twine &Name)
Change the name of the value.
const Value * stripInBoundsConstantOffsets() const
Strip off pointer casts and all-constant inbounds GEPs.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
const Value * stripInBoundsOffsets(function_ref< void(const Value *)> Func=[](const Value *) {}) const
Strip off pointer casts and inbounds GEPs.
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
void dump() const
Support for debugging, callable in GDB: V->dump()
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
NodeAddr< PhiNode * > Phi
This is an optimization pass for GlobalISel generic memory operations.
bool stripDebugInfo(Function &F)
Function::ProfileCount ProfileCount
bool verifyFunction(const Function &F, raw_ostream *OS=nullptr)
Check a function for errors, useful for use when debugging a pass.
auto successors(const MachineBasicBlock *BB)
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
void findDbgUsers(SmallVectorImpl< DbgVariableIntrinsic * > &DbgInsts, Value *V, SmallVectorImpl< DPValue * > *DPValues=nullptr)
Finds the debug info intrinsics describing a value.
BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
auto predecessors(const MachineBasicBlock *BB)
void updateLoopMetadataDebugLocations(Instruction &I, function_ref< Metadata *(Metadata *)> Updater)
Update the debug locations contained within the MD_loop metadata attached to the instruction I,...
Representative of a block.
Distribution of unscaled probability weight.