64 #define DEBUG_TYPE "partial-inlining"
67 "Number of callsites functions partially inlined into.");
68 STATISTIC(NumColdOutlinePartialInlined,
"Number of times functions with "
69 "cold outlined regions were partially "
70 "inlined into its caller(s).");
72 "Number of cold single entry/exit regions found.");
74 "Number of cold single entry/exit regions outlined.");
84 cl::desc(
"Disable multi-region partial inlining"));
90 cl::desc(
"Force outline regions with live exits"));
96 cl::desc(
"Mark outline function calls with ColdCC"));
109 cl::desc(
"Minimum ratio comparing relative sizes of each "
110 "outline candidate and original function"));
115 cl::desc(
"Minimum block executions to consider "
116 "its BranchProbabilityInfo valid"));
121 cl::desc(
"Minimum BranchProbability to consider a region cold."));
125 cl::desc(
"Max number of blocks to be partially inlined"));
131 cl::desc(
"Max number of partial inlining. The default is unlimited"));
139 cl::desc(
"Relative frequency of outline region to "
144 cl::desc(
"A debug option to add additional penalty to the computed one."));
148 struct FunctionOutliningInfo {
149 FunctionOutliningInfo() =
default;
153 unsigned getNumInlinedBlocks()
const {
return Entries.size() + 1; }
169 struct FunctionOutliningMultiRegionInfo {
170 FunctionOutliningMultiRegionInfo() =
default;
173 struct OutlineRegionInfo {
178 ExitBlock(ExitBlock), ReturnBlock(ReturnBlock) {}
188 struct PartialInlinerImpl {
197 : GetAssumptionCache(GetAC), LookupAssumptionCache(LookupAC),
198 GetTTI(GTTI), GetBFI(GBFI), GetTLI(GTLI), PSI(ProfSI) {}
208 std::pair<bool, Function *> unswitchFunction(
Function &
F);
214 struct FunctionCloner {
217 FunctionCloner(
Function *
F, FunctionOutliningInfo *OI,
221 FunctionCloner(
Function *
F, FunctionOutliningMultiRegionInfo *OMRI,
231 void normalizeReturnBlock()
const;
234 bool doMultiRegionFunctionOutlining();
241 Function *doSingleRegionFunctionOutlining();
246 typedef std::pair<Function *, BasicBlock *> FuncBodyCallerPair;
252 bool IsFunctionInlined =
false;
256 std::unique_ptr<FunctionOutliningInfo> ClonedOI =
nullptr;
258 std::unique_ptr<FunctionOutliningMultiRegionInfo> ClonedOMRI =
nullptr;
259 std::unique_ptr<BlockFrequencyInfo> ClonedFuncBFI =
nullptr;
266 int NumPartialInlining = 0;
279 getOutliningCallBBRelativeFreq(FunctionCloner &Cloner)
const;
283 bool shouldPartialInline(
CallBase &CB, FunctionCloner &Cloner,
290 bool tryPartialInline(FunctionCloner &Cloner);
295 computeCallsiteToProfCountMap(
Function *DuplicateFunction,
298 bool isLimitReached()
const {
304 if (isa<CallInst>(U) || isa<InvokeInst>(U))
305 return cast<CallBase>(U);
312 return getSupportedCallBase(
User);
315 std::tuple<DebugLoc, BasicBlock *> getOneDebugLoc(
Function &
F)
const {
319 return std::make_tuple(DLoc, Block);
328 std::tuple<InstructionCost, InstructionCost>
329 computeOutliningCosts(FunctionCloner &Cloner)
const;
337 std::unique_ptr<FunctionOutliningInfo>
340 std::unique_ptr<FunctionOutliningMultiRegionInfo>
341 computeOutliningColdRegionsInfo(
Function &
F,
345 struct PartialInlinerLegacyPass :
public ModulePass {
359 bool runOnModule(
Module &M)
override {
365 &getAnalysis<TargetTransformInfoWrapperPass>();
367 getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
382 return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
F);
385 return PartialInlinerImpl(GetAssumptionCache, LookupAssumptionCache, GetTTI,
393 std::unique_ptr<FunctionOutliningMultiRegionInfo>
394 PartialInlinerImpl::computeOutliningColdRegionsInfo(
401 std::unique_ptr<BlockFrequencyInfo> ScopedBFI;
405 BFI = ScopedBFI.get();
411 return std::unique_ptr<FunctionOutliningMultiRegionInfo>();
413 std::unique_ptr<FunctionOutliningMultiRegionInfo> OutliningInfo =
414 std::make_unique<FunctionOutliningMultiRegionInfo>();
419 for (
auto *Block : BlockList) {
426 <<
"Region dominated by "
427 <<
ore::NV(
"Block", BlockList.front()->getName())
428 <<
" has more than one region exit edge.";
441 return BFI->getBlockProfileCount(
BB).value_or(0);
449 OverallFunctionCost += computeBBInlineCost(&
BB, FTTI);
451 LLVM_DEBUG(
dbgs() <<
"OverallFunctionCost = " << OverallFunctionCost
460 bool ColdCandidateFound =
false;
462 std::vector<BasicBlock *> DFS;
464 DFS.push_back(CurrEntry);
465 VisitedMap[CurrEntry] =
true;
473 while (!DFS.empty()) {
474 auto *ThisBB = DFS.back();
485 VisitedMap[*
SI] =
true;
489 if (SuccProb > MinBranchProbability)
492 LLVM_DEBUG(
dbgs() <<
"Found cold edge: " << ThisBB->getName() <<
"->"
494 <<
"\nBranch Probability = " << SuccProb <<
"\n";);
497 DT.getDescendants(*
SI, DominateVector);
498 assert(!DominateVector.empty() &&
499 "SI should be reachable and have at least itself as descendant");
502 if (!DominateVector.front()->hasNPredecessors(1)) {
504 <<
" doesn't have a single predecessor in the "
505 "dominator tree\n";);
511 if (!(ExitBlock = IsSingleExit(DominateVector))) {
513 <<
" doesn't have a unique successor\n";);
518 for (
auto *
BB : DominateVector)
519 OutlineRegionCost += computeBBInlineCost(
BB, &GetTTI(*
BB->getParent()));
529 <<
" inline cost-savings smaller than "
530 <<
ore::NV(
"Cost", MinOutlineRegionCost);
533 LLVM_DEBUG(
dbgs() <<
"ABORT: Outline region cost is smaller than "
534 << MinOutlineRegionCost <<
"\n";);
542 for (
auto *
BB : DominateVector)
543 VisitedMap[
BB] =
true;
547 FunctionOutliningMultiRegionInfo::OutlineRegionInfo RegInfo(
548 DominateVector, DominateVector.front(), ExitBlock, ReturnBlock);
549 OutliningInfo->ORI.push_back(RegInfo);
551 << DominateVector.front()->getName() <<
"\n";);
552 ColdCandidateFound =
true;
553 NumColdRegionsFound++;
557 if (ColdCandidateFound)
558 return OutliningInfo;
560 return std::unique_ptr<FunctionOutliningMultiRegionInfo>();
563 std::unique_ptr<FunctionOutliningInfo>
564 PartialInlinerImpl::computeOutliningInfo(
Function &
F)
const {
567 if (!
BR ||
BR->isUnconditional())
568 return std::unique_ptr<FunctionOutliningInfo>();
577 return isa<ReturnInst>(TI);
581 if (IsReturnBlock(Succ1))
582 return std::make_tuple(Succ1, Succ2);
583 if (IsReturnBlock(Succ2))
584 return std::make_tuple(Succ2, Succ1);
586 return std::make_tuple<BasicBlock *, BasicBlock *>(
nullptr,
nullptr);
591 if (IsSuccessor(Succ1, Succ2))
592 return std::make_tuple(Succ1, Succ2);
593 if (IsSuccessor(Succ2, Succ1))
594 return std::make_tuple(Succ2, Succ1);
596 return std::make_tuple<BasicBlock *, BasicBlock *>(
nullptr,
nullptr);
599 std::unique_ptr<FunctionOutliningInfo> OutliningInfo =
600 std::make_unique<FunctionOutliningInfo>();
603 bool CandidateFound =
false;
618 std::tie(ReturnBlock, NonReturnBlock) = GetReturnBlock(Succ1, Succ2);
621 OutliningInfo->Entries.push_back(CurrEntry);
622 OutliningInfo->ReturnBlock = ReturnBlock;
623 OutliningInfo->NonReturnBlock = NonReturnBlock;
624 CandidateFound =
true;
629 std::tie(CommSucc,
OtherSucc) = GetCommonSucc(Succ1, Succ2);
634 OutliningInfo->Entries.push_back(CurrEntry);
639 return std::unique_ptr<FunctionOutliningInfo>();
643 assert(OutliningInfo->Entries[0] == &
F.front() &&
644 "Function Entry must be the first in Entries vector");
653 if (!Entries.count(Pred))
658 auto CheckAndNormalizeCandidate =
659 [Entries, HasNonEntryPred](FunctionOutliningInfo *OutliningInfo) {
662 if (Entries.count(Succ))
664 if (Succ == OutliningInfo->ReturnBlock)
665 OutliningInfo->ReturnBlockPreds.push_back(
E);
666 else if (Succ != OutliningInfo->NonReturnBlock)
670 if (HasNonEntryPred(
E))
676 if (!CheckAndNormalizeCandidate(OutliningInfo.get()))
677 return std::unique_ptr<FunctionOutliningInfo>();
682 BasicBlock *Cand = OutliningInfo->NonReturnBlock;
686 if (HasNonEntryPred(Cand))
693 std::tie(ReturnBlock, NonReturnBlock) = GetReturnBlock(Succ1, Succ2);
694 if (!ReturnBlock || ReturnBlock != OutliningInfo->ReturnBlock)
701 OutliningInfo->Entries.push_back(Cand);
702 OutliningInfo->NonReturnBlock = NonReturnBlock;
703 OutliningInfo->ReturnBlockPreds.push_back(Cand);
704 Entries.insert(Cand);
707 return OutliningInfo;
712 if (
F.hasProfileData())
715 for (
auto *
E : OI.Entries) {
717 if (!
BR ||
BR->isUnconditional())
720 if (
BR->extractProfMetadata(
T,
F))
727 FunctionCloner &Cloner)
const {
728 BasicBlock *OutliningCallBB = Cloner.OutlinedFunctions.back().second;
730 Cloner.ClonedFuncBFI->getBlockFreq(&Cloner.ClonedFunc->getEntryBlock());
731 auto OutliningCallFreq =
732 Cloner.ClonedFuncBFI->getBlockFreq(OutliningCallBB);
736 if (OutliningCallFreq.getFrequency() > EntryFreq.getFrequency())
737 OutliningCallFreq = EntryFreq;
740 OutliningCallFreq.getFrequency(), EntryFreq.getFrequency());
743 return OutlineRegionRelFreq;
758 return OutlineRegionRelFreq;
763 return OutlineRegionRelFreq;
766 bool PartialInlinerImpl::shouldPartialInline(
772 assert(Callee == Cloner.ClonedFunc);
778 auto &CalleeTTI = GetTTI(*Callee);
779 bool RemarksEnabled =
780 Callee->getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
784 GetTLI, GetBFI, &PSI, RemarksEnabled ? &ORE :
nullptr);
789 <<
NV(
"Callee", Cloner.OrigFunc)
790 <<
" should always be fully inlined, not partially";
798 <<
NV(
"Callee", Cloner.OrigFunc) <<
" not partially inlined into "
799 <<
NV(
"Caller", Caller)
800 <<
" because it should never be inlined (cost=never)";
808 <<
NV(
"Callee", Cloner.OrigFunc) <<
" not partially inlined into "
809 <<
NV(
"Caller", Caller) <<
" because too costly to inline (cost="
810 <<
NV(
"Cost", IC.
getCost()) <<
", threshold="
822 if (NormWeightedSavings < WeightedOutliningRcost) {
826 <<
NV(
"Callee", Cloner.OrigFunc) <<
" not partially inlined into "
827 <<
NV(
"Caller", Caller) <<
" runtime overhead (overhead="
828 <<
NV(
"Overhead", (
unsigned)WeightedOutliningRcost.
getFrequency())
830 <<
NV(
"Savings", (
unsigned)NormWeightedSavings.getFrequency())
832 <<
" of making the outlined call is too high";
840 <<
NV(
"Callee", Cloner.OrigFunc) <<
" can be partially inlined into "
841 <<
NV(
"Caller", Caller) <<
" with cost=" <<
NV(
"Cost", IC.
getCost())
855 const DataLayout &
DL =
BB->getParent()->getParent()->getDataLayout();
858 switch (
I.getOpcode()) {
859 case Instruction::BitCast:
860 case Instruction::PtrToInt:
861 case Instruction::IntToPtr:
862 case Instruction::Alloca:
863 case Instruction::PHI:
865 case Instruction::GetElementPtr:
866 if (cast<GetElementPtrInst>(&
I)->hasAllZeroIndices())
873 if (
I.isLifetimeStartOrEnd())
876 if (
auto *II = dyn_cast<IntrinsicInst>(&
I)) {
880 for (
Value *Val : II->args())
881 Tys.push_back(Val->getType());
883 if (
auto *FPMO = dyn_cast<FPMathOperator>(II))
884 FMF = FPMO->getFastMathFlags();
891 if (
CallInst *CI = dyn_cast<CallInst>(&
I)) {
911 std::tuple<InstructionCost, InstructionCost>
912 PartialInlinerImpl::computeOutliningCosts(FunctionCloner &Cloner)
const {
914 for (
auto FuncBBPair : Cloner.OutlinedFunctions) {
915 Function *OutlinedFunc = FuncBBPair.first;
916 BasicBlock* OutliningCallBB = FuncBBPair.second;
919 auto *OutlinedFuncTTI = &GetTTI(*OutlinedFunc);
920 OutliningFuncCallCost +=
921 computeBBInlineCost(OutliningCallBB, OutlinedFuncTTI);
925 OutlinedFunctionCost += computeBBInlineCost(&
BB, OutlinedFuncTTI);
927 assert(OutlinedFunctionCost >= Cloner.OutlinedRegionCost &&
928 "Outlined function cost should be no less than the outlined region");
933 OutlinedFunctionCost -=
937 OutliningFuncCallCost +
938 (OutlinedFunctionCost - Cloner.OutlinedRegionCost) +
941 return std::make_tuple(OutliningFuncCallCost, OutliningRuntimeOverhead);
947 void PartialInlinerImpl::computeCallsiteToProfCountMap(
953 std::unique_ptr<BlockFrequencyInfo> TempBFI;
963 CurrentCallerBFI = TempBFI.get();
966 CurrentCallerBFI = &(GetBFI(*Caller));
972 if (isa<BlockAddress>(
User))
976 if (CurrentCaller != Caller) {
978 ComputeCurrBFI(Caller);
980 assert(CurrentCallerBFI &&
"CallerBFI is not set");
985 CallSiteToProfCountMap[
User] = *Count;
987 CallSiteToProfCountMap[
User] = 0;
991 PartialInlinerImpl::FunctionCloner::FunctionCloner(
995 : OrigFunc(
F), ORE(ORE), LookupAC(LookupAC), GetTTI(GetTTI) {
996 ClonedOI = std::make_unique<FunctionOutliningInfo>();
1002 ClonedOI->ReturnBlock = cast<BasicBlock>(VMap[OI->ReturnBlock]);
1003 ClonedOI->NonReturnBlock = cast<BasicBlock>(VMap[OI->NonReturnBlock]);
1005 ClonedOI->Entries.push_back(cast<BasicBlock>(VMap[
BB]));
1009 ClonedOI->ReturnBlockPreds.push_back(NewE);
1013 F->replaceAllUsesWith(ClonedFunc);
1016 PartialInlinerImpl::FunctionCloner::FunctionCloner(
1017 Function *
F, FunctionOutliningMultiRegionInfo *OI,
1021 : OrigFunc(
F), ORE(ORE), LookupAC(LookupAC), GetTTI(GetTTI) {
1022 ClonedOMRI = std::make_unique<FunctionOutliningMultiRegionInfo>();
1030 for (FunctionOutliningMultiRegionInfo::OutlineRegionInfo
RegionInfo :
1034 Region.push_back(cast<BasicBlock>(VMap[
BB]));
1040 NewReturnBlock = cast<BasicBlock>(VMap[
RegionInfo.ReturnBlock]);
1041 FunctionOutliningMultiRegionInfo::OutlineRegionInfo MappedRegionInfo(
1042 Region, NewEntryBlock, NewExitBlock, NewReturnBlock);
1043 ClonedOMRI->ORI.push_back(MappedRegionInfo);
1047 F->replaceAllUsesWith(ClonedFunc);
1050 void PartialInlinerImpl::FunctionCloner::normalizeReturnBlock()
const {
1054 while (
I !=
BB->end()) {
1055 PHINode *Phi = dyn_cast<PHINode>(
I);
1075 BasicBlock *PreReturn = ClonedOI->ReturnBlock;
1077 PHINode *FirstPhi = GetFirstPHI(PreReturn);
1078 unsigned NumPredsFromEntries = ClonedOI->ReturnBlockPreds.size();
1084 Value *CommonValue = PN->getIncomingValue(0);
1085 if (
all_of(PN->incoming_values(),
1086 [&](
Value *V) { return V == CommonValue; }))
1091 ClonedOI->ReturnBlock = ClonedOI->ReturnBlock->splitBasicBlock(
1092 ClonedOI->ReturnBlock->getFirstNonPHI()->getIterator());
1096 while (
I != PreReturn->
end()) {
1097 PHINode *OldPhi = dyn_cast<PHINode>(
I);
1102 PHINode::Create(OldPhi->
getType(), NumPredsFromEntries + 1,
"",
Ins);
1104 Ins = ClonedOI->ReturnBlock->getFirstNonPHI();
1107 for (
BasicBlock *
E : ClonedOI->ReturnBlockPreds) {
1116 if (
auto *OldPhiVal = IsTrivialPhi(OldPhi)) {
1118 DeadPhis.push_back(OldPhi);
1122 for (
auto *
DP : DeadPhis)
1123 DP->eraseFromParent();
1125 for (
auto *
E : ClonedOI->ReturnBlockPreds)
1126 E->getTerminator()->replaceUsesOfWith(PreReturn, ClonedOI->ReturnBlock);
1129 bool PartialInlinerImpl::FunctionCloner::doMultiRegionFunctionOutlining() {
1131 auto ComputeRegionCost =
1135 Cost += computeBBInlineCost(
BB, &GetTTI(*
BB->getParent()));
1139 assert(ClonedOMRI &&
"Expecting OutlineInfo for multi region outline");
1141 if (ClonedOMRI->ORI.empty())
1157 for (FunctionOutliningMultiRegionInfo::OutlineRegionInfo
RegionInfo :
1163 ClonedFuncBFI.get(), &BPI,
1164 LookupAC(*
RegionInfo.EntryBlock->getParent()),
1167 CE.findInputsOutputs(Inputs, Outputs, Sinks);
1170 dbgs() <<
"inputs: " << Inputs.
size() <<
"\n";
1171 dbgs() <<
"outputs: " << Outputs.
size() <<
"\n";
1172 for (
Value *value : Inputs)
1173 dbgs() <<
"value used in func: " << *value <<
"\n";
1175 dbgs() <<
"instr used in func: " << *
output <<
"\n";
1182 if (
Function *OutlinedFunc =
CE.extractCodeRegion(CEAC)) {
1183 CallBase *OCS = PartialInlinerImpl::getOneCallSiteTo(*OutlinedFunc);
1186 OutlinedFunctions.push_back(std::make_pair(OutlinedFunc,OutliningCallBB));
1187 NumColdRegionsOutlined++;
1188 OutlinedRegionCost += CurrentOutlinedRegionCost;
1198 <<
"Failed to extract region at block "
1203 return !OutlinedFunctions.empty();
1207 PartialInlinerImpl::FunctionCloner::doSingleRegionFunctionOutlining() {
1211 return BB == ClonedOI->ReturnBlock ||
1215 assert(ClonedOI &&
"Expecting OutlineInfo for single region outline");
1226 std::vector<BasicBlock *> ToExtract;
1227 auto *ClonedFuncTTI = &GetTTI(*ClonedFunc);
1228 ToExtract.push_back(ClonedOI->NonReturnBlock);
1229 OutlinedRegionCost += PartialInlinerImpl::computeBBInlineCost(
1230 ClonedOI->NonReturnBlock, ClonedFuncTTI);
1232 if (!ToBeInlined(&
BB) && &
BB != ClonedOI->NonReturnBlock) {
1233 ToExtract.push_back(&
BB);
1238 OutlinedRegionCost += computeBBInlineCost(&
BB, ClonedFuncTTI);
1245 ClonedFuncBFI.get(), &BPI, LookupAC(*ClonedFunc),
1251 PartialInlinerImpl::getOneCallSiteTo(*OutlinedFunc)->
getParent();
1253 OutlinedFunctions.push_back(std::make_pair(OutlinedFunc, OutliningCallBB));
1257 &ToExtract.front()->front())
1258 <<
"Failed to extract region at block "
1259 <<
ore::NV(
"Block", ToExtract.front());
1262 return OutlinedFunc;
1265 PartialInlinerImpl::FunctionCloner::~FunctionCloner() {
1269 ClonedFunc->eraseFromParent();
1270 if (!IsFunctionInlined) {
1273 for (
auto FuncBBPair : OutlinedFunctions) {
1275 Func->eraseFromParent();
1280 std::pair<bool, Function *> PartialInlinerImpl::unswitchFunction(
Function &
F) {
1281 if (
F.hasAddressTaken())
1282 return {
false,
nullptr};
1285 if (
F.hasFnAttribute(Attribute::AlwaysInline))
1286 return {
false,
nullptr};
1288 if (
F.hasFnAttribute(Attribute::NoInline))
1289 return {
false,
nullptr};
1292 return {
false,
nullptr};
1294 if (
F.users().empty())
1295 return {
false,
nullptr};
1303 std::unique_ptr<FunctionOutliningMultiRegionInfo> OMRI =
1304 computeOutliningColdRegionsInfo(
F, ORE);
1306 FunctionCloner Cloner(&
F, OMRI.get(), ORE, LookupAssumptionCache, GetTTI);
1314 bool DidOutline = Cloner.doMultiRegionFunctionOutlining();
1318 dbgs() <<
">>>>>> Outlined (Cloned) Function >>>>>>\n";
1319 Cloner.ClonedFunc->print(
dbgs());
1320 dbgs() <<
"<<<<<< Outlined (Cloned) Function <<<<<<\n";
1323 if (tryPartialInline(Cloner))
1324 return {
true,
nullptr};
1332 std::unique_ptr<FunctionOutliningInfo> OI = computeOutliningInfo(
F);
1334 return {
false,
nullptr};
1336 FunctionCloner Cloner(&
F, OI.get(), ORE, LookupAssumptionCache, GetTTI);
1337 Cloner.normalizeReturnBlock();
1339 Function *OutlinedFunction = Cloner.doSingleRegionFunctionOutlining();
1341 if (!OutlinedFunction)
1342 return {
false,
nullptr};
1344 if (tryPartialInline(Cloner))
1345 return {
true, OutlinedFunction};
1347 return {
false,
nullptr};
1350 bool PartialInlinerImpl::tryPartialInline(FunctionCloner &Cloner) {
1351 if (Cloner.OutlinedFunctions.empty())
1356 int NonWeightedRcost;
1358 auto OutliningCosts = computeOutliningCosts(Cloner);
1360 std::get<1>(OutliningCosts).
isValid() &&
"Expected valid costs");
1362 SizeCost = *std::get<0>(OutliningCosts).getValue();
1363 NonWeightedRcost = *std::get<1>(OutliningCosts).getValue();
1368 if (Cloner.ClonedOI)
1369 RelativeToEntryFreq = getOutliningCallBBRelativeFreq(Cloner);
1378 WeightedRcost =
BlockFrequency(NonWeightedRcost) * RelativeToEntryFreq;
1388 std::tie(DLoc, Block) = getOneDebugLoc(*Cloner.ClonedFunc);
1389 OrigFuncORE.emit([&]() {
1392 <<
ore::NV(
"Function", Cloner.OrigFunc)
1393 <<
" not partially inlined into callers (Original Size = "
1394 <<
ore::NV(
"OutlinedRegionOriginalSize", Cloner.OutlinedRegionCost)
1395 <<
", Size of call sequence to outlined function = "
1396 <<
ore::NV(
"NewSize", SizeCost) <<
")";
1401 assert(Cloner.OrigFunc->users().empty() &&
1402 "F's users should all be replaced!");
1404 std::vector<User *>
Users(Cloner.ClonedFunc->user_begin(),
1405 Cloner.ClonedFunc->user_end());
1408 auto CalleeEntryCount = Cloner.OrigFunc->getEntryCount();
1409 if (CalleeEntryCount)
1410 computeCallsiteToProfCountMap(Cloner.ClonedFunc, CallSiteToProfCountMap);
1413 (CalleeEntryCount ? CalleeEntryCount->getCount() : 0);
1415 bool AnyInline =
false;
1418 if (isa<BlockAddress>(
User))
1423 if (isLimitReached())
1427 if (!shouldPartialInline(*CB, Cloner, WeightedRcost, CallerORE))
1433 OR <<
ore::NV(
"Callee", Cloner.OrigFunc) <<
" partially inlined into "
1440 (Cloner.ClonedOI ? Cloner.OutlinedFunctions.back().first
1448 if (CalleeEntryCountV && CallSiteToProfCountMap.
count(
User)) {
1450 CalleeEntryCountV -=
std::min(CalleeEntryCountV, CallSiteCount);
1454 NumPartialInlining++;
1456 if (Cloner.ClonedOI)
1457 NumPartialInlined++;
1459 NumColdOutlinePartialInlined++;
1463 Cloner.IsFunctionInlined =
true;
1464 if (CalleeEntryCount)
1466 CalleeEntryCountV, CalleeEntryCount->getType()));
1468 OrigFuncORE.emit([&]() {
1470 <<
"Partially inlined into at least one caller";
1481 std::vector<Function *> Worklist;
1484 if (!
F.use_empty() && !
F.isDeclaration())
1485 Worklist.push_back(&
F);
1487 bool Changed =
false;
1488 while (!Worklist.empty()) {
1490 Worklist.pop_back();
1495 bool Recursive =
false;
1498 if (
I->getParent()->getParent() == CurrFunc) {
1505 std::pair<bool, Function *>
Result = unswitchFunction(*CurrFunc);
1507 Worklist.push_back(
Result.second);
1517 "Partial Inliner",
false,
false)
1526 return new PartialInlinerLegacyPass();
1555 if (PartialInlinerImpl(GetAssumptionCache, LookupAssumptionCache, GetTTI,
1556 GetTLI, PSI, GetBFI)