129#define DEBUG_TYPE "wholeprogramdevirt"
131STATISTIC(NumDevirtTargets,
"Number of whole program devirtualization targets");
132STATISTIC(NumSingleImpl,
"Number of single implementation devirtualizations");
134STATISTIC(NumUniformRetVal,
"Number of uniform return value optimizations");
135STATISTIC(NumUniqueRetVal,
"Number of unique return value optimizations");
137 "Number of 1 bit virtual constant propagations");
138STATISTIC(NumVirtConstProp,
"Number of virtual constant propagations");
140 "Controls how many calls should be devirtualized.");
146 cl::desc(
"Print index-based devirtualization messages"));
154 cl::desc(
"Enable whole program visibility"));
159 "disable-whole-program-visibility",
cl::Hidden,
160 cl::desc(
"Disable whole program visibility (overrides enabling options)"));
165 cl::desc(
"Prevent function(s) from being devirtualized"),
188 "wholeprogramdevirt-keep-unreachable-function",
189 cl::desc(
"Regard unreachable functions as possible devirtualize targets."),
194 std::vector<GlobPattern> Patterns;
195 template <
class T>
void init(
const T &StringList) {
196 for (
const auto &S : StringList)
198 Patterns.push_back(std::move(*Pat));
200 bool match(StringRef S) {
201 for (
const GlobPattern &
P : Patterns)
214 bool IsAfter, uint64_t
Size) {
216 uint64_t MinByte = 0;
219 MinByte = std::max(MinByte,
Target.minAfterBytes());
221 MinByte = std::max(MinByte,
Target.minBeforeBytes());
244 std::vector<ArrayRef<uint8_t>> Used;
247 :
Target.TM->Bits->Before.BytesUsed;
248 uint64_t
Offset = IsAfter ? MinByte -
Target.minAfterBytes()
249 : MinByte -
Target.minBeforeBytes();
259 for (
unsigned I = 0;; ++
I) {
261 for (
auto &&
B : Used)
264 if (BitsUsed != 0xff)
270 for (
unsigned I = 0;; ++
I) {
271 for (
auto &&
B : Used) {
273 while ((
I + Byte) <
B.size() && Byte < (
Size / 8)) {
289 unsigned BitWidth, int64_t &OffsetByte, uint64_t &OffsetBit) {
291 OffsetByte = -(AllocBefore / 8 + 1);
293 OffsetByte = -((AllocBefore + 7) / 8 + (
BitWidth + 7) / 8);
294 OffsetBit = AllocBefore % 8;
298 Target.setBeforeBit(AllocBefore);
306 unsigned BitWidth, int64_t &OffsetByte, uint64_t &OffsetBit) {
308 OffsetByte = AllocAfter / 8;
310 OffsetByte = (AllocAfter + 7) / 8;
311 OffsetBit = AllocAfter % 8;
315 Target.setAfterBit(AllocAfter);
344 const VTableSlot &RHS) {
345 return LHS.TypeID == RHS.TypeID && LHS.ByteOffset == RHS.ByteOffset;
356 return LHS.TypeID == RHS.TypeID && LHS.ByteOffset == RHS.ByteOffset;
380 if (!Summary->isLive())
383 if (!FS->fflags().MustBeUnreachable)
398struct VirtualCallSite {
405 unsigned *NumUnsafeUses =
nullptr;
408 emitRemark(
const StringRef OptName,
const StringRef TargetName,
409 function_ref<OptimizationRemarkEmitter &(
Function &)> OREGetter) {
415 OREGetter(*F).emit(OptimizationRemark(
DEBUG_TYPE, OptName, DLoc,
Block)
416 <<
NV(
"Optimization", OptName)
417 <<
": devirtualized a call to "
418 <<
NV(
"FunctionName", TargetName));
421 void replaceAndErase(
422 const StringRef OptName,
const StringRef TargetName,
bool RemarksEnabled,
423 function_ref<OptimizationRemarkEmitter &(
Function &)> OREGetter,
430 II->getUnwindDest()->removePredecessor(
II->getParent());
447 std::vector<VirtualCallSite> CallSites;
456 bool AllCallSitesDevirted =
true;
465 std::vector<FunctionSummary *> SummaryTypeCheckedLoadUsers;
469 std::vector<FunctionSummary *> SummaryTypeTestAssumeUsers;
471 bool isExported()
const {
472 return !SummaryTypeCheckedLoadUsers.empty() ||
473 !SummaryTypeTestAssumeUsers.empty();
476 void addSummaryTypeCheckedLoadUser(FunctionSummary *FS) {
477 SummaryTypeCheckedLoadUsers.push_back(FS);
478 AllCallSitesDevirted =
false;
481 void addSummaryTypeTestAssumeUser(FunctionSummary *FS) {
482 SummaryTypeTestAssumeUsers.push_back(FS);
483 AllCallSitesDevirted =
false;
486 void markDevirt() { AllCallSitesDevirted =
true; }
490struct VTableSlotInfo {
497 std::map<std::vector<uint64_t>,
CallSiteInfo> ConstCSInfo;
499 void addCallSite(
Value *VTable, CallBase &CB,
unsigned *NumUnsafeUses);
505CallSiteInfo &VTableSlotInfo::findCallSiteInfo(CallBase &CB) {
506 std::vector<uint64_t>
Args;
508 if (!CBType || CBType->getBitWidth() > 64 || CB.
arg_empty())
512 if (!CI || CI->getBitWidth() > 64)
514 Args.push_back(CI->getZExtValue());
516 return ConstCSInfo[
Args];
519void VTableSlotInfo::addCallSite(
Value *VTable, CallBase &CB,
520 unsigned *NumUnsafeUses) {
521 auto &CSI = findCallSiteInfo(CB);
522 CSI.AllCallSitesDevirted =
false;
523 CSI.CallSites.push_back({
VTable, CB, NumUnsafeUses});
527 const IPOOptions &Opts;
532 ModuleSummaryIndex *
const ExportSummary;
533 const ModuleSummaryIndex *
const ImportSummary;
535 IntegerType *
const Int8Ty;
537 IntegerType *
const Int32Ty;
538 IntegerType *
const Int64Ty;
545 const bool RemarksEnabled;
546 std::function<OptimizationRemarkEmitter &(
Function &)> OREGetter;
547 MapVector<VTableSlot, VTableSlotInfo> CallSlots;
552 SmallPtrSet<CallBase *, 8> OptimizedCalls;
566 std::map<CallInst *, unsigned> NumUnsafeUsesForTypeTest;
567 PatternList FunctionsToSkip;
569 const bool DevirtSpeculatively;
571 ModuleSummaryIndex *ExportSummary,
572 const ModuleSummaryIndex *ImportSummary,
573 bool DevirtSpeculatively)
576 ExportSummary(ExportSummary), ImportSummary(ImportSummary),
583 RemarksEnabled(areRemarksEnabled()),
584 OREGetter([&](
Function &
F) -> OptimizationRemarkEmitter & {
588 DevirtSpeculatively ||
590 assert(!(ExportSummary && ImportSummary));
594 bool areRemarksEnabled();
597 scanTypeTestUsers(
Function *TypeTestFunc,
598 DenseMap<
Metadata *, std::set<TypeMemberInfo>> &TypeIdMap);
599 void scanTypeCheckedLoadUsers(
Function *TypeCheckedLoadFunc);
601 void buildTypeIdentifierMap(
602 std::vector<VTableBits> &Bits,
603 DenseMap<
Metadata *, std::set<TypeMemberInfo>> &TypeIdMap);
605 bool tryFindVirtualCallTargets(
606 std::vector<VirtualCallTarget> &TargetsForSlot,
607 const std::set<TypeMemberInfo> &TypeMemberInfos,
uint64_t ByteOffset,
608 ModuleSummaryIndex *ExportSummary,
bool &HasPublicTarget);
610 void applySingleImplDevirt(VTableSlotInfo &SlotInfo, Constant *TheFn,
611 bool HasPublicTarget,
bool &IsExported);
612 bool trySingleImplDevirt(ModuleSummaryIndex *ExportSummary,
614 VTableSlotInfo &SlotInfo,
615 WholeProgramDevirtResolution *Res,
616 bool HasPublicTarget);
618 void applyICallBranchFunnel(VTableSlotInfo &SlotInfo,
Function &JT,
621 VTableSlotInfo &SlotInfo,
622 WholeProgramDevirtResolution *Res, VTableSlot Slot);
624 bool tryEvaluateFunctionsWithArgs(
626 ArrayRef<uint64_t> Args);
628 void applyUniformRetValOpt(
CallSiteInfo &CSInfo, StringRef FnName,
632 WholeProgramDevirtResolution::ByArg *Res);
636 std::string getGlobalName(VTableSlot Slot, ArrayRef<uint64_t> Args,
639 bool shouldExportConstantsAsAbsoluteSymbols();
644 void exportGlobal(VTableSlot Slot, ArrayRef<uint64_t> Args, StringRef Name,
646 void exportConstant(VTableSlot Slot, ArrayRef<uint64_t> Args, StringRef Name,
647 uint32_t Const, uint32_t &Storage);
651 Constant *importGlobal(VTableSlot Slot, ArrayRef<uint64_t> Args,
653 Constant *importConstant(VTableSlot Slot, ArrayRef<uint64_t> Args,
654 StringRef Name, IntegerType *IntTy,
657 Constant *getMemberAddr(
const TypeMemberInfo *M);
659 void applyUniqueRetValOpt(
CallSiteInfo &CSInfo, StringRef FnName,
bool IsOne,
660 Constant *UniqueMemberAddr);
661 bool tryUniqueRetValOpt(
unsigned BitWidth,
664 WholeProgramDevirtResolution::ByArg *Res,
665 VTableSlot Slot, ArrayRef<uint64_t> Args);
667 void applyVirtualConstProp(
CallSiteInfo &CSInfo, StringRef FnName,
668 Constant *Byte, Constant *Bit);
670 VTableSlotInfo &SlotInfo,
671 WholeProgramDevirtResolution *Res, VTableSlot Slot);
673 void rebuildGlobal(VTableBits &
B);
676 void importResolution(VTableSlot Slot, VTableSlotInfo &SlotInfo);
680 void removeRedundantTypeTests();
687 static ValueInfo lookUpFunctionValueInfo(
Function *TheFn,
688 ModuleSummaryIndex *ExportSummary);
699 ModuleSummaryIndex *ExportSummary);
704 bool DevirtSpeculatively);
708 ModuleSummaryIndex &ExportSummary;
711 std::set<GlobalValue::GUID> &ExportedGUIDs;
715 std::map<ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap;
720 DenseSet<StringRef> *ExternallyVisibleSymbolNamesPtr;
722 MapVector<VTableSlotSummary, VTableSlotInfo> CallSlots;
724 PatternList FunctionsToSkip;
728 const bool DevirtSpeculatively;
731 ModuleSummaryIndex &ExportSummary,
732 std::set<GlobalValue::GUID> &ExportedGUIDs,
733 std::map<ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap,
734 DenseSet<StringRef> *ExternallyVisibleSymbolNamesPtr)
735 : ExportSummary(ExportSummary), ExportedGUIDs(ExportedGUIDs),
736 LocalWPDTargetsMap(LocalWPDTargetsMap),
737 ExternallyVisibleSymbolNamesPtr(ExternallyVisibleSymbolNamesPtr),
738 DevirtSpeculatively(ExportSummary.withSpeculativeDevirt()) {
742 bool tryFindVirtualCallTargets(std::vector<ValueInfo> &TargetsForSlot,
744 uint64_t ByteOffset,
bool &HasPublicTarget);
747 VTableSlotSummary &SlotSummary,
748 VTableSlotInfo &SlotInfo,
749 WholeProgramDevirtResolution *Res,
750 std::set<ValueInfo> &DevirtTargets,
751 bool HasPublicTarget);
759 const bool DevirtualizeSpeculatively =
763 if (!DevirtModule::runForTesting(M,
MAM, DevirtualizeSpeculatively))
771 std::optional<ModuleSummaryIndex> Index;
798 if (
TypeID.ends_with(
".virtual"))
804 if (!
TypeID.consume_front(
"_ZTS"))
812 std::string TypeInfo = (
"_ZTI" +
TypeID).str();
813 return IsVisibleToRegularObj(TypeInfo);
822 for (
auto *
Type : Types)
825 IsVisibleToRegularObj);
834 Module &M,
bool WholeProgramVisibilityEnabledInLTO,
836 bool ValidateAllVtablesHaveTypeInfos,
854 !(ValidateAllVtablesHaveTypeInfos &&
861 bool WholeProgramVisibilityEnabledInLTO,
862 bool DevirtSpeculativelyInIndex) {
866 if (!PublicTypeTestFunc)
874 TypeTestFunc, {CI->getArgOperand(0), CI->getArgOperand(1)}, {},
"",
876 CI->replaceAllUsesWith(NewCI);
877 CI->eraseFromParent();
879 }
else if (!DevirtSpeculativelyInIndex &&
888 CI->replaceAllUsesWith(
True);
889 CI->eraseFromParent();
900 for (
const auto &TypeID : Index.typeIdCompatibleVtableMap()) {
903 VisibleToRegularObjSymbols.
insert(
P.VTableVI.getGUID());
916 for (
auto &
P : Index) {
919 if (DynamicExportSymbols.
count(
P.first))
925 if (VisibleToRegularObjSymbols.
count(
P.first))
927 for (
auto &S :
P.second.getSummaryList()) {
939 std::map<
ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap,
947 Summary.setWithSpeculativeDevirt();
948 DevirtIndex(Summary, ExportedGUIDs, LocalWPDTargetsMap,
949 ExternallyVisibleSymbolNamesPtr)
956 std::map<
ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap,
958 for (
auto &
T : LocalWPDTargetsMap) {
961 assert(VI.getSummaryList().size() == 1 &&
962 "Devirt of local target has more than one copy");
963 auto &S = VI.getSummaryList()[0];
964 if (!IsExported(S->modulePath(), VI))
968 for (
auto &SlotSummary :
T.second) {
969 auto *TIdSum = Summary.getTypeIdSummary(SlotSummary.TypeID);
971 auto WPDRes = TIdSum->WPDRes.find(SlotSummary.ByteOffset);
972 assert(WPDRes != TIdSum->WPDRes.end());
973 if (ExternallyVisibleSymbolNamesPtr)
974 ExternallyVisibleSymbolNamesPtr->
insert(WPDRes->second.SingleImplName);
976 WPDRes->second.SingleImplName,
977 Summary.getModuleHash(S->modulePath()));
988 const auto &ModPaths = Summary->modulePaths();
993 "combined summary should contain Regular LTO module");
998 bool DevirtSpeculatively) {
1000 StringRef ReadSummary = Opts.wholeprogramdevirt_read_summary;
1001 StringRef WriteSummary = Opts.wholeprogramdevirt_write_summary;
1003 std::unique_ptr<ModuleSummaryIndex>
Summary =
1004 std::make_unique<ModuleSummaryIndex>(
false);
1008 if (!ReadSummary.
empty()) {
1009 ExitOnError ExitOnErr(
1010 (
"-wholeprogramdevirt-read-summary: " + ReadSummary +
": ").str());
1011 auto ReadSummaryFile =
1013 if (Expected<std::unique_ptr<ModuleSummaryIndex>> SummaryOrErr =
1015 Summary = std::move(*SummaryOrErr);
1020 yaml::Input
In(ReadSummaryFile->getBuffer());
1031 DevirtSpeculatively)
1034 if (!WriteSummary.
empty()) {
1035 ExitOnError ExitOnErr(
1036 (
"-wholeprogramdevirt-write-summary: " + WriteSummary +
": ").str());
1045 yaml::Output
Out(OS);
1053void DevirtModule::buildTypeIdentifierMap(
1054 std::vector<VTableBits> &Bits,
1055 DenseMap<
Metadata *, std::set<TypeMemberInfo>> &TypeIdMap) {
1056 DenseMap<GlobalVariable *, VTableBits *> GVToBits;
1057 Bits.reserve(
M.global_size());
1059 for (GlobalVariable &GV :
M.globals()) {
1067 Bits.emplace_back();
1068 Bits.back().GV = &GV;
1069 Bits.back().ObjectSize =
1071 BitsPtr = &
Bits.back();
1074 for (MDNode *
Type : Types) {
1087bool DevirtModule::tryFindVirtualCallTargets(
1088 std::vector<VirtualCallTarget> &TargetsForSlot,
1089 const std::set<TypeMemberInfo> &TypeMemberInfos,
uint64_t ByteOffset,
1090 ModuleSummaryIndex *ExportSummary,
bool &HasPublicTarget) {
1091 HasPublicTarget =
false;
1094 if (!TM.Bits->GV->isConstant())
1097 if (TM.Bits->GV->getVCallVisibility() ==
1099 HasPublicTarget =
true;
1100 if (!DevirtSpeculatively)
1106 if (TM.Bits->GV->hasLocalLinkage() && TM.Bits->GV->hasZeroLiveUses())
1117 if (FunctionsToSkip.match(Fn->
getName()))
1122 if (Fn->
getName() ==
"__cxa_pure_virtual")
1135 if (!GA->isInterposable() && !GA->getAliaseeObject()->isInterposable())
1137 TargetsForSlot.push_back({GV, &TM});
1143 if (HasPublicTarget) {
1157 return !TargetsForSlot.empty();
1160bool DevirtIndex::tryFindVirtualCallTargets(
1161 std::vector<ValueInfo> &TargetsForSlot,
1163 bool &HasPublicTarget) {
1164 HasPublicTarget =
false;
1166 for (
const TypeIdOffsetVtableInfo &
P : TIdInfo) {
1177 if (
P.VTableVI.hasLocal() &&
P.VTableVI.getSummaryList().size() > 1)
1179 const GlobalVarSummary *
VS =
nullptr;
1180 for (
const auto &S :
P.VTableVI.getSummaryList()) {
1182 if (!CurVS->vTableFuncs().empty() ||
1195 HasPublicTarget =
true;
1196 if (!DevirtSpeculatively)
1208 for (
auto VTP :
VS->vTableFuncs()) {
1209 if (VTP.VTableOffset !=
P.AddressPointOffset + ByteOffset)
1215 TargetsForSlot.push_back(VTP.FuncVI);
1220 return !TargetsForSlot.empty();
1223void DevirtModule::applySingleImplDevirt(VTableSlotInfo &SlotInfo,
1224 Constant *TheFn,
bool HasPublicTarget,
1231 for (
auto &&VCallSite : CSInfo.CallSites) {
1232 if (!OptimizedCalls.
insert(&VCallSite.CB).second)
1240 VCallSite.emitRemark(
"single-impl",
1243 auto &CB = VCallSite.CB;
1256 MDBuilder(
M.getContext()).createUnlikelyBranchWeights());
1257 Builder.SetInsertPoint(ThenTerm);
1260 auto *CallTrap = Builder.CreateCall(TrapFn);
1271 MDBuilder MDB(
M.getContext());
1274 CB, InstrProfValueKind::IPVK_IndirectCallTarget,
1275 std::numeric_limits<uint32_t>::max(), TotalCount);
1276 MDNode *Weights =
nullptr;
1277 if (ValueData.
empty() || TotalCount == 0) {
1278 Weights = MDB.createLikelyBranchWeights();
1281 auto It =
llvm::find_if(ValueData, [&](
const InstrProfValueData &VD) {
1282 return VD.Value == TargetGUID;
1284 if (It == ValueData.
end()) {
1285 Weights = MDB.createLikelyBranchWeights();
1288 TotalCount - std::min(TotalCount, It->Count);
1289 assert((FallbackCount != 0 || It->Count != 0) &&
1290 "devirt branch weights are both zero.");
1292 MDB.createBranchWeights(
fitWeights({It->Count, FallbackCount}));
1303 NewInst.
setMetadata(LLVMContext::MD_prof,
nullptr);
1304 NewInst.
setMetadata(LLVMContext::MD_callees,
nullptr);
1314 assert(!HasPublicTarget);
1327 CallsWithPtrAuthBundleRemoved.
push_back(&CB);
1331 if (VCallSite.NumUnsafeUses)
1332 --*VCallSite.NumUnsafeUses;
1335 if (CSInfo.isExported())
1337 if (!HasPublicTarget &&
1339 CSInfo.markDevirt();
1341 Apply(SlotInfo.CSInfo);
1342 for (
auto &
P : SlotInfo.ConstCSInfo)
1348 if (Callee.getSummaryList().empty())
1355 bool IsExported =
false;
1356 auto &S = Callee.getSummaryList()[0];
1358 auto AddCalls = [&](CallSiteInfo &CSInfo) {
1359 for (
auto *FS : CSInfo.SummaryTypeCheckedLoadUsers) {
1360 FS->addCall({Callee, CI});
1361 IsExported |= S->modulePath() != FS->modulePath();
1363 for (
auto *FS : CSInfo.SummaryTypeTestAssumeUsers) {
1364 FS->addCall({Callee, CI});
1365 IsExported |= S->modulePath() != FS->modulePath();
1368 AddCalls(SlotInfo.CSInfo);
1369 for (
auto &
P : SlotInfo.ConstCSInfo)
1374bool DevirtModule::trySingleImplDevirt(
1375 ModuleSummaryIndex *ExportSummary,
1377 WholeProgramDevirtResolution *Res,
bool HasPublicTarget) {
1380 auto *TheFn = TargetsForSlot[0].Fn;
1381 for (
auto &&Target : TargetsForSlot)
1387 TargetsForSlot[0].WasDevirt =
true;
1389 bool IsExported =
false;
1390 applySingleImplDevirt(SlotInfo, TheFn, HasPublicTarget, IsExported);
1397 if (TheFn->hasLocalLinkage()) {
1398 std::string NewName = (TheFn->
getName() +
".llvm.merged").str();
1403 if (Comdat *
C = TheFn->getComdat()) {
1404 if (
C->getName() == TheFn->
getName()) {
1405 Comdat *NewC =
M.getOrInsertComdat(NewName);
1407 for (GlobalObject &GO :
M.global_objects())
1408 if (GO.getComdat() ==
C)
1417 if (ValueInfo TheFnVI = ExportSummary->
getValueInfo(TheFn->getGUID()))
1422 if (HasPublicTarget)
1433 VTableSlotSummary &SlotSummary,
1434 VTableSlotInfo &SlotInfo,
1435 WholeProgramDevirtResolution *Res,
1436 std::set<ValueInfo> &DevirtTargets,
1437 bool HasPublicTarget) {
1440 auto TheFn = TargetsForSlot[0];
1441 for (
auto &&Target : TargetsForSlot)
1442 if (TheFn != Target)
1446 auto Size = TheFn.getSummaryList().size();
1452 if (FunctionsToSkip.match(TheFn.name()))
1457 if (TheFn.hasLocal() &&
Size > 1)
1462 DevirtTargets.insert(TheFn);
1464 auto &S = TheFn.getSummaryList()[0];
1465 bool IsExported =
addCalls(SlotInfo, TheFn);
1467 ExportedGUIDs.insert(TheFn.getGUID());
1471 if (HasPublicTarget)
1481 if (ExternallyVisibleSymbolNamesPtr)
1482 ExternallyVisibleSymbolNamesPtr->insert(TheFn.name());
1484 TheFn.name(), ExportSummary.
getModuleHash(S->modulePath()));
1486 LocalWPDTargetsMap[TheFn].push_back(SlotSummary);
1500void DevirtModule::tryICallBranchFunnel(
1502 WholeProgramDevirtResolution *Res, VTableSlot Slot) {
1503 Triple
T(
M.getTargetTriple());
1507 if (TargetsForSlot.
size() > Opts.wholeprogramdevirt_branch_funnel_threshold)
1510 bool HasNonDevirt = !SlotInfo.CSInfo.AllCallSitesDevirted;
1512 for (
auto &
P : SlotInfo.ConstCSInfo)
1513 if (!
P.second.AllCallSitesDevirted) {
1514 HasNonDevirt =
true;
1532 for (
auto &
T : TargetsForSlot) {
1533 if (
T.TM->Bits->GV->hasAvailableExternallyLinkage())
1542 M.getDataLayout().getProgramAddressSpace(),
1543 getGlobalName(Slot, {},
"branch_funnel"), &
M);
1547 M.getDataLayout().getProgramAddressSpace(),
1548 "branch_funnel", &M);
1552 std::vector<Value *> JTArgs;
1554 for (
auto &
T : TargetsForSlot) {
1555 JTArgs.push_back(getMemberAddr(
T.TM));
1556 JTArgs.push_back(
T.Fn);
1561 &M, llvm::Intrinsic::icall_branch_funnel, {});
1567 bool IsExported =
false;
1568 applyICallBranchFunnel(SlotInfo, *JT, IsExported);
1578void DevirtModule::applyICallBranchFunnel(VTableSlotInfo &SlotInfo,
1580 DenseMap<Function *, double> FunctionEntryCounts;
1582 if (CSInfo.isExported())
1584 if (CSInfo.AllCallSitesDevirted)
1587 std::map<CallBase *, CallBase *> CallBases;
1588 for (
auto &&VCallSite : CSInfo.CallSites) {
1589 CallBase &CB = VCallSite.CB;
1591 if (CallBases.find(&CB) != CallBases.end()) {
1608 VCallSite.emitRemark(
"branch-funnel", JT.
getName(), OREGetter);
1612 std::vector<Type *> NewArgs;
1613 NewArgs.push_back(Int8PtrTy);
1615 FunctionType *NewFT =
1619 std::vector<Value *>
Args;
1620 Args.push_back(VCallSite.VTable);
1623 CallBase *NewCS =
nullptr;
1629 auto EC = BFI.getBlockFreq(&
F.getEntryBlock());
1630 auto CC =
F.getEntryCount();
1631 double CallCount = 0.0;
1632 if (
EC.getFrequency() != 0 &&
CC && *
CC != 0) {
1634 static_cast<double>(
1635 BFI.getBlockFreq(CB.
getParent()).getFrequency()) /
1637 CallCount = CallFreq * *
CC;
1639 FunctionEntryCounts[&JT] += CallCount;
1642 NewCS = IRB.CreateCall(NewFT, &JT, Args);
1650 std::vector<AttributeSet> NewArgAttrs;
1653 M.getContext(), Attribute::Nest)}));
1654 for (
unsigned I = 0;
I + 2 <
Attrs.getNumAttrSets(); ++
I)
1655 NewArgAttrs.push_back(
Attrs.getParamAttrs(
I));
1657 AttributeList::get(
M.getContext(),
Attrs.getFnAttrs(),
1658 Attrs.getRetAttrs(), NewArgAttrs));
1660 CallBases[&CB] = NewCS;
1663 if (VCallSite.NumUnsafeUses)
1664 --*VCallSite.NumUnsafeUses;
1671 for (
auto &[Old, New] : CallBases) {
1672 Old->replaceAllUsesWith(New);
1673 Old->eraseFromParent();
1676 Apply(SlotInfo.CSInfo);
1677 for (
auto &
P : SlotInfo.ConstCSInfo)
1679 for (
auto &[
F,
C] : FunctionEntryCounts) {
1681 "Unexpected entry count for funnel that was freshly synthesized");
1682 F->setEntryCount(
static_cast<uint64_t>(std::round(
C)));
1686bool DevirtModule::tryEvaluateFunctionsWithArgs(
1688 ArrayRef<uint64_t> Args) {
1702 Evaluator Eval(
M.getDataLayout(),
nullptr);
1706 for (
unsigned I = 0;
I !=
Args.size(); ++
I) {
1711 EvalArgs.
push_back(ConstantInt::get(ArgTy, Args[
I]));
1715 if (!Eval.EvaluateFunction(Fn, RetVal, EvalArgs) ||
1723void DevirtModule::applyUniformRetValOpt(
CallSiteInfo &CSInfo, StringRef FnName,
1725 for (
auto Call : CSInfo.CallSites) {
1729 Call.replaceAndErase(
1730 "uniform-ret-val", FnName, RemarksEnabled, OREGetter,
1733 CSInfo.markDevirt();
1736bool DevirtModule::tryUniformRetValOpt(
1738 WholeProgramDevirtResolution::ByArg *Res) {
1741 uint64_t TheRetVal = TargetsForSlot[0].RetVal;
1743 if (
Target.RetVal != TheRetVal)
1746 if (CSInfo.isExported()) {
1748 Res->
Info = TheRetVal;
1751 applyUniformRetValOpt(CSInfo, TargetsForSlot[0].Fn->
getName(), TheRetVal);
1753 for (
auto &&Target : TargetsForSlot)
1758std::string DevirtModule::getGlobalName(VTableSlot Slot,
1759 ArrayRef<uint64_t> Args,
1761 std::string FullName =
"__typeid_";
1762 raw_string_ostream OS(FullName);
1763 OS << cast<MDString>(
Slot.TypeID)->getString() <<
'_' <<
Slot.ByteOffset;
1770bool DevirtModule::shouldExportConstantsAsAbsoluteSymbols() {
1771 Triple
T(
M.getTargetTriple());
1775void DevirtModule::exportGlobal(VTableSlot Slot, ArrayRef<uint64_t> Args,
1776 StringRef Name, Constant *
C) {
1778 getGlobalName(Slot, Args, Name),
C, &M);
1782void DevirtModule::exportConstant(VTableSlot Slot, ArrayRef<uint64_t> Args,
1783 StringRef Name, uint32_t Const,
1784 uint32_t &Storage) {
1785 if (shouldExportConstantsAsAbsoluteSymbols()) {
1795Constant *DevirtModule::importGlobal(VTableSlot Slot, ArrayRef<uint64_t> Args,
1797 GlobalVariable *GV =
1798 M.getOrInsertGlobal(getGlobalName(Slot, Args, Name), Int8Arr0Ty);
1803Constant *DevirtModule::importConstant(VTableSlot Slot, ArrayRef<uint64_t> Args,
1804 StringRef Name, IntegerType *IntTy,
1806 if (!shouldExportConstantsAsAbsoluteSymbols())
1807 return ConstantInt::get(IntTy, Storage);
1809 Constant *
C = importGlobal(Slot, Args, Name);
1815 if (GV->
hasMetadata(LLVMContext::MD_absolute_symbol))
1825 if (AbsWidth ==
IntPtrTy->getBitWidth()) {
1829 SetAbsRange(0, 1ull << AbsWidth);
1834void DevirtModule::applyUniqueRetValOpt(
CallSiteInfo &CSInfo, StringRef FnName,
1836 Constant *UniqueMemberAddr) {
1837 for (
auto &&
Call : CSInfo.CallSites) {
1843 B.CreateBitCast(UniqueMemberAddr,
Call.VTable->
getType()));
1846 Call.replaceAndErase(
"unique-ret-val", FnName, RemarksEnabled, OREGetter,
1849 CSInfo.markDevirt();
1854 ConstantInt::get(Int64Ty,
M->Offset));
1857bool DevirtModule::tryUniqueRetValOpt(
1859 CallSiteInfo &CSInfo, WholeProgramDevirtResolution::ByArg *Res,
1860 VTableSlot Slot, ArrayRef<uint64_t> Args) {
1862 auto tryUniqueRetValOptFor = [&](
bool IsOne) {
1865 if (
Target.RetVal == (IsOne ? 1 : 0)) {
1868 UniqueMember =
Target.TM;
1876 Constant *UniqueMemberAddr = getMemberAddr(UniqueMember);
1877 if (CSInfo.isExported()) {
1881 exportGlobal(Slot, Args,
"unique_member", UniqueMemberAddr);
1885 applyUniqueRetValOpt(CSInfo, TargetsForSlot[0].Fn->
getName(), IsOne,
1890 for (
auto &&Target : TargetsForSlot)
1897 if (tryUniqueRetValOptFor(
true))
1899 if (tryUniqueRetValOptFor(
false))
1905void DevirtModule::applyVirtualConstProp(
CallSiteInfo &CSInfo, StringRef FnName,
1906 Constant *Byte, Constant *Bit) {
1907 for (
auto Call : CSInfo.CallSites) {
1912 Value *Addr =
B.CreatePtrAddWithSExtOrTrunc(
Call.VTable, Byte);
1913 if (RetType->getBitWidth() == 1) {
1915 Value *BitsAndBit =
B.CreateAnd(Bits, Bit);
1916 auto IsBitSet =
B.CreateICmpNE(BitsAndBit, ConstantInt::get(Int8Ty, 0));
1917 NumVirtConstProp1Bit++;
1918 Call.replaceAndErase(
"virtual-const-prop-1-bit", FnName, RemarksEnabled,
1919 OREGetter, IsBitSet);
1921 Value *Val =
B.CreateLoad(RetType, Addr);
1923 Call.replaceAndErase(
"virtual-const-prop", FnName, RemarksEnabled,
1927 CSInfo.markDevirt();
1930bool DevirtModule::tryVirtualConstProp(
1932 WholeProgramDevirtResolution *Res, VTableSlot Slot) {
1943 unsigned BitWidth = RetType->getBitWidth();
1955 Align TypeAlignment =
M.getDataLayout().getABIIntegerTypeAlignment(
BitWidth);
1988 GlobalVariable *GV =
Target.TM->Bits->GV;
1989 Align TableAlignment =
M.getDataLayout().getValueOrABITypeAlignment(
1991 if (TypeAlignment > TableAlignment)
1995 for (
auto &&CSByConstantArg : SlotInfo.ConstCSInfo) {
1996 if (!tryEvaluateFunctionsWithArgs(TargetsForSlot, CSByConstantArg.first))
1999 WholeProgramDevirtResolution::ByArg *ResByArg =
nullptr;
2001 ResByArg = &Res->
ResByArg[CSByConstantArg.first];
2003 if (tryUniformRetValOpt(TargetsForSlot, CSByConstantArg.second, ResByArg))
2006 if (tryUniqueRetValOpt(
BitWidth, TargetsForSlot, CSByConstantArg.second,
2007 ResByArg, Slot, CSByConstantArg.first))
2022 uint64_t TotalPaddingBefore = 0, TotalPaddingAfter = 0;
2023 for (
auto &&Target : TargetsForSlot) {
2024 TotalPaddingBefore += std::max<int64_t>(
2025 (AllocBefore + 7) / 8 -
Target.allocatedBeforeBytes() - 1, 0);
2026 TotalPaddingAfter += std::max<int64_t>(
2027 (AllocAfter + 7) / 8 -
Target.allocatedAfterBytes() - 1, 0);
2032 if (std::min(TotalPaddingBefore, TotalPaddingAfter) > 128)
2039 if (TotalPaddingBefore <= TotalPaddingAfter)
2055 for (
auto &&Target : TargetsForSlot)
2059 if (CSByConstantArg.second.isExported()) {
2061 ResByArg->
Byte = OffsetByte;
2062 exportConstant(Slot, CSByConstantArg.first,
"bit", 1ULL << OffsetBit,
2068 Constant *BitConst = ConstantInt::get(Int8Ty, 1ULL << OffsetBit);
2069 applyVirtualConstProp(CSByConstantArg.second,
2070 TargetsForSlot[0].Fn->getName(), ByteConst, BitConst);
2076 if (
B.Before.Bytes.empty() &&
B.After.Bytes.empty())
2082 B.GV->getAlign(),
B.GV->getValueType());
2083 B.Before.Bytes.resize(
alignTo(
B.Before.Bytes.size(), Alignment));
2086 for (
size_t I = 0,
Size =
B.Before.Bytes.size();
I !=
Size / 2; ++
I)
2093 B.GV->getInitializer(),
2096 new GlobalVariable(M, NewInit->getType(),
B.GV->isConstant(),
2098 NewGV->setSection(
B.GV->getSection());
2099 NewGV->setComdat(
B.GV->getComdat());
2100 NewGV->setAlignment(
B.GV->getAlign());
2104 NewGV->copyMetadata(
B.GV,
B.Before.Bytes.size());
2109 B.GV->getInitializer()->getType(), 0,
B.GV->getLinkage(),
"",
2111 M.getDataLayout(), NewInit->getType(), NewGV,
2112 {ConstantInt::get(Int32Ty, 0), ConstantInt::get(Int32Ty, 1)},
2115 Alias->setVisibility(
B.GV->getVisibility());
2116 Alias->takeName(
B.GV);
2118 B.GV->replaceAllUsesWith(Alias);
2119 B.GV->eraseFromParent();
2122bool DevirtModule::areRemarksEnabled() {
2123 const auto &FL =
M.getFunctionList();
2128 return DI.isEnabled();
2158 if (
Value *
Merge = GetMergeUser(U, &TypeTest))
2161 while (!Worklist.
empty()) {
2163 if (!VisitedMerges.
insert(
V).second)
2166 for (
User *U :
V->users()) {
2168 if (Assume->getArgOperand(0) ==
V) {
2170 "assume must not be both direct and merged");
2182void DevirtModule::scanTypeTestUsers(
2184 DenseMap<
Metadata *, std::set<TypeMemberInfo>> &TypeIdMap) {
2187 SmallPtrSet<Value *, 8> VisitedMerges;
2202 auto &DT =
FAM.
getResult<DominatorTreeAnalysis>(*CI->getFunction());
2208 if (!Assumes.
empty()) {
2209 Value *Ptr = CI->getArgOperand(0)->stripPointerCasts();
2210 for (DevirtCallSite
Call : DevirtCalls)
2211 CallSlots[{TypeId,
Call.Offset}].addCallSite(Ptr,
Call.CB,
nullptr);
2214 auto RemoveTypeTestAssumes = [&]() {
2222 for (
auto *Assume : Assumes)
2223 Assume->eraseFromParent();
2226 if (CI->use_empty())
2227 CI->eraseFromParent();
2242 if (!TypeIdMap.count(TypeId))
2243 RemoveTypeTestAssumes();
2255 const TypeIdSummary *TidSummary =
2258 RemoveTypeTestAssumes();
2267void DevirtModule::scanTypeCheckedLoadUsers(
Function *TypeCheckedLoadFunc) {
2276 Value *Ptr = CI->getArgOperand(0);
2278 Value *TypeIdValue = CI->getArgOperand(2);
2284 bool HasNonCallUses =
false;
2285 auto &DT =
FAM.
getResult<DominatorTreeAnalysis>(*CI->getFunction());
2287 HasNonCallUses, CI, DT);
2296 (LoadedPtrs.
size() == 1 && !HasNonCallUses) ? LoadedPtrs[0] : CI);
2298 Value *LoadedValue =
nullptr;
2300 Intrinsic::type_checked_load_relative) {
2302 &M, Intrinsic::load_relative, {Int32Ty});
2303 LoadedValue = LoadB.CreateCall(LoadRelFunc, {Ptr,
Offset});
2306 LoadedValue = LoadB.CreateLoad(Int8PtrTy,
GEP);
2309 for (Instruction *LoadedPtr : LoadedPtrs) {
2310 LoadedPtr->replaceAllUsesWith(LoadedValue);
2311 LoadedPtr->eraseFromParent();
2315 IRBuilder<> CallB((Preds.
size() == 1 && !HasNonCallUses) ? Preds[0] : CI);
2316 CallInst *TypeTestCall = CallB.CreateCall(TypeTestFunc, {Ptr, TypeIdValue});
2318 for (Instruction *
Pred : Preds) {
2319 Pred->replaceAllUsesWith(TypeTestCall);
2320 Pred->eraseFromParent();
2327 if (!CI->use_empty()) {
2330 Pair =
B.CreateInsertValue(Pair, LoadedValue, {0});
2331 Pair =
B.CreateInsertValue(Pair, TypeTestCall, {1});
2336 auto &NumUnsafeUses = NumUnsafeUsesForTypeTest[TypeTestCall];
2337 NumUnsafeUses = DevirtCalls.
size();
2344 for (DevirtCallSite
Call : DevirtCalls) {
2345 CallSlots[{TypeId,
Call.Offset}].addCallSite(Ptr,
Call.CB,
2349 CI->eraseFromParent();
2353void DevirtModule::importResolution(VTableSlot Slot, VTableSlotInfo &SlotInfo) {
2357 const TypeIdSummary *TidSummary =
2361 auto ResI = TidSummary->
WPDRes.find(
Slot.ByteOffset);
2362 if (ResI == TidSummary->
WPDRes.end())
2364 const WholeProgramDevirtResolution &Res = ResI->second;
2371 Value *SingleImplVal =
2376 if (!
A->isInterposable() && !
A->getAliaseeObject()->isInterposable())
2377 SingleImplVal =
A->getAliaseeObject();
2381 bool IsExported =
false;
2382 bool IsSpeculative =
2384 applySingleImplDevirt(SlotInfo, SingleImpl, IsSpeculative, IsExported);
2388 for (
auto &CSByConstantArg : SlotInfo.ConstCSInfo) {
2389 auto I = Res.
ResByArg.find(CSByConstantArg.first);
2392 auto &ResByArg =
I->second;
2399 applyUniformRetValOpt(CSByConstantArg.second,
"", ResByArg.
Info);
2403 importGlobal(Slot, CSByConstantArg.first,
"unique_member");
2404 applyUniqueRetValOpt(CSByConstantArg.second,
"", ResByArg.
Info,
2410 Constant *
Bit = importConstant(Slot, CSByConstantArg.first,
"bit", Int8Ty,
2412 applyVirtualConstProp(CSByConstantArg.second,
"", Byte, Bit);
2424 M.getOrInsertFunction(getGlobalName(Slot, {},
"branch_funnel"),
2427 bool IsExported =
false;
2428 applyICallBranchFunnel(SlotInfo, *JT, IsExported);
2433void DevirtModule::removeRedundantTypeTests() {
2435 for (
auto &&U : NumUnsafeUsesForTypeTest) {
2436 if (
U.second == 0) {
2437 U.first->replaceAllUsesWith(
True);
2438 U.first->eraseFromParent();
2444DevirtModule::lookUpFunctionValueInfo(
Function *TheFn,
2445 ModuleSummaryIndex *ExportSummary) {
2446 assert((ExportSummary !=
nullptr) &&
2447 "Caller guarantees ExportSummary is not nullptr");
2449 const auto TheFnGUID = TheFn->
getGUID();
2450 const auto TheFnGUIDWithExportedName =
2453 ValueInfo TheFnVI = ExportSummary->
getValueInfo(TheFnGUID);
2462 if ((!TheFnVI) && (TheFnGUID != TheFnGUIDWithExportedName)) {
2463 TheFnVI = ExportSummary->
getValueInfo(TheFnGUIDWithExportedName);
2468bool DevirtModule::mustBeUnreachableFunction(
2469 Function *
const F, ModuleSummaryIndex *ExportSummary) {
2473 if (!
F->isDeclaration()) {
2479 return ExportSummary &&
2481 DevirtModule::lookUpFunctionValueInfo(
F, ExportSummary));
2484bool DevirtModule::run() {
2493 Function *PublicTypeTestFunc =
nullptr;
2496 if (DevirtSpeculatively)
2497 PublicTypeTestFunc =
2504 &M, Intrinsic::type_checked_load_relative);
2511 if (!ExportSummary &&
2512 (((!PublicTypeTestFunc || PublicTypeTestFunc->
use_empty()) &&
2513 (!TypeTestFunc || TypeTestFunc->
use_empty())) ||
2514 !AssumeFunc || AssumeFunc->
use_empty()) &&
2515 (!TypeCheckedLoadFunc || TypeCheckedLoadFunc->
use_empty()) &&
2516 (!TypeCheckedLoadRelativeFunc ||
2517 TypeCheckedLoadRelativeFunc->
use_empty()))
2521 std::vector<VTableBits>
Bits;
2522 DenseMap<Metadata *, std::set<TypeMemberInfo>> TypeIdMap;
2523 buildTypeIdentifierMap(Bits, TypeIdMap);
2525 if (PublicTypeTestFunc && AssumeFunc)
2526 scanTypeTestUsers(PublicTypeTestFunc, TypeIdMap);
2528 if (TypeTestFunc && AssumeFunc)
2529 scanTypeTestUsers(TypeTestFunc, TypeIdMap);
2531 if (TypeCheckedLoadFunc)
2532 scanTypeCheckedLoadUsers(TypeCheckedLoadFunc);
2534 if (TypeCheckedLoadRelativeFunc)
2535 scanTypeCheckedLoadUsers(TypeCheckedLoadRelativeFunc);
2537 if (ImportSummary) {
2538 for (
auto &S : CallSlots)
2539 importResolution(S.first, S.second);
2541 removeRedundantTypeTests();
2546 for (GlobalVariable &GV :
M.globals())
2554 if (TypeIdMap.
empty())
2558 if (ExportSummary) {
2559 DenseMap<GlobalValue::GUID, TinyPtrVector<Metadata *>> MetadataByGUID;
2560 for (
auto &
P : TypeIdMap) {
2563 TypeId->getString())]
2567 for (
auto &
P : *ExportSummary) {
2568 for (
auto &S :
P.second.getSummaryList()) {
2573 for (FunctionSummary::VFuncId VF :
FS->type_test_assume_vcalls()) {
2574 for (
Metadata *MD : MetadataByGUID[VF.GUID]) {
2575 CallSlots[{MD, VF.Offset}].CSInfo.addSummaryTypeTestAssumeUser(FS);
2578 for (FunctionSummary::VFuncId VF :
FS->type_checked_load_vcalls()) {
2579 for (
Metadata *MD : MetadataByGUID[VF.GUID]) {
2580 CallSlots[{MD, VF.Offset}].CSInfo.addSummaryTypeCheckedLoadUser(FS);
2583 for (
const FunctionSummary::ConstVCall &VC :
2584 FS->type_test_assume_const_vcalls()) {
2585 for (
Metadata *MD : MetadataByGUID[
VC.VFunc.GUID]) {
2586 CallSlots[{MD,
VC.VFunc.Offset}]
2587 .ConstCSInfo[
VC.Args]
2588 .addSummaryTypeTestAssumeUser(FS);
2591 for (
const FunctionSummary::ConstVCall &VC :
2592 FS->type_checked_load_const_vcalls()) {
2593 for (
Metadata *MD : MetadataByGUID[
VC.VFunc.GUID]) {
2594 CallSlots[{MD,
VC.VFunc.Offset}]
2595 .ConstCSInfo[
VC.Args]
2596 .addSummaryTypeCheckedLoadUser(FS);
2604 bool DidVirtualConstProp =
false;
2605 std::map<std::string, GlobalValue *> DevirtTargets;
2606 for (
auto &S : CallSlots) {
2610 std::vector<VirtualCallTarget> TargetsForSlot;
2611 WholeProgramDevirtResolution *Res =
nullptr;
2612 const std::set<TypeMemberInfo> &TypeMemberInfos = TypeIdMap[S.first.TypeID];
2614 TypeMemberInfos.size())
2621 Res = &ExportSummary
2622 ->getOrInsertTypeIdSummary(
2624 .WPDRes[S.first.ByteOffset];
2626 bool HasPublicTarget =
true;
2627 if (tryFindVirtualCallTargets(TargetsForSlot, TypeMemberInfos,
2628 S.first.ByteOffset, ExportSummary,
2630 bool SingleImplDevirt = trySingleImplDevirt(
2631 ExportSummary, TargetsForSlot, S.second, Res, HasPublicTarget);
2632 if (!SingleImplDevirt && !HasPublicTarget) {
2633 DidVirtualConstProp |=
2634 tryVirtualConstProp(TargetsForSlot, S.second, Res, S.first);
2636 tryICallBranchFunnel(TargetsForSlot, S.second, Res, S.first);
2641 for (
const auto &
T : TargetsForSlot)
2643 DevirtTargets[std::string(
T.Fn->getName())] =
T.Fn;
2653 auto AddTypeTestsForTypeCheckedLoads = [&](
CallSiteInfo &CSI) {
2654 if (!CSI.AllCallSitesDevirted)
2655 for (
auto *FS : CSI.SummaryTypeCheckedLoadUsers)
2656 FS->addTypeTest(GUID);
2658 AddTypeTestsForTypeCheckedLoads(S.second.CSInfo);
2659 for (
auto &CCS : S.second.ConstCSInfo)
2660 AddTypeTestsForTypeCheckedLoads(CCS.second);
2664 if (RemarksEnabled) {
2666 for (
const auto &DT : DevirtTargets) {
2667 GlobalValue *GV = DT.second;
2676 using namespace ore;
2677 OREGetter(*F).emit(OptimizationRemark(
DEBUG_TYPE,
"Devirtualized",
F)
2678 <<
"devirtualized " <<
NV(
"FunctionName", DT.first));
2682 NumDevirtTargets += DevirtTargets.size();
2684 removeRedundantTypeTests();
2688 if (DidVirtualConstProp)
2695 for (GlobalVariable &GV :
M.globals())
2698 for (
auto *CI : CallsWithPtrAuthBundleRemoved)
2699 CI->eraseFromParent();
2704void DevirtIndex::run() {
2705 if (ExportSummary.typeIdCompatibleVtableMap().empty())
2710 assert(!ExportSummary.withInternalizeAndPromote() &&
2711 "Expect index-based WPD to run before internalization and promotion");
2713 DenseMap<GlobalValue::GUID, std::vector<StringRef>> NameByGUID;
2714 for (
const auto &
P : ExportSummary.typeIdCompatibleVtableMap()) {
2723 ExportSummary.getOrInsertTypeIdSummary(
P.first);
2727 for (
auto &
P : ExportSummary) {
2728 for (
auto &S :
P.second.getSummaryList()) {
2733 for (FunctionSummary::VFuncId VF :
FS->type_test_assume_vcalls()) {
2734 for (StringRef Name : NameByGUID[VF.GUID]) {
2735 CallSlots[{
Name, VF.Offset}].CSInfo.addSummaryTypeTestAssumeUser(FS);
2738 for (FunctionSummary::VFuncId VF :
FS->type_checked_load_vcalls()) {
2739 for (StringRef Name : NameByGUID[VF.GUID]) {
2740 CallSlots[{
Name, VF.Offset}].CSInfo.addSummaryTypeCheckedLoadUser(FS);
2743 for (
const FunctionSummary::ConstVCall &VC :
2744 FS->type_test_assume_const_vcalls()) {
2745 for (StringRef Name : NameByGUID[
VC.VFunc.GUID]) {
2746 CallSlots[{
Name,
VC.VFunc.Offset}]
2747 .ConstCSInfo[
VC.Args]
2748 .addSummaryTypeTestAssumeUser(FS);
2751 for (
const FunctionSummary::ConstVCall &VC :
2752 FS->type_checked_load_const_vcalls()) {
2753 for (StringRef Name : NameByGUID[
VC.VFunc.GUID]) {
2754 CallSlots[{
Name,
VC.VFunc.Offset}]
2755 .ConstCSInfo[
VC.Args]
2756 .addSummaryTypeCheckedLoadUser(FS);
2762 std::set<ValueInfo> DevirtTargets;
2764 for (
auto &S : CallSlots) {
2768 std::vector<ValueInfo> TargetsForSlot;
2769 auto TidSummary = ExportSummary.getTypeIdCompatibleVtableSummary(S.first.TypeID);
2773 WholeProgramDevirtResolution *Res =
2774 &ExportSummary.getTypeIdSummary(S.first.TypeID)
2775 ->WPDRes[S.first.ByteOffset];
2777 bool HasPublicTarget =
true;
2778 if (tryFindVirtualCallTargets(TargetsForSlot, *TidSummary,
2779 S.first.ByteOffset, HasPublicTarget)) {
2781 if (!trySingleImplDevirt(TargetsForSlot, S.first, S.second, Res,
2782 DevirtTargets, HasPublicTarget))
2790 for (
const auto &DT : DevirtTargets)
2791 errs() <<
"Devirtualized call to " << DT <<
"\n";
2793 NumDevirtTargets += DevirtTargets.size();
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
AMDGPU Register Bank Select
This is the interface for LLVM's primary stateless and local alias analysis.
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
This file defines DenseMapInfo traits for DenseMap.
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
Provides passes for computing function attributes based on interprocedural analyses.
static void emitRemark(const Function &F, OptimizationRemarkEmitter &ORE, bool Skip)
Module.h This file contains the declarations for the Module class.
This header defines various interfaces for pass management in LLVM.
Machine Check Debug Module
This file implements a map that provides insertion order iteration.
static bool mustBeUnreachableFunction(const Function &F)
This is the interface to build a ModuleSummaryIndex for a module.
uint64_t IntrinsicInst * II
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This file contains the declarations for profiling metadata utility functions.
SmallVector< MachineOperand, 4 > Cond
Func getContext().diagnose(DiagnosticInfoUnsupported(Func
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static void findAssumesThroughMergesForTypeTest(SmallVectorImpl< CallInst * > &Assumes, CallInst &TypeTest, SmallPtrSetImpl< Value * > &VisitedMerges)
Find assumes whose conditions depend on this type test through phi or select nodes.
static bool typeIDVisibleToRegularObj(StringRef TypeID, function_ref< bool(StringRef)> IsVisibleToRegularObj)
static bool addCalls(VTableSlotInfo &SlotInfo, const ValueInfo &Callee)
static cl::opt< bool > WholeProgramDevirtKeepUnreachableFunction("wholeprogramdevirt-keep-unreachable-function", cl::desc("Regard unreachable functions as possible devirtualize targets."), cl::Hidden, cl::init(true))
With Clang, a pure virtual class's deleting destructor is emitted as a llvm.trap intrinsic followed b...
static bool skipUpdateDueToValidation(GlobalVariable &GV, function_ref< bool(StringRef)> IsVisibleToRegularObj)
static Error checkCombinedSummaryForTesting(const IPOOptions &Opts, ModuleSummaryIndex *Summary)
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
static LLVM_ABI AttributeSet get(LLVMContext &C, const AttrBuilder &B)
LLVM_ABI StringRef getValueAsString() const
Return the attribute's value as a string.
bool isValid() const
Return true if the attribute is any kind of attribute.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
void setCallingConv(CallingConv::ID CC)
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
CallingConv::ID getCallingConv() const
Value * getCalledOperand() const
void setAttributes(AttributeList A)
Set the attributes for this call.
FunctionType * getFunctionType() const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void setCalledOperand(Value *V)
static LLVM_ABI CallBase * removeOperandBundle(CallBase *CB, uint32_t ID, InsertPosition InsertPt=nullptr)
Create a clone of CB with operand bundle ID removed.
AttributeList getAttributes() const
Return the attributes for this call.
LLVM_ABI Function * getCaller()
Helper to get the caller (the parent function).
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
void setSelectionKind(SelectionKind Val)
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
static LLVM_ABI Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getPtrAdd(Constant *Ptr, Constant *Offset, GEPNoWrapFlags NW=GEPNoWrapFlags::none(), std::optional< ConstantRange > InRange=std::nullopt, Type *OnlyIfReduced=nullptr)
Create a getelementptr i8, ptr, offset constant expression.
static LLVM_ABI Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static LLVM_SUPPRESS_DEPRECATED_DECLARATIONS_PUSH Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList, GEPNoWrapFlags NW=GEPNoWrapFlags::none(), std::optional< ConstantRange > InRange=std::nullopt, Type *OnlyIfReducedTy=nullptr)
Getelementptr form.
static LLVM_ABI ConstantInt * getTrue(LLVMContext &Context)
static ConstantInt * getSigned(IntegerType *Ty, int64_t V, bool ImplicitTrunc=false)
Return a ConstantInt with the specified value for the specified type.
static Constant * getAnon(ArrayRef< Constant * > V, bool Packed=false)
Return an anonymous struct that has the specified elements.
const Constant * stripPointerCasts() const
static LLVM_ABI Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
static bool shouldExecute(CounterInfo &Counter)
Implements a dense probed hash-table based set.
Subclass of Error for the sole purpose of identifying the success path in the type system.
Lightweight error class with error context and mandatory checking.
Tagged union holding either a T or a Error.
Type * getParamType(unsigned i) const
Parameter type accessors.
ArrayRef< Type * > params() const
Type * getReturnType() const
static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
const BasicBlock & front() const
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
std::optional< uint64_t > getEntryCount() const
Get the entry count for this function.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
adds the attribute to the list of attributes for the given arg.
Type * getReturnType() const
Returns the type of the ret val.
unsigned getInstructionCount() const
Returns the number of non-debug IR instructions in this function.
static GEPNoWrapFlags inBounds()
static LLVM_ABI Expected< GlobPattern > create(StringRef Pat, std::optional< size_t > MaxSubPatterns={}, bool SlashAgnostic=false)
static LLVM_ABI GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set a particular kind of metadata attachment.
bool hasMetadata() const
Return true if this GlobalObject has any metadata attached to it.
LLVM_ABI VCallVisibility getVCallVisibility() const
LLVM_ABI bool eraseMetadata(unsigned KindID)
Erase all metadata attachments with the given kind.
@ VCallVisibilityLinkageUnit
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this GlobalObject.
LLVM_ABI void setVCallVisibilityMetadata(VCallVisibility Visibility)
static LLVM_ABI GUID getGUIDAssumingExternalLinkage(StringRef GlobalName)
Return a 64-bit global unique ID constructed from the name of a global symbol.
static bool isLocalLinkage(LinkageTypes Linkage)
LLVM_ABI bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
static bool isAvailableExternallyLinkage(LinkageTypes Linkage)
LLVM_ABI const GlobalObject * getAliaseeObject() const
LLVM_ABI GUID getGUID() const
Return a 64-bit global unique ID for this value.
@ HiddenVisibility
The GV is hidden.
void setVisibility(VisibilityTypes V)
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ InternalLinkage
Rename collisions when linking (static functions).
@ ExternalLinkage
Externally visible function.
Type * getValueType() const
LLVM_ABI bool isInterposable(bool CheckNoIPA=true) const
Return true if this global's definition can be substituted with an arbitrary definition at link time ...
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
MaybeAlign getAlign() const
Returns the alignment of the given variable.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
iterator_range< user_iterator > users()
LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
uint64_t getBitMask() const
Return a bitmask with ones set for all of the bits that can be set by an unsigned version of this typ...
This is an important class for using LLVM in a threaded context.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFile(const Twine &Filename, bool IsText=false, bool RequiresNullTerminator=true, bool IsVolatile=false, std::optional< Align > Alignment=std::nullopt)
Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful,...
bool doesNotAccessMemory() const
Whether this function accesses no memory.
Class to hold module path string table and global value map, and encapsulate methods for operating on...
const TypeIdSummary * getTypeIdSummary(StringRef TypeId) const
This returns either a pointer to the type id summary (if present in the summary map) or null (if not ...
ValueInfo getValueInfo(const GlobalValueSummaryMapTy::value_type &R) const
Return a ValueInfo for the index value_type (convenient when iterating index).
const ModuleHash & getModuleHash(const StringRef ModPath) const
Get the module SHA1 hash recorded for the given module path.
static constexpr const char * getRegularLTOModuleName()
bool partiallySplitLTOUnits() const
static std::string getGlobalNameForLocal(StringRef Name, ModuleHash ModHash)
Convenience method for creating a promoted global name for the given value name of a local,...
bool withSpeculativeDevirt() const
A Module instance is used to store all the information related to an LLVM module.
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Analysis providing profile information.
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, InsertPosition InsertBefore=nullptr)
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
LLVM_NO_SANITIZE_CFI_UNRELATED_CAST iterator begin()
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
constexpr bool empty() const
Check if the string is empty.
bool contains(StringRef Other) const
Return true if the given string is a substring of *this, and false otherwise.
bool ends_with(StringRef Suffix) const
Check if this string ends with the given Suffix.
Target - Wrapper for Target specific information.
The TimeTraceScope is a helper class to call the begin and end functions of the time trace profiler.
The instances of the Type class are immutable: once they are created, they are never changed.
static LLVM_ABI Type * getVoidTy(LLVMContext &C)
bool isVoidTy() const
Return true if this is 'void'.
static UncondBrInst * Create(BasicBlock *Target, InsertPosition InsertBefore=nullptr)
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI void setName(const Twine &Name)
Change the name of the value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVM_ABI const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
LLVM_ABI bool eraseMetadata(unsigned KindID)
Erase all metadata attachments with the given kind.
iterator_range< use_iterator > uses()
LLVM_ABI StringRef getName() const
Return a constant reference to the value's name.
std::pair< iterator, bool > insert(const ValueT &V)
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
self_iterator getIterator()
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
@ BasicBlock
Various leaf nodes.
LLVM_ABI Function * getDeclarationIfExists(const Module *M, ID id)
Look up the Function declaration of the intrinsic id in the Module M and return it if it exists.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
bool match(Val *V, const Pattern &P)
initializer< Ty > init(const Ty &Val)
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
DiagnosticInfoOptimizationBase::Argument NV
friend class Instruction
Iterator for Instructions in a `BasicBlock.
@ OF_TextWithCRLF
The file should be opened in text mode and use a carriage linefeed '\r '.
LLVM_ABI uint64_t findLowestOffset(ArrayRef< VirtualCallTarget > Targets, bool IsAfter, uint64_t Size)
LLVM_ABI void setAfterReturnValues(MutableArrayRef< VirtualCallTarget > Targets, uint64_t AllocAfter, unsigned BitWidth, int64_t &OffsetByte, uint64_t &OffsetBit)
LLVM_ABI void setBeforeReturnValues(MutableArrayRef< VirtualCallTarget > Targets, uint64_t AllocBefore, unsigned BitWidth, int64_t &OffsetByte, uint64_t &OffsetBit)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
LLVM_ABI MemoryEffects computeFunctionBodyMemoryAccess(Function &F, AAResults &AAR)
Returns the memory access properties of this copy of the function.
static cl::opt< bool > DisableWholeProgramVisibility("disable-whole-program-visibility", cl::Hidden, cl::desc("Disable whole program visibility (overrides enabling options)"))
Provide a way to force disable whole program for debugging or workarounds, when enabled via the linke...
static cl::opt< bool > WholeProgramVisibility("whole-program-visibility", cl::Hidden, cl::desc("Enable whole program visibility"))
Provide a way to force enable whole program visibility in tests.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI void updatePublicTypeTestCalls(Module &M, bool WholeProgramVisibilityEnabledInLTO, bool DevirtSpeculativelyInIndex=false)
Convert llvm.public.type.test calls to llvm.type.test when whole program visibility is available,...
PassSummaryAction
What to do with the summary when running passes that operate on it.
@ Export
Export information to summary.
@ Import
Import information from summary.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy
Provide the FunctionAnalysisManager to Module proxy.
LLVM_ABI ModuleSummaryIndex buildModuleSummaryIndex(const Module &M, std::function< BlockFrequencyInfo *(const Function &F)> GetBFICallback, ProfileSummaryInfo *PSI, std::function< const StackSafetyInfo *(const Function &F)> GetSSICallback=[](const Function &F) -> const StackSafetyInfo *{ return nullptr;})
Direct function to compute a ModuleSummaryIndex from a given module.
Error createStringError(std::error_code EC, char const *Fmt, const Ts &... Vals)
Create formatted StringError object.
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
LLVM_ABI bool hasWholeProgramVisibility(bool WholeProgramVisibilityEnabledInLTO)
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
LLVM_ABI void writeIndexToFile(const ModuleSummaryIndex &Index, raw_ostream &Out, const ModuleToSummariesForIndexTy *ModuleToSummariesForIndex=nullptr, const GVSummaryPtrSet *DecSummaries=nullptr)
Write the specified module summary index to the given raw output stream, where it will be written in ...
LLVM_ABI Expected< std::unique_ptr< ModuleSummaryIndex > > getModuleSummaryIndex(MemoryBufferRef Buffer)
Parse the specified bitcode buffer, returning the module summary index.
const IPOOptions & getIPOOptions(const LLVMContext &C)
LLVM_ABI void updateIndexWPDForExports(ModuleSummaryIndex &Summary, function_ref< bool(StringRef, ValueInfo)> isExported, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap, DenseSet< StringRef > *ExternallyVisibleSymbolNamesPtr=nullptr)
Call after cross-module importing to update the recorded single impl devirt target names for any loca...
LLVM_ABI SmallVector< uint32_t > fitWeights(ArrayRef< uint64_t > Weights)
Push the weights right to fit in uint32_t.
LLVM_ABI void getVisibleToRegularObjVtableGUIDs(ModuleSummaryIndex &Index, DenseSet< GlobalValue::GUID > &VisibleToRegularObjSymbols, function_ref< bool(StringRef)> IsVisibleToRegularObj)
Based on typeID string, get all associated vtable GUIDS that are visible to regular objects.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
IRBuilder(LLVMContext &, FolderTy, InserterTy) -> IRBuilder< FolderTy, InserterTy >
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
LLVM_ABI void findDevirtualizableCallsForTypeCheckedLoad(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< Instruction * > &LoadedPtrs, SmallVectorImpl< Instruction * > &Preds, bool &HasNonCallUses, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.checked.load, find all devirtualizable call sites based on t...
LLVM_ABI CallBase & versionCallSite(CallBase &CB, Value *Callee, MDNode *BranchWeights)
Predicate and clone the given call site.
LLVM_ABI bool AreStatisticsEnabled()
Check if statistics are enabled.
LLVM_ABI SmallVector< InstrProfValueData, 4 > getValueProfDataFromInst(const Instruction &Inst, InstrProfValueKind ValueKind, uint32_t MaxNumValueData, uint64_t &TotalC, bool GetNoICPValue=false)
Extract the value profile data from Inst and returns them if Inst is annotated with value profile dat...
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
LLVM_ABI void setExplicitlyUnknownFunctionEntryCount(Function &F, StringRef PassName)
Analogous to setExplicitlyUnknownBranchWeights, but for functions and their entry counts.
MutableArrayRef(T &OneElt) -> MutableArrayRef< T >
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
LLVM_ABI void runWholeProgramDevirtOnIndex(ModuleSummaryIndex &Summary, std::set< GlobalValue::GUID > &ExportedGUIDs, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap, DenseSet< StringRef > *ExternallyVisibleSymbolNamesPtr, const LLVMContext *Ctx)
Perform index-based whole program devirtualization on the Summary index.
static cl::list< std::string > SkipFunctionNames("wholeprogramdevirt-skip", cl::desc("Prevent function(s) from being devirtualized"), cl::Hidden, cl::CommaSeparated)
Provide way to prevent certain function from being devirtualized.
Expected< T > errorOrToExpected(ErrorOr< T > &&EO)
Convert an ErrorOr<T> to an Expected<T>.
ArrayRef(const T &OneElt) -> ArrayRef< T >
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
LLVM_ABI Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
LLVM_ABI Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
std::vector< TypeIdOffsetVtableInfo > TypeIdCompatibleVtableInfo
List of vtable definitions decorated by a particular type identifier, and their corresponding offsets...
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
static cl::opt< bool > PrintSummaryDevirt("wholeprogramdevirt-print-index-based", cl::Hidden, cl::desc("Print index-based devirtualization messages"))
void consumeError(Error Err)
Consume a Error without doing anything.
LLVM_ABI void findDevirtualizableCallsForTypeTest(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< CallInst * > &Assumes, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.test, find all devirtualizable call sites based on the call ...
LLVM_ABI void updateVCallVisibilityInModule(Module &M, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols, bool ValidateAllVtablesHaveTypeInfos, function_ref< bool(StringRef)> IsVisibleToRegularObj)
If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...
AnalysisManager< Module > ModuleAnalysisManager
Convenience typedef for the Module analysis manager.
LLVM_ABI std::pair< Function *, Constant * > getFunctionAtVTableOffset(GlobalVariable *GV, uint64_t Offset, Module &M)
Given a vtable and a specified offset, returns the function and the trivial pointer at the specified ...
LLVM_ABI void updateVCallVisibilityInIndex(ModuleSummaryIndex &Index, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols, const DenseSet< GlobalValue::GUID > &VisibleToRegularObjSymbols)
If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Class to accumulate and hold information about a callee.
static unsigned getHashValue(const VTableSlotSummary &I)
static bool isEqual(const VTableSlotSummary &LHS, const VTableSlotSummary &RHS)
static bool isEqual(const VTableSlot &LHS, const VTableSlot &RHS)
static unsigned getHashValue(const VTableSlot &I)
An information struct used to provide DenseMap with the various necessary components for a given valu...
The following data structures summarize type metadata information.
std::map< uint64_t, WholeProgramDevirtResolution > WPDRes
Mapping from byte offset to whole-program devirt resolution for that (typeid, byte offset) pair.
@ Unsat
Unsatisfiable type (i.e. no global has this type metadata)
enum llvm::TypeTestResolution::Kind TheKind
Struct that holds a reference to a particular GUID in a global value summary.
ArrayRef< std::unique_ptr< GlobalValueSummary > > getSummaryList() const
const ModuleSummaryIndex * ImportSummary
ModuleSummaryIndex * ExportSummary
LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &)
@ UniformRetVal
Uniform return value optimization.
@ VirtualConstProp
Virtual constant propagation.
@ UniqueRetVal
Unique return value optimization.
uint64_t Info
Additional information for the resolution:
enum llvm::WholeProgramDevirtResolution::ByArg::Kind TheKind
enum llvm::WholeProgramDevirtResolution::Kind TheKind
std::map< std::vector< uint64_t >, ByArg > ResByArg
Resolutions for calls with all constant integer arguments (excluding the first argument,...
std::string SingleImplName
@ SingleImpl
Single implementation devirtualization.
@ Speculative
Speculative devirtualization.
@ BranchFunnel
When retpoline mitigation is enabled, use a branch funnel that is defined in the merged module.
LLVM_ABI VirtualCallTarget(GlobalValue *Fn, const TypeMemberInfo *TM)
const TypeMemberInfo * TM