109using namespace wholeprogramdevirt;
111#define DEBUG_TYPE "wholeprogramdevirt"
113STATISTIC(NumDevirtTargets,
"Number of whole program devirtualization targets");
114STATISTIC(NumSingleImpl,
"Number of single implementation devirtualizations");
116STATISTIC(NumUniformRetVal,
"Number of uniform return value optimizations");
117STATISTIC(NumUniqueRetVal,
"Number of unique return value optimizations");
119 "Number of 1 bit virtual constant propagations");
120STATISTIC(NumVirtConstProp,
"Number of virtual constant propagations");
123 "wholeprogramdevirt-summary-action",
124 cl::desc(
"What to do with the summary when running this pass"),
126 clEnumValN(PassSummaryAction::Import,
"import",
127 "Import typeid resolutions from summary and globals"),
128 clEnumValN(PassSummaryAction::Export,
"export",
129 "Export typeid resolutions to summary and globals")),
133 "wholeprogramdevirt-read-summary",
135 "Read summary from given bitcode or YAML file before running pass"),
139 "wholeprogramdevirt-write-summary",
140 cl::desc(
"Write summary to given bitcode or YAML file after running pass. "
141 "Output file format is deduced from extension: *.bc means writing "
142 "bitcode, otherwise YAML"),
148 cl::desc(
"Maximum number of call targets per "
149 "call site to enable branch funnels"));
153 cl::desc(
"Print index-based devirtualization messages"));
161 cl::desc(
"Enable whole program visibility"));
166 "disable-whole-program-visibility",
cl::Hidden,
167 cl::desc(
"Disable whole program visibility (overrides enabling options)"));
172 cl::desc(
"Prevent function(s) from being devirtualized"),
183 cl::desc(
"Type of checking for incorrect devirtualizations"),
187 "Fallback to indirect when incorrect")));
191 std::vector<GlobPattern> Patterns;
192 template <
class T>
void init(
const T &StringList) {
193 for (
const auto &S : StringList)
195 Patterns.push_back(std::move(*Pat));
216 MinByte = std::max(MinByte,
Target.minAfterBytes());
218 MinByte = std::max(MinByte,
Target.minBeforeBytes());
241 std::vector<ArrayRef<uint8_t>> Used;
244 :
Target.TM->Bits->Before.BytesUsed;
246 : MinByte -
Target.minBeforeBytes();
256 for (
unsigned I = 0;; ++
I) {
257 uint8_t BitsUsed = 0;
258 for (
auto &&
B : Used)
261 if (BitsUsed != 0xff)
267 for (
unsigned I = 0;; ++
I) {
268 for (
auto &&
B : Used) {
270 while ((
I + Byte) <
B.size() && Byte < (
Size / 8)) {
276 return (MinByte +
I) * 8;
286 OffsetByte = -(AllocBefore / 8 + 1);
288 OffsetByte = -((AllocBefore + 7) / 8 + (
BitWidth + 7) / 8);
289 OffsetBit = AllocBefore % 8;
293 Target.setBeforeBit(AllocBefore);
303 OffsetByte = AllocAfter / 8;
305 OffsetByte = (AllocAfter + 7) / 8;
306 OffsetBit = AllocAfter % 8;
310 Target.setAfterBit(AllocAfter);
349 const VTableSlot &
RHS) {
350 return LHS.TypeID ==
RHS.TypeID &&
LHS.ByteOffset ==
RHS.ByteOffset;
369 return LHS.TypeID ==
RHS.TypeID &&
LHS.ByteOffset ==
RHS.ByteOffset;
396 if (
auto *FS = dyn_cast<FunctionSummary>(
Summary->getBaseObject())) {
397 if (!
FS->fflags().MustBeUnreachable)
411struct VirtualCallSite {
418 unsigned *NumUnsafeUses =
nullptr;
429 <<
NV(
"Optimization", OptName)
430 <<
": devirtualized a call to "
431 <<
NV(
"FunctionName", TargetName));
434 void replaceAndErase(
439 emitRemark(OptName, TargetName, OREGetter);
441 if (
auto *II = dyn_cast<InvokeInst>(&CB)) {
443 II->getUnwindDest()->removePredecessor(II->getParent());
460 std::vector<VirtualCallSite> CallSites;
465 bool AllCallSitesDevirted =
true;
472 bool SummaryHasTypeTestAssumeUsers =
false;
481 std::vector<FunctionSummary *> SummaryTypeCheckedLoadUsers;
482 std::vector<FunctionSummary *> SummaryTypeTestAssumeUsers;
484 bool isExported()
const {
485 return SummaryHasTypeTestAssumeUsers ||
486 !SummaryTypeCheckedLoadUsers.empty();
490 SummaryTypeCheckedLoadUsers.push_back(FS);
491 AllCallSitesDevirted =
false;
495 SummaryTypeTestAssumeUsers.push_back(FS);
496 SummaryHasTypeTestAssumeUsers =
true;
497 AllCallSitesDevirted =
false;
501 AllCallSitesDevirted =
true;
504 SummaryTypeCheckedLoadUsers.clear();
509struct VTableSlotInfo {
516 std::map<std::vector<uint64_t>, CallSiteInfo> ConstCSInfo;
518 void addCallSite(
Value *VTable,
CallBase &CB,
unsigned *NumUnsafeUses);
521 CallSiteInfo &findCallSiteInfo(
CallBase &CB);
524CallSiteInfo &VTableSlotInfo::findCallSiteInfo(
CallBase &CB) {
525 std::vector<uint64_t>
Args;
526 auto *CBType = dyn_cast<IntegerType>(CB.
getType());
527 if (!CBType || CBType->getBitWidth() > 64 || CB.
arg_empty())
530 auto *CI = dyn_cast<ConstantInt>(
Arg);
531 if (!CI || CI->getBitWidth() > 64)
533 Args.push_back(CI->getZExtValue());
535 return ConstCSInfo[
Args];
539 unsigned *NumUnsafeUses) {
540 auto &CSI = findCallSiteInfo(CB);
541 CSI.AllCallSitesDevirted =
false;
542 CSI.CallSites.push_back({
VTable, CB, NumUnsafeUses});
581 std::map<CallInst *, unsigned> NumUnsafeUsesForTypeTest;
582 PatternList FunctionsToSkip;
589 :
M(
M), AARGetter(AARGetter), LookupDomTree(LookupDomTree),
590 ExportSummary(ExportSummary), ImportSummary(ImportSummary),
591 Int8Ty(
Type::getInt8Ty(
M.getContext())),
592 Int8PtrTy(
Type::getInt8PtrTy(
M.getContext())),
594 Int64Ty(
Type::getInt64Ty(
M.getContext())),
595 IntPtrTy(
M.getDataLayout().getIntPtrType(
M.getContext(), 0)),
597 RemarksEnabled(areRemarksEnabled()), OREGetter(OREGetter) {
598 assert(!(ExportSummary && ImportSummary));
602 bool areRemarksEnabled();
605 scanTypeTestUsers(
Function *TypeTestFunc,
607 void scanTypeCheckedLoadUsers(
Function *TypeCheckedLoadFunc);
609 void buildTypeIdentifierMap(
610 std::vector<VTableBits> &Bits,
614 tryFindVirtualCallTargets(std::vector<VirtualCallTarget> &TargetsForSlot,
615 const std::set<TypeMemberInfo> &TypeMemberInfos,
619 void applySingleImplDevirt(VTableSlotInfo &SlotInfo,
Constant *TheFn,
623 VTableSlotInfo &SlotInfo,
626 void applyICallBranchFunnel(VTableSlotInfo &SlotInfo,
Constant *JT,
629 VTableSlotInfo &SlotInfo,
632 bool tryEvaluateFunctionsWithArgs(
636 void applyUniformRetValOpt(CallSiteInfo &CSInfo,
StringRef FnName,
639 CallSiteInfo &CSInfo,
647 bool shouldExportConstantsAsAbsoluteSymbols();
667 void applyUniqueRetValOpt(CallSiteInfo &CSInfo,
StringRef FnName,
bool IsOne,
669 bool tryUniqueRetValOpt(
unsigned BitWidth,
671 CallSiteInfo &CSInfo,
675 void applyVirtualConstProp(CallSiteInfo &CSInfo,
StringRef FnName,
678 VTableSlotInfo &SlotInfo,
684 void importResolution(VTableSlot Slot, VTableSlotInfo &SlotInfo);
688 void removeRedundantTypeTests();
721 std::set<GlobalValue::GUID> &ExportedGUIDs;
725 std::map<ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap;
729 PatternList FunctionsToSkip;
733 std::set<GlobalValue::GUID> &ExportedGUIDs,
734 std::map<
ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap)
735 : ExportSummary(ExportSummary), ExportedGUIDs(ExportedGUIDs),
736 LocalWPDTargetsMap(LocalWPDTargetsMap) {
740 bool tryFindVirtualCallTargets(std::vector<ValueInfo> &TargetsForSlot,
746 VTableSlotInfo &SlotInfo,
748 std::set<ValueInfo> &DevirtTargets);
766 if (UseCommandLine) {
767 if (!DevirtModule::runForTesting(M, AARGetter, OREGetter, LookupDomTree))
771 if (!DevirtModule(M, AARGetter, OREGetter, LookupDomTree, ExportSummary,
790 Module &M,
bool WholeProgramVisibilityEnabledInLTO,
798 if (GV.hasMetadata(LLVMContext::MD_type) &&
802 !DynamicExportSymbols.
count(GV.getGUID()))
808 bool WholeProgramVisibilityEnabledInLTO) {
811 if (!PublicTypeTestFunc)
817 auto *CI = cast<CallInst>(U.getUser());
819 TypeTestFunc, {CI->getArgOperand(0), CI->getArgOperand(1)},
820 std::nullopt,
"", CI);
822 CI->eraseFromParent();
827 auto *CI = cast<CallInst>(U.getUser());
828 CI->replaceAllUsesWith(True);
829 CI->eraseFromParent();
845 if (DynamicExportSymbols.
count(
P.first))
847 for (
auto &S :
P.second.SummaryList) {
848 auto *GVar = dyn_cast<GlobalVarSummary>(S.get());
859 std::map<
ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap) {
860 DevirtIndex(Summary, ExportedGUIDs, LocalWPDTargetsMap).run();
866 std::map<
ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap) {
867 for (
auto &
T : LocalWPDTargetsMap) {
870 assert(
VI.getSummaryList().size() == 1 &&
871 "Devirt of local target has more than one copy");
872 auto &S =
VI.getSummaryList()[0];
873 if (!isExported(S->modulePath(),
VI))
877 for (
auto &SlotSummary :
T.second) {
878 auto *TIdSum = Summary.getTypeIdSummary(SlotSummary.TypeID);
880 auto WPDRes = TIdSum->WPDRes.find(SlotSummary.ByteOffset);
881 assert(WPDRes != TIdSum->WPDRes.end());
883 WPDRes->second.SingleImplName,
884 Summary.getModuleHash(S->modulePath()));
896 const auto &ModPaths = Summary->modulePaths();
901 "combined summary should contain Regular LTO module");
905bool DevirtModule::runForTesting(
909 std::unique_ptr<ModuleSummaryIndex>
Summary =
910 std::make_unique<ModuleSummaryIndex>(
false);
917 auto ReadSummaryFile =
919 if (
Expected<std::unique_ptr<ModuleSummaryIndex>> SummaryOrErr =
921 Summary = std::move(*SummaryOrErr);
926 yaml::Input
In(ReadSummaryFile->getBuffer());
933 DevirtModule(M, AARGetter, OREGetter, LookupDomTree,
951 yaml::Output Out(
OS);
959void DevirtModule::buildTypeIdentifierMap(
960 std::vector<VTableBits> &Bits,
963 Bits.reserve(
M.global_size());
967 GV.getMetadata(LLVMContext::MD_type, Types);
968 if (GV.isDeclaration() ||
Types.empty())
974 Bits.back().GV = &GV;
975 Bits.back().ObjectSize =
976 M.getDataLayout().getTypeAllocSize(GV.getInitializer()->getType());
977 BitsPtr = &
Bits.back();
985 cast<ConstantAsMetadata>(
Type->getOperand(0))->getValue())
993bool DevirtModule::tryFindVirtualCallTargets(
994 std::vector<VirtualCallTarget> &TargetsForSlot,
995 const std::set<TypeMemberInfo> &TypeMemberInfos,
uint64_t ByteOffset,
998 if (!
TM.Bits->GV->isConstant())
1003 if (
TM.Bits->GV->getVCallVisibility() ==
1008 TM.Offset + ByteOffset, M);
1012 auto C =
Ptr->stripPointerCasts();
1014 auto Fn = dyn_cast<Function>(
C);
1015 auto A = dyn_cast<GlobalAlias>(
C);
1017 Fn = dyn_cast<Function>(
A->getAliasee());
1022 if (FunctionsToSkip.match(Fn->getName()))
1027 if (Fn->getName() ==
"__cxa_pure_virtual")
1037 auto GV = dyn_cast<GlobalValue>(
C);
1039 TargetsForSlot.push_back({GV, &
TM});
1043 return !TargetsForSlot.empty();
1046bool DevirtIndex::tryFindVirtualCallTargets(
1061 bool LocalFound =
false;
1062 for (
const auto &S :
P.VTableVI.getSummaryList()) {
1068 auto *CurVS = cast<GlobalVarSummary>(S->getBaseObject());
1069 if (!CurVS->vTableFuncs().empty() ||
1091 for (
auto VTP :
VS->vTableFuncs()) {
1092 if (VTP.VTableOffset !=
P.AddressPointOffset + ByteOffset)
1098 TargetsForSlot.push_back(VTP.FuncVI);
1103 return !TargetsForSlot.empty();
1106void DevirtModule::applySingleImplDevirt(VTableSlotInfo &SlotInfo,
1107 Constant *TheFn,
bool &IsExported) {
1112 auto Apply = [&](CallSiteInfo &CSInfo) {
1113 for (
auto &&VCallSite : CSInfo.CallSites) {
1114 if (!OptimizedCalls.
insert(&VCallSite.CB).second)
1118 VCallSite.emitRemark(
"single-impl",
1121 auto &CB = VCallSite.CB;
1134 Builder.SetInsertPoint(ThenTerm);
1136 auto *CallTrap =
Builder.CreateCall(TrapFn);
1154 NewInst.
setMetadata(LLVMContext::MD_prof,
nullptr);
1155 NewInst.
setMetadata(LLVMContext::MD_callees,
nullptr);
1174 if (VCallSite.NumUnsafeUses)
1175 --*VCallSite.NumUnsafeUses;
1177 if (CSInfo.isExported())
1179 CSInfo.markDevirt();
1181 Apply(SlotInfo.CSInfo);
1182 for (
auto &
P : SlotInfo.ConstCSInfo)
1188 if (
Callee.getSummaryList().empty())
1195 bool IsExported =
false;
1196 auto &S =
Callee.getSummaryList()[0];
1198 auto AddCalls = [&](CallSiteInfo &CSInfo) {
1199 for (
auto *FS : CSInfo.SummaryTypeCheckedLoadUsers) {
1200 FS->addCall({
Callee, CI});
1201 IsExported |= S->modulePath() != FS->modulePath();
1203 for (
auto *FS : CSInfo.SummaryTypeTestAssumeUsers) {
1204 FS->addCall({
Callee, CI});
1205 IsExported |= S->modulePath() != FS->modulePath();
1209 for (
auto &
P : SlotInfo.ConstCSInfo)
1214bool DevirtModule::trySingleImplDevirt(
1220 auto *TheFn = TargetsForSlot[0].Fn;
1221 for (
auto &&
Target : TargetsForSlot)
1227 TargetsForSlot[0].WasDevirt =
true;
1229 bool IsExported =
false;
1230 applySingleImplDevirt(SlotInfo, TheFn, IsExported);
1237 if (TheFn->hasLocalLinkage()) {
1238 std::string NewName = (TheFn->
getName() +
".llvm.merged").str();
1243 if (
Comdat *
C = TheFn->getComdat()) {
1244 if (
C->getName() == TheFn->
getName()) {
1245 Comdat *NewC =
M.getOrInsertComdat(NewName);
1248 if (GO.getComdat() ==
C)
1270 VTableSlotInfo &SlotInfo,
1272 std::set<ValueInfo> &DevirtTargets) {
1275 auto TheFn = TargetsForSlot[0];
1276 for (
auto &&
Target : TargetsForSlot)
1281 auto Size = TheFn.getSummaryList().size();
1287 if (FunctionsToSkip.match(TheFn.name()))
1292 for (
const auto &S : TheFn.getSummaryList())
1298 DevirtTargets.insert(TheFn);
1300 auto &S = TheFn.getSummaryList()[0];
1301 bool IsExported =
AddCalls(SlotInfo, TheFn);
1303 ExportedGUIDs.insert(TheFn.getGUID());
1314 TheFn.name(), ExportSummary.
getModuleHash(S->modulePath()));
1316 LocalWPDTargetsMap[TheFn].push_back(SlotSummary);
1330void DevirtModule::tryICallBranchFunnel(
1340 bool HasNonDevirt = !SlotInfo.CSInfo.AllCallSitesDevirted;
1342 for (
auto &
P : SlotInfo.ConstCSInfo)
1343 if (!
P.second.AllCallSitesDevirted) {
1344 HasNonDevirt =
true;
1354 if (isa<MDString>(
Slot.TypeID)) {
1356 M.getDataLayout().getProgramAddressSpace(),
1357 getGlobalName(Slot, {},
"branch_funnel"), &M);
1361 M.getDataLayout().getProgramAddressSpace(),
1362 "branch_funnel", &M);
1364 JT->addParamAttr(0, Attribute::Nest);
1366 std::vector<Value *> JTArgs;
1367 JTArgs.push_back(
JT->arg_begin());
1368 for (
auto &
T : TargetsForSlot) {
1369 JTArgs.push_back(getMemberAddr(
T.TM));
1370 JTArgs.push_back(
T.Fn);
1381 bool IsExported =
false;
1382 applyICallBranchFunnel(SlotInfo, JT, IsExported);
1387void DevirtModule::applyICallBranchFunnel(VTableSlotInfo &SlotInfo,
1389 auto Apply = [&](CallSiteInfo &CSInfo) {
1390 if (CSInfo.isExported())
1392 if (CSInfo.AllCallSitesDevirted)
1395 std::map<CallBase *, CallBase *> CallBases;
1396 for (
auto &&VCallSite : CSInfo.CallSites) {
1399 if (CallBases.find(&CB) != CallBases.end()) {
1416 VCallSite.emitRemark(
"branch-funnel",
1417 JT->stripPointerCasts()->getName(), OREGetter);
1421 std::vector<Type *> NewArgs;
1422 NewArgs.push_back(Int8PtrTy);
1430 std::vector<Value *>
Args;
1431 Args.push_back(IRB.CreateBitCast(VCallSite.VTable, Int8PtrTy));
1435 if (isa<CallInst>(CB))
1436 NewCS = IRB.CreateCall(NewFT, IRB.CreateBitCast(JT, NewFTPtr), Args);
1438 NewCS = IRB.CreateInvoke(NewFT, IRB.CreateBitCast(JT, NewFTPtr),
1439 cast<InvokeInst>(CB).getNormalDest(),
1440 cast<InvokeInst>(CB).getUnwindDest(), Args);
1444 std::vector<AttributeSet> NewArgAttrs;
1447 M.getContext(), Attribute::Nest)}));
1448 for (
unsigned I = 0;
I + 2 <
Attrs.getNumAttrSets(); ++
I)
1449 NewArgAttrs.push_back(
Attrs.getParamAttrs(
I));
1452 Attrs.getRetAttrs(), NewArgAttrs));
1454 CallBases[&CB] = NewCS;
1457 if (VCallSite.NumUnsafeUses)
1458 --*VCallSite.NumUnsafeUses;
1465 std::for_each(CallBases.begin(), CallBases.end(), [](
auto &CBs) {
1466 CBs.first->replaceAllUsesWith(CBs.second);
1467 CBs.first->eraseFromParent();
1470 Apply(SlotInfo.CSInfo);
1471 for (
auto &
P : SlotInfo.ConstCSInfo)
1475bool DevirtModule::tryEvaluateFunctionsWithArgs(
1484 auto Fn = dyn_cast<Function>(
Target.Fn);
1488 if (Fn->arg_size() !=
Args.size() + 1)
1495 for (
unsigned I = 0;
I !=
Args.size(); ++
I) {
1497 dyn_cast<IntegerType>(Fn->getFunctionType()->getParamType(
I + 1));
1504 if (!Eval.EvaluateFunction(Fn, RetVal, EvalArgs) ||
1505 !isa<ConstantInt>(RetVal))
1507 Target.RetVal = cast<ConstantInt>(RetVal)->getZExtValue();
1512void DevirtModule::applyUniformRetValOpt(CallSiteInfo &CSInfo,
StringRef FnName,
1514 for (
auto Call : CSInfo.CallSites) {
1518 Call.replaceAndErase(
1519 "uniform-ret-val", FnName, RemarksEnabled, OREGetter,
1522 CSInfo.markDevirt();
1525bool DevirtModule::tryUniformRetValOpt(
1530 uint64_t TheRetVal = TargetsForSlot[0].RetVal;
1532 if (
Target.RetVal != TheRetVal)
1535 if (CSInfo.isExported()) {
1537 Res->
Info = TheRetVal;
1540 applyUniformRetValOpt(CSInfo, TargetsForSlot[0].Fn->getName(), TheRetVal);
1542 for (
auto &&
Target : TargetsForSlot)
1547std::string DevirtModule::getGlobalName(VTableSlot Slot,
1550 std::string FullName =
"__typeid_";
1552 OS << cast<MDString>(
Slot.TypeID)->getString() <<
'_' <<
Slot.ByteOffset;
1559bool DevirtModule::shouldExportConstantsAsAbsoluteSymbols() {
1567 getGlobalName(Slot, Args,
Name),
C, &M);
1574 if (shouldExportConstantsAsAbsoluteSymbols()) {
1587 M.getOrInsertGlobal(getGlobalName(Slot, Args,
Name), Int8Arr0Ty);
1588 auto *GV = dyn_cast<GlobalVariable>(
C);
1597 if (!shouldExportConstantsAsAbsoluteSymbols())
1601 auto *GV = cast<GlobalVariable>(
C->stripPointerCasts());
1606 if (GV->hasMetadata(LLVMContext::MD_absolute_symbol))
1612 GV->setMetadata(LLVMContext::MD_absolute_symbol,
1617 SetAbsRange(~0ull, ~0ull);
1619 SetAbsRange(0, 1ull << AbsWidth);
1623void DevirtModule::applyUniqueRetValOpt(CallSiteInfo &CSInfo,
StringRef FnName,
1626 for (
auto &&Call : CSInfo.CallSites) {
1632 B.CreateBitCast(UniqueMemberAddr,
Call.VTable->getType()));
1633 Cmp =
B.CreateZExt(Cmp,
Call.CB.getType());
1635 Call.replaceAndErase(
"unique-ret-val", FnName, RemarksEnabled, OREGetter,
1638 CSInfo.markDevirt();
1647bool DevirtModule::tryUniqueRetValOpt(
1652 auto tryUniqueRetValOptFor = [&](
bool IsOne) {
1655 if (
Target.RetVal == (IsOne ? 1 : 0)) {
1658 UniqueMember =
Target.TM;
1666 Constant *UniqueMemberAddr = getMemberAddr(UniqueMember);
1667 if (CSInfo.isExported()) {
1671 exportGlobal(Slot, Args,
"unique_member", UniqueMemberAddr);
1675 applyUniqueRetValOpt(CSInfo, TargetsForSlot[0].Fn->getName(), IsOne,
1680 for (
auto &&
Target : TargetsForSlot)
1687 if (tryUniqueRetValOptFor(
true))
1689 if (tryUniqueRetValOptFor(
false))
1695void DevirtModule::applyVirtualConstProp(CallSiteInfo &CSInfo,
StringRef FnName,
1697 for (
auto Call : CSInfo.CallSites) {
1700 auto *RetType = cast<IntegerType>(
Call.CB.getType());
1703 B.CreateGEP(Int8Ty,
B.CreateBitCast(
Call.VTable, Int8PtrTy), Byte);
1704 if (RetType->getBitWidth() == 1) {
1706 Value *BitsAndBit =
B.CreateAnd(Bits, Bit);
1708 NumVirtConstProp1Bit++;
1709 Call.replaceAndErase(
"virtual-const-prop-1-bit", FnName, RemarksEnabled,
1710 OREGetter, IsBitSet);
1712 Value *ValAddr =
B.CreateBitCast(
Addr, RetType->getPointerTo());
1713 Value *Val =
B.CreateLoad(RetType, ValAddr);
1715 Call.replaceAndErase(
"virtual-const-prop", FnName, RemarksEnabled,
1719 CSInfo.markDevirt();
1722bool DevirtModule::tryVirtualConstProp(
1728 auto Fn = dyn_cast<Function>(TargetsForSlot[0].Fn);
1732 auto RetType = dyn_cast<IntegerType>(Fn->getReturnType());
1735 unsigned BitWidth = RetType->getBitWidth();
1753 auto Fn = dyn_cast<Function>(
Target.Fn);
1757 if (Fn->isDeclaration() ||
1759 .doesNotAccessMemory() ||
1760 Fn->arg_empty() || !Fn->arg_begin()->use_empty() ||
1761 Fn->getReturnType() != RetType)
1765 for (
auto &&CSByConstantArg : SlotInfo.ConstCSInfo) {
1766 if (!tryEvaluateFunctionsWithArgs(TargetsForSlot, CSByConstantArg.first))
1771 ResByArg = &Res->
ResByArg[CSByConstantArg.first];
1773 if (tryUniformRetValOpt(TargetsForSlot, CSByConstantArg.second, ResByArg))
1776 if (tryUniqueRetValOpt(
BitWidth, TargetsForSlot, CSByConstantArg.second,
1777 ResByArg, Slot, CSByConstantArg.first))
1789 uint64_t TotalPaddingBefore = 0, TotalPaddingAfter = 0;
1790 for (
auto &&
Target : TargetsForSlot) {
1791 TotalPaddingBefore += std::max<int64_t>(
1792 (AllocBefore + 7) / 8 -
Target.allocatedBeforeBytes() - 1, 0);
1793 TotalPaddingAfter += std::max<int64_t>(
1794 (AllocAfter + 7) / 8 -
Target.allocatedAfterBytes() - 1, 0);
1799 if (std::min(TotalPaddingBefore, TotalPaddingAfter) > 128)
1806 if (TotalPaddingBefore <= TotalPaddingAfter)
1814 for (
auto &&
Target : TargetsForSlot)
1818 if (CSByConstantArg.second.isExported()) {
1820 exportConstant(Slot, CSByConstantArg.first,
"byte", OffsetByte,
1822 exportConstant(Slot, CSByConstantArg.first,
"bit", 1ULL << OffsetBit,
1829 applyVirtualConstProp(CSByConstantArg.second,
1830 TargetsForSlot[0].Fn->getName(), ByteConst, BitConst);
1836 if (
B.Before.Bytes.empty() &&
B.After.Bytes.empty())
1841 Align Alignment =
M.getDataLayout().getValueOrABITypeAlignment(
1842 B.GV->getAlign(),
B.GV->getValueType());
1843 B.Before.Bytes.resize(
alignTo(
B.Before.Bytes.size(), Alignment));
1846 for (
size_t I = 0,
Size =
B.Before.Bytes.size();
I !=
Size / 2; ++
I)
1853 B.GV->getInitializer(),
1858 NewGV->setSection(
B.GV->getSection());
1859 NewGV->setComdat(
B.GV->getComdat());
1860 NewGV->setAlignment(
B.GV->getAlign());
1864 NewGV->copyMetadata(
B.GV,
B.Before.Bytes.size());
1869 B.GV->getInitializer()->getType(), 0,
B.GV->getLinkage(),
"",
1871 NewInit->getType(), NewGV,
1873 ConstantInt::get(Int32Ty, 1)}),
1875 Alias->setVisibility(
B.GV->getVisibility());
1876 Alias->takeName(
B.GV);
1878 B.GV->replaceAllUsesWith(Alias);
1879 B.GV->eraseFromParent();
1882bool DevirtModule::areRemarksEnabled() {
1883 const auto &FL =
M.getFunctionList();
1888 return DI.isEnabled();
1893void DevirtModule::scanTypeTestUsers(
1902 auto *CI = dyn_cast<CallInst>(
U.getUser());
1909 auto &DT = LookupDomTree(*CI->getFunction());
1913 cast<MetadataAsValue>(CI->getArgOperand(1))->getMetadata();
1915 if (!Assumes.
empty()) {
1916 Value *
Ptr = CI->getArgOperand(0)->stripPointerCasts();
1918 CallSlots[{TypeId,
Call.Offset}].addCallSite(
Ptr,
Call.CB,
nullptr);
1921 auto RemoveTypeTestAssumes = [&]() {
1923 for (
auto *Assume : Assumes)
1924 Assume->eraseFromParent();
1927 if (CI->use_empty())
1928 CI->eraseFromParent();
1943 if (!TypeIdMap.count(TypeId))
1944 RemoveTypeTestAssumes();
1955 else if (ImportSummary && isa<MDString>(TypeId)) {
1959 RemoveTypeTestAssumes();
1968void DevirtModule::scanTypeCheckedLoadUsers(
Function *TypeCheckedLoadFunc) {
1972 auto *CI = dyn_cast<CallInst>(
U.getUser());
1978 Value *TypeIdValue = CI->getArgOperand(2);
1979 Metadata *TypeId = cast<MetadataAsValue>(TypeIdValue)->getMetadata();
1984 bool HasNonCallUses =
false;
1985 auto &DT = LookupDomTree(*CI->getFunction());
1987 HasNonCallUses, CI, DT);
1996 (LoadedPtrs.
size() == 1 && !HasNonCallUses) ? LoadedPtrs[0] : CI);
1999 Value *LoadedValue = LoadB.CreateLoad(Int8PtrTy, GEPPtr);
2003 LoadedPtr->eraseFromParent();
2007 IRBuilder<> CallB((Preds.
size() == 1 && !HasNonCallUses) ? Preds[0] : CI);
2008 CallInst *TypeTestCall = CallB.CreateCall(TypeTestFunc, {
Ptr, TypeIdValue});
2012 Pred->eraseFromParent();
2019 if (!CI->use_empty()) {
2022 Pair =
B.CreateInsertValue(Pair, LoadedValue, {0});
2023 Pair =
B.CreateInsertValue(Pair, TypeTestCall, {1});
2024 CI->replaceAllUsesWith(Pair);
2028 auto &NumUnsafeUses = NumUnsafeUsesForTypeTest[TypeTestCall];
2029 NumUnsafeUses = DevirtCalls.
size();
2037 CallSlots[{TypeId,
Call.Offset}].addCallSite(
Ptr,
Call.CB,
2041 CI->eraseFromParent();
2045void DevirtModule::importResolution(VTableSlot Slot, VTableSlotInfo &SlotInfo) {
2046 auto *TypeId = dyn_cast<MDString>(
Slot.TypeID);
2053 auto ResI = TidSummary->
WPDRes.find(
Slot.ByteOffset);
2054 if (ResI == TidSummary->
WPDRes.end())
2068 bool IsExported =
false;
2069 applySingleImplDevirt(SlotInfo, SingleImpl, IsExported);
2073 for (
auto &CSByConstantArg : SlotInfo.ConstCSInfo) {
2074 auto I = Res.
ResByArg.find(CSByConstantArg.first);
2077 auto &ResByArg =
I->second;
2084 applyUniformRetValOpt(CSByConstantArg.second,
"", ResByArg.
Info);
2088 importGlobal(Slot, CSByConstantArg.first,
"unique_member");
2089 applyUniqueRetValOpt(CSByConstantArg.second,
"", ResByArg.
Info,
2094 Constant *
Byte = importConstant(Slot, CSByConstantArg.first,
"byte",
2096 Constant *
Bit = importConstant(Slot, CSByConstantArg.first,
"bit", Int8Ty,
2098 applyVirtualConstProp(CSByConstantArg.second,
"", Byte, Bit);
2110 M.getOrInsertFunction(getGlobalName(Slot, {},
"branch_funnel"),
2113 bool IsExported =
false;
2114 applyICallBranchFunnel(SlotInfo, JT, IsExported);
2119void DevirtModule::removeRedundantTypeTests() {
2121 for (
auto &&U : NumUnsafeUsesForTypeTest) {
2122 if (
U.second == 0) {
2123 U.first->replaceAllUsesWith(True);
2124 U.first->eraseFromParent();
2130DevirtModule::lookUpFunctionValueInfo(
Function *TheFn,
2132 assert((ExportSummary !=
nullptr) &&
2133 "Caller guarantees ExportSummary is not nullptr");
2135 const auto TheFnGUID = TheFn->
getGUID();
2147 if ((!TheFnVI) && (TheFnGUID != TheFnGUIDWithExportedName)) {
2148 TheFnVI = ExportSummary->
getValueInfo(TheFnGUIDWithExportedName);
2153bool DevirtModule::mustBeUnreachableFunction(
2156 if (!
F->isDeclaration()) {
2159 return isa<UnreachableInst>(
F->getEntryBlock().getTerminator());
2162 return ExportSummary &&
2164 DevirtModule::lookUpFunctionValueInfo(
F, ExportSummary));
2167bool DevirtModule::run() {
2185 if (!ExportSummary &&
2186 (!TypeTestFunc || TypeTestFunc->
use_empty() || !AssumeFunc ||
2188 (!TypeCheckedLoadFunc || TypeCheckedLoadFunc->
use_empty()))
2192 std::vector<VTableBits>
Bits;
2194 buildTypeIdentifierMap(Bits, TypeIdMap);
2196 if (TypeTestFunc && AssumeFunc)
2197 scanTypeTestUsers(TypeTestFunc, TypeIdMap);
2199 if (TypeCheckedLoadFunc)
2200 scanTypeCheckedLoadUsers(TypeCheckedLoadFunc);
2202 if (ImportSummary) {
2203 for (
auto &S : CallSlots)
2204 importResolution(S.first, S.second);
2206 removeRedundantTypeTests();
2212 GV.eraseMetadata(LLVMContext::MD_vcall_visibility);
2219 if (TypeIdMap.
empty())
2223 if (ExportSummary) {
2225 for (
auto &
P : TypeIdMap) {
2226 if (
auto *TypeId = dyn_cast<MDString>(
P.first))
2231 for (
auto &
P : *ExportSummary) {
2232 for (
auto &S :
P.second.SummaryList) {
2233 auto *
FS = dyn_cast<FunctionSummary>(S.get());
2238 for (
Metadata *MD : MetadataByGUID[VF.GUID]) {
2239 CallSlots[{MD, VF.Offset}].CSInfo.addSummaryTypeTestAssumeUser(FS);
2243 for (
Metadata *MD : MetadataByGUID[VF.GUID]) {
2244 CallSlots[{MD, VF.Offset}].CSInfo.addSummaryTypeCheckedLoadUser(FS);
2248 FS->type_test_assume_const_vcalls()) {
2249 for (
Metadata *MD : MetadataByGUID[
VC.VFunc.GUID]) {
2250 CallSlots[{MD,
VC.VFunc.Offset}]
2251 .ConstCSInfo[
VC.Args]
2252 .addSummaryTypeTestAssumeUser(FS);
2256 FS->type_checked_load_const_vcalls()) {
2257 for (
Metadata *MD : MetadataByGUID[
VC.VFunc.GUID]) {
2258 CallSlots[{MD,
VC.VFunc.Offset}]
2259 .ConstCSInfo[
VC.Args]
2260 .addSummaryTypeCheckedLoadUser(FS);
2268 bool DidVirtualConstProp =
false;
2269 std::map<std::string, GlobalValue *> DevirtTargets;
2270 for (
auto &S : CallSlots) {
2274 std::vector<VirtualCallTarget> TargetsForSlot;
2276 const std::set<TypeMemberInfo> &TypeMemberInfos = TypeIdMap[S.first.TypeID];
2277 if (ExportSummary && isa<MDString>(S.first.TypeID) &&
2278 TypeMemberInfos.size())
2285 Res = &ExportSummary
2286 ->getOrInsertTypeIdSummary(
2287 cast<MDString>(S.first.TypeID)->getString())
2288 .WPDRes[S.first.ByteOffset];
2289 if (tryFindVirtualCallTargets(TargetsForSlot, TypeMemberInfos,
2290 S.first.ByteOffset, ExportSummary)) {
2292 if (!trySingleImplDevirt(ExportSummary, TargetsForSlot, S.second, Res)) {
2293 DidVirtualConstProp |=
2294 tryVirtualConstProp(TargetsForSlot, S.second, Res, S.first);
2296 tryICallBranchFunnel(TargetsForSlot, S.second, Res, S.first);
2301 for (
const auto &
T : TargetsForSlot)
2303 DevirtTargets[std::string(
T.Fn->getName())] =
T.Fn;
2310 if (ExportSummary && isa<MDString>(S.first.TypeID)) {
2313 for (
auto *FS : S.second.CSInfo.SummaryTypeCheckedLoadUsers)
2314 FS->addTypeTest(GUID);
2315 for (
auto &CCS : S.second.ConstCSInfo)
2316 for (
auto *FS : CCS.second.SummaryTypeCheckedLoadUsers)
2317 FS->addTypeTest(GUID);
2321 if (RemarksEnabled) {
2323 for (
const auto &DT : DevirtTargets) {
2325 auto F = dyn_cast<Function>(GV);
2327 auto A = dyn_cast<GlobalAlias>(GV);
2328 assert(
A && isa<Function>(
A->getAliasee()));
2329 F = dyn_cast<Function>(
A->getAliasee());
2333 using namespace ore;
2336 <<
NV(
"FunctionName", DT.first));
2340 NumDevirtTargets += DevirtTargets.size();
2342 removeRedundantTypeTests();
2346 if (DidVirtualConstProp)
2359void DevirtIndex::run() {
2360 if (ExportSummary.typeIdCompatibleVtableMap().empty())
2364 for (
const auto &
P : ExportSummary.typeIdCompatibleVtableMap()) {
2372 ExportSummary.getOrInsertTypeIdSummary(
P.first);
2376 for (
auto &
P : ExportSummary) {
2377 for (
auto &S :
P.second.SummaryList) {
2378 auto *
FS = dyn_cast<FunctionSummary>(S.get());
2384 CallSlots[{
Name, VF.Offset}].CSInfo.addSummaryTypeTestAssumeUser(FS);
2389 CallSlots[{
Name, VF.Offset}].CSInfo.addSummaryTypeCheckedLoadUser(FS);
2393 FS->type_test_assume_const_vcalls()) {
2395 CallSlots[{
Name,
VC.VFunc.Offset}]
2396 .ConstCSInfo[
VC.Args]
2397 .addSummaryTypeTestAssumeUser(FS);
2401 FS->type_checked_load_const_vcalls()) {
2403 CallSlots[{
Name,
VC.VFunc.Offset}]
2404 .ConstCSInfo[
VC.Args]
2405 .addSummaryTypeCheckedLoadUser(FS);
2411 std::set<ValueInfo> DevirtTargets;
2413 for (
auto &S : CallSlots) {
2417 std::vector<ValueInfo> TargetsForSlot;
2418 auto TidSummary = ExportSummary.getTypeIdCompatibleVtableSummary(S.first.TypeID);
2423 &ExportSummary.getTypeIdSummary(S.first.TypeID)
2424 ->WPDRes[S.first.ByteOffset];
2425 if (tryFindVirtualCallTargets(TargetsForSlot, *TidSummary,
2426 S.first.ByteOffset)) {
2428 if (!trySingleImplDevirt(TargetsForSlot, S.first, S.second, Res,
2437 for (
const auto &DT : DevirtTargets)
2438 errs() <<
"Devirtualized call to " << DT <<
"\n";
2440 NumDevirtTargets += DevirtTargets.size();
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
static const Function * getParent(const Value *V)
This is the interface for LLVM's primary stateless and local alias analysis.
SmallVector< MachineOperand, 4 > Cond
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file defines DenseMapInfo traits for DenseMap.
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
Provides passes for computing function attributes based on interprocedural analyses.
static cl::opt< std::string > ClReadSummary("lowertypetests-read-summary", cl::desc("Read summary from given YAML file before running pass"), cl::Hidden)
static cl::opt< PassSummaryAction > ClSummaryAction("lowertypetests-summary-action", cl::desc("What to do with the summary when running this pass"), cl::values(clEnumValN(PassSummaryAction::None, "none", "Do nothing"), clEnumValN(PassSummaryAction::Import, "import", "Import typeid resolutions from summary and globals"), clEnumValN(PassSummaryAction::Export, "export", "Export typeid resolutions to summary and globals")), cl::Hidden)
static cl::opt< std::string > ClWriteSummary("lowertypetests-write-summary", cl::desc("Write summary to given YAML file after running pass"), cl::Hidden)
This file implements a map that provides insertion order iteration.
static bool mustBeUnreachableFunction(const Function &F)
Module.h This file contains the declarations for the Module class.
FunctionAnalysisManager FAM
const char LLVMTargetMachineRef TM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static cl::opt< bool > DisableWholeProgramVisibility("disable-whole-program-visibility", cl::Hidden, cl::desc("Disable whole program visibility (overrides enabling options)"))
Provide a way to force disable whole program for debugging or workarounds, when enabled via the linke...
WPDCheckMode
Mechanism to add runtime checking of devirtualization decisions, optionally trapping or falling back ...
static cl::opt< PassSummaryAction > ClSummaryAction("wholeprogramdevirt-summary-action", cl::desc("What to do with the summary when running this pass"), cl::values(clEnumValN(PassSummaryAction::None, "none", "Do nothing"), clEnumValN(PassSummaryAction::Import, "import", "Import typeid resolutions from summary and globals"), clEnumValN(PassSummaryAction::Export, "export", "Export typeid resolutions to summary and globals")), cl::Hidden)
static cl::opt< bool > WholeProgramVisibility("whole-program-visibility", cl::Hidden, cl::desc("Enable whole program visibility"))
Provide a way to force enable whole program visibility in tests.
static Error checkCombinedSummaryForTesting(ModuleSummaryIndex *Summary)
static cl::list< std::string > SkipFunctionNames("wholeprogramdevirt-skip", cl::desc("Prevent function(s) from being devirtualized"), cl::Hidden, cl::CommaSeparated)
Provide way to prevent certain function from being devirtualized.
static cl::opt< std::string > ClWriteSummary("wholeprogramdevirt-write-summary", cl::desc("Write summary to given bitcode or YAML file after running pass. " "Output file format is deduced from extension: *.bc means writing " "bitcode, otherwise YAML"), cl::Hidden)
static cl::opt< unsigned > ClThreshold("wholeprogramdevirt-branch-funnel-threshold", cl::Hidden, cl::init(10), cl::desc("Maximum number of call targets per " "call site to enable branch funnels"))
static cl::opt< WPDCheckMode > DevirtCheckMode("wholeprogramdevirt-check", cl::Hidden, cl::desc("Type of checking for incorrect devirtualizations"), cl::values(clEnumValN(WPDCheckMode::None, "none", "No checking"), clEnumValN(WPDCheckMode::Trap, "trap", "Trap when incorrect"), clEnumValN(WPDCheckMode::Fallback, "fallback", "Fallback to indirect when incorrect")))
static cl::opt< std::string > ClReadSummary("wholeprogramdevirt-read-summary", cl::desc("Read summary from given bitcode or YAML file before running pass"), cl::Hidden)
static bool AddCalls(VTableSlotInfo &SlotInfo, const ValueInfo &Callee)
static cl::opt< bool > PrintSummaryDevirt("wholeprogramdevirt-print-index-based", cl::Hidden, cl::desc("Print index-based devirtualization messages"))
A manager for alias analyses.
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
static AttributeSet get(LLVMContext &C, const AttrBuilder &B)
StringRef getValueAsString() const
Return the attribute's value as a string.
bool isValid() const
Return true if the attribute is any kind of attribute.
LLVM Basic Block Representation.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void setCallingConv(CallingConv::ID CC)
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
CallingConv::ID getCallingConv() const
Value * getCalledOperand() const
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
FunctionType * getFunctionType() const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void setCalledOperand(Value *V)
AttributeList getAttributes() const
Return the parameter attributes for this call.
Function * getCaller()
Helper to get the caller (the parent function).
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
void setSelectionKind(SelectionKind Val)
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
static Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList, bool InBounds=false, std::optional< unsigned > InRangeIndex=std::nullopt, Type *OnlyIfReducedTy=nullptr)
Getelementptr form.
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static ConstantInt * getTrue(LLVMContext &Context)
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static Constant * getAnon(ArrayRef< Constant * > V, bool Packed=false)
Return an anonymous struct that has the specified elements.
This is an important base class in LLVM.
const Constant * stripPointerCasts() const
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Implements a dense probed hash-table based set.
Analysis pass which computes a DominatorTree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Subclass of Error for the sole purpose of identifying the success path in the type system.
Lightweight error class with error context and mandatory checking.
This class evaluates LLVM IR, producing the Constant representing each SSA instruction.
Helper for check-and-exit error handling.
Tagged union holding either a T or a Error.
Function summary information to aid decisions and implementation of importing.
ArrayRef< Type * > params() const
Type * getReturnType() const
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
static Expected< GlobPattern > create(StringRef Pat)
static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
@ VCallVisibilityLinkageUnit
static bool isLocalLinkage(LinkageTypes Linkage)
static bool isAvailableExternallyLinkage(LinkageTypes Linkage)
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
@ HiddenVisibility
The GV is hidden.
static GUID getGUID(StringRef GlobalName)
Return a 64-bit global unique ID constructed from global value name (i.e.
void setVisibility(VisibilityTypes V)
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ InternalLinkage
Rename collisions when linking (static functions).
@ ExternalLinkage
Externally visible function.
Global variable summary information to aid decisions and implementation of importing.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const BasicBlock * getParent() const
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Class to represent integer types.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight)
Return metadata containing two branch weights.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
This class implements a map that also provides access to all stored values in a deterministic order.
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFile(const Twine &Filename, bool IsText=false, bool RequiresNullTerminator=true, bool IsVolatile=false, std::optional< Align > Alignment=std::nullopt)
Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful,...
Class to hold module path string table and global value map, and encapsulate methods for operating on...
const TypeIdSummary * getTypeIdSummary(StringRef TypeId) const
This returns either a pointer to the type id summary (if present in the summary map) or null (if not ...
ValueInfo getValueInfo(const GlobalValueSummaryMapTy::value_type &R) const
Return a ValueInfo for the index value_type (convenient when iterating index).
const ModuleHash & getModuleHash(const StringRef ModPath) const
Get the module SHA1 hash recorded for the given module path.
static constexpr const char * getRegularLTOModuleName()
bool partiallySplitLTOUnits() const
static std::string getGlobalNameForLocal(StringRef Name, ModuleHash ModHash)
Convenience method for creating a promoted global name for the given value name of a local,...
A Module instance is used to store all the information related to an LLVM module.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, Instruction *InsertBefore=nullptr)
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
bool contains(StringRef Other) const
Return true if the given string is a substring of *this, and false otherwise.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
The instances of the Type class are immutable: once they are created, they are never changed.
TypeID
Definitions of all of the base types for the Type system.
static Type * getVoidTy(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
bool eraseMetadata(unsigned KindID)
Erase all metadata attachments with the given kind.
iterator_range< use_iterator > uses()
StringRef getName() const
Return a constant reference to the value's name.
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
An efficient, type-erasing, non-owning reference to a callable.
A raw_ostream that writes to a file descriptor.
A raw_ostream that writes to an std::string.
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
@ C
The default llvm calling convention, compatible with C.
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
bool match(Val *V, const Pattern &P)
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
DiagnosticInfoOptimizationBase::Argument NV
@ OF_TextWithCRLF
The file should be opened in text mode and use a carriage linefeed '\r '.
uint64_t findLowestOffset(ArrayRef< VirtualCallTarget > Targets, bool IsAfter, uint64_t Size)
void setAfterReturnValues(MutableArrayRef< VirtualCallTarget > Targets, uint64_t AllocAfter, unsigned BitWidth, int64_t &OffsetByte, uint64_t &OffsetBit)
void setBeforeReturnValues(MutableArrayRef< VirtualCallTarget > Targets, uint64_t AllocBefore, unsigned BitWidth, int64_t &OffsetByte, uint64_t &OffsetBit)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
MemoryEffects computeFunctionBodyMemoryAccess(Function &F, AAResults &AAR)
Returns the memory access properties of this copy of the function.
void updateVCallVisibilityInModule(Module &M, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols)
If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...
void writeIndexToFile(const ModuleSummaryIndex &Index, raw_ostream &Out, const std::map< std::string, GVSummaryMapTy > *ModuleToSummariesForIndex=nullptr)
Write the specified module summary index to the given raw output stream, where it will be written in ...
@ Export
Export information to summary.
@ Import
Import information from summary.
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Error createStringError(std::error_code EC, char const *Fmt, const Ts &... Vals)
Create formatted StringError object.
bool hasWholeProgramVisibility(bool WholeProgramVisibilityEnabledInLTO)
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Expected< std::unique_ptr< ModuleSummaryIndex > > getModuleSummaryIndex(MemoryBufferRef Buffer)
Parse the specified bitcode buffer, returning the module summary index.
void updatePublicTypeTestCalls(Module &M, bool WholeProgramVisibilityEnabledInLTO)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void findDevirtualizableCallsForTypeCheckedLoad(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< Instruction * > &LoadedPtrs, SmallVectorImpl< Instruction * > &Preds, bool &HasNonCallUses, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.checked.load, find all devirtualizable call sites based on t...
CallBase & versionCallSite(CallBase &CB, Value *Callee, MDNode *BranchWeights)
Predicate and clone the given call site.
bool AreStatisticsEnabled()
Check if statistics are enabled.
void updateIndexWPDForExports(ModuleSummaryIndex &Summary, function_ref< bool(StringRef, ValueInfo)> isExported, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap)
Call after cross-module importing to update the recorded single impl devirt target names for any loca...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
void updateVCallVisibilityInIndex(ModuleSummaryIndex &Index, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols)
If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...
void runWholeProgramDevirtOnIndex(ModuleSummaryIndex &Summary, std::set< GlobalValue::GUID > &ExportedGUIDs, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap)
Perform index-based whole program devirtualization on the Summary index.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Expected< T > errorOrToExpected(ErrorOr< T > &&EO)
Convert an ErrorOr<T> to an Expected<T>.
constexpr unsigned BitWidth
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Instruction * SplitBlockAndInsertIfThen(Value *Cond, Instruction *SplitBefore, bool Unreachable, MDNode *BranchWeights, DominatorTree *DT, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
std::vector< TypeIdOffsetVtableInfo > TypeIdCompatibleVtableInfo
List of vtable definitions decorated by a particular type identifier, and their corresponding offsets...
void consumeError(Error Err)
Consume a Error without doing anything.
void findDevirtualizableCallsForTypeTest(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< CallInst * > &Assumes, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.test, find all devirtualizable call sites based on the call ...
Constant * getPointerAtOffset(Constant *I, uint64_t Offset, Module &M, Constant *TopLevelGlobal=nullptr)
Processes a Constant recursively looking into elements of arrays, structs and expressions to find a t...
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Class to accumulate and hold information about a callee.
static unsigned getHashValue(const VTableSlotSummary &I)
static bool isEqual(const VTableSlotSummary &LHS, const VTableSlotSummary &RHS)
static VTableSlotSummary getTombstoneKey()
static VTableSlotSummary getEmptyKey()
static VTableSlot getTombstoneKey()
static VTableSlot getEmptyKey()
static bool isEqual(const VTableSlot &LHS, const VTableSlot &RHS)
static unsigned getHashValue(const VTableSlot &I)
An information struct used to provide DenseMap with the various necessary components for a given valu...
A call site that could be devirtualized.
A specification for a virtual function call with all constant integer arguments.
An "identifier" for a virtual function.
The following data structures summarize type metadata information.
std::map< uint64_t, WholeProgramDevirtResolution > WPDRes
Mapping from byte offset to whole-program devirt resolution for that (typeid, byte offset) pair.
@ Unsat
Unsatisfiable type (i.e. no global has this type metadata)
enum llvm::TypeTestResolution::Kind TheKind
Struct that holds a reference to a particular GUID in a global value summary.
ArrayRef< std::unique_ptr< GlobalValueSummary > > getSummaryList() const
PreservedAnalyses run(Module &M, ModuleAnalysisManager &)
@ UniformRetVal
Uniform return value optimization.
@ VirtualConstProp
Virtual constant propagation.
@ UniqueRetVal
Unique return value optimization.
uint64_t Info
Additional information for the resolution:
enum llvm::WholeProgramDevirtResolution::ByArg::Kind TheKind
enum llvm::WholeProgramDevirtResolution::Kind TheKind
std::map< std::vector< uint64_t >, ByArg > ResByArg
Resolutions for calls with all constant integer arguments (excluding the first argument,...
std::string SingleImplName
@ SingleImpl
Single implementation devirtualization.
@ BranchFunnel
When retpoline mitigation is enabled, use a branch funnel that is defined in the merged module.
VirtualCallTarget(GlobalValue *Fn, const TypeMemberInfo *TM)