105using namespace wholeprogramdevirt;
107#define DEBUG_TYPE "wholeprogramdevirt"
109STATISTIC(NumDevirtTargets,
"Number of whole program devirtualization targets");
110STATISTIC(NumSingleImpl,
"Number of single implementation devirtualizations");
112STATISTIC(NumUniformRetVal,
"Number of uniform return value optimizations");
113STATISTIC(NumUniqueRetVal,
"Number of unique return value optimizations");
115 "Number of 1 bit virtual constant propagations");
116STATISTIC(NumVirtConstProp,
"Number of virtual constant propagations");
119 "wholeprogramdevirt-summary-action",
120 cl::desc(
"What to do with the summary when running this pass"),
122 clEnumValN(PassSummaryAction::Import,
"import",
123 "Import typeid resolutions from summary and globals"),
124 clEnumValN(PassSummaryAction::Export,
"export",
125 "Export typeid resolutions to summary and globals")),
129 "wholeprogramdevirt-read-summary",
131 "Read summary from given bitcode or YAML file before running pass"),
135 "wholeprogramdevirt-write-summary",
136 cl::desc(
"Write summary to given bitcode or YAML file after running pass. "
137 "Output file format is deduced from extension: *.bc means writing "
138 "bitcode, otherwise YAML"),
144 cl::desc(
"Maximum number of call targets per "
145 "call site to enable branch funnels"));
149 cl::desc(
"Print index-based devirtualization messages"));
157 cl::desc(
"Enable whole program visibility"));
162 "disable-whole-program-visibility",
cl::Hidden,
163 cl::desc(
"Disable whole program visibility (overrides enabling options)"));
168 cl::desc(
"Prevent function(s) from being devirtualized"),
179 cl::desc(
"Type of checking for incorrect devirtualizations"),
183 "Fallback to indirect when incorrect")));
187 std::vector<GlobPattern> Patterns;
188 template <
class T>
void init(
const T &StringList) {
189 for (
const auto &S : StringList)
191 Patterns.push_back(std::move(*Pat));
212 MinByte = std::max(MinByte,
Target.minAfterBytes());
214 MinByte = std::max(MinByte,
Target.minBeforeBytes());
237 std::vector<ArrayRef<uint8_t>> Used;
240 :
Target.TM->Bits->Before.BytesUsed;
242 : MinByte -
Target.minBeforeBytes();
252 for (
unsigned I = 0;; ++
I) {
253 uint8_t BitsUsed = 0;
254 for (
auto &&
B : Used)
257 if (BitsUsed != 0xff)
263 for (
unsigned I = 0;; ++
I) {
264 for (
auto &&
B : Used) {
266 while ((
I + Byte) <
B.size() && Byte < (
Size / 8)) {
272 return (MinByte +
I) * 8;
282 OffsetByte = -(AllocBefore / 8 + 1);
284 OffsetByte = -((AllocBefore + 7) / 8 + (
BitWidth + 7) / 8);
285 OffsetBit = AllocBefore % 8;
289 Target.setBeforeBit(AllocBefore);
299 OffsetByte = AllocAfter / 8;
301 OffsetByte = (AllocAfter + 7) / 8;
302 OffsetBit = AllocAfter % 8;
306 Target.setAfterBit(AllocAfter);
345 const VTableSlot &
RHS) {
346 return LHS.TypeID ==
RHS.TypeID &&
LHS.ByteOffset ==
RHS.ByteOffset;
365 return LHS.TypeID ==
RHS.TypeID &&
LHS.ByteOffset ==
RHS.ByteOffset;
388 if (!Summary->isLive())
390 if (
auto *FS = dyn_cast<FunctionSummary>(Summary->getBaseObject())) {
391 if (!FS->fflags().MustBeUnreachable)
406struct VirtualCallSite {
413 unsigned *NumUnsafeUses =
nullptr;
424 <<
NV(
"Optimization", OptName)
425 <<
": devirtualized a call to "
426 <<
NV(
"FunctionName", TargetName));
429 void replaceAndErase(
436 if (
auto *
II = dyn_cast<InvokeInst>(&CB)) {
438 II->getUnwindDest()->removePredecessor(
II->getParent());
455 std::vector<VirtualCallSite> CallSites;
460 bool AllCallSitesDevirted =
true;
467 bool SummaryHasTypeTestAssumeUsers =
false;
476 std::vector<FunctionSummary *> SummaryTypeCheckedLoadUsers;
477 std::vector<FunctionSummary *> SummaryTypeTestAssumeUsers;
479 bool isExported()
const {
480 return SummaryHasTypeTestAssumeUsers ||
481 !SummaryTypeCheckedLoadUsers.empty();
485 SummaryTypeCheckedLoadUsers.push_back(FS);
486 AllCallSitesDevirted =
false;
490 SummaryTypeTestAssumeUsers.push_back(FS);
491 SummaryHasTypeTestAssumeUsers =
true;
492 AllCallSitesDevirted =
false;
496 AllCallSitesDevirted =
true;
499 SummaryTypeCheckedLoadUsers.clear();
504struct VTableSlotInfo {
511 std::map<std::vector<uint64_t>, CallSiteInfo> ConstCSInfo;
513 void addCallSite(
Value *VTable,
CallBase &CB,
unsigned *NumUnsafeUses);
516 CallSiteInfo &findCallSiteInfo(
CallBase &CB);
519CallSiteInfo &VTableSlotInfo::findCallSiteInfo(
CallBase &CB) {
520 std::vector<uint64_t>
Args;
521 auto *CBType = dyn_cast<IntegerType>(CB.
getType());
522 if (!CBType || CBType->getBitWidth() > 64 || CB.
arg_empty())
525 auto *CI = dyn_cast<ConstantInt>(Arg);
526 if (!CI || CI->getBitWidth() > 64)
528 Args.push_back(CI->getZExtValue());
530 return ConstCSInfo[
Args];
534 unsigned *NumUnsafeUses) {
535 auto &CSI = findCallSiteInfo(CB);
536 CSI.AllCallSitesDevirted =
false;
537 CSI.CallSites.push_back({
VTable, CB, NumUnsafeUses});
580 std::map<CallInst *, unsigned> NumUnsafeUsesForTypeTest;
581 PatternList FunctionsToSkip;
588 :
M(
M), AARGetter(AARGetter), LookupDomTree(LookupDomTree),
589 ExportSummary(ExportSummary), ImportSummary(ImportSummary),
590 Int8Ty(
Type::getInt8Ty(
M.getContext())),
592 Int32Ty(
Type::getInt32Ty(
M.getContext())),
593 Int64Ty(
Type::getInt64Ty(
M.getContext())),
594 IntPtrTy(
M.getDataLayout().getIntPtrType(
M.getContext(), 0)),
596 RemarksEnabled(areRemarksEnabled()), OREGetter(OREGetter) {
597 assert(!(ExportSummary && ImportSummary));
601 bool areRemarksEnabled();
604 scanTypeTestUsers(
Function *TypeTestFunc,
606 void scanTypeCheckedLoadUsers(
Function *TypeCheckedLoadFunc);
608 void buildTypeIdentifierMap(
609 std::vector<VTableBits> &Bits,
613 tryFindVirtualCallTargets(std::vector<VirtualCallTarget> &TargetsForSlot,
614 const std::set<TypeMemberInfo> &TypeMemberInfos,
618 void applySingleImplDevirt(VTableSlotInfo &SlotInfo,
Constant *TheFn,
622 VTableSlotInfo &SlotInfo,
625 void applyICallBranchFunnel(VTableSlotInfo &SlotInfo,
Constant *JT,
628 VTableSlotInfo &SlotInfo,
631 bool tryEvaluateFunctionsWithArgs(
635 void applyUniformRetValOpt(CallSiteInfo &CSInfo,
StringRef FnName,
638 CallSiteInfo &CSInfo,
646 bool shouldExportConstantsAsAbsoluteSymbols();
666 void applyUniqueRetValOpt(CallSiteInfo &CSInfo,
StringRef FnName,
bool IsOne,
668 bool tryUniqueRetValOpt(
unsigned BitWidth,
670 CallSiteInfo &CSInfo,
674 void applyVirtualConstProp(CallSiteInfo &CSInfo,
StringRef FnName,
677 VTableSlotInfo &SlotInfo,
683 void importResolution(VTableSlot Slot, VTableSlotInfo &SlotInfo);
687 void removeRedundantTypeTests();
720 std::set<GlobalValue::GUID> &ExportedGUIDs;
724 std::map<ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap;
728 PatternList FunctionsToSkip;
732 std::set<GlobalValue::GUID> &ExportedGUIDs,
733 std::map<
ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap)
734 : ExportSummary(ExportSummary), ExportedGUIDs(ExportedGUIDs),
735 LocalWPDTargetsMap(LocalWPDTargetsMap) {
739 bool tryFindVirtualCallTargets(std::vector<ValueInfo> &TargetsForSlot,
745 VTableSlotInfo &SlotInfo,
747 std::set<ValueInfo> &DevirtTargets);
765 if (UseCommandLine) {
766 if (!DevirtModule::runForTesting(M, AARGetter, OREGetter, LookupDomTree))
770 if (!DevirtModule(M, AARGetter, OREGetter, LookupDomTree, ExportSummary,
790 if (
TypeID.ends_with(
".virtual"))
796 if (!
TypeID.consume_front(
"_ZTS"))
804 std::string typeInfo = (
"_ZTI" +
TypeID).str();
805 return IsVisibleToRegularObj(typeInfo);
814 for (
auto Type : Types)
815 if (
auto *
TypeID = dyn_cast<MDString>(
Type->getOperand(1).get()))
817 IsVisibleToRegularObj);
826 Module &M,
bool WholeProgramVisibilityEnabledInLTO,
828 bool ValidateAllVtablesHaveTypeInfos,
845 !(ValidateAllVtablesHaveTypeInfos &&
852 bool WholeProgramVisibilityEnabledInLTO) {
855 if (!PublicTypeTestFunc)
861 auto *CI = cast<CallInst>(U.getUser());
863 TypeTestFunc, {CI->getArgOperand(0), CI->getArgOperand(1)},
865 CI->replaceAllUsesWith(NewCI);
866 CI->eraseFromParent();
871 auto *CI = cast<CallInst>(U.getUser());
872 CI->replaceAllUsesWith(True);
873 CI->eraseFromParent();
884 for (
const auto &typeID :
Index.typeIdCompatibleVtableMap()) {
887 VisibleToRegularObjSymbols.
insert(
P.VTableVI.getGUID());
903 if (DynamicExportSymbols.
count(
P.first))
905 for (
auto &S :
P.second.SummaryList) {
906 auto *GVar = dyn_cast<GlobalVarSummary>(S.get());
914 if (VisibleToRegularObjSymbols.
count(
P.first))
923 std::map<
ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap) {
924 DevirtIndex(Summary, ExportedGUIDs, LocalWPDTargetsMap).run();
930 std::map<
ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap) {
931 for (
auto &
T : LocalWPDTargetsMap) {
934 assert(VI.getSummaryList().size() == 1 &&
935 "Devirt of local target has more than one copy");
936 auto &S = VI.getSummaryList()[0];
937 if (!isExported(S->modulePath(), VI))
941 for (
auto &SlotSummary :
T.second) {
942 auto *TIdSum = Summary.getTypeIdSummary(SlotSummary.TypeID);
944 auto WPDRes = TIdSum->WPDRes.find(SlotSummary.ByteOffset);
945 assert(WPDRes != TIdSum->WPDRes.end());
947 WPDRes->second.SingleImplName,
948 Summary.getModuleHash(S->modulePath()));
958 const auto &ModPaths = Summary->modulePaths();
963 "combined summary should contain Regular LTO module");
967bool DevirtModule::runForTesting(
971 std::unique_ptr<ModuleSummaryIndex>
Summary =
972 std::make_unique<ModuleSummaryIndex>(
false);
979 auto ReadSummaryFile =
981 if (
Expected<std::unique_ptr<ModuleSummaryIndex>> SummaryOrErr =
983 Summary = std::move(*SummaryOrErr);
988 yaml::Input
In(ReadSummaryFile->getBuffer());
995 DevirtModule(M, AARGetter, OREGetter, LookupDomTree,
1013 yaml::Output Out(
OS);
1021void DevirtModule::buildTypeIdentifierMap(
1022 std::vector<VTableBits> &Bits,
1025 Bits.reserve(
M.global_size());
1035 Bits.emplace_back();
1036 Bits.back().GV = &GV;
1037 Bits.back().ObjectSize =
1039 BitsPtr = &
Bits.back();
1047 cast<ConstantAsMetadata>(
Type->getOperand(0))->getValue())
1055bool DevirtModule::tryFindVirtualCallTargets(
1056 std::vector<VirtualCallTarget> &TargetsForSlot,
1057 const std::set<TypeMemberInfo> &TypeMemberInfos,
uint64_t ByteOffset,
1060 if (!
TM.Bits->GV->isConstant())
1065 if (
TM.Bits->GV->getVCallVisibility() ==
1077 if (FunctionsToSkip.match(Fn->
getName()))
1082 if (Fn->
getName() ==
"__cxa_pure_virtual")
1092 auto GV = dyn_cast<GlobalValue>(
C);
1094 TargetsForSlot.push_back({GV, &
TM});
1098 return !TargetsForSlot.empty();
1101bool DevirtIndex::tryFindVirtualCallTargets(
1102 std::vector<ValueInfo> &TargetsForSlot,
1116 bool LocalFound =
false;
1117 for (
const auto &S :
P.VTableVI.getSummaryList()) {
1123 auto *CurVS = cast<GlobalVarSummary>(S->getBaseObject());
1124 if (!CurVS->vTableFuncs().empty() ||
1146 for (
auto VTP :
VS->vTableFuncs()) {
1147 if (VTP.VTableOffset !=
P.AddressPointOffset + ByteOffset)
1153 TargetsForSlot.push_back(VTP.FuncVI);
1158 return !TargetsForSlot.empty();
1161void DevirtModule::applySingleImplDevirt(VTableSlotInfo &SlotInfo,
1162 Constant *TheFn,
bool &IsExported) {
1167 auto Apply = [&](CallSiteInfo &CSInfo) {
1168 for (
auto &&VCallSite : CSInfo.CallSites) {
1169 if (!OptimizedCalls.
insert(&VCallSite.CB).second)
1173 VCallSite.emitRemark(
"single-impl",
1176 auto &CB = VCallSite.CB;
1189 Builder.SetInsertPoint(ThenTerm);
1191 auto *CallTrap = Builder.CreateCall(TrapFn);
1208 NewInst.
setMetadata(LLVMContext::MD_prof,
nullptr);
1209 NewInst.
setMetadata(LLVMContext::MD_callees,
nullptr);
1231 CallsWithPtrAuthBundleRemoved.
push_back(&CB);
1236 if (VCallSite.NumUnsafeUses)
1237 --*VCallSite.NumUnsafeUses;
1239 if (CSInfo.isExported())
1241 CSInfo.markDevirt();
1243 Apply(SlotInfo.CSInfo);
1244 for (
auto &
P : SlotInfo.ConstCSInfo)
1250 if (Callee.getSummaryList().empty())
1257 bool IsExported =
false;
1258 auto &S = Callee.getSummaryList()[0];
1261 auto AddCalls = [&](CallSiteInfo &CSInfo) {
1262 for (
auto *FS : CSInfo.SummaryTypeCheckedLoadUsers) {
1263 FS->addCall({Callee, CI});
1264 IsExported |= S->modulePath() != FS->modulePath();
1266 for (
auto *FS : CSInfo.SummaryTypeTestAssumeUsers) {
1267 FS->addCall({Callee, CI});
1268 IsExported |= S->modulePath() != FS->modulePath();
1272 for (
auto &
P : SlotInfo.ConstCSInfo)
1277bool DevirtModule::trySingleImplDevirt(
1283 auto *TheFn = TargetsForSlot[0].Fn;
1284 for (
auto &&
Target : TargetsForSlot)
1290 TargetsForSlot[0].WasDevirt =
true;
1292 bool IsExported =
false;
1293 applySingleImplDevirt(SlotInfo, TheFn, IsExported);
1300 if (TheFn->hasLocalLinkage()) {
1301 std::string NewName = (TheFn->
getName() +
".llvm.merged").str();
1306 if (
Comdat *
C = TheFn->getComdat()) {
1307 if (
C->getName() == TheFn->
getName()) {
1308 Comdat *NewC =
M.getOrInsertComdat(NewName);
1311 if (GO.getComdat() ==
C)
1333 VTableSlotInfo &SlotInfo,
1335 std::set<ValueInfo> &DevirtTargets) {
1338 auto TheFn = TargetsForSlot[0];
1339 for (
auto &&
Target : TargetsForSlot)
1344 auto Size = TheFn.getSummaryList().size();
1350 if (FunctionsToSkip.match(TheFn.name()))
1355 for (
const auto &S : TheFn.getSummaryList())
1361 DevirtTargets.insert(TheFn);
1363 auto &S = TheFn.getSummaryList()[0];
1364 bool IsExported =
AddCalls(SlotInfo, TheFn);
1366 ExportedGUIDs.insert(TheFn.getGUID());
1377 TheFn.name(), ExportSummary.
getModuleHash(S->modulePath()));
1379 LocalWPDTargetsMap[TheFn].push_back(SlotSummary);
1393void DevirtModule::tryICallBranchFunnel(
1403 bool HasNonDevirt = !SlotInfo.CSInfo.AllCallSitesDevirted;
1405 for (
auto &
P : SlotInfo.ConstCSInfo)
1406 if (!
P.second.AllCallSitesDevirted) {
1407 HasNonDevirt =
true;
1417 if (isa<MDString>(
Slot.TypeID)) {
1419 M.getDataLayout().getProgramAddressSpace(),
1420 getGlobalName(Slot, {},
"branch_funnel"), &M);
1424 M.getDataLayout().getProgramAddressSpace(),
1425 "branch_funnel", &M);
1427 JT->addParamAttr(0, Attribute::Nest);
1429 std::vector<Value *> JTArgs;
1430 JTArgs.push_back(
JT->arg_begin());
1431 for (
auto &
T : TargetsForSlot) {
1432 JTArgs.push_back(getMemberAddr(
T.TM));
1433 JTArgs.push_back(
T.Fn);
1444 bool IsExported =
false;
1445 applyICallBranchFunnel(SlotInfo, JT, IsExported);
1450void DevirtModule::applyICallBranchFunnel(VTableSlotInfo &SlotInfo,
1452 auto Apply = [&](CallSiteInfo &CSInfo) {
1453 if (CSInfo.isExported())
1455 if (CSInfo.AllCallSitesDevirted)
1458 std::map<CallBase *, CallBase *> CallBases;
1459 for (
auto &&VCallSite : CSInfo.CallSites) {
1462 if (CallBases.find(&CB) != CallBases.end()) {
1479 VCallSite.emitRemark(
"branch-funnel",
1480 JT->stripPointerCasts()->getName(), OREGetter);
1484 std::vector<Type *> NewArgs;
1485 NewArgs.push_back(Int8PtrTy);
1493 std::vector<Value *>
Args;
1494 Args.push_back(VCallSite.VTable);
1498 if (isa<CallInst>(CB))
1499 NewCS = IRB.CreateCall(NewFT, IRB.CreateBitCast(JT, NewFTPtr), Args);
1501 NewCS = IRB.CreateInvoke(NewFT, IRB.CreateBitCast(JT, NewFTPtr),
1502 cast<InvokeInst>(CB).getNormalDest(),
1503 cast<InvokeInst>(CB).getUnwindDest(), Args);
1507 std::vector<AttributeSet> NewArgAttrs;
1510 M.getContext(), Attribute::Nest)}));
1511 for (
unsigned I = 0;
I + 2 <
Attrs.getNumAttrSets(); ++
I)
1512 NewArgAttrs.push_back(
Attrs.getParamAttrs(
I));
1515 Attrs.getRetAttrs(), NewArgAttrs));
1517 CallBases[&CB] = NewCS;
1520 if (VCallSite.NumUnsafeUses)
1521 --*VCallSite.NumUnsafeUses;
1528 for (
auto &[Old, New] : CallBases) {
1530 Old->eraseFromParent();
1533 Apply(SlotInfo.CSInfo);
1534 for (
auto &
P : SlotInfo.ConstCSInfo)
1538bool DevirtModule::tryEvaluateFunctionsWithArgs(
1547 auto Fn = dyn_cast<Function>(
Target.Fn);
1558 for (
unsigned I = 0;
I !=
Args.size(); ++
I) {
1563 EvalArgs.
push_back(ConstantInt::get(ArgTy, Args[
I]));
1567 if (!Eval.EvaluateFunction(Fn, RetVal, EvalArgs) ||
1568 !isa<ConstantInt>(RetVal))
1570 Target.RetVal = cast<ConstantInt>(RetVal)->getZExtValue();
1575void DevirtModule::applyUniformRetValOpt(CallSiteInfo &CSInfo,
StringRef FnName,
1577 for (
auto Call : CSInfo.CallSites) {
1581 Call.replaceAndErase(
1582 "uniform-ret-val", FnName, RemarksEnabled, OREGetter,
1583 ConstantInt::get(cast<IntegerType>(
Call.CB.getType()), TheRetVal));
1585 CSInfo.markDevirt();
1588bool DevirtModule::tryUniformRetValOpt(
1593 uint64_t TheRetVal = TargetsForSlot[0].RetVal;
1595 if (
Target.RetVal != TheRetVal)
1598 if (CSInfo.isExported()) {
1600 Res->
Info = TheRetVal;
1603 applyUniformRetValOpt(CSInfo, TargetsForSlot[0].Fn->
getName(), TheRetVal);
1605 for (
auto &&
Target : TargetsForSlot)
1610std::string DevirtModule::getGlobalName(VTableSlot Slot,
1613 std::string FullName =
"__typeid_";
1615 OS << cast<MDString>(
Slot.TypeID)->getString() <<
'_' <<
Slot.ByteOffset;
1622bool DevirtModule::shouldExportConstantsAsAbsoluteSymbols() {
1630 getGlobalName(Slot, Args,
Name),
C, &M);
1637 if (shouldExportConstantsAsAbsoluteSymbols()) {
1650 M.getOrInsertGlobal(getGlobalName(Slot, Args,
Name), Int8Arr0Ty);
1651 auto *GV = dyn_cast<GlobalVariable>(
C);
1660 if (!shouldExportConstantsAsAbsoluteSymbols())
1661 return ConstantInt::get(IntTy, Storage);
1664 auto *GV = cast<GlobalVariable>(
C->stripPointerCasts());
1669 if (GV->
hasMetadata(LLVMContext::MD_absolute_symbol))
1680 SetAbsRange(~0ull, ~0ull);
1682 SetAbsRange(0, 1ull << AbsWidth);
1686void DevirtModule::applyUniqueRetValOpt(CallSiteInfo &CSInfo,
StringRef FnName,
1689 for (
auto &&Call : CSInfo.CallSites) {
1695 B.CreateBitCast(UniqueMemberAddr,
Call.VTable->getType()));
1696 Cmp =
B.CreateZExt(Cmp,
Call.CB.getType());
1698 Call.replaceAndErase(
"unique-ret-val", FnName, RemarksEnabled, OREGetter,
1701 CSInfo.markDevirt();
1706 ConstantInt::get(Int64Ty,
M->Offset));
1709bool DevirtModule::tryUniqueRetValOpt(
1714 auto tryUniqueRetValOptFor = [&](
bool IsOne) {
1717 if (
Target.RetVal == (IsOne ? 1 : 0)) {
1720 UniqueMember =
Target.TM;
1728 Constant *UniqueMemberAddr = getMemberAddr(UniqueMember);
1729 if (CSInfo.isExported()) {
1733 exportGlobal(Slot, Args,
"unique_member", UniqueMemberAddr);
1737 applyUniqueRetValOpt(CSInfo, TargetsForSlot[0].Fn->
getName(), IsOne,
1742 for (
auto &&
Target : TargetsForSlot)
1749 if (tryUniqueRetValOptFor(
true))
1751 if (tryUniqueRetValOptFor(
false))
1757void DevirtModule::applyVirtualConstProp(CallSiteInfo &CSInfo,
StringRef FnName,
1759 for (
auto Call : CSInfo.CallSites) {
1762 auto *RetType = cast<IntegerType>(
Call.CB.getType());
1765 if (RetType->getBitWidth() == 1) {
1767 Value *BitsAndBit =
B.CreateAnd(Bits, Bit);
1768 auto IsBitSet =
B.CreateICmpNE(BitsAndBit, ConstantInt::get(Int8Ty, 0));
1769 NumVirtConstProp1Bit++;
1770 Call.replaceAndErase(
"virtual-const-prop-1-bit", FnName, RemarksEnabled,
1771 OREGetter, IsBitSet);
1775 Call.replaceAndErase(
"virtual-const-prop", FnName, RemarksEnabled,
1779 CSInfo.markDevirt();
1782bool DevirtModule::tryVirtualConstProp(
1788 auto Fn = dyn_cast<Function>(TargetsForSlot[0].Fn);
1795 unsigned BitWidth = RetType->getBitWidth();
1813 auto Fn = dyn_cast<Function>(
Target.Fn);
1819 .doesNotAccessMemory() ||
1825 for (
auto &&CSByConstantArg : SlotInfo.ConstCSInfo) {
1826 if (!tryEvaluateFunctionsWithArgs(TargetsForSlot, CSByConstantArg.first))
1831 ResByArg = &Res->
ResByArg[CSByConstantArg.first];
1833 if (tryUniformRetValOpt(TargetsForSlot, CSByConstantArg.second, ResByArg))
1836 if (tryUniqueRetValOpt(
BitWidth, TargetsForSlot, CSByConstantArg.second,
1837 ResByArg, Slot, CSByConstantArg.first))
1849 uint64_t TotalPaddingBefore = 0, TotalPaddingAfter = 0;
1850 for (
auto &&
Target : TargetsForSlot) {
1851 TotalPaddingBefore += std::max<int64_t>(
1852 (AllocBefore + 7) / 8 -
Target.allocatedBeforeBytes() - 1, 0);
1853 TotalPaddingAfter += std::max<int64_t>(
1854 (AllocAfter + 7) / 8 -
Target.allocatedAfterBytes() - 1, 0);
1859 if (std::min(TotalPaddingBefore, TotalPaddingAfter) > 128)
1866 if (TotalPaddingBefore <= TotalPaddingAfter)
1874 for (
auto &&
Target : TargetsForSlot)
1878 if (CSByConstantArg.second.isExported()) {
1880 exportConstant(Slot, CSByConstantArg.first,
"byte", OffsetByte,
1882 exportConstant(Slot, CSByConstantArg.first,
"bit", 1ULL << OffsetBit,
1887 Constant *ByteConst = ConstantInt::get(Int32Ty, OffsetByte);
1888 Constant *BitConst = ConstantInt::get(Int8Ty, 1ULL << OffsetBit);
1889 applyVirtualConstProp(CSByConstantArg.second,
1890 TargetsForSlot[0].Fn->getName(), ByteConst, BitConst);
1896 if (
B.Before.Bytes.empty() &&
B.After.Bytes.empty())
1901 Align Alignment =
M.getDataLayout().getValueOrABITypeAlignment(
1902 B.GV->getAlign(),
B.GV->getValueType());
1903 B.Before.Bytes.resize(
alignTo(
B.Before.Bytes.size(), Alignment));
1906 for (
size_t I = 0,
Size =
B.Before.Bytes.size();
I !=
Size / 2; ++
I)
1913 B.GV->getInitializer(),
1918 NewGV->setSection(
B.GV->getSection());
1919 NewGV->setComdat(
B.GV->getComdat());
1920 NewGV->setAlignment(
B.GV->getAlign());
1924 NewGV->copyMetadata(
B.GV,
B.Before.Bytes.size());
1929 B.GV->getInitializer()->getType(), 0,
B.GV->getLinkage(),
"",
1931 NewInit->getType(), NewGV,
1933 ConstantInt::get(Int32Ty, 1)}),
1935 Alias->setVisibility(
B.GV->getVisibility());
1936 Alias->takeName(
B.GV);
1938 B.GV->replaceAllUsesWith(Alias);
1939 B.GV->eraseFromParent();
1942bool DevirtModule::areRemarksEnabled() {
1943 const auto &FL =
M.getFunctionList();
1948 return DI.isEnabled();
1953void DevirtModule::scanTypeTestUsers(
1962 auto *CI = dyn_cast<CallInst>(
U.getUser());
1969 auto &DT = LookupDomTree(*CI->getFunction());
1973 cast<MetadataAsValue>(CI->getArgOperand(1))->getMetadata();
1975 if (!Assumes.
empty()) {
1976 Value *
Ptr = CI->getArgOperand(0)->stripPointerCasts();
1978 CallSlots[{TypeId,
Call.Offset}].addCallSite(
Ptr,
Call.CB,
nullptr);
1981 auto RemoveTypeTestAssumes = [&]() {
1983 for (
auto *Assume : Assumes)
1984 Assume->eraseFromParent();
1987 if (CI->use_empty())
1988 CI->eraseFromParent();
2003 if (!TypeIdMap.count(TypeId))
2004 RemoveTypeTestAssumes();
2015 else if (ImportSummary && isa<MDString>(TypeId)) {
2019 RemoveTypeTestAssumes();
2028void DevirtModule::scanTypeCheckedLoadUsers(
Function *TypeCheckedLoadFunc) {
2032 auto *CI = dyn_cast<CallInst>(
U.getUser());
2038 Value *TypeIdValue = CI->getArgOperand(2);
2039 Metadata *TypeId = cast<MetadataAsValue>(TypeIdValue)->getMetadata();
2044 bool HasNonCallUses =
false;
2045 auto &DT = LookupDomTree(*CI->getFunction());
2047 HasNonCallUses, CI, DT);
2056 (LoadedPtrs.
size() == 1 && !HasNonCallUses) ? LoadedPtrs[0] : CI);
2058 Value *LoadedValue =
nullptr;
2060 Intrinsic::type_checked_load_relative) {
2062 LoadedValue = LoadB.CreateLoad(Int32Ty,
GEP);
2063 LoadedValue = LoadB.CreateSExt(LoadedValue, IntPtrTy);
2064 GEP = LoadB.CreatePtrToInt(
GEP, IntPtrTy);
2065 LoadedValue = LoadB.CreateAdd(
GEP, LoadedValue);
2066 LoadedValue = LoadB.CreateIntToPtr(LoadedValue, Int8PtrTy);
2069 LoadedValue = LoadB.CreateLoad(Int8PtrTy,
GEP);
2074 LoadedPtr->eraseFromParent();
2078 IRBuilder<> CallB((Preds.
size() == 1 && !HasNonCallUses) ? Preds[0] : CI);
2079 CallInst *TypeTestCall = CallB.CreateCall(TypeTestFunc, {
Ptr, TypeIdValue});
2083 Pred->eraseFromParent();
2090 if (!CI->use_empty()) {
2093 Pair =
B.CreateInsertValue(Pair, LoadedValue, {0});
2094 Pair =
B.CreateInsertValue(Pair, TypeTestCall, {1});
2095 CI->replaceAllUsesWith(Pair);
2099 auto &NumUnsafeUses = NumUnsafeUsesForTypeTest[TypeTestCall];
2100 NumUnsafeUses = DevirtCalls.
size();
2108 CallSlots[{TypeId,
Call.Offset}].addCallSite(
Ptr,
Call.CB,
2112 CI->eraseFromParent();
2116void DevirtModule::importResolution(VTableSlot Slot, VTableSlotInfo &SlotInfo) {
2117 auto *TypeId = dyn_cast<MDString>(
Slot.TypeID);
2124 auto ResI = TidSummary->
WPDRes.find(
Slot.ByteOffset);
2125 if (ResI == TidSummary->
WPDRes.end())
2139 bool IsExported =
false;
2140 applySingleImplDevirt(SlotInfo, SingleImpl, IsExported);
2144 for (
auto &CSByConstantArg : SlotInfo.ConstCSInfo) {
2145 auto I = Res.
ResByArg.find(CSByConstantArg.first);
2148 auto &ResByArg =
I->second;
2155 applyUniformRetValOpt(CSByConstantArg.second,
"", ResByArg.
Info);
2159 importGlobal(Slot, CSByConstantArg.first,
"unique_member");
2160 applyUniqueRetValOpt(CSByConstantArg.second,
"", ResByArg.
Info,
2165 Constant *
Byte = importConstant(Slot, CSByConstantArg.first,
"byte",
2166 Int32Ty, ResByArg.
Byte);
2167 Constant *
Bit = importConstant(Slot, CSByConstantArg.first,
"bit", Int8Ty,
2169 applyVirtualConstProp(CSByConstantArg.second,
"", Byte, Bit);
2181 M.getOrInsertFunction(getGlobalName(Slot, {},
"branch_funnel"),
2184 bool IsExported =
false;
2185 applyICallBranchFunnel(SlotInfo, JT, IsExported);
2190void DevirtModule::removeRedundantTypeTests() {
2192 for (
auto &&U : NumUnsafeUsesForTypeTest) {
2193 if (
U.second == 0) {
2194 U.first->replaceAllUsesWith(True);
2195 U.first->eraseFromParent();
2201DevirtModule::lookUpFunctionValueInfo(
Function *TheFn,
2203 assert((ExportSummary !=
nullptr) &&
2204 "Caller guarantees ExportSummary is not nullptr");
2206 const auto TheFnGUID = TheFn->
getGUID();
2218 if ((!TheFnVI) && (TheFnGUID != TheFnGUIDWithExportedName)) {
2219 TheFnVI = ExportSummary->
getValueInfo(TheFnGUIDWithExportedName);
2224bool DevirtModule::mustBeUnreachableFunction(
2227 if (!
F->isDeclaration()) {
2230 return isa<UnreachableInst>(
F->getEntryBlock().getTerminator());
2233 return ExportSummary &&
2235 DevirtModule::lookUpFunctionValueInfo(
F, ExportSummary));
2238bool DevirtModule::run() {
2251 Function *TypeCheckedLoadRelativeFunc =
2258 if (!ExportSummary &&
2259 (!TypeTestFunc || TypeTestFunc->
use_empty() || !AssumeFunc ||
2261 (!TypeCheckedLoadFunc || TypeCheckedLoadFunc->
use_empty()) &&
2262 (!TypeCheckedLoadRelativeFunc ||
2263 TypeCheckedLoadRelativeFunc->
use_empty()))
2267 std::vector<VTableBits>
Bits;
2269 buildTypeIdentifierMap(Bits, TypeIdMap);
2271 if (TypeTestFunc && AssumeFunc)
2272 scanTypeTestUsers(TypeTestFunc, TypeIdMap);
2274 if (TypeCheckedLoadFunc)
2275 scanTypeCheckedLoadUsers(TypeCheckedLoadFunc);
2277 if (TypeCheckedLoadRelativeFunc)
2278 scanTypeCheckedLoadUsers(TypeCheckedLoadRelativeFunc);
2280 if (ImportSummary) {
2281 for (
auto &S : CallSlots)
2282 importResolution(S.first, S.second);
2284 removeRedundantTypeTests();
2297 if (TypeIdMap.
empty())
2301 if (ExportSummary) {
2303 for (
auto &
P : TypeIdMap) {
2304 if (
auto *TypeId = dyn_cast<MDString>(
P.first))
2309 for (
auto &
P : *ExportSummary) {
2310 for (
auto &S :
P.second.SummaryList) {
2311 auto *
FS = dyn_cast<FunctionSummary>(S.get());
2316 for (
Metadata *MD : MetadataByGUID[VF.GUID]) {
2317 CallSlots[{MD, VF.Offset}].CSInfo.addSummaryTypeTestAssumeUser(FS);
2321 for (
Metadata *MD : MetadataByGUID[VF.GUID]) {
2322 CallSlots[{MD, VF.Offset}].CSInfo.addSummaryTypeCheckedLoadUser(FS);
2326 FS->type_test_assume_const_vcalls()) {
2327 for (
Metadata *MD : MetadataByGUID[
VC.VFunc.GUID]) {
2328 CallSlots[{MD,
VC.VFunc.Offset}]
2329 .ConstCSInfo[
VC.Args]
2330 .addSummaryTypeTestAssumeUser(FS);
2334 FS->type_checked_load_const_vcalls()) {
2335 for (
Metadata *MD : MetadataByGUID[
VC.VFunc.GUID]) {
2336 CallSlots[{MD,
VC.VFunc.Offset}]
2337 .ConstCSInfo[
VC.Args]
2338 .addSummaryTypeCheckedLoadUser(FS);
2346 bool DidVirtualConstProp =
false;
2347 std::map<std::string, GlobalValue *> DevirtTargets;
2348 for (
auto &S : CallSlots) {
2352 std::vector<VirtualCallTarget> TargetsForSlot;
2354 const std::set<TypeMemberInfo> &TypeMemberInfos = TypeIdMap[S.first.TypeID];
2355 if (ExportSummary && isa<MDString>(S.first.TypeID) &&
2356 TypeMemberInfos.size())
2363 Res = &ExportSummary
2364 ->getOrInsertTypeIdSummary(
2365 cast<MDString>(S.first.TypeID)->getString())
2366 .WPDRes[S.first.ByteOffset];
2367 if (tryFindVirtualCallTargets(TargetsForSlot, TypeMemberInfos,
2368 S.first.ByteOffset, ExportSummary)) {
2370 if (!trySingleImplDevirt(ExportSummary, TargetsForSlot, S.second, Res)) {
2371 DidVirtualConstProp |=
2372 tryVirtualConstProp(TargetsForSlot, S.second, Res, S.first);
2374 tryICallBranchFunnel(TargetsForSlot, S.second, Res, S.first);
2379 for (
const auto &
T : TargetsForSlot)
2381 DevirtTargets[std::string(
T.Fn->getName())] =
T.Fn;
2388 if (ExportSummary && isa<MDString>(S.first.TypeID)) {
2391 for (
auto *FS : S.second.CSInfo.SummaryTypeCheckedLoadUsers)
2392 FS->addTypeTest(GUID);
2393 for (
auto &CCS : S.second.ConstCSInfo)
2394 for (
auto *FS : CCS.second.SummaryTypeCheckedLoadUsers)
2395 FS->addTypeTest(GUID);
2399 if (RemarksEnabled) {
2401 for (
const auto &DT : DevirtTargets) {
2403 auto F = dyn_cast<Function>(GV);
2405 auto A = dyn_cast<GlobalAlias>(GV);
2406 assert(
A && isa<Function>(
A->getAliasee()));
2407 F = dyn_cast<Function>(
A->getAliasee());
2411 using namespace ore;
2414 <<
NV(
"FunctionName", DT.first));
2418 NumDevirtTargets += DevirtTargets.size();
2420 removeRedundantTypeTests();
2424 if (DidVirtualConstProp)
2434 for (
auto *CI : CallsWithPtrAuthBundleRemoved)
2435 CI->eraseFromParent();
2440void DevirtIndex::run() {
2441 if (ExportSummary.typeIdCompatibleVtableMap().empty())
2445 for (
const auto &
P : ExportSummary.typeIdCompatibleVtableMap()) {
2453 ExportSummary.getOrInsertTypeIdSummary(
P.first);
2457 for (
auto &
P : ExportSummary) {
2458 for (
auto &S :
P.second.SummaryList) {
2459 auto *
FS = dyn_cast<FunctionSummary>(S.get());
2465 CallSlots[{
Name, VF.Offset}].CSInfo.addSummaryTypeTestAssumeUser(FS);
2470 CallSlots[{
Name, VF.Offset}].CSInfo.addSummaryTypeCheckedLoadUser(FS);
2474 FS->type_test_assume_const_vcalls()) {
2476 CallSlots[{
Name,
VC.VFunc.Offset}]
2477 .ConstCSInfo[
VC.Args]
2478 .addSummaryTypeTestAssumeUser(FS);
2482 FS->type_checked_load_const_vcalls()) {
2484 CallSlots[{
Name,
VC.VFunc.Offset}]
2485 .ConstCSInfo[
VC.Args]
2486 .addSummaryTypeCheckedLoadUser(FS);
2492 std::set<ValueInfo> DevirtTargets;
2494 for (
auto &S : CallSlots) {
2498 std::vector<ValueInfo> TargetsForSlot;
2499 auto TidSummary = ExportSummary.getTypeIdCompatibleVtableSummary(S.first.TypeID);
2504 &ExportSummary.getTypeIdSummary(S.first.TypeID)
2505 ->WPDRes[S.first.ByteOffset];
2506 if (tryFindVirtualCallTargets(TargetsForSlot, *TidSummary,
2507 S.first.ByteOffset)) {
2509 if (!trySingleImplDevirt(TargetsForSlot, S.first, S.second, Res,
2518 for (
const auto &DT : DevirtTargets)
2519 errs() <<
"Devirtualized call to " << DT <<
"\n";
2521 NumDevirtTargets += DevirtTargets.size();
This is the interface for LLVM's primary stateless and local alias analysis.
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
This file contains the declarations for the subclasses of Constant, which represent the different fla...
This file defines DenseMapInfo traits for DenseMap.
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
Provides passes for computing function attributes based on interprocedural analyses.
static void emitRemark(const Function &F, OptimizationRemarkEmitter &ORE, bool Skip)
static cl::opt< std::string > ClReadSummary("lowertypetests-read-summary", cl::desc("Read summary from given YAML file before running pass"), cl::Hidden)
static cl::opt< PassSummaryAction > ClSummaryAction("lowertypetests-summary-action", cl::desc("What to do with the summary when running this pass"), cl::values(clEnumValN(PassSummaryAction::None, "none", "Do nothing"), clEnumValN(PassSummaryAction::Import, "import", "Import typeid resolutions from summary and globals"), clEnumValN(PassSummaryAction::Export, "export", "Export typeid resolutions to summary and globals")), cl::Hidden)
static cl::opt< std::string > ClWriteSummary("lowertypetests-write-summary", cl::desc("Write summary to given YAML file after running pass"), cl::Hidden)
This file implements a map that provides insertion order iteration.
static bool mustBeUnreachableFunction(const Function &F)
Module.h This file contains the declarations for the Module class.
uint64_t IntrinsicInst * II
FunctionAnalysisManager FAM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static cl::opt< bool > DisableWholeProgramVisibility("disable-whole-program-visibility", cl::Hidden, cl::desc("Disable whole program visibility (overrides enabling options)"))
Provide a way to force disable whole program for debugging or workarounds, when enabled via the linke...
WPDCheckMode
Mechanism to add runtime checking of devirtualization decisions, optionally trapping or falling back ...
static cl::opt< PassSummaryAction > ClSummaryAction("wholeprogramdevirt-summary-action", cl::desc("What to do with the summary when running this pass"), cl::values(clEnumValN(PassSummaryAction::None, "none", "Do nothing"), clEnumValN(PassSummaryAction::Import, "import", "Import typeid resolutions from summary and globals"), clEnumValN(PassSummaryAction::Export, "export", "Export typeid resolutions to summary and globals")), cl::Hidden)
static cl::opt< bool > WholeProgramVisibility("whole-program-visibility", cl::Hidden, cl::desc("Enable whole program visibility"))
Provide a way to force enable whole program visibility in tests.
static bool typeIDVisibleToRegularObj(StringRef TypeID, function_ref< bool(StringRef)> IsVisibleToRegularObj)
static Error checkCombinedSummaryForTesting(ModuleSummaryIndex *Summary)
static cl::list< std::string > SkipFunctionNames("wholeprogramdevirt-skip", cl::desc("Prevent function(s) from being devirtualized"), cl::Hidden, cl::CommaSeparated)
Provide way to prevent certain function from being devirtualized.
static cl::opt< std::string > ClWriteSummary("wholeprogramdevirt-write-summary", cl::desc("Write summary to given bitcode or YAML file after running pass. " "Output file format is deduced from extension: *.bc means writing " "bitcode, otherwise YAML"), cl::Hidden)
static cl::opt< unsigned > ClThreshold("wholeprogramdevirt-branch-funnel-threshold", cl::Hidden, cl::init(10), cl::desc("Maximum number of call targets per " "call site to enable branch funnels"))
static cl::opt< WPDCheckMode > DevirtCheckMode("wholeprogramdevirt-check", cl::Hidden, cl::desc("Type of checking for incorrect devirtualizations"), cl::values(clEnumValN(WPDCheckMode::None, "none", "No checking"), clEnumValN(WPDCheckMode::Trap, "trap", "Trap when incorrect"), clEnumValN(WPDCheckMode::Fallback, "fallback", "Fallback to indirect when incorrect")))
static cl::opt< std::string > ClReadSummary("wholeprogramdevirt-read-summary", cl::desc("Read summary from given bitcode or YAML file before running pass"), cl::Hidden)
static bool skipUpdateDueToValidation(GlobalVariable &GV, function_ref< bool(StringRef)> IsVisibleToRegularObj)
static bool AddCalls(VTableSlotInfo &SlotInfo, const ValueInfo &Callee)
static cl::opt< bool > PrintSummaryDevirt("wholeprogramdevirt-print-index-based", cl::Hidden, cl::desc("Print index-based devirtualization messages"))
A manager for alias analyses.
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
static AttributeSet get(LLVMContext &C, const AttrBuilder &B)
StringRef getValueAsString() const
Return the attribute's value as a string.
bool isValid() const
Return true if the attribute is any kind of attribute.
LLVM Basic Block Representation.
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
void setCallingConv(CallingConv::ID CC)
std::optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
CallingConv::ID getCallingConv() const
Value * getCalledOperand() const
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
FunctionType * getFunctionType() const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void setCalledOperand(Value *V)
static CallBase * removeOperandBundle(CallBase *CB, uint32_t ID, InsertPosition InsertPt=nullptr)
Create a clone of CB with operand bundle ID removed.
AttributeList getAttributes() const
Return the parameter attributes for this call.
Function * getCaller()
Helper to get the caller (the parent function).
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
void setSelectionKind(SelectionKind Val)
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
static Constant * getIntToPtr(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getInBoundsGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList)
Create an "inbounds" getelementptr.
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
static Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList, GEPNoWrapFlags NW=GEPNoWrapFlags::none(), std::optional< ConstantRange > InRange=std::nullopt, Type *OnlyIfReducedTy=nullptr)
Getelementptr form.
static ConstantInt * getTrue(LLVMContext &Context)
static Constant * getAnon(ArrayRef< Constant * > V, bool Packed=false)
Return an anonymous struct that has the specified elements.
This is an important base class in LLVM.
const Constant * stripPointerCasts() const
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Implements a dense probed hash-table based set.
Analysis pass which computes a DominatorTree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Subclass of Error for the sole purpose of identifying the success path in the type system.
Lightweight error class with error context and mandatory checking.
This class evaluates LLVM IR, producing the Constant representing each SSA instruction.
Helper for check-and-exit error handling.
Tagged union holding either a T or a Error.
Function summary information to aid decisions and implementation of importing.
Type * getParamType(unsigned i) const
Parameter type accessors.
ArrayRef< Type * > params() const
Type * getReturnType() const
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
const BasicBlock & front() const
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Intrinsic::ID getIntrinsicID() const LLVM_READONLY
getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...
Type * getReturnType() const
Returns the type of the ret val.
This class implements a glob pattern matcher similar to the one found in bash, but with some key diff...
static Expected< GlobPattern > create(StringRef Pat, std::optional< size_t > MaxSubPatterns={})
static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
bool hasMetadata() const
Return true if this value has any metadata attached to it.
void setMetadata(unsigned KindID, MDNode *Node)
Set a particular kind of metadata attachment.
VCallVisibility getVCallVisibility() const
bool eraseMetadata(unsigned KindID)
Erase all metadata attachments with the given kind.
@ VCallVisibilityLinkageUnit
MDNode * getMetadata(unsigned KindID) const
Get the current metadata attachments for the given kind, if any.
void setVCallVisibilityMetadata(VCallVisibility Visibility)
static bool isLocalLinkage(LinkageTypes Linkage)
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
static bool isAvailableExternallyLinkage(LinkageTypes Linkage)
static GUID getGUID(StringRef GlobalName)
Return a 64-bit global unique ID constructed from global value name (i.e.
GUID getGUID() const
Return a 64-bit global unique ID constructed from global value name (i.e.
@ HiddenVisibility
The GV is hidden.
void setVisibility(VisibilityTypes V)
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ InternalLinkage
Rename collisions when linking (static functions).
@ ExternalLinkage
Externally visible function.
Global variable summary information to aid decisions and implementation of importing.
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Class to represent integer types.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
MDNode * createLikelyBranchWeights()
Return metadata containing two branch weights, with significant bias towards true destination.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
This class implements a map that also provides access to all stored values in a deterministic order.
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFile(const Twine &Filename, bool IsText=false, bool RequiresNullTerminator=true, bool IsVolatile=false, std::optional< Align > Alignment=std::nullopt)
Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful,...
Class to hold module path string table and global value map, and encapsulate methods for operating on...
const TypeIdSummary * getTypeIdSummary(StringRef TypeId) const
This returns either a pointer to the type id summary (if present in the summary map) or null (if not ...
ValueInfo getValueInfo(const GlobalValueSummaryMapTy::value_type &R) const
Return a ValueInfo for the index value_type (convenient when iterating index).
const ModuleHash & getModuleHash(const StringRef ModPath) const
Get the module SHA1 hash recorded for the given module path.
static constexpr const char * getRegularLTOModuleName()
bool partiallySplitLTOUnits() const
static std::string getGlobalNameForLocal(StringRef Name, ModuleHash ModHash)
Convenience method for creating a promoted global name for the given value name of a local,...
A Module instance is used to store all the information related to an LLVM module.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, InsertPosition InsertBefore=nullptr)
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
bool contains(StringRef Other) const
Return true if the given string is a substring of *this, and false otherwise.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
The instances of the Type class are immutable: once they are created, they are never changed.
TypeID
Definitions of all of the base types for the Type system.
static Type * getVoidTy(LLVMContext &C)
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
bool eraseMetadata(unsigned KindID)
Erase all metadata attachments with the given kind.
iterator_range< use_iterator > uses()
StringRef getName() const
Return a constant reference to the value's name.
std::pair< iterator, bool > insert(const ValueT &V)
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
self_iterator getIterator()
A raw_ostream that writes to a file descriptor.
A raw_ostream that writes to an std::string.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
@ C
The default llvm calling convention, compatible with C.
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
bool match(Val *V, const Pattern &P)
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
DiagnosticInfoOptimizationBase::Argument NV
@ OF_TextWithCRLF
The file should be opened in text mode and use a carriage linefeed '\r '.
uint64_t findLowestOffset(ArrayRef< VirtualCallTarget > Targets, bool IsAfter, uint64_t Size)
void setAfterReturnValues(MutableArrayRef< VirtualCallTarget > Targets, uint64_t AllocAfter, unsigned BitWidth, int64_t &OffsetByte, uint64_t &OffsetBit)
void setBeforeReturnValues(MutableArrayRef< VirtualCallTarget > Targets, uint64_t AllocBefore, unsigned BitWidth, int64_t &OffsetByte, uint64_t &OffsetBit)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
MemoryEffects computeFunctionBodyMemoryAccess(Function &F, AAResults &AAR)
Returns the memory access properties of this copy of the function.
@ Export
Export information to summary.
@ Import
Import information from summary.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Error createStringError(std::error_code EC, char const *Fmt, const Ts &... Vals)
Create formatted StringError object.
bool hasWholeProgramVisibility(bool WholeProgramVisibilityEnabledInLTO)
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Expected< std::unique_ptr< ModuleSummaryIndex > > getModuleSummaryIndex(MemoryBufferRef Buffer)
Parse the specified bitcode buffer, returning the module summary index.
void updatePublicTypeTestCalls(Module &M, bool WholeProgramVisibilityEnabledInLTO)
void getVisibleToRegularObjVtableGUIDs(ModuleSummaryIndex &Index, DenseSet< GlobalValue::GUID > &VisibleToRegularObjSymbols, function_ref< bool(StringRef)> IsVisibleToRegularObj)
Based on typeID string, get all associated vtable GUIDS that are visible to regular objects.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void writeIndexToFile(const ModuleSummaryIndex &Index, raw_ostream &Out, const std::map< std::string, GVSummaryMapTy > *ModuleToSummariesForIndex=nullptr, const GVSummaryPtrSet *DecSummaries=nullptr)
Write the specified module summary index to the given raw output stream, where it will be written in ...
void findDevirtualizableCallsForTypeCheckedLoad(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< Instruction * > &LoadedPtrs, SmallVectorImpl< Instruction * > &Preds, bool &HasNonCallUses, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.checked.load, find all devirtualizable call sites based on t...
CallBase & versionCallSite(CallBase &CB, Value *Callee, MDNode *BranchWeights)
Predicate and clone the given call site.
bool AreStatisticsEnabled()
Check if statistics are enabled.
void updateIndexWPDForExports(ModuleSummaryIndex &Summary, function_ref< bool(StringRef, ValueInfo)> isExported, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap)
Call after cross-module importing to update the recorded single impl devirt target names for any loca...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
void runWholeProgramDevirtOnIndex(ModuleSummaryIndex &Summary, std::set< GlobalValue::GUID > &ExportedGUIDs, std::map< ValueInfo, std::vector< VTableSlotSummary > > &LocalWPDTargetsMap)
Perform index-based whole program devirtualization on the Summary index.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Expected< T > errorOrToExpected(ErrorOr< T > &&EO)
Convert an ErrorOr<T> to an Expected<T>.
constexpr unsigned BitWidth
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
std::vector< TypeIdOffsetVtableInfo > TypeIdCompatibleVtableInfo
List of vtable definitions decorated by a particular type identifier, and their corresponding offsets...
void consumeError(Error Err)
Consume a Error without doing anything.
void findDevirtualizableCallsForTypeTest(SmallVectorImpl< DevirtCallSite > &DevirtCalls, SmallVectorImpl< CallInst * > &Assumes, const CallInst *CI, DominatorTree &DT)
Given a call to the intrinsic @llvm.type.test, find all devirtualizable call sites based on the call ...
void updateVCallVisibilityInModule(Module &M, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols, bool ValidateAllVtablesHaveTypeInfos, function_ref< bool(StringRef)> IsVisibleToRegularObj)
If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...
std::pair< Function *, Constant * > getFunctionAtVTableOffset(GlobalVariable *GV, uint64_t Offset, Module &M)
Given a vtable and a specified offset, returns the function and the trivial pointer at the specified ...
void updateVCallVisibilityInIndex(ModuleSummaryIndex &Index, bool WholeProgramVisibilityEnabledInLTO, const DenseSet< GlobalValue::GUID > &DynamicExportSymbols, const DenseSet< GlobalValue::GUID > &VisibleToRegularObjSymbols)
If whole program visibility asserted, then upgrade all public vcall visibility metadata on vtable def...
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Class to accumulate and hold information about a callee.
static unsigned getHashValue(const VTableSlotSummary &I)
static bool isEqual(const VTableSlotSummary &LHS, const VTableSlotSummary &RHS)
static VTableSlotSummary getTombstoneKey()
static VTableSlotSummary getEmptyKey()
static VTableSlot getTombstoneKey()
static VTableSlot getEmptyKey()
static bool isEqual(const VTableSlot &LHS, const VTableSlot &RHS)
static unsigned getHashValue(const VTableSlot &I)
An information struct used to provide DenseMap with the various necessary components for a given valu...
A call site that could be devirtualized.
A specification for a virtual function call with all constant integer arguments.
An "identifier" for a virtual function.
The following data structures summarize type metadata information.
std::map< uint64_t, WholeProgramDevirtResolution > WPDRes
Mapping from byte offset to whole-program devirt resolution for that (typeid, byte offset) pair.
@ Unsat
Unsatisfiable type (i.e. no global has this type metadata)
enum llvm::TypeTestResolution::Kind TheKind
Struct that holds a reference to a particular GUID in a global value summary.
ArrayRef< std::unique_ptr< GlobalValueSummary > > getSummaryList() const
PreservedAnalyses run(Module &M, ModuleAnalysisManager &)
@ UniformRetVal
Uniform return value optimization.
@ VirtualConstProp
Virtual constant propagation.
@ UniqueRetVal
Unique return value optimization.
uint64_t Info
Additional information for the resolution:
enum llvm::WholeProgramDevirtResolution::ByArg::Kind TheKind
enum llvm::WholeProgramDevirtResolution::Kind TheKind
std::map< std::vector< uint64_t >, ByArg > ResByArg
Resolutions for calls with all constant integer arguments (excluding the first argument,...
std::string SingleImplName
@ SingleImpl
Single implementation devirtualization.
@ BranchFunnel
When retpoline mitigation is enabled, use a branch funnel that is defined in the merged module.
VirtualCallTarget(GlobalValue *Fn, const TypeMemberInfo *TM)