32 #include "llvm/Config/llvm-config.h"
88 cl::desc(
"Number of metadatas above which we emit an index "
89 "to enable lazy-loading"));
92 cl::desc(
"The threshold (unit M) for flushing LLVM bitcode."));
96 cl::desc(
"Write relative block frequency to function summary "));
111 VST_BBENTRY_6_ABBREV,
115 CONSTANTS_INTEGER_ABBREV,
116 CONSTANTS_CE_CAST_Abbrev,
117 CONSTANTS_NULL_Abbrev,
121 FUNCTION_INST_UNOP_ABBREV,
122 FUNCTION_INST_UNOP_FLAGS_ABBREV,
123 FUNCTION_INST_BINOP_ABBREV,
124 FUNCTION_INST_BINOP_FLAGS_ABBREV,
125 FUNCTION_INST_CAST_ABBREV,
126 FUNCTION_INST_RET_VOID_ABBREV,
127 FUNCTION_INST_RET_VAL_ABBREV,
128 FUNCTION_INST_UNREACHABLE_ABBREV,
129 FUNCTION_INST_GEP_ABBREV,
134 class BitcodeWriterBase {
145 : Stream(Stream), StrtabBuilder(StrtabBuilder) {}
148 void writeModuleVersion();
151 void BitcodeWriterBase::writeModuleVersion() {
158 class ModuleBitcodeWriterBase :
public BitcodeWriterBase {
172 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
175 unsigned GlobalValueId;
186 bool ShouldPreserveUseListOrder,
188 : BitcodeWriterBase(Stream, StrtabBuilder),
M(
M),
198 for (
const auto &GUIDSummaryLists : *
Index)
200 for (
auto &Summary : GUIDSummaryLists.second.SummaryList)
201 if (
auto FS = dyn_cast<FunctionSummary>(
Summary.get()))
206 for (
auto &CallEdge :
FS->calls())
207 if (!CallEdge.first.haveGVs() || !CallEdge.first.getValue())
208 assignValueId(CallEdge.first.getGUID());
212 void writePerModuleGlobalValueSummary();
215 void writePerModuleFunctionSummaryRecord(
217 unsigned ValueID,
unsigned FSCallsAbbrev,
unsigned FSCallsProfileAbbrev,
218 unsigned CallsiteAbbrev,
unsigned AllocAbbrev,
const Function &
F);
221 unsigned FSModRefsAbbrev,
222 unsigned FSModVTableRefsAbbrev);
225 GUIDToValueIdMap[ValGUID] = ++GlobalValueId;
229 const auto &VMI = GUIDToValueIdMap.find(ValGUID);
232 assert(VMI != GUIDToValueIdMap.end() &&
233 "GUID does not have assigned value Id");
239 if (!
VI.haveGVs() || !
VI.getValue())
240 return getValueId(
VI.getGUID());
244 std::map<GlobalValue::GUID, unsigned> &valueIds() {
return GUIDToValueIdMap; }
248 class ModuleBitcodeWriter :
public ModuleBitcodeWriterBase {
272 : ModuleBitcodeWriterBase(
M, StrtabBuilder, Stream,
273 ShouldPreserveUseListOrder,
Index),
274 Buffer(Buffer), GenerateHash(GenerateHash), ModHash(ModHash),
275 BitcodeStartBit(Stream.GetCurrentBitNo()) {}
281 uint64_t bitcodeStartBit() {
return BitcodeStartBit; }
285 void writeAttributeGroupTable();
286 void writeAttributeTable();
287 void writeTypeTable();
289 void writeValueSymbolTableForwardDecl();
290 void writeModuleInfo();
295 unsigned createDILocationAbbrev();
298 unsigned createGenericDINodeAbbrev();
367 unsigned createNamedMetadataAbbrev();
369 unsigned createMetadataStringsAbbrev();
374 std::vector<unsigned> *MDAbbrevs =
nullptr,
375 std::vector<uint64_t> *IndexPos =
nullptr);
376 void writeModuleMetadata();
377 void writeFunctionMetadata(
const Function &
F);
378 void writeFunctionMetadataAttachment(
const Function &
F);
381 void writeModuleMetadataKinds();
382 void writeOperandBundleTags();
383 void writeSyncScopeNames();
384 void writeConstants(
unsigned FirstVal,
unsigned LastVal,
bool isGlobal);
385 void writeModuleConstants();
386 bool pushValueAndType(
const Value *V,
unsigned InstID,
388 void writeOperandBundles(
const CallBase &CB,
unsigned InstID);
389 void pushValue(
const Value *V,
unsigned InstID,
391 void pushValueSigned(
const Value *V,
unsigned InstID,
393 void writeInstruction(
const Instruction &
I,
unsigned InstID,
396 void writeGlobalValueSymbolTable(
399 void writeUseListBlock(
const Function *
F);
403 void writeBlockInfo();
404 void writeModuleHash(
size_t BlockStartPos);
407 return unsigned(SSID);
410 unsigned getEncodedAlign(
MaybeAlign Alignment) {
return encode(Alignment); }
414 class IndexBitcodeWriter :
public BitcodeWriterBase {
420 const std::map<std::string, GVSummaryMapTy> *ModuleToSummariesForIndex;
424 std::map<GlobalValue::GUID, unsigned> GUIDToValueIdMap;
429 std::vector<unsigned> StackIdIndices;
432 unsigned GlobalValueId = 0;
440 const std::map<std::string, GVSummaryMapTy>
441 *ModuleToSummariesForIndex =
nullptr)
442 : BitcodeWriterBase(Stream, StrtabBuilder),
Index(
Index),
443 ModuleToSummariesForIndex(ModuleToSummariesForIndex) {
448 forEachSummary([&](GVInfo
I,
bool IsAliasee) {
449 GUIDToValueIdMap[
I.first] = ++GlobalValueId;
452 auto *
FS = dyn_cast<FunctionSummary>(
I.second);
458 for (
auto &CI :
FS->callsites())
459 for (
auto Idx : CI.StackIdIndices)
460 StackIdIndices.push_back(Idx);
461 for (
auto &AI :
FS->allocs())
462 for (
auto &MIB : AI.MIBs)
463 for (
auto Idx : MIB.StackIdIndices)
464 StackIdIndices.push_back(Idx);
467 StackIdIndices.erase(
468 std::unique(StackIdIndices.begin(), StackIdIndices.end()),
469 StackIdIndices.end());
473 using GVInfo = std::pair<GlobalValue::GUID, GlobalValueSummary *>;
478 template<
typename Functor>
479 void forEachSummary(Functor Callback) {
480 if (ModuleToSummariesForIndex) {
481 for (
auto &M : *ModuleToSummariesForIndex)
482 for (
auto &Summary :
M.second) {
483 Callback(Summary,
false);
487 if (
auto *AS = dyn_cast<AliasSummary>(
Summary.getSecond()))
488 Callback({AS->getAliaseeGUID(), &AS->getAliasee()},
true);
491 for (
auto &Summaries :
Index)
492 for (
auto &Summary : Summaries.second.SummaryList)
493 Callback({Summaries.first,
Summary.get()},
false);
501 template <
typename Functor>
void forEachModule(Functor Callback) {
502 if (ModuleToSummariesForIndex) {
503 for (
const auto &M : *ModuleToSummariesForIndex) {
504 const auto &MPI =
Index.modulePaths().find(
M.first);
505 if (MPI ==
Index.modulePaths().end()) {
509 assert(ModuleToSummariesForIndex->size() == 1);
515 for (
const auto &MPSE :
Index.modulePaths())
524 void writeModStrings();
525 void writeCombinedGlobalValueSummary();
528 auto VMI = GUIDToValueIdMap.find(ValGUID);
529 if (VMI == GUIDToValueIdMap.end())
534 std::map<GlobalValue::GUID, unsigned> &valueIds() {
return GUIDToValueIdMap; }
570 case Instruction::Sub:
575 case Instruction::FDiv:
578 case Instruction::FRem:
644 case Attribute::Alignment:
646 case Attribute::AllocAlign:
648 case Attribute::AllocSize:
650 case Attribute::AlwaysInline:
652 case Attribute::Builtin:
654 case Attribute::ByVal:
658 case Attribute::InAlloca:
662 case Attribute::DisableSanitizerInstrumentation:
664 case Attribute::FnRetThunkExtern:
668 case Attribute::ElementType:
670 case Attribute::InlineHint:
672 case Attribute::InReg:
676 case Attribute::MinSize:
678 case Attribute::AllocatedPointer:
680 case Attribute::AllocKind:
682 case Attribute::Memory:
684 case Attribute::Naked:
686 case Attribute::Nest:
688 case Attribute::NoAlias:
690 case Attribute::NoBuiltin:
692 case Attribute::NoCallback:
694 case Attribute::NoCapture:
696 case Attribute::NoDuplicate:
698 case Attribute::NoFree:
700 case Attribute::NoImplicitFloat:
702 case Attribute::NoInline:
704 case Attribute::NoRecurse:
706 case Attribute::NoMerge:
708 case Attribute::NonLazyBind:
710 case Attribute::NonNull:
712 case Attribute::Dereferenceable:
714 case Attribute::DereferenceableOrNull:
716 case Attribute::NoRedZone:
718 case Attribute::NoReturn:
720 case Attribute::NoSync:
722 case Attribute::NoCfCheck:
724 case Attribute::NoProfile:
726 case Attribute::SkipProfile:
728 case Attribute::NoUnwind:
730 case Attribute::NoSanitizeBounds:
732 case Attribute::NoSanitizeCoverage:
734 case Attribute::NullPointerIsValid:
736 case Attribute::OptForFuzzing:
738 case Attribute::OptimizeForSize:
740 case Attribute::OptimizeNone:
742 case Attribute::ReadNone:
744 case Attribute::ReadOnly:
746 case Attribute::Returned:
748 case Attribute::ReturnsTwice:
750 case Attribute::SExt:
752 case Attribute::Speculatable:
754 case Attribute::StackAlignment:
756 case Attribute::StackProtect:
758 case Attribute::StackProtectReq:
760 case Attribute::StackProtectStrong:
762 case Attribute::SafeStack:
764 case Attribute::ShadowCallStack:
766 case Attribute::StrictFP:
768 case Attribute::StructRet:
770 case Attribute::SanitizeAddress:
772 case Attribute::SanitizeHWAddress:
774 case Attribute::SanitizeThread:
776 case Attribute::SanitizeMemory:
778 case Attribute::SpeculativeLoadHardening:
780 case Attribute::SwiftError:
782 case Attribute::SwiftSelf:
784 case Attribute::SwiftAsync:
786 case Attribute::UWTable:
788 case Attribute::VScaleRange:
790 case Attribute::WillReturn:
792 case Attribute::WriteOnly:
794 case Attribute::ZExt:
796 case Attribute::ImmArg:
798 case Attribute::SanitizeMemTag:
800 case Attribute::Preallocated:
802 case Attribute::NoUndef:
804 case Attribute::ByRef:
806 case Attribute::MustProgress:
808 case Attribute::PresplitCoroutine:
822 void ModuleBitcodeWriter::writeAttributeGroupTable() {
823 const std::vector<ValueEnumerator::IndexAndAttrSet> &AttrGrps =
825 if (AttrGrps.empty())
return;
831 unsigned AttrListIndex = Pair.first;
834 Record.push_back(AttrListIndex);
837 if (Attr.isEnumAttribute()) {
840 }
else if (Attr.isIntAttribute()) {
843 Record.push_back(Attr.getValueAsInt());
844 }
else if (Attr.isStringAttribute()) {
856 assert(Attr.isTypeAttribute());
857 Type *Ty = Attr.getValueAsType();
858 Record.push_back(Ty ? 6 : 5);
872 void ModuleBitcodeWriter::writeAttributeTable() {
880 for (
unsigned i :
AL.indexes()) {
894 void ModuleBitcodeWriter::writeTypeTable() {
903 auto Abbv = std::make_shared<BitCodeAbbrev>();
910 Abbv = std::make_shared<BitCodeAbbrev>();
916 Abbv = std::make_shared<BitCodeAbbrev>();
924 Abbv = std::make_shared<BitCodeAbbrev>();
932 Abbv = std::make_shared<BitCodeAbbrev>();
939 Abbv = std::make_shared<BitCodeAbbrev>();
947 Abbv = std::make_shared<BitCodeAbbrev>();
954 TypeVals.push_back(TypeList.size());
959 for (
Type *
T : TypeList) {
963 switch (
T->getTypeID()) {
980 TypeVals.push_back(cast<IntegerType>(
T)->
getBitWidth());
990 AbbrevToUse = OpaquePtrAbbrev;
997 AbbrevToUse = PtrAbbrev;
1005 TypeVals.push_back(FT->
isVarArg());
1009 AbbrevToUse = FunctionAbbrev;
1015 TypeVals.push_back(
ST->isPacked());
1017 for (
Type *ET :
ST->elements())
1020 if (
ST->isLiteral()) {
1022 AbbrevToUse = StructAnonAbbrev;
1024 if (
ST->isOpaque()) {
1028 AbbrevToUse = StructNamedAbbrev;
1032 if (!
ST->getName().empty())
1044 AbbrevToUse = ArrayAbbrev;
1055 if (isa<ScalableVectorType>(VT))
1056 TypeVals.push_back(
true);
1064 TypeVals.push_back(
TET->getNumTypeParameters());
1065 for (
Type *InnerTy :
TET->type_params())
1066 TypeVals.push_back(VE.
getTypeID(InnerTy));
1067 for (
unsigned IntParam :
TET->int_params())
1068 TypeVals.push_back(IntParam);
1076 Stream.
EmitRecord(Code, TypeVals, AbbrevToUse);
1117 RawFlags |= Flags.ReadNone;
1118 RawFlags |= (Flags.ReadOnly << 1);
1119 RawFlags |= (Flags.NoRecurse << 2);
1120 RawFlags |= (Flags.ReturnDoesNotAlias << 3);
1121 RawFlags |= (Flags.NoInline << 4);
1122 RawFlags |= (Flags.AlwaysInline << 5);
1123 RawFlags |= (Flags.NoUnwind << 6);
1124 RawFlags |= (Flags.MayThrow << 7);
1125 RawFlags |= (Flags.HasUnknownCall << 8);
1126 RawFlags |= (Flags.MustBeUnreachable << 9);
1135 RawFlags |= Flags.NotEligibleToImport;
1136 RawFlags |= (Flags.Live << 1);
1137 RawFlags |= (Flags.DSOLocal << 2);
1138 RawFlags |= (Flags.CanAutoHide << 3);
1143 RawFlags = (RawFlags << 4) | Flags.Linkage;
1145 RawFlags |= (Flags.Visibility << 8);
1151 uint64_t RawFlags = Flags.MaybeReadOnly | (Flags.MaybeWriteOnly << 1) |
1152 (Flags.Constant << 2) | Flags.VCallVisibility << 3;
1186 switch (
C.getSelectionKind()) {
1210 size_t ModuleBitcodeWriter::addToStrtab(
StringRef Str) {
1213 return StrtabBuilder.
add(Str);
1216 void ModuleBitcodeWriter::writeComdats() {
1220 Vals.push_back(addToStrtab(
C->getName()));
1221 Vals.push_back(
C->getName().size());
1231 void ModuleBitcodeWriter::writeValueSymbolTableForwardDecl() {
1236 auto Abbv = std::make_shared<BitCodeAbbrev>();
1258 bool isChar6 =
true;
1259 for (
char C : Str) {
1262 if ((
unsigned char)
C & 128)
1272 "Sanitizer Metadata is too large for naive serialization.");
1275 return Meta.NoAddress | (Meta.NoHWAddress << 1) |
1276 (Meta.Memtag << 2) | (Meta.IsDynInit << 3);
1282 void ModuleBitcodeWriter::writeModuleInfo() {
1284 if (!
M.getTargetTriple().empty())
1287 const std::string &
DL =
M.getDataLayoutStr();
1290 if (!
M.getModuleInlineAsm().empty())
1296 std::map<std::string, unsigned> SectionMap;
1297 std::map<std::string, unsigned> GCMap;
1299 unsigned MaxGlobalType = 0;
1300 const auto UpdateMaxAlignment = [&MaxAlignment](
const MaybeAlign A) {
1302 MaxAlignment = !MaxAlignment ? *
A :
std::max(*MaxAlignment, *A);
1305 UpdateMaxAlignment(GV.getAlign());
1307 if (GV.hasSection()) {
1309 unsigned &Entry = SectionMap[std::string(GV.getSection())];
1313 Entry = SectionMap.size();
1318 UpdateMaxAlignment(
F.getAlign());
1319 if (
F.hasSection()) {
1321 unsigned &Entry = SectionMap[std::string(
F.getSection())];
1325 Entry = SectionMap.size();
1330 unsigned &Entry = GCMap[
F.getGC()];
1334 Entry = GCMap.size();
1340 unsigned SimpleGVarAbbrev = 0;
1341 if (!
M.global_empty()) {
1343 auto Abbv = std::make_shared<BitCodeAbbrev>();
1357 unsigned MaxEncAlignment = getEncodedAlign(MaxAlignment);
1361 if (SectionMap.empty())
1381 auto Abbv = std::make_shared<BitCodeAbbrev>();
1384 Abbv->Add(AbbrevOpToUse);
1387 for (
const auto P :
M.getSourceFileName())
1388 Vals.push_back((
unsigned char)
P);
1397 unsigned AbbrevToUse = 0;
1403 Vals.push_back(addToStrtab(GV.getName()));
1404 Vals.push_back(GV.getName().size());
1405 Vals.push_back(VE.
getTypeID(GV.getValueType()));
1406 Vals.push_back(GV.getType()->getAddressSpace() << 2 | 2 | GV.isConstant());
1407 Vals.push_back(GV.isDeclaration() ? 0 :
1410 Vals.push_back(getEncodedAlign(GV.getAlign()));
1411 Vals.push_back(GV.hasSection() ? SectionMap[std::string(GV.getSection())]
1413 if (GV.isThreadLocal() ||
1416 GV.isExternallyInitialized() ||
1418 GV.hasComdat() || GV.hasAttributes() || GV.isDSOLocal() ||
1419 GV.hasPartition() || GV.hasSanitizerMetadata()) {
1423 Vals.push_back(GV.isExternallyInitialized());
1425 Vals.push_back(GV.hasComdat() ? VE.
getComdatID(GV.getComdat()) : 0);
1430 Vals.push_back(GV.isDSOLocal());
1431 Vals.push_back(addToStrtab(GV.getPartition()));
1432 Vals.push_back(GV.getPartition().size());
1435 GV.getSanitizerMetadata())
1438 AbbrevToUse = SimpleGVarAbbrev;
1451 Vals.push_back(addToStrtab(
F.getName()));
1452 Vals.push_back(
F.getName().size());
1453 Vals.push_back(VE.
getTypeID(
F.getFunctionType()));
1454 Vals.push_back(
F.getCallingConv());
1455 Vals.push_back(
F.isDeclaration());
1458 Vals.push_back(getEncodedAlign(
F.getAlign()));
1459 Vals.push_back(
F.hasSection() ? SectionMap[std::string(
F.getSection())]
1462 Vals.push_back(
F.hasGC() ? GCMap[
F.getGC()] : 0);
1464 Vals.push_back(
F.hasPrologueData() ? (VE.
getValueID(
F.getPrologueData()) + 1)
1467 Vals.push_back(
F.hasComdat() ? VE.
getComdatID(
F.getComdat()) : 0);
1468 Vals.push_back(
F.hasPrefixData() ? (VE.
getValueID(
F.getPrefixData()) + 1)
1471 F.hasPersonalityFn() ? (VE.
getValueID(
F.getPersonalityFn()) + 1) : 0);
1473 Vals.push_back(
F.isDSOLocal());
1474 Vals.push_back(
F.getAddressSpace());
1475 Vals.push_back(addToStrtab(
F.getPartition()));
1476 Vals.push_back(
F.getPartition().size());
1478 unsigned AbbrevToUse = 0;
1488 Vals.push_back(addToStrtab(
A.getName()));
1489 Vals.push_back(
A.getName().size());
1490 Vals.push_back(VE.
getTypeID(
A.getValueType()));
1491 Vals.push_back(
A.getType()->getAddressSpace());
1498 Vals.push_back(
A.isDSOLocal());
1499 Vals.push_back(addToStrtab(
A.getPartition()));
1500 Vals.push_back(
A.getPartition().size());
1502 unsigned AbbrevToUse = 0;
1511 Vals.push_back(addToStrtab(
I.getName()));
1512 Vals.push_back(
I.getName().size());
1513 Vals.push_back(VE.
getTypeID(
I.getValueType()));
1514 Vals.push_back(
I.getType()->getAddressSpace());
1518 Vals.push_back(
I.isDSOLocal());
1519 Vals.push_back(addToStrtab(
I.getPartition()));
1520 Vals.push_back(
I.getPartition().size());
1525 writeValueSymbolTableForwardDecl();
1531 if (
const auto *OBO = dyn_cast<OverflowingBinaryOperator>(V)) {
1532 if (OBO->hasNoSignedWrap())
1534 if (OBO->hasNoUnsignedWrap())
1536 }
else if (
const auto *PEO = dyn_cast<PossiblyExactOperator>(V)) {
1539 }
else if (
const auto *FPMO = dyn_cast<FPMathOperator>(V)) {
1540 if (FPMO->hasAllowReassoc())
1542 if (FPMO->hasNoNaNs())
1544 if (FPMO->hasNoInfs())
1546 if (FPMO->hasNoSignedZeros())
1548 if (FPMO->hasAllowReciprocal())
1550 if (FPMO->hasAllowContract())
1552 if (FPMO->hasApproxFunc())
1559 void ModuleBitcodeWriter::writeValueAsMetadata(
1572 for (
unsigned i = 0,
e =
N->getNumOperands();
i !=
e; ++
i) {
1574 assert(!(MD && isa<LocalAsMetadata>(MD)) &&
1575 "Unexpected function-local metadata");
1584 unsigned ModuleBitcodeWriter::createDILocationAbbrev() {
1587 auto Abbv = std::make_shared<BitCodeAbbrev>();
1602 Abbrev = createDILocationAbbrev();
1604 Record.push_back(
N->isDistinct());
1605 Record.push_back(
N->getLine());
1606 Record.push_back(
N->getColumn());
1609 Record.push_back(
N->isImplicitCode());
1615 unsigned ModuleBitcodeWriter::createGenericDINodeAbbrev() {
1618 auto Abbv = std::make_shared<BitCodeAbbrev>();
1633 Abbrev = createGenericDINodeAbbrev();
1635 Record.push_back(
N->isDistinct());
1636 Record.push_back(
N->getTag());
1639 for (
auto &
I :
N->operands())
1674 if ((int64_t)V >= 0)
1675 Vals.push_back(V << 1);
1677 Vals.push_back((-V << 1) | 1);
1685 unsigned NumWords = A.getActiveWords();
1686 const uint64_t *RawData = A.getRawData();
1687 for (
unsigned i = 0;
i < NumWords;
i++)
1695 Record.push_back(IsBigInt | (
N->isUnsigned() << 1) |
N->isDistinct());
1696 Record.push_back(
N->getValue().getBitWidth());
1707 Record.push_back(
N->isDistinct());
1708 Record.push_back(
N->getTag());
1710 Record.push_back(
N->getSizeInBits());
1711 Record.push_back(
N->getAlignInBits());
1712 Record.push_back(
N->getEncoding());
1713 Record.push_back(
N->getFlags());
1722 Record.push_back(
N->isDistinct());
1723 Record.push_back(
N->getTag());
1728 Record.push_back(
N->getSizeInBits());
1729 Record.push_back(
N->getAlignInBits());
1730 Record.push_back(
N->getEncoding());
1739 Record.push_back(
N->isDistinct());
1740 Record.push_back(
N->getTag());
1743 Record.push_back(
N->getLine());
1746 Record.push_back(
N->getSizeInBits());
1747 Record.push_back(
N->getAlignInBits());
1748 Record.push_back(
N->getOffsetInBits());
1749 Record.push_back(
N->getFlags());
1754 if (
const auto &DWARFAddressSpace =
N->getDWARFAddressSpace())
1755 Record.push_back(*DWARFAddressSpace + 1);
1768 const unsigned IsNotUsedInOldTypeRef = 0x2;
1769 Record.push_back(IsNotUsedInOldTypeRef | (
unsigned)
N->isDistinct());
1770 Record.push_back(
N->getTag());
1773 Record.push_back(
N->getLine());
1776 Record.push_back(
N->getSizeInBits());
1777 Record.push_back(
N->getAlignInBits());
1778 Record.push_back(
N->getOffsetInBits());
1779 Record.push_back(
N->getFlags());
1781 Record.push_back(
N->getRuntimeLang());
1799 const unsigned HasNoOldTypeRefs = 0x2;
1800 Record.push_back(HasNoOldTypeRefs | (
unsigned)
N->isDistinct());
1801 Record.push_back(
N->getFlags());
1812 Record.push_back(
N->isDistinct());
1815 if (
N->getRawChecksum()) {
1816 Record.push_back(
N->getRawChecksum()->Kind);
1824 auto Source =
N->getRawSource();
1835 assert(
N->isDistinct() &&
"Expected distinct compile units");
1837 Record.push_back(
N->getSourceLanguage());
1840 Record.push_back(
N->isOptimized());
1842 Record.push_back(
N->getRuntimeVersion());
1844 Record.push_back(
N->getEmissionKind());
1850 Record.push_back(
N->getDWOId());
1852 Record.push_back(
N->getSplitDebugInlining());
1853 Record.push_back(
N->getDebugInfoForProfiling());
1854 Record.push_back((
unsigned)
N->getNameTableKind());
1855 Record.push_back(
N->getRangesBaseAddress());
1866 const uint64_t HasUnitFlag = 1 << 1;
1867 const uint64_t HasSPFlagsFlag = 1 << 2;
1868 Record.push_back(
uint64_t(
N->isDistinct()) | HasUnitFlag | HasSPFlagsFlag);
1873 Record.push_back(
N->getLine());
1875 Record.push_back(
N->getScopeLine());
1877 Record.push_back(
N->getSPFlags());
1878 Record.push_back(
N->getVirtualIndex());
1879 Record.push_back(
N->getFlags());
1884 Record.push_back(
N->getThisAdjustment());
1896 Record.push_back(
N->isDistinct());
1899 Record.push_back(
N->getLine());
1900 Record.push_back(
N->getColumn());
1909 Record.push_back(
N->isDistinct());
1912 Record.push_back(
N->getDiscriminator());
1921 Record.push_back(
N->isDistinct());
1926 Record.push_back(
N->getLineNo());
1935 Record.push_back(
N->isDistinct() |
N->getExportSymbols() << 1);
1946 Record.push_back(
N->isDistinct());
1947 Record.push_back(
N->getMacinfoType());
1948 Record.push_back(
N->getLine());
1959 Record.push_back(
N->isDistinct());
1960 Record.push_back(
N->getMacinfoType());
1961 Record.push_back(
N->getLine());
1972 Record.reserve(
N->getArgs().size());
1983 Record.push_back(
N->isDistinct());
1984 for (
auto &
I :
N->operands())
1986 Record.push_back(
N->getLineNo());
1987 Record.push_back(
N->getIsDecl());
1997 Record.push_back(
N->isDistinct());
2005 Record.push_back(
N->isDistinct());
2008 Record.push_back(
N->isDefault());
2017 Record.push_back(
N->isDistinct());
2018 Record.push_back(
N->getTag());
2021 Record.push_back(
N->isDefault());
2037 Record.push_back(
N->getLine());
2039 Record.push_back(
N->isLocalToUnit());
2040 Record.push_back(
N->isDefinition());
2043 Record.push_back(
N->getAlignInBits());
2066 const uint64_t HasAlignmentFlag = 1 << 1;
2071 Record.push_back(
N->getLine());
2073 Record.push_back(
N->getArg());
2074 Record.push_back(
N->getFlags());
2075 Record.push_back(
N->getAlignInBits());
2089 Record.push_back(
N->getLine());
2098 Record.reserve(
N->getElements().size() + 1);
2101 Record.append(
N->elements_begin(),
N->elements_end());
2110 Record.push_back(
N->isDistinct());
2121 Record.push_back(
N->isDistinct());
2124 Record.push_back(
N->getLine());
2127 Record.push_back(
N->getAttributes());
2137 Record.push_back(
N->isDistinct());
2138 Record.push_back(
N->getTag());
2141 Record.push_back(
N->getLine());
2150 unsigned ModuleBitcodeWriter::createNamedMetadataAbbrev() {
2151 auto Abbv = std::make_shared<BitCodeAbbrev>();
2158 void ModuleBitcodeWriter::writeNamedMetadata(
2160 if (
M.named_metadata_empty())
2163 unsigned Abbrev = createNamedMetadataAbbrev();
2167 Record.append(Str.bytes_begin(), Str.bytes_end());
2172 for (
const MDNode *
N : NMD.operands())
2179 unsigned ModuleBitcodeWriter::createMetadataStringsAbbrev() {
2180 auto Abbv = std::make_shared<BitCodeAbbrev>();
2192 void ModuleBitcodeWriter::writeMetadataStrings(
2194 if (Strings.empty())
2199 Record.push_back(Strings.size());
2206 W.EmitVBR(cast<MDString>(MD)->getLength(), 6);
2211 Record.push_back(Blob.size());
2215 Blob.
append(cast<MDString>(MD)->getString());
2224 #define HANDLE_MDNODE_LEAF(CLASS) CLASS##AbbrevID,
2225 #include "llvm/IR/Metadata.def"
2229 void ModuleBitcodeWriter::writeMetadataRecords(
2231 std::vector<unsigned> *MDAbbrevs, std::vector<uint64_t> *IndexPos) {
2236 #define HANDLE_MDNODE_LEAF(CLASS) unsigned CLASS##Abbrev = 0;
2237 #include "llvm/IR/Metadata.def"
2242 if (
const MDNode *
N = dyn_cast<MDNode>(MD)) {
2243 assert(
N->isResolved() &&
"Expected forward references to be resolved");
2245 switch (
N->getMetadataID()) {
2248 #define HANDLE_MDNODE_LEAF(CLASS) \
2249 case Metadata::CLASS##Kind: \
2251 write##CLASS(cast<CLASS>(N), Record, \
2252 (*MDAbbrevs)[MetadataAbbrev::CLASS##AbbrevID]); \
2254 write##CLASS(cast<CLASS>(N), Record, CLASS##Abbrev); \
2256 #include "llvm/IR/Metadata.def"
2259 writeValueAsMetadata(cast<ValueAsMetadata>(MD),
Record);
2263 void ModuleBitcodeWriter::writeModuleMetadata() {
2264 if (!VE.
hasMDs() &&
M.named_metadata_empty())
2272 std::vector<unsigned> MDAbbrevs;
2275 MDAbbrevs[MetadataAbbrev::DILocationAbbrevID] = createDILocationAbbrev();
2276 MDAbbrevs[MetadataAbbrev::GenericDINodeAbbrevID] =
2277 createGenericDINodeAbbrev();
2279 auto Abbv = std::make_shared<BitCodeAbbrev>();
2285 Abbv = std::make_shared<BitCodeAbbrev>();
2311 std::vector<uint64_t> IndexPos;
2326 uint64_t PreviousValue = IndexOffsetRecordBitPos;
2327 for (
auto &Elt : IndexPos) {
2328 auto EltDelta = Elt - PreviousValue;
2329 PreviousValue = Elt;
2338 writeNamedMetadata(
Record);
2340 auto AddDeclAttachedMetadata = [&](
const GlobalObject &GO) {
2343 pushGlobalMetadataAttachment(
Record, GO);
2347 if (
F.isDeclaration() &&
F.hasMetadata())
2348 AddDeclAttachedMetadata(
F);
2352 if (GV.hasMetadata())
2353 AddDeclAttachedMetadata(GV);
2358 void ModuleBitcodeWriter::writeFunctionMetadata(
const Function &
F) {
2369 void ModuleBitcodeWriter::pushGlobalMetadataAttachment(
2374 for (
const auto &
I : MDs) {
2380 void ModuleBitcodeWriter::writeFunctionMetadataAttachment(
const Function &
F) {
2385 if (
F.hasMetadata()) {
2386 pushGlobalMetadataAttachment(
Record,
F);
2397 I.getAllMetadataOtherThanDebugLoc(MDs);
2400 if (MDs.empty())
continue;
2404 for (
unsigned i = 0,
e = MDs.size();
i !=
e; ++
i) {
2405 Record.push_back(MDs[
i].first);
2415 void ModuleBitcodeWriter::writeModuleMetadataKinds() {
2421 M.getMDKindNames(Names);
2423 if (Names.empty())
return;
2427 for (
unsigned MDKindID = 0,
e = Names.size(); MDKindID !=
e; ++MDKindID) {
2428 Record.push_back(MDKindID);
2439 void ModuleBitcodeWriter::writeOperandBundleTags() {
2447 M.getOperandBundleTags(Tags);
2456 for (
auto Tag : Tags) {
2466 void ModuleBitcodeWriter::writeSyncScopeNames() {
2468 M.getContext().getSyncScopeNames(SSNs);
2475 for (
auto SSN : SSNs) {
2476 Record.append(SSN.begin(), SSN.end());
2484 void ModuleBitcodeWriter::writeConstants(
unsigned FirstVal,
unsigned LastVal,
2486 if (FirstVal == LastVal)
return;
2490 unsigned AggregateAbbrev = 0;
2491 unsigned String8Abbrev = 0;
2492 unsigned CString7Abbrev = 0;
2493 unsigned CString6Abbrev = 0;
2497 auto Abbv = std::make_shared<BitCodeAbbrev>();
2504 Abbv = std::make_shared<BitCodeAbbrev>();
2510 Abbv = std::make_shared<BitCodeAbbrev>();
2516 Abbv = std::make_shared<BitCodeAbbrev>();
2526 Type *LastTy =
nullptr;
2527 for (
unsigned i = FirstVal;
i != LastVal; ++
i) {
2528 const Value *V = Vals[
i].first;
2534 CONSTANTS_SETTYPE_ABBREV);
2538 if (
const InlineAsm *
IA = dyn_cast<InlineAsm>(V)) {
2541 unsigned(
IA->hasSideEffects()) |
unsigned(
IA->isAlignStack()) << 1 |
2542 unsigned(
IA->getDialect() & 1) << 2 |
unsigned(
IA->canThrow()) << 3);
2545 const std::string &AsmStr =
IA->getAsmString();
2546 Record.push_back(AsmStr.size());
2547 Record.append(AsmStr.begin(), AsmStr.end());
2550 const std::string &ConstraintStr =
IA->getConstraintString();
2551 Record.push_back(ConstraintStr.size());
2552 Record.append(ConstraintStr.begin(), ConstraintStr.end());
2558 unsigned Code = -1U;
2559 unsigned AbbrevToUse = 0;
2560 if (
C->isNullValue()) {
2562 }
else if (isa<PoisonValue>(
C)) {
2564 }
else if (isa<UndefValue>(
C)) {
2567 if (
IV->getBitWidth() <= 64) {
2571 AbbrevToUse = CONSTANTS_INTEGER_ABBREV;
2576 }
else if (
const ConstantFP *CFP = dyn_cast<ConstantFP>(
C)) {
2578 Type *Ty = CFP->getType();
2581 Record.push_back(CFP->getValueAPF().bitcastToAPInt().getZExtValue());
2585 APInt api = CFP->getValueAPF().bitcastToAPInt();
2587 Record.push_back((
p[1] << 48) | (
p[0] >> 16));
2588 Record.push_back(
p[0] & 0xffffLL);
2590 APInt api = CFP->getValueAPF().bitcastToAPInt();
2595 assert(0 &&
"Unknown FP type!");
2597 }
else if (isa<ConstantDataSequential>(
C) &&
2598 cast<ConstantDataSequential>(
C)->isString()) {
2601 unsigned NumElts = Str->getNumElements();
2603 if (Str->isCString()) {
2608 AbbrevToUse = String8Abbrev;
2612 for (
unsigned i = 0;
i != NumElts; ++
i) {
2613 unsigned char V = Str->getElementAsInteger(
i);
2615 isCStr7 &= (V & 128) == 0;
2621 AbbrevToUse = CString6Abbrev;
2623 AbbrevToUse = CString7Abbrev;
2625 dyn_cast<ConstantDataSequential>(
C)) {
2627 Type *EltTy = CDS->getElementType();
2628 if (isa<IntegerType>(EltTy)) {
2629 for (
unsigned i = 0,
e = CDS->getNumElements();
i !=
e; ++
i)
2630 Record.push_back(CDS->getElementAsInteger(
i));
2632 for (
unsigned i = 0,
e = CDS->getNumElements();
i !=
e; ++
i)
2634 CDS->getElementAsAPFloat(
i).bitcastToAPInt().getLimitedValue());
2636 }
else if (isa<ConstantAggregate>(
C)) {
2638 for (
const Value *
Op :
C->operands())
2640 AbbrevToUse = AggregateAbbrev;
2641 }
else if (
const ConstantExpr *CE = dyn_cast<ConstantExpr>(
C)) {
2642 switch (
CE->getOpcode()) {
2649 AbbrevToUse = CONSTANTS_CE_CAST_Abbrev;
2651 assert(
CE->getNumOperands() == 2 &&
"Unknown constant expr!");
2661 case Instruction::FNeg: {
2662 assert(
CE->getNumOperands() == 1 &&
"Unknown constant expr!");
2671 case Instruction::GetElementPtr: {
2673 const auto *GO = cast<GEPOperator>(
C);
2675 if (std::optional<unsigned> Idx = GO->getInRangeIndex()) {
2677 Record.push_back((*Idx << 1) | GO->isInBounds());
2678 }
else if (GO->isInBounds())
2680 for (
unsigned i = 0,
e =
CE->getNumOperands();
i !=
e; ++
i) {
2692 case Instruction::ExtractElement:
2699 case Instruction::InsertElement:
2706 case Instruction::ShuffleVector:
2711 if (
C->getType() ==
C->getOperand(0)->getType()) {
2721 case Instruction::ICmp:
2722 case Instruction::FCmp:
2727 Record.push_back(
CE->getPredicate());
2730 }
else if (
const BlockAddress *BA = dyn_cast<BlockAddress>(
C)) {
2735 }
else if (
const auto *Equiv = dyn_cast<DSOLocalEquivalent>(
C)) {
2739 }
else if (
const auto *
NC = dyn_cast<NoCFIValue>(
C)) {
2756 void ModuleBitcodeWriter::writeModuleConstants() {
2761 for (
unsigned i = 0,
e = Vals.size();
i !=
e; ++
i) {
2762 if (!isa<GlobalValue>(Vals[
i].first)) {
2763 writeConstants(
i, Vals.size(),
true);
2777 bool ModuleBitcodeWriter::pushValueAndType(
const Value *V,
unsigned InstID,
2781 Vals.push_back(InstID -
ValID);
2782 if (
ValID >= InstID) {
2789 void ModuleBitcodeWriter::writeOperandBundles(
const CallBase &CS,
2796 Record.push_back(
C.getOperandBundleTagID(Bundle.getTagName()));
2798 for (
auto &Input : Bundle.Inputs)
2799 pushValueAndType(Input, InstID,
Record);
2808 void ModuleBitcodeWriter::pushValue(
const Value *V,
unsigned InstID,
2811 Vals.push_back(InstID -
ValID);
2814 void ModuleBitcodeWriter::pushValueSigned(
const Value *V,
unsigned InstID,
2817 int64_t diff = ((int32_t)InstID - (int32_t)
ValID);
2822 void ModuleBitcodeWriter::writeInstruction(
const Instruction &
I,
2826 unsigned AbbrevToUse = 0;
2828 switch (
I.getOpcode()) {
2832 if (!pushValueAndType(
I.getOperand(0), InstID, Vals))
2833 AbbrevToUse = FUNCTION_INST_CAST_ABBREV;
2837 assert(isa<BinaryOperator>(
I) &&
"Unknown instruction!");
2839 if (!pushValueAndType(
I.getOperand(0), InstID, Vals))
2840 AbbrevToUse = FUNCTION_INST_BINOP_ABBREV;
2841 pushValue(
I.getOperand(1), InstID, Vals);
2845 if (AbbrevToUse == FUNCTION_INST_BINOP_ABBREV)
2846 AbbrevToUse = FUNCTION_INST_BINOP_FLAGS_ABBREV;
2847 Vals.push_back(Flags);
2851 case Instruction::FNeg: {
2853 if (!pushValueAndType(
I.getOperand(0), InstID, Vals))
2854 AbbrevToUse = FUNCTION_INST_UNOP_ABBREV;
2858 if (AbbrevToUse == FUNCTION_INST_UNOP_ABBREV)
2859 AbbrevToUse = FUNCTION_INST_UNOP_FLAGS_ABBREV;
2860 Vals.push_back(Flags);
2864 case Instruction::GetElementPtr: {
2866 AbbrevToUse = FUNCTION_INST_GEP_ABBREV;
2867 auto &GEPInst = cast<GetElementPtrInst>(
I);
2868 Vals.push_back(GEPInst.isInBounds());
2869 Vals.push_back(VE.
getTypeID(GEPInst.getSourceElementType()));
2870 for (
unsigned i = 0,
e =
I.getNumOperands();
i !=
e; ++
i)
2871 pushValueAndType(
I.getOperand(
i), InstID, Vals);
2874 case Instruction::ExtractValue: {
2876 pushValueAndType(
I.getOperand(0), InstID, Vals);
2881 case Instruction::InsertValue: {
2883 pushValueAndType(
I.getOperand(0), InstID, Vals);
2884 pushValueAndType(
I.getOperand(1), InstID, Vals);
2891 pushValueAndType(
I.getOperand(1), InstID, Vals);
2892 pushValue(
I.getOperand(2), InstID, Vals);
2893 pushValueAndType(
I.getOperand(0), InstID, Vals);
2896 Vals.push_back(Flags);
2899 case Instruction::ExtractElement:
2901 pushValueAndType(
I.getOperand(0), InstID, Vals);
2902 pushValueAndType(
I.getOperand(1), InstID, Vals);
2904 case Instruction::InsertElement:
2906 pushValueAndType(
I.getOperand(0), InstID, Vals);
2907 pushValue(
I.getOperand(1), InstID, Vals);
2908 pushValueAndType(
I.getOperand(2), InstID, Vals);
2910 case Instruction::ShuffleVector:
2912 pushValueAndType(
I.getOperand(0), InstID, Vals);
2913 pushValue(
I.getOperand(1), InstID, Vals);
2914 pushValue(cast<ShuffleVectorInst>(
I).getShuffleMaskForBitcode(), InstID,
2917 case Instruction::ICmp:
2918 case Instruction::FCmp: {
2921 pushValueAndType(
I.getOperand(0), InstID, Vals);
2922 pushValue(
I.getOperand(1), InstID, Vals);
2926 Vals.push_back(Flags);
2933 unsigned NumOperands =
I.getNumOperands();
2934 if (NumOperands == 0)
2935 AbbrevToUse = FUNCTION_INST_RET_VOID_ABBREV;
2936 else if (NumOperands == 1) {
2937 if (!pushValueAndType(
I.getOperand(0), InstID, Vals))
2938 AbbrevToUse = FUNCTION_INST_RET_VAL_ABBREV;
2940 for (
unsigned i = 0,
e = NumOperands;
i !=
e; ++
i)
2941 pushValueAndType(
I.getOperand(
i), InstID, Vals);
2945 case Instruction::Br:
2956 case Instruction::Switch:
2960 Vals.push_back(VE.
getTypeID(
SI.getCondition()->getType()));
2961 pushValue(
SI.getCondition(), InstID, Vals);
2963 for (
auto Case :
SI.cases()) {
2964 Vals.push_back(VE.
getValueID(Case.getCaseValue()));
2965 Vals.push_back(VE.
getValueID(Case.getCaseSuccessor()));
2969 case Instruction::IndirectBr:
2971 Vals.push_back(VE.
getTypeID(
I.getOperand(0)->getType()));
2973 pushValue(
I.getOperand(0), InstID, Vals);
2974 for (
unsigned i = 1,
e =
I.getNumOperands();
i !=
e; ++
i)
2978 case Instruction::Invoke: {
2984 writeOperandBundles(*II, InstID);
2993 pushValueAndType(Callee, InstID, Vals);
2997 pushValue(
I.getOperand(
i), InstID, Vals);
3002 pushValueAndType(
I.getOperand(
i), InstID, Vals);
3006 case Instruction::Resume:
3008 pushValueAndType(
I.getOperand(0), InstID, Vals);
3010 case Instruction::CleanupRet: {
3012 const auto &CRI = cast<CleanupReturnInst>(
I);
3013 pushValue(CRI.getCleanupPad(), InstID, Vals);
3014 if (CRI.hasUnwindDest())
3015 Vals.push_back(VE.
getValueID(CRI.getUnwindDest()));
3018 case Instruction::CatchRet: {
3020 const auto &CRI = cast<CatchReturnInst>(
I);
3021 pushValue(CRI.getCatchPad(), InstID, Vals);
3022 Vals.push_back(VE.
getValueID(CRI.getSuccessor()));
3025 case Instruction::CleanupPad:
3026 case Instruction::CatchPad: {
3027 const auto &FuncletPad = cast<FuncletPadInst>(
I);
3030 pushValue(FuncletPad.getParentPad(), InstID, Vals);
3032 unsigned NumArgOperands = FuncletPad.arg_size();
3033 Vals.push_back(NumArgOperands);
3034 for (
unsigned Op = 0;
Op != NumArgOperands; ++
Op)
3035 pushValueAndType(FuncletPad.getArgOperand(
Op), InstID, Vals);
3038 case Instruction::CatchSwitch: {
3040 const auto &CatchSwitch = cast<CatchSwitchInst>(
I);
3042 pushValue(CatchSwitch.getParentPad(), InstID, Vals);
3044 unsigned NumHandlers = CatchSwitch.getNumHandlers();
3045 Vals.push_back(NumHandlers);
3046 for (
const BasicBlock *CatchPadBB : CatchSwitch.handlers())
3049 if (CatchSwitch.hasUnwindDest())
3050 Vals.push_back(VE.
getValueID(CatchSwitch.getUnwindDest()));
3053 case Instruction::CallBr: {
3059 writeOperandBundles(*CBI, InstID);
3074 pushValueAndType(Callee, InstID, Vals);
3078 pushValue(
I.getOperand(
i), InstID, Vals);
3083 pushValueAndType(
I.getOperand(
i), InstID, Vals);
3087 case Instruction::Unreachable:
3089 AbbrevToUse = FUNCTION_INST_UNREACHABLE_ABBREV;
3093 const PHINode &PN = cast<PHINode>(
I);
3107 Vals64.push_back(Flags);
3110 Stream.
EmitRecord(Code, Vals64, AbbrevToUse);
3115 case Instruction::LandingPad: {
3126 pushValueAndType(LP.
getClause(
I), InstID, Vals);
3131 case Instruction::Alloca: {
3135 Vals.push_back(VE.
getTypeID(
I.getOperand(0)->getType()));
3139 unsigned EncodedAlign = getEncodedAlign(AI.
getAlign());
3140 Bitfield::set<APV::AlignLower>(
3142 Bitfield::set<APV::AlignUpper>(
Record,
3145 Bitfield::set<APV::ExplicitType>(
Record,
true);
3150 if (AS !=
M.getDataLayout().getAllocaAddrSpace())
3156 if (cast<LoadInst>(
I).isAtomic()) {
3158 pushValueAndType(
I.getOperand(0), InstID, Vals);
3161 if (!pushValueAndType(
I.getOperand(0), InstID, Vals))
3162 AbbrevToUse = FUNCTION_INST_LOAD_ABBREV;
3165 Vals.push_back(getEncodedAlign(cast<LoadInst>(
I).
getAlign()));
3166 Vals.push_back(cast<LoadInst>(
I).isVolatile());
3167 if (cast<LoadInst>(
I).isAtomic()) {
3169 Vals.push_back(getEncodedSyncScopeID(cast<LoadInst>(
I).getSyncScopeID()));
3173 if (cast<StoreInst>(
I).isAtomic())
3177 pushValueAndType(
I.getOperand(1), InstID, Vals);
3178 pushValueAndType(
I.getOperand(0), InstID, Vals);
3179 Vals.push_back(getEncodedAlign(cast<StoreInst>(
I).
getAlign()));
3180 Vals.push_back(cast<StoreInst>(
I).isVolatile());
3181 if (cast<StoreInst>(
I).isAtomic()) {
3184 getEncodedSyncScopeID(cast<StoreInst>(
I).getSyncScopeID()));
3187 case Instruction::AtomicCmpXchg:
3189 pushValueAndType(
I.getOperand(0), InstID, Vals);
3190 pushValueAndType(
I.getOperand(1), InstID, Vals);
3191 pushValue(
I.getOperand(2), InstID, Vals);
3192 Vals.push_back(cast<AtomicCmpXchgInst>(
I).isVolatile());
3196 getEncodedSyncScopeID(cast<AtomicCmpXchgInst>(
I).getSyncScopeID()));
3199 Vals.push_back(cast<AtomicCmpXchgInst>(
I).isWeak());
3200 Vals.push_back(getEncodedAlign(cast<AtomicCmpXchgInst>(
I).
getAlign()));
3202 case Instruction::AtomicRMW:
3204 pushValueAndType(
I.getOperand(0), InstID, Vals);
3205 pushValueAndType(
I.getOperand(1), InstID, Vals);
3208 Vals.push_back(cast<AtomicRMWInst>(
I).isVolatile());
3211 getEncodedSyncScopeID(cast<AtomicRMWInst>(
I).getSyncScopeID()));
3212 Vals.push_back(getEncodedAlign(cast<AtomicRMWInst>(
I).
getAlign()));
3214 case Instruction::Fence:
3217 Vals.push_back(getEncodedSyncScopeID(cast<FenceInst>(
I).getSyncScopeID()));
3224 writeOperandBundles(CI, InstID);
3238 Vals.push_back(Flags);
3259 case Instruction::VAArg:
3261 Vals.push_back(VE.
getTypeID(
I.getOperand(0)->getType()));
3262 pushValue(
I.getOperand(0), InstID, Vals);
3265 case Instruction::Freeze:
3267 pushValueAndType(
I.getOperand(0), InstID, Vals);
3277 void ModuleBitcodeWriter::writeGlobalValueSymbolTable(
3283 VSTOffset -= bitcodeStartBit();
3284 assert((VSTOffset & 31) == 0 &&
"VST block not 32-bit aligned");
3288 Stream.
BackpatchWord(VSTOffsetPlaceholder, VSTOffset / 32 + 1);
3292 auto Abbv = std::make_shared<BitCodeAbbrev>();
3301 if (
F.isDeclaration())
3308 uint64_t BitcodeIndex = FunctionToBitcodeIndex[&
F] - bitcodeStartBit();
3309 assert((BitcodeIndex & 31) == 0 &&
"function block not 32-bit aligned");
3313 Record[1] = BitcodeIndex / 32 + 1;
3322 void ModuleBitcodeWriter::writeFunctionLevelValueSymbolTable(
3337 unsigned AbbrevToUse = VST_ENTRY_8_ABBREV;
3343 if (isa<BasicBlock>(
Name.getValue())) {
3346 AbbrevToUse = VST_BBENTRY_6_ABBREV;
3350 AbbrevToUse = VST_ENTRY_6_ABBREV;
3352 AbbrevToUse = VST_ENTRY_7_ABBREV;
3355 for (
const auto P :
Name.getKey())
3356 NameVals.push_back((
unsigned char)
P);
3359 Stream.
EmitRecord(Code, NameVals, AbbrevToUse);
3366 void ModuleBitcodeWriter::writeUseList(
UseListOrder &&Order) {
3367 assert(Order.Shuffle.size() >= 2 &&
"Shuffle too small");
3369 if (isa<BasicBlock>(Order.V))
3379 void ModuleBitcodeWriter::writeUseListBlock(
const Function *
F) {
3381 "Expected to be preserving use-list order");
3383 auto hasMore = [&]() {
3399 void ModuleBitcodeWriter::writeFunction(
3418 unsigned CstStart, CstEnd;
3420 writeConstants(CstStart, CstEnd,
false);
3423 writeFunctionMetadata(
F);
3426 unsigned InstID = CstEnd;
3428 bool NeedsMetadataAttachment =
F.hasMetadata();
3436 writeInstruction(
I, InstID, Vals);
3438 if (!
I.getType()->isVoidTy())
3442 NeedsMetadataAttachment |=
I.hasMetadataOtherThanDebugLoc();
3455 Vals.push_back(
DL->getLine());
3456 Vals.push_back(
DL->getColumn());
3459 Vals.push_back(
DL->isImplicitCode());
3469 while (!Worklist.empty()) {
3470 Value *V = Worklist.pop_back_val();
3472 if (
auto *
I = dyn_cast<Instruction>(U)) {
3476 }
else if (isa<Constant>(U) && !isa<GlobalValue>(U) &&
3477 Visited.insert(U).second)
3478 Worklist.push_back(U);
3484 if (!BlockAddressUsers.
empty()) {
3493 if (
auto *Symtab =
F.getValueSymbolTable())
3494 writeFunctionLevelValueSymbolTable(*Symtab);
3496 if (NeedsMetadataAttachment)
3497 writeFunctionMetadataAttachment(
F);
3499 writeUseListBlock(&
F);
3505 void ModuleBitcodeWriter::writeBlockInfo() {
3512 auto Abbv = std::make_shared<BitCodeAbbrev>();
3523 auto Abbv = std::make_shared<BitCodeAbbrev>();
3533 auto Abbv = std::make_shared<BitCodeAbbrev>();
3543 auto Abbv = std::make_shared<BitCodeAbbrev>();
3549 VST_BBENTRY_6_ABBREV)
3554 auto Abbv = std::make_shared<BitCodeAbbrev>();
3559 CONSTANTS_SETTYPE_ABBREV)
3564 auto Abbv = std::make_shared<BitCodeAbbrev>();
3568 CONSTANTS_INTEGER_ABBREV)
3573 auto Abbv = std::make_shared<BitCodeAbbrev>();
3581 CONSTANTS_CE_CAST_Abbrev)
3585 auto Abbv = std::make_shared<BitCodeAbbrev>();
3588 CONSTANTS_NULL_Abbrev)
3595 auto Abbv = std::make_shared<BitCodeAbbrev>();
3603 FUNCTION_INST_LOAD_ABBREV)
3607 auto Abbv = std::make_shared<BitCodeAbbrev>();
3612 FUNCTION_INST_UNOP_ABBREV)
3616 auto Abbv = std::make_shared<BitCodeAbbrev>();
3622 FUNCTION_INST_UNOP_FLAGS_ABBREV)
3626 auto Abbv = std::make_shared<BitCodeAbbrev>();
3632 FUNCTION_INST_BINOP_ABBREV)
3636 auto Abbv = std::make_shared<BitCodeAbbrev>();
3643 FUNCTION_INST_BINOP_FLAGS_ABBREV)
3647 auto Abbv = std::make_shared<BitCodeAbbrev>();
3654 FUNCTION_INST_CAST_ABBREV)
3659 auto Abbv = std::make_shared<BitCodeAbbrev>();
3662 FUNCTION_INST_RET_VOID_ABBREV)
3666 auto Abbv = std::make_shared<BitCodeAbbrev>();
3670 FUNCTION_INST_RET_VAL_ABBREV)
3674 auto Abbv = std::make_shared<BitCodeAbbrev>();
3677 FUNCTION_INST_UNREACHABLE_ABBREV)
3681 auto Abbv = std::make_shared<BitCodeAbbrev>();
3689 FUNCTION_INST_GEP_ABBREV)
3698 void IndexBitcodeWriter::writeModStrings() {
3704 auto Abbv = std::make_shared<BitCodeAbbrev>();
3712 Abbv = std::make_shared<BitCodeAbbrev>();
3720 Abbv = std::make_shared<BitCodeAbbrev>();
3728 Abbv = std::make_shared<BitCodeAbbrev>();
3739 [&](
const StringMapEntry<std::pair<uint64_t, ModuleHash>> &MPSE) {
3741 const auto &
Value = MPSE.getValue();
3743 unsigned AbbrevToUse = Abbrev8Bit;
3745 AbbrevToUse = Abbrev6Bit;
3747 AbbrevToUse = Abbrev7Bit;
3749 Vals.push_back(
Value.first);
3756 const auto &Hash =
Value.second;
3758 Vals.
assign(Hash.begin(), Hash.end());
3770 template <
typename Fn>
3774 if (!
FS->type_tests().empty())
3779 auto WriteVFuncIdVec = [&](
uint64_t Ty,
3784 for (
auto &VF : VFs) {
3785 Record.push_back(VF.GUID);
3786 Record.push_back(VF.Offset);
3792 FS->type_test_assume_vcalls());
3794 FS->type_checked_load_vcalls());
3796 auto WriteConstVCallVec = [&](
uint64_t Ty,
3798 for (
auto &
VC : VCs) {
3808 FS->type_test_assume_const_vcalls());
3810 FS->type_checked_load_const_vcalls());
3814 assert(Range.getLower().getNumWords() == 1);
3815 assert(Range.getUpper().getNumWords() == 1);
3820 if (!
FS->paramAccesses().empty()) {
3822 for (
auto &
Arg :
FS->paramAccesses()) {
3823 size_t UndoSize =
Record.size();
3825 WriteRange(
Arg.Use);
3827 for (
auto &Call :
Arg.Calls) {
3828 Record.push_back(Call.ParamNo);
3829 std::optional<unsigned> ValueID = GetValueID(Call.Callee);
3836 Record.push_back(*ValueID);
3837 WriteRange(Call.Offsets);
3848 std::set<GlobalValue::GUID> &ReferencedTypeIds) {
3849 if (!
FS->type_tests().empty())
3850 for (
auto &TT :
FS->type_tests())
3851 ReferencedTypeIds.insert(TT);
3853 auto GetReferencedTypesFromVFuncIdVec =
3855 for (
auto &VF : VFs)
3856 ReferencedTypeIds.insert(VF.GUID);
3859 GetReferencedTypesFromVFuncIdVec(
FS->type_test_assume_vcalls());
3860 GetReferencedTypesFromVFuncIdVec(
FS->type_checked_load_vcalls());
3862 auto GetReferencedTypesFromConstVCallVec =
3864 for (
auto &
VC : VCs)
3865 ReferencedTypeIds.insert(
VC.VFunc.GUID);
3868 GetReferencedTypesFromConstVCallVec(
FS->type_test_assume_const_vcalls());
3869 GetReferencedTypesFromConstVCallVec(
FS->type_checked_load_const_vcalls());
3875 NameVals.push_back(
args.size());
3878 NameVals.push_back(ByArg.
TheKind);
3879 NameVals.push_back(ByArg.
Info);
3880 NameVals.push_back(ByArg.
Byte);
3881 NameVals.push_back(ByArg.
Bit);
3887 NameVals.push_back(
Id);
3889 NameVals.push_back(Wpd.
TheKind);
3893 NameVals.push_back(Wpd.
ResByArg.size());
3900 const std::string &
Id,
3902 NameVals.push_back(StrtabBuilder.
add(
Id));
3903 NameVals.push_back(
Id.size());
3905 NameVals.push_back(Summary.TTRes.TheKind);
3906 NameVals.push_back(Summary.TTRes.SizeM1BitWidth);
3907 NameVals.push_back(Summary.TTRes.AlignLog2);
3908 NameVals.push_back(Summary.TTRes.SizeM1);
3909 NameVals.push_back(Summary.TTRes.BitMask);
3910 NameVals.push_back(Summary.TTRes.InlineBits);
3912 for (
auto &
W : Summary.WPDRes)
3921 NameVals.push_back(StrtabBuilder.
add(
Id));
3922 NameVals.push_back(
Id.size());
3924 for (
auto &
P : Summary) {
3925 NameVals.push_back(
P.AddressPointOffset);
3926 NameVals.push_back(VE.
getValueID(
P.VTableVI.getValue()));
3932 unsigned AllocAbbrev,
bool PerModule,
3937 for (
auto &CI :
FS->callsites()) {
3941 assert(!PerModule || (CI.Clones.size() == 1 && CI.Clones[0] == 0));
3942 Record.push_back(GetValueID(CI.Callee));
3944 Record.push_back(CI.StackIdIndices.size());
3945 Record.push_back(CI.Clones.size());
3947 for (
auto Id : CI.StackIdIndices)
3948 Record.push_back(GetStackIndex(
Id));
3950 for (
auto V : CI.Clones)
3958 for (
auto &AI :
FS->allocs()) {
3962 assert(!PerModule || (AI.Versions.size() == 1 && AI.Versions[0] == 0));
3964 Record.push_back(AI.MIBs.size());
3965 Record.push_back(AI.Versions.size());
3967 for (
auto &MIB : AI.MIBs) {
3968 Record.push_back((uint8_t)MIB.AllocType);
3969 Record.push_back(MIB.StackIdIndices.size());
3970 for (
auto Id : MIB.StackIdIndices)
3971 Record.push_back(GetStackIndex(
Id));
3974 for (
auto V : AI.Versions)
3984 void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
3986 unsigned ValueID,
unsigned FSCallsAbbrev,
unsigned FSCallsProfileAbbrev,
3987 unsigned CallsiteAbbrev,
unsigned AllocAbbrev,
const Function &
F) {
3988 NameVals.push_back(ValueID);
3993 Stream,
FS, [&](
const ValueInfo &
VI) -> std::optional<unsigned> {
3998 Stream,
FS, CallsiteAbbrev, AllocAbbrev,
4001 [&](
unsigned I) {
return I; });
4003 auto SpecialRefCnts =
FS->specialRefCounts();
4005 NameVals.push_back(
FS->instCount());
4007 NameVals.push_back(
FS->refs().size());
4008 NameVals.push_back(SpecialRefCnts.first);
4009 NameVals.push_back(SpecialRefCnts.second);
4011 for (
auto &RI :
FS->refs())
4012 NameVals.push_back(VE.
getValueID(RI.getValue()));
4014 bool HasProfileData =
4016 for (
auto &ECI :
FS->calls()) {
4017 NameVals.push_back(getValueId(ECI.first));
4019 NameVals.push_back(
static_cast<uint8_t
>(ECI.second.Hotness));
4021 NameVals.push_back(ECI.second.RelBlockFreq);
4024 unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev);
4037 void ModuleBitcodeWriterBase::writeModuleLevelReferences(
4039 unsigned FSModRefsAbbrev,
unsigned FSModVTableRefsAbbrev) {
4041 if (!
VI ||
VI.getSummaryList().empty()) {
4047 auto *
Summary =
VI.getSummaryList()[0].get();
4053 auto VTableFuncs =
VS->vTableFuncs();
4054 if (!VTableFuncs.empty())
4055 NameVals.push_back(
VS->refs().size());
4057 unsigned SizeBeforeRefs = NameVals.size();
4058 for (
auto &RI :
VS->refs())
4059 NameVals.push_back(VE.
getValueID(RI.getValue()));
4064 if (VTableFuncs.empty())
4069 for (
auto &
P : VTableFuncs) {
4070 NameVals.push_back(VE.
getValueID(
P.FuncVI.getValue()));
4071 NameVals.push_back(
P.VTableOffset);
4075 FSModVTableRefsAbbrev);
4082 void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
4085 bool IsThinLTO =
true;
4087 mdconst::extract_or_null<ConstantInt>(
M.getModuleFlag(
"ThinLTO")))
4088 IsThinLTO = MD->getZExtValue();
4100 if (
Index->enableSplitLTOUnit())
4109 for (
const auto &GVI : valueIds()) {
4114 if (!
Index->stackIds().empty()) {
4115 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4125 auto Abbv = std::make_shared<BitCodeAbbrev>();
4140 Abbv = std::make_shared<BitCodeAbbrev>();
4158 Abbv = std::make_shared<BitCodeAbbrev>();
4167 Abbv = std::make_shared<BitCodeAbbrev>();
4178 Abbv = std::make_shared<BitCodeAbbrev>();
4186 Abbv = std::make_shared<BitCodeAbbrev>();
4195 Abbv = std::make_shared<BitCodeAbbrev>();
4203 Abbv = std::make_shared<BitCodeAbbrev>();
4220 if (!
VI ||
VI.getSummaryList().empty()) {
4226 auto *
Summary =
VI.getSummaryList()[0].get();
4227 writePerModuleFunctionSummaryRecord(NameVals, Summary, VE.
getValueID(&
F),
4228 FSCallsAbbrev, FSCallsProfileAbbrev,
4229 CallsiteAbbrev, AllocAbbrev,
F);
4235 writeModuleLevelReferences(
G, NameVals, FSModRefsAbbrev,
4236 FSModVTableRefsAbbrev);
4239 auto *Aliasee =
A.getAliaseeObject();
4242 if (!Aliasee->hasName() || isa<GlobalIFunc>(Aliasee))
4246 NameVals.push_back(AliasId);
4250 NameVals.push_back(AliaseeId);
4255 for (
auto &
S :
Index->typeIdCompatibleVtableMap()) {
4259 TypeIdCompatibleVtableAbbrev);
4270 void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
4279 for (
const auto &GVI : valueIds()) {
4284 if (!StackIdIndices.empty()) {
4285 auto StackIdAbbv = std::make_shared<BitCodeAbbrev>();
4293 std::vector<uint64_t> StackIds;
4294 for (
auto &
I : StackIdIndices)
4295 StackIds.push_back(
Index.getStackIdAtIndex(
I));
4300 auto Abbv = std::make_shared<BitCodeAbbrev>();
4317 Abbv = std::make_shared<BitCodeAbbrev>();
4334 Abbv = std::make_shared<BitCodeAbbrev>();
4344 Abbv = std::make_shared<BitCodeAbbrev>();
4352 Abbv = std::make_shared<BitCodeAbbrev>();
4362 Abbv = std::make_shared<BitCodeAbbrev>();
4383 std::set<GlobalValue::GUID> ReferencedTypeIds;
4397 NameVals.push_back(
S.getOriginalName());
4402 std::set<GlobalValue::GUID> DefOrUseGUIDs;
4403 forEachSummary([&](GVInfo
I,
bool IsAliasee) {
4406 DefOrUseGUIDs.insert(
I.first);
4408 DefOrUseGUIDs.insert(
VI.getGUID());
4410 auto ValueId = getValueId(
I.first);
4412 SummaryToValueIdMap[
S] = *ValueId;
4420 if (
auto *AS = dyn_cast<AliasSummary>(
S)) {
4423 Aliases.push_back(AS);
4427 if (
auto *
VS = dyn_cast<GlobalVarSummary>(
S)) {
4428 NameVals.push_back(*ValueId);
4429 NameVals.push_back(
Index.getModuleId(
VS->modulePath()));
4432 for (
auto &RI :
VS->refs()) {
4433 auto RefValueId = getValueId(RI.getGUID());
4436 NameVals.push_back(*RefValueId);
4443 MaybeEmitOriginalName(*
S);
4447 auto GetValueId = [&](
const ValueInfo &
VI) -> std::optional<unsigned> {
4449 return std::nullopt;
4450 return getValueId(
VI.getGUID());
4453 auto *
FS = cast<FunctionSummary>(
S);
4458 Stream,
FS, CallsiteAbbrev, AllocAbbrev,
4461 std::optional<unsigned> ValueID = GetValueId(
VI);
4475 return std::distance(StackIdIndices.begin(), Lower);
4478 NameVals.push_back(*ValueId);
4479 NameVals.push_back(
Index.getModuleId(
FS->modulePath()));
4481 NameVals.push_back(
FS->instCount());
4483 NameVals.push_back(
FS->entryCount());
4486 NameVals.push_back(0);
4487 NameVals.push_back(0);
4488 NameVals.push_back(0);
4490 unsigned Count = 0, RORefCnt = 0, WORefCnt = 0;
4491 for (
auto &RI :
FS->refs()) {
4492 auto RefValueId = getValueId(RI.getGUID());
4495 NameVals.push_back(*RefValueId);
4496 if (RI.isReadOnly())
4498 else if (RI.isWriteOnly())
4502 NameVals[6] = Count;
4503 NameVals[7] = RORefCnt;
4504 NameVals[8] = WORefCnt;
4506 bool HasProfileData =
false;
4507 for (
auto &EI :
FS->calls()) {
4514 for (
auto &EI :
FS->calls()) {
4517 std::optional<unsigned> CallValueId = GetValueId(EI.first);
4520 NameVals.push_back(*CallValueId);
4522 NameVals.push_back(
static_cast<uint8_t
>(EI.second.Hotness));
4525 unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev);
4532 MaybeEmitOriginalName(*
S);
4535 for (
auto *AS : Aliases) {
4536 auto AliasValueId = SummaryToValueIdMap[AS];
4538 NameVals.push_back(AliasValueId);
4541 auto AliaseeValueId = SummaryToValueIdMap[&AS->
getAliasee()];
4543 NameVals.push_back(AliaseeValueId);
4548 MaybeEmitOriginalName(*AS);
4550 if (
auto *
FS = dyn_cast<FunctionSummary>(&AS->
getAliasee()))
4554 if (!
Index.cfiFunctionDefs().empty()) {
4555 for (
auto &
S :
Index.cfiFunctionDefs()) {
4556 if (DefOrUseGUIDs.count(
4558 NameVals.push_back(StrtabBuilder.
add(
S));
4559 NameVals.push_back(
S.size());
4562 if (!NameVals.empty()) {
4568 if (!
Index.cfiFunctionDecls().empty()) {
4569 for (
auto &
S :
Index.cfiFunctionDecls()) {
4570 if (DefOrUseGUIDs.count(
4572 NameVals.push_back(StrtabBuilder.
add(
S));
4573 NameVals.push_back(
S.size());
4576 if (!NameVals.empty()) {
4584 for (
auto &
T : ReferencedTypeIds) {
4585 auto TidIter =
Index.typeIds().equal_range(
T);
4586 for (
auto It = TidIter.first; It != TidIter.second; ++It) {
4606 auto Abbv = std::make_shared<BitCodeAbbrev>();
4612 "LLVM" LLVM_VERSION_STRING, StringAbbrev);
4615 Abbv = std::make_shared<BitCodeAbbrev>();
4624 void ModuleBitcodeWriter::writeModuleHash(
size_t BlockStartPos) {
4630 Buffer.size() - BlockStartPos));
4631 std::array<uint8_t, 20> Hash = Hasher.result();
4632 for (
int Pos = 0; Pos < 20; Pos += 4) {
4649 size_t BlockStartPos = Buffer.size();
4651 writeModuleVersion();
4660 writeAttributeGroupTable();