21 #include "llvm/ADT/DenseMap.h" 22 #include "llvm/ADT/MapVector.h" 23 #include "llvm/ADT/SmallString.h" 24 #include "llvm/Bitstream/BitstreamReader.h" 25 #include "llvm/Bitstream/BitstreamWriter.h" 26 #include "llvm/Support/DJB.h" 27 #include "llvm/Support/FileSystem.h" 28 #include "llvm/Support/LockFileManager.h" 29 #include "llvm/Support/MemoryBuffer.h" 30 #include "llvm/Support/OnDiskHashTable.h" 31 #include "llvm/Support/Path.h" 32 #include "llvm/Support/TimeProfiler.h" 34 using namespace clang;
35 using namespace serialization;
43 GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
72 class IdentifierIndexReaderTrait {
74 typedef StringRef external_key_type;
75 typedef StringRef internal_key_type;
77 typedef unsigned hash_value_type;
78 typedef unsigned offset_type;
80 static bool EqualKey(
const internal_key_type&
a,
const internal_key_type&
b) {
84 static hash_value_type
ComputeHash(
const internal_key_type& a) {
85 return llvm::djbHash(a);
88 static std::pair<unsigned, unsigned>
89 ReadKeyDataLength(
const unsigned char*&
d) {
90 using namespace llvm::support;
91 unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(
d);
92 unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(
d);
93 return std::make_pair(KeyLen, DataLen);
96 static const internal_key_type&
97 GetInternalKey(
const external_key_type& x) {
return x; }
99 static const external_key_type&
100 GetExternalKey(
const internal_key_type& x) {
return x; }
102 static internal_key_type ReadKey(
const unsigned char* d,
unsigned n) {
103 return StringRef((
const char *)d, n);
106 static data_type ReadData(
const internal_key_type& k,
107 const unsigned char* d,
109 using namespace llvm::support;
112 while (DataLen > 0) {
113 unsigned ID = endian::readNext<uint32_t, little, unaligned>(
d);
114 Result.push_back(ID);
122 typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait>
123 IdentifierIndexTable;
127 GlobalModuleIndex::GlobalModuleIndex(std::unique_ptr<llvm::MemoryBuffer> Buffer,
128 llvm::BitstreamCursor
Cursor)
129 : Buffer(std::move(Buffer)), IdentifierIndex(), NumIdentifierLookups(),
130 NumIdentifierLookupHits() {
132 report_fatal_error(
"Module index '" + Buffer->getBufferIdentifier() +
133 "' failed: " +
toString(std::move(Err)));
136 llvm::TimeTraceScope TimeScope(
"Module LoadIndex", StringRef(
""));
138 bool InGlobalIndexBlock =
false;
141 llvm::BitstreamEntry Entry;
145 Fail(Res.takeError());
147 switch (Entry.Kind) {
151 case llvm::BitstreamEntry::EndBlock:
152 if (InGlobalIndexBlock) {
153 InGlobalIndexBlock =
false;
160 case llvm::BitstreamEntry::Record:
162 if (InGlobalIndexBlock)
167 case llvm::BitstreamEntry::SubBlock:
168 if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
170 Fail(std::move(Err));
171 InGlobalIndexBlock =
true;
173 Fail(std::move(Err));
180 Cursor.readRecord(Entry.ID, Record, &Blob);
181 if (!MaybeIndexRecord)
182 Fail(MaybeIndexRecord.takeError());
185 switch (IndexRecord) {
194 unsigned ID = Record[Idx++];
197 if (ID == Modules.size())
198 Modules.push_back(ModuleInfo());
200 Modules.resize(ID + 1);
204 Modules[
ID].Size = Record[Idx++];
205 Modules[
ID].ModTime = Record[Idx++];
208 unsigned NameLen = Record[Idx++];
209 Modules[
ID].FileName.assign(Record.begin() + Idx,
210 Record.begin() + Idx + NameLen);
214 unsigned NumDeps = Record[Idx++];
215 Modules[
ID].Dependencies.insert(Modules[ID].Dependencies.end(),
216 Record.begin() + Idx,
217 Record.begin() + Idx + NumDeps);
221 assert(Idx == Record.size() &&
"More module info?");
226 StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName);
228 ModuleName = ModuleName.rsplit(
'-').first;
229 UnresolvedModules[ModuleName] =
ID;
233 case IDENTIFIER_INDEX:
237 (
const unsigned char *)Blob.data() + Record[0],
238 (
const unsigned char *)Blob.data() +
sizeof(uint32_t),
239 (
const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
247 delete static_cast<IdentifierIndexTable *
>(IdentifierIndex);
250 std::pair<GlobalModuleIndex *, llvm::Error>
257 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
258 llvm::MemoryBuffer::getFile(IndexPath.c_str());
260 return std::make_pair(
nullptr,
261 llvm::errorCodeToError(BufferOrErr.getError()));
262 std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
265 llvm::BitstreamCursor
Cursor(*Buffer);
268 for (
unsigned char C : {
'B',
'C',
'G',
'I'}) {
271 return std::make_pair(
272 nullptr, llvm::createStringError(std::errc::illegal_byte_sequence,
273 "expected signature BCGI"));
275 return std::make_pair(
nullptr, Res.takeError());
279 llvm::Error::success());
285 for (
unsigned I = 0, N = Modules.size(); I != N; ++I) {
287 ModuleFiles.push_back(MF);
295 llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
296 = ModulesByFile.find(File);
297 if (Known == ModulesByFile.end())
301 Dependencies.clear();
303 for (
unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
305 Dependencies.push_back(MF);
313 if (!IdentifierIndex)
317 ++NumIdentifierLookups;
318 IdentifierIndexTable &Table
319 = *
static_cast<IdentifierIndexTable *
>(IdentifierIndex);
320 IdentifierIndexTable::iterator Known = Table.find(Name);
321 if (Known == Table.end()) {
326 for (
unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
327 if (
ModuleFile *MF = Modules[ModuleIDs[I]].File)
331 ++NumIdentifierLookupHits;
338 llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
339 if (Known == UnresolvedModules.end()) {
344 ModuleInfo &Info = Modules[Known->second];
352 ModulesByFile[File] = Known->second;
358 UnresolvedModules.erase(Known);
363 std::fprintf(stderr,
"*** Global Module Index Statistics:\n");
364 if (NumIdentifierLookups) {
365 fprintf(stderr,
" %u / %u identifier lookups succeeded (%f%%)\n",
366 NumIdentifierLookupHits, NumIdentifierLookups,
367 (
double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
369 std::fprintf(stderr,
"\n");
373 llvm::errs() <<
"*** Global Module Index Dump:\n";
374 llvm::errs() <<
"Module files:\n";
375 for (
auto &MI : Modules) {
376 llvm::errs() <<
"** " << MI.FileName <<
"\n";
380 llvm::errs() <<
"\n";
382 llvm::errs() <<
"\n";
401 struct ImportedModuleFileInfo {
403 time_t StoredModTime;
406 : StoredSize(Size), StoredModTime(ModTime), StoredSignature(Sig) {}
410 class GlobalModuleIndexBuilder {
415 typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
418 ModuleFilesMap ModuleFiles;
422 typedef std::multimap<const FileEntry *, ImportedModuleFileInfo>
423 ImportedModuleFilesMap;
426 ImportedModuleFilesMap ImportedModuleFiles;
430 typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
434 InterestingIdentifierMap InterestingIdentifiers;
437 void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
441 llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
442 = ModuleFiles.find(File);
443 if (Known != ModuleFiles.end())
444 return Known->second;
446 unsigned NewID = ModuleFiles.size();
453 explicit GlobalModuleIndexBuilder(
455 : FileMgr(FileMgr), PCHContainerRdr(PCHContainerRdr) {}
462 bool writeIndex(llvm::BitstreamWriter &Stream);
467 llvm::BitstreamWriter &Stream,
470 Record.push_back(ID);
471 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
474 if (!Name || Name[0] == 0)
return;
477 Record.push_back(*Name++);
478 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
482 llvm::BitstreamWriter &Stream,
485 Record.push_back(ID);
487 Record.push_back(*Name++);
488 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
492 GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
494 Stream.EnterBlockInfoBlock();
496 #define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record) 497 #define RECORD(X) emitRecordID(X, #X, Stream, Record) 498 BLOCK(GLOBAL_INDEX_BLOCK);
509 class InterestingASTIdentifierLookupTrait
514 typedef std::pair<StringRef, bool> data_type;
516 data_type ReadData(
const internal_key_type& k,
517 const unsigned char*
d,
521 using namespace llvm::support;
522 unsigned RawID = endian::readNext<uint32_t, little, unaligned>(
d);
523 bool IsInteresting = RawID & 0x01;
524 return std::make_pair(k, IsInteresting);
532 auto Buffer = FileMgr.getBufferForFile(File,
true);
534 return llvm::createStringError(Buffer.getError(),
535 "failed getting buffer for module file");
538 llvm::BitstreamCursor InStream(PCHContainerRdr.ExtractPCH(**Buffer));
541 for (
unsigned char C : {
'C',
'P',
'C',
'H'})
544 return llvm::createStringError(std::errc::illegal_byte_sequence,
545 "expected signature CPCH");
547 return Res.takeError();
551 unsigned ID = getModuleFileInfo(File).ID;
554 enum { Other, ControlBlock, ASTBlock, DiagnosticOptionsBlock }
State = Other;
559 return MaybeEntry.takeError();
560 llvm::BitstreamEntry Entry = MaybeEntry.get();
562 switch (Entry.Kind) {
567 case llvm::BitstreamEntry::Record:
569 if (State == Other) {
573 return Skipped.takeError();
579 case llvm::BitstreamEntry::SubBlock:
585 State = ControlBlock;
603 State = DiagnosticOptionsBlock;
612 case llvm::BitstreamEntry::EndBlock:
622 return MaybeCode.takeError();
623 unsigned Code = MaybeCode.get();
626 if (State == ControlBlock && Code ==
IMPORTS) {
628 unsigned Idx = 0, N = Record.size();
639 off_t StoredSize = (off_t)Record[Idx++];
640 time_t StoredModTime = (time_t)Record[Idx++];
645 {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
646 (uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
647 (uint32_t)Record[Idx++]}}};
651 Idx += Record[Idx] + 1;
654 unsigned Length = Record[Idx++];
656 Record.begin() + Idx + Length);
661 = FileMgr.getFile(ImportedFile,
false,
665 return llvm::createStringError(std::errc::bad_file_descriptor,
666 "imported file \"%s\" not found",
667 ImportedFile.c_str());
671 ImportedModuleFiles.insert(std::make_pair(
672 DependsOnFile, ImportedModuleFileInfo(StoredSize, StoredModTime,
676 unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
677 getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
685 typedef llvm::OnDiskIterableChainedHashTable<
686 InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
687 std::unique_ptr<InterestingIdentifierTable> Table(
689 (
const unsigned char *)Blob.data() + Record[0],
690 (
const unsigned char *)Blob.data() +
sizeof(uint32_t),
691 (
const unsigned char *)Blob.data()));
692 for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
693 DEnd = Table->data_end();
695 std::pair<StringRef, bool> Ident = *D;
697 InterestingIdentifiers[Ident.first].push_back(ID);
699 (
void)InterestingIdentifiers[Ident.first];
704 if (State == DiagnosticOptionsBlock && Code ==
SIGNATURE)
705 getModuleFileInfo(File).Signature = {
706 {{(uint32_t)Record[0], (uint32_t)Record[1], (uint32_t)Record[2],
707 (uint32_t)Record[3], (uint32_t)Record[4]}}};
712 return llvm::Error::success();
719 class IdentifierIndexWriterTrait {
721 typedef StringRef key_type;
722 typedef StringRef key_type_ref;
725 typedef unsigned hash_value_type;
726 typedef unsigned offset_type;
728 static hash_value_type
ComputeHash(key_type_ref Key) {
729 return llvm::djbHash(Key);
732 std::pair<unsigned,unsigned>
733 EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
734 using namespace llvm::support;
735 endian::Writer LE(Out, little);
736 unsigned KeyLen = Key.size();
737 unsigned DataLen = Data.size() * 4;
738 LE.write<uint16_t>(KeyLen);
739 LE.write<uint16_t>(DataLen);
740 return std::make_pair(KeyLen, DataLen);
743 void EmitKey(raw_ostream& Out, key_type_ref Key,
unsigned KeyLen) {
744 Out.write(Key.data(), KeyLen);
747 void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
749 using namespace llvm::support;
750 for (
unsigned I = 0, N = Data.size(); I != N; ++I)
751 endian::write<uint32_t>(Out, Data[I], little);
757 bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
758 for (
auto MapEntry : ImportedModuleFiles) {
759 auto *File = MapEntry.first;
760 ImportedModuleFileInfo &Info = MapEntry.second;
761 if (getModuleFileInfo(File).Signature) {
762 if (getModuleFileInfo(File).Signature != Info.StoredSignature)
765 }
else if (Info.StoredSize != File->
getSize() ||
771 using namespace llvm;
772 llvm::TimeTraceScope TimeScope(
"Module WriteIndex", StringRef(
""));
775 Stream.Emit((
unsigned)
'B', 8);
776 Stream.Emit((
unsigned)
'C', 8);
777 Stream.Emit((
unsigned)
'G', 8);
778 Stream.Emit((
unsigned)
'I', 8);
782 emitBlockInfoBlock(Stream);
784 Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
789 Stream.EmitRecord(INDEX_METADATA, Record);
792 for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
793 MEnd = ModuleFiles.end();
796 Record.push_back(M->second.ID);
797 Record.push_back(M->first->getSize());
798 Record.push_back(M->first->getModificationTime());
801 StringRef Name(M->first->getName());
802 Record.push_back(Name.size());
803 Record.append(Name.begin(), Name.end());
806 Record.push_back(M->second.Dependencies.size());
807 Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
808 Stream.EmitRecord(MODULE, Record);
813 llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
814 IdentifierIndexWriterTrait Trait;
817 for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
818 IEnd = InterestingIdentifiers.end();
820 Generator.insert(I->first(), I->second, Trait);
825 uint32_t BucketOffset;
827 using namespace llvm::support;
828 llvm::raw_svector_ostream Out(IdentifierTable);
830 endian::write<uint32_t>(Out, 0, little);
831 BucketOffset = Generator.Emit(Out, Trait);
835 auto Abbrev = std::make_shared<BitCodeAbbrev>();
836 Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
837 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
838 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
839 unsigned IDTableAbbrev = Stream.EmitAbbrev(std::move(Abbrev));
842 uint64_t Record[] = {IDENTIFIER_INDEX, BucketOffset};
843 Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable);
860 llvm::LockFileManager Locked(IndexPath);
862 case llvm::LockFileManager::LFS_Error:
863 return llvm::createStringError(std::errc::io_error,
"LFS error");
865 case llvm::LockFileManager::LFS_Owned:
869 case llvm::LockFileManager::LFS_Shared:
872 return llvm::createStringError(std::errc::device_or_resource_busy,
873 "someone else is building the index");
877 GlobalModuleIndexBuilder Builder(FileMgr, PCHContainerRdr);
881 for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
885 if (llvm::sys::path::extension(D->path()) !=
".pcm") {
889 if (llvm::sys::path::extension(D->path()) ==
".pcm.lock")
890 return llvm::createStringError(std::errc::device_or_resource_busy,
891 "someone else is building the index");
902 if (
llvm::Error Err = Builder.loadModuleFile(ModuleFile))
909 llvm::BitstreamWriter OutputStream(OutputBuffer);
910 if (Builder.writeIndex(OutputStream))
911 return llvm::createStringError(std::errc::io_error,
912 "failed writing index");
918 if (llvm::sys::fs::createUniqueFile(IndexPath +
"-%%%%%%%%", TmpFD,
920 return llvm::createStringError(std::errc::io_error,
921 "failed creating unique file");
924 llvm::raw_fd_ostream Out(TmpFD,
true);
926 return llvm::createStringError(Out.error(),
"failed outputting to stream");
929 Out.write(OutputBuffer.data(), OutputBuffer.size());
932 return llvm::createStringError(Out.error(),
"failed writing to stream");
938 if (std::error_code Err = llvm::sys::fs::rename(IndexTmpPath, IndexPath)) {
941 return llvm::createStringError(Err,
"failed renaming file \"%s\" to \"%s\"",
942 IndexTmpPath.c_str(), IndexPath.c_str());
945 return llvm::Error::success();
951 IdentifierIndexTable::key_iterator Current;
954 IdentifierIndexTable::key_iterator
End;
957 explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
958 Current = Idx.key_begin();
962 StringRef Next()
override {
966 StringRef Result = *Current;
974 IdentifierIndexTable &Table =
975 *
static_cast<IdentifierIndexTable *
>(IdentifierIndex);
976 return new GlobalIndexIdentifierIterator(Table);
Implements support for file system lookup, file system caching, and directory search management...
Defines the clang::FileManager interface and associated types.
time_t getModificationTime() const
Specialize PointerLikeTypeTraits to allow LazyGenerationalUpdatePtr to be placed into a PointerUnion...
std::string ModuleName
The name of the module.
static const unsigned CurrentVersion
The global index file version.
A block with unhashed content.
void printStats()
Print statistics to standard error.
static void emitRecordID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
static llvm::Error writeIndex(FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr, llvm::StringRef Path)
Write a global index into the given.
static std::string toString(const clang::SanitizerSet &Sanitizers)
Produce a string containing comma-separated names of sanitizers in Sanitizers set.
llvm::SmallPtrSet< ModuleFile *, 4 > HitSet
A set of module files in which we found a result.
The signature of a module, which is a hash of the AST content.
This abstract interface provides operations for unwrapping containers for serialized ASTs (precompile...
const FileEntry * getFile(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Lookup, cache, and verify the specified file (real or virtual).
bool lookupIdentifier(llvm::StringRef Name, HitSet &Hits)
Look for all of the module files with information about the given identifier, e.g., a global function, variable, or type with that name.
void dump()
Print debugging view to standard error.
static void emitBlockID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
IdentifierIterator * createIdentifierIterator() const
Returns an iterator for identifiers stored in the index table.
Implements an efficient mapping from strings to IdentifierInfo nodes.
Base class for the trait describing the on-disk hash table for the identifiers in an AST file...
Information about a module that has been loaded by the ASTReader.
An iterator that walks over all of the known identifiers in the lookup table.
static const char *const IndexFileName
The name of the global index file.
Record code for the identifier table.
IndexRecordTypes
Describes the record types in the index.
Cached information about one file (either on disk or in the virtual file system). ...
static std::pair< GlobalModuleIndex *, llvm::Error > readIndex(llvm::StringRef Path)
Read a global index file for the given directory.
void getKnownModules(llvm::SmallVectorImpl< ModuleFile *> &ModuleFiles)
Retrieve the set of modules that have up-to-date indexes.
The AST block, which acts as a container around the full AST block.
A global index for a set of module files, providing information about the identifiers within those mo...
unsigned ComputeHash(Selector Sel)
Record code for the signature that identifiers this AST file.
Record code for the list of other AST files imported by this AST file.
Dataflow Directional Tag Classes.
void getModuleDependencies(ModuleFile *File, llvm::SmallVectorImpl< ModuleFile *> &Dependencies)
Retrieve the set of module files on which the given module file directly depends. ...
Dump information about a module file.
The control block, which contains all of the information that needs to be validated prior to committi...
const FileEntry * File
The file entry for the module file.
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
bool loadedModuleFile(ModuleFile *File)
Note that the given module file has been loaded.