21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/ADT/MapVector.h"
23 #include "llvm/ADT/SmallString.h"
24 #include "llvm/ADT/StringExtras.h"
25 #include "llvm/Bitcode/BitstreamReader.h"
26 #include "llvm/Bitcode/BitstreamWriter.h"
27 #include "llvm/Support/FileSystem.h"
28 #include "llvm/Support/LockFileManager.h"
29 #include "llvm/Support/MemoryBuffer.h"
30 #include "llvm/Support/OnDiskHashTable.h"
31 #include "llvm/Support/Path.h"
33 using namespace clang;
34 using namespace serialization;
42 GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
71 class IdentifierIndexReaderTrait {
73 typedef StringRef external_key_type;
74 typedef StringRef internal_key_type;
76 typedef unsigned hash_value_type;
77 typedef unsigned offset_type;
79 static bool EqualKey(
const internal_key_type& a,
const internal_key_type& b) {
83 static hash_value_type
ComputeHash(
const internal_key_type& a) {
84 return llvm::HashString(a);
87 static std::pair<unsigned, unsigned>
88 ReadKeyDataLength(
const unsigned char*& d) {
89 using namespace llvm::support;
90 unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
91 unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
92 return std::make_pair(KeyLen, DataLen);
95 static const internal_key_type&
96 GetInternalKey(
const external_key_type& x) {
return x; }
98 static const external_key_type&
99 GetExternalKey(
const internal_key_type& x) {
return x; }
101 static internal_key_type ReadKey(
const unsigned char* d,
unsigned n) {
102 return StringRef((
const char *)d, n);
105 static data_type ReadData(
const internal_key_type& k,
106 const unsigned char* d,
108 using namespace llvm::support;
111 while (DataLen > 0) {
112 unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
122 IdentifierIndexTable;
126 GlobalModuleIndex::GlobalModuleIndex(std::unique_ptr<llvm::MemoryBuffer>
Buffer,
127 llvm::BitstreamCursor
Cursor)
128 : Buffer(std::move(Buffer)), IdentifierIndex(), NumIdentifierLookups(),
129 NumIdentifierLookupHits() {
131 bool InGlobalIndexBlock =
false;
134 llvm::BitstreamEntry Entry = Cursor.advance();
136 switch (Entry.Kind) {
140 case llvm::BitstreamEntry::EndBlock:
141 if (InGlobalIndexBlock) {
142 InGlobalIndexBlock =
false;
149 case llvm::BitstreamEntry::Record:
151 if (InGlobalIndexBlock)
156 case llvm::BitstreamEntry::SubBlock:
157 if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
158 if (Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
161 InGlobalIndexBlock =
true;
162 }
else if (Cursor.SkipBlock()) {
179 unsigned ID = Record[Idx++];
182 if (ID == Modules.size())
183 Modules.push_back(ModuleInfo());
185 Modules.resize(ID + 1);
189 Modules[
ID].Size = Record[Idx++];
190 Modules[
ID].ModTime = Record[Idx++];
193 unsigned NameLen = Record[Idx++];
194 Modules[
ID].FileName.assign(Record.begin() + Idx,
195 Record.begin() + Idx + NameLen);
199 unsigned NumDeps = Record[Idx++];
200 Modules[
ID].Dependencies.insert(Modules[ID].Dependencies.end(),
201 Record.begin() + Idx,
202 Record.begin() + Idx + NumDeps);
206 assert(Idx == Record.size() &&
"More module info?");
211 StringRef ModuleName = llvm::sys::path::stem(Modules[ID].
FileName);
213 ModuleName = ModuleName.rsplit(
'-').first;
214 UnresolvedModules[ModuleName] =
ID;
218 case IDENTIFIER_INDEX:
222 (
const unsigned char *)Blob.data() + Record[0],
223 (
const unsigned char *)Blob.data() +
sizeof(uint32_t),
224 (
const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
231 GlobalModuleIndex::~GlobalModuleIndex() {
232 delete static_cast<IdentifierIndexTable *
>(IdentifierIndex);
235 std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode>
242 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
243 llvm::MemoryBuffer::getFile(IndexPath.c_str());
246 std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
249 llvm::BitstreamCursor
Cursor(*Buffer);
252 if (Cursor.Read(8) !=
'B' ||
253 Cursor.Read(8) !=
'C' ||
254 Cursor.Read(8) !=
'G' ||
255 Cursor.Read(8) !=
'I') {
266 for (
unsigned I = 0, N = Modules.size();
I != N; ++
I) {
267 if (ModuleFile *MF = Modules[
I].File)
268 ModuleFiles.push_back(MF);
276 llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
277 = ModulesByFile.find(File);
278 if (Known == ModulesByFile.end())
282 Dependencies.clear();
284 for (
unsigned I = 0, N = StoredDependencies.size();
I != N; ++
I) {
285 if (ModuleFile *MF = Modules[
I].File)
286 Dependencies.push_back(MF);
294 if (!IdentifierIndex)
298 ++NumIdentifierLookups;
299 IdentifierIndexTable &Table
300 = *
static_cast<IdentifierIndexTable *
>(IdentifierIndex);
301 IdentifierIndexTable::iterator Known = Table.find(Name);
302 if (Known == Table.end()) {
307 for (
unsigned I = 0, N = ModuleIDs.size();
I != N; ++
I) {
308 if (ModuleFile *MF = Modules[ModuleIDs[
I]].File)
312 ++NumIdentifierLookupHits;
318 StringRef
Name = File->ModuleName;
319 llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
320 if (Known == UnresolvedModules.end()) {
325 ModuleInfo &Info = Modules[Known->second];
330 if (File->File->getSize() == Info.Size &&
331 File->File->getModificationTime() == Info.ModTime) {
333 ModulesByFile[File] = Known->second;
339 UnresolvedModules.erase(Known);
344 std::fprintf(stderr,
"*** Global Module Index Statistics:\n");
345 if (NumIdentifierLookups) {
346 fprintf(stderr,
" %u / %u identifier lookups succeeded (%f%%)\n",
347 NumIdentifierLookupHits, NumIdentifierLookups,
348 (
double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
350 std::fprintf(stderr,
"\n");
354 llvm::errs() <<
"*** Global Module Index Dump:\n";
355 llvm::errs() <<
"Module files:\n";
356 for (
auto &MI : Modules) {
357 llvm::errs() <<
"** " << MI.FileName <<
"\n";
361 llvm::errs() <<
"\n";
363 llvm::errs() <<
"\n";
382 struct ImportedModuleFileInfo {
384 time_t StoredModTime;
387 : StoredSize(Size), StoredModTime(ModTime), StoredSignature(Sig) {}
391 class GlobalModuleIndexBuilder {
396 typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
399 ModuleFilesMap ModuleFiles;
403 typedef std::multimap<const FileEntry *, ImportedModuleFileInfo>
404 ImportedModuleFilesMap;
407 ImportedModuleFilesMap ImportedModuleFiles;
411 typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
415 InterestingIdentifierMap InterestingIdentifiers;
418 void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
422 llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
423 = ModuleFiles.find(File);
424 if (Known != ModuleFiles.end())
425 return Known->second;
427 unsigned NewID = ModuleFiles.size();
434 explicit GlobalModuleIndexBuilder(
436 : FileMgr(FileMgr), PCHContainerRdr(PCHContainerRdr) {}
441 bool loadModuleFile(
const FileEntry *File);
445 bool writeIndex(llvm::BitstreamWriter &Stream);
450 llvm::BitstreamWriter &Stream,
453 Record.push_back(ID);
454 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
457 if (!Name || Name[0] == 0)
return;
460 Record.push_back(*Name++);
461 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
465 llvm::BitstreamWriter &Stream,
468 Record.push_back(ID);
470 Record.push_back(*Name++);
471 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
475 GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
477 Stream.EnterBlockInfoBlock();
479 #define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
480 #define RECORD(X) emitRecordID(X, #X, Stream, Record)
481 BLOCK(GLOBAL_INDEX_BLOCK);
492 class InterestingASTIdentifierLookupTrait
497 typedef std::pair<StringRef, bool> data_type;
499 data_type ReadData(
const internal_key_type& k,
500 const unsigned char* d,
504 using namespace llvm::support;
505 unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
506 bool IsInteresting = RawID & 0x01;
507 return std::make_pair(k, IsInteresting);
512 bool GlobalModuleIndexBuilder::loadModuleFile(
const FileEntry *File) {
515 auto Buffer = FileMgr.getBufferForFile(File,
true);
521 llvm::BitstreamCursor InStream(PCHContainerRdr.ExtractPCH(**Buffer));
524 if (InStream.Read(8) !=
'C' ||
525 InStream.Read(8) !=
'P' ||
526 InStream.Read(8) !=
'C' ||
527 InStream.Read(8) !=
'H') {
533 unsigned ID = getModuleFileInfo(File).ID;
536 enum { Other, ControlBlock, ASTBlock, DiagnosticOptionsBlock }
State = Other;
539 llvm::BitstreamEntry Entry = InStream.advance();
540 switch (Entry.Kind) {
545 case llvm::BitstreamEntry::Record:
547 if (
State == Other) {
548 InStream.skipRecord(Entry.ID);
555 case llvm::BitstreamEntry::SubBlock:
561 State = ControlBlock;
579 State = DiagnosticOptionsBlock;
583 if (InStream.SkipBlock())
588 case llvm::BitstreamEntry::EndBlock:
596 unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob);
601 unsigned Idx = 0, N = Record.size();
612 off_t StoredSize = (off_t)Record[Idx++];
613 time_t StoredModTime = (time_t)Record[Idx++];
618 {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
619 (uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
620 (uint32_t)Record[Idx++]}}};
623 unsigned Length = Record[Idx++];
624 SmallString<128> ImportedFile(Record.begin() + Idx,
625 Record.begin() + Idx +
Length);
630 = FileMgr.getFile(ImportedFile,
false,
638 ImportedModuleFiles.insert(std::make_pair(
639 DependsOnFile, ImportedModuleFileInfo(StoredSize, StoredModTime,
643 unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
644 getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
653 InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
654 std::unique_ptr<InterestingIdentifierTable> Table(
656 (
const unsigned char *)Blob.data() + Record[0],
657 (
const unsigned char *)Blob.data() +
sizeof(uint32_t),
658 (
const unsigned char *)Blob.data()));
659 for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
660 DEnd = Table->data_end();
662 std::pair<StringRef, bool> Ident = *D;
664 InterestingIdentifiers[Ident.first].push_back(ID);
666 (
void)InterestingIdentifiers[Ident.first];
672 getModuleFileInfo(File).Signature = {
673 {{(uint32_t)Record[0], (uint32_t)Record[1], (uint32_t)Record[2],
674 (uint32_t)Record[3], (uint32_t)Record[4]}}};
686 class IdentifierIndexWriterTrait {
688 typedef StringRef key_type;
689 typedef StringRef key_type_ref;
692 typedef unsigned hash_value_type;
693 typedef unsigned offset_type;
695 static hash_value_type
ComputeHash(key_type_ref Key) {
696 return llvm::HashString(Key);
699 std::pair<unsigned,unsigned>
700 EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
701 using namespace llvm::support;
702 endian::Writer<little> LE(Out);
703 unsigned KeyLen = Key.size();
704 unsigned DataLen = Data.size() * 4;
705 LE.write<uint16_t>(KeyLen);
706 LE.write<uint16_t>(DataLen);
707 return std::make_pair(KeyLen, DataLen);
710 void EmitKey(raw_ostream& Out, key_type_ref Key,
unsigned KeyLen) {
711 Out.write(Key.data(), KeyLen);
714 void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
716 using namespace llvm::support;
717 for (
unsigned I = 0, N = Data.size();
I != N; ++
I)
718 endian::Writer<little>(Out).write<uint32_t>(Data[
I]);
724 bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
725 for (
auto MapEntry : ImportedModuleFiles) {
726 auto *File = MapEntry.first;
727 ImportedModuleFileInfo &Info = MapEntry.second;
728 if (getModuleFileInfo(File).Signature) {
729 if (getModuleFileInfo(File).Signature != Info.StoredSignature)
732 }
else if (Info.StoredSize != File->
getSize() ||
738 using namespace llvm;
741 Stream.Emit((
unsigned)
'B', 8);
742 Stream.Emit((
unsigned)
'C', 8);
743 Stream.Emit((
unsigned)
'G', 8);
744 Stream.Emit((
unsigned)
'I', 8);
748 emitBlockInfoBlock(Stream);
750 Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
755 Stream.EmitRecord(INDEX_METADATA, Record);
758 for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
759 MEnd = ModuleFiles.end();
762 Record.push_back(M->second.ID);
763 Record.push_back(M->first->getSize());
764 Record.push_back(M->first->getModificationTime());
767 StringRef
Name(M->first->getName());
768 Record.push_back(
Name.size());
769 Record.append(
Name.begin(),
Name.end());
772 Record.push_back(M->second.Dependencies.size());
773 Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
774 Stream.EmitRecord(MODULE, Record);
779 llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
780 IdentifierIndexWriterTrait Trait;
783 for (InterestingIdentifierMap::iterator
I = InterestingIdentifiers.begin(),
784 IEnd = InterestingIdentifiers.end();
786 Generator.insert(
I->first(),
I->second, Trait);
791 uint32_t BucketOffset;
793 using namespace llvm::support;
794 llvm::raw_svector_ostream Out(IdentifierTable);
796 endian::Writer<little>(Out).write<uint32_t>(0);
797 BucketOffset = Generator.Emit(Out, Trait);
801 auto Abbrev = std::make_shared<BitCodeAbbrev>();
802 Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
803 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
804 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
805 unsigned IDTableAbbrev = Stream.EmitAbbrev(std::move(Abbrev));
808 uint64_t Record[] = {IDENTIFIER_INDEX, BucketOffset};
809 Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable);
826 llvm::LockFileManager Locked(IndexPath);
828 case llvm::LockFileManager::LFS_Error:
831 case llvm::LockFileManager::LFS_Owned:
835 case llvm::LockFileManager::LFS_Shared:
842 GlobalModuleIndexBuilder
Builder(FileMgr, PCHContainerRdr);
846 for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
850 if (llvm::sys::path::extension(D->path()) !=
".pcm") {
854 if (llvm::sys::path::extension(D->path()) ==
".pcm.lock")
866 if (Builder.loadModuleFile(ModuleFile))
873 llvm::BitstreamWriter OutputStream(OutputBuffer);
874 if (Builder.writeIndex(OutputStream))
881 if (llvm::sys::fs::createUniqueFile(IndexPath +
"-%%%%%%%%", TmpFD,
886 llvm::raw_fd_ostream Out(TmpFD,
true);
891 Out.write(OutputBuffer.data(), OutputBuffer.size());
897 llvm::sys::fs::remove(IndexPath);
900 if (llvm::sys::fs::rename(IndexTmpPath, IndexPath)) {
902 llvm::sys::fs::remove(IndexTmpPath);
913 IdentifierIndexTable::key_iterator
Current;
916 IdentifierIndexTable::key_iterator
End;
919 explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
924 StringRef
Next()
override {
936 IdentifierIndexTable &Table =
937 *
static_cast<IdentifierIndexTable *
>(IdentifierIndex);
938 return new GlobalIndexIdentifierIterator(Table);
void getKnownModules(SmallVectorImpl< ModuleFile * > &ModuleFiles)
Retrieve the set of modules that have up-to-date indexes.
Implements support for file system lookup, file system caching, and directory search management...
Defines the clang::FileManager interface and associated types.
std::unique_ptr< llvm::MemoryBuffer > Buffer
void getModuleDependencies(ModuleFile *File, SmallVectorImpl< ModuleFile * > &Dependencies)
Retrieve the set of module files on which the given module file directly depends. ...
static const unsigned CurrentVersion
The global index file version.
A block with unhashed content.
return(__x >> __y)|(__x<< (32-__y))
void printStats()
Print statistics to standard error.
static void emitRecordID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
llvm::SmallPtrSet< ModuleFile *, 4 > HitSet
A set of module files in which we found a result.
The signature of a module, which is a hash of the AST content.
IdentifierIterator * createIdentifierIterator() const
Returns an iterator for identifiers stored in the index table.
ErrorCode
An error code returned when trying to read an index.
This abstract interface provides operations for unwrapping containers for serialized ASTs (precompile...
detail::InMemoryDirectory::const_iterator I
static ErrorCode writeIndex(FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr, StringRef Path)
Write a global index into the given.
const FileEntry * getFile(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Lookup, cache, and verify the specified file (real or virtual).
void dump()
Print debugging view to standard error.
static void emitBlockID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
Implements an efficient mapping from strings to IdentifierInfo nodes.
Base class for the trait describing the on-disk hash table for the identifiers in an AST file...
An iterator that walks over all of the known identifiers in the lookup table.
static const char *const IndexFileName
The name of the global index file.
The result type of a method or function.
Record code for the identifier table.
IndexRecordTypes
Describes the record types in the index.
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr * > VL, ArrayRef< Expr * > PL, ArrayRef< Expr * > IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
Cached information about one file (either on disk or in the virtual file system). ...
The AST block, which acts as a container around the full AST block.
A global index for a set of module files, providing information about the identifiers within those mo...
unsigned ComputeHash(Selector Sel)
Record code for the signature that identifiers this AST file.
if(T->getSizeExpr()) TRY_TO(TraverseStmt(T-> getSizeExpr()))
Record code for the list of other AST files imported by this AST file.
There was an unspecified I/O error reading or writing the index.
static std::pair< GlobalModuleIndex *, ErrorCode > readIndex(StringRef Path)
Read a global index file for the given directory.
time_t getModificationTime() const
BoundNodesTreeBuilder *const Builder
Dump information about a module file.
The control block, which contains all of the information that needs to be validated prior to committi...
bool lookupIdentifier(StringRef Name, HitSet &Hits)
Look for all of the module files with information about the given identifier, e.g., a global function, variable, or type with that name.
bool loadedModuleFile(ModuleFile *File)
Note that the given module file has been loaded.