21 #include "llvm/ADT/DenseMap.h" 22 #include "llvm/ADT/MapVector.h" 23 #include "llvm/ADT/SmallString.h" 24 #include "llvm/Bitcode/BitstreamReader.h" 25 #include "llvm/Bitcode/BitstreamWriter.h" 26 #include "llvm/Support/DJB.h" 27 #include "llvm/Support/FileSystem.h" 28 #include "llvm/Support/LockFileManager.h" 29 #include "llvm/Support/MemoryBuffer.h" 30 #include "llvm/Support/OnDiskHashTable.h" 31 #include "llvm/Support/Path.h" 33 using namespace clang;
34 using namespace serialization;
42 GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
71 class IdentifierIndexReaderTrait {
73 typedef StringRef external_key_type;
74 typedef StringRef internal_key_type;
76 typedef unsigned hash_value_type;
77 typedef unsigned offset_type;
79 static bool EqualKey(
const internal_key_type& a,
const internal_key_type& b) {
83 static hash_value_type
ComputeHash(
const internal_key_type& a) {
84 return llvm::djbHash(a);
87 static std::pair<unsigned, unsigned>
88 ReadKeyDataLength(
const unsigned char*& d) {
89 using namespace llvm::support;
90 unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
91 unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
92 return std::make_pair(KeyLen, DataLen);
95 static const internal_key_type&
96 GetInternalKey(
const external_key_type& x) {
return x; }
98 static const external_key_type&
99 GetExternalKey(
const internal_key_type& x) {
return x; }
101 static internal_key_type ReadKey(
const unsigned char* d,
unsigned n) {
102 return StringRef((
const char *)d, n);
105 static data_type ReadData(
const internal_key_type& k,
106 const unsigned char* d,
108 using namespace llvm::support;
111 while (DataLen > 0) {
112 unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
113 Result.push_back(ID);
121 typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait>
122 IdentifierIndexTable;
126 GlobalModuleIndex::GlobalModuleIndex(std::unique_ptr<llvm::MemoryBuffer> Buffer,
127 llvm::BitstreamCursor
Cursor)
128 : Buffer(std::move(Buffer)), IdentifierIndex(), NumIdentifierLookups(),
129 NumIdentifierLookupHits() {
131 bool InGlobalIndexBlock =
false;
134 llvm::BitstreamEntry Entry =
Cursor.advance();
136 switch (Entry.Kind) {
140 case llvm::BitstreamEntry::EndBlock:
141 if (InGlobalIndexBlock) {
142 InGlobalIndexBlock =
false;
149 case llvm::BitstreamEntry::Record:
151 if (InGlobalIndexBlock)
156 case llvm::BitstreamEntry::SubBlock:
157 if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
158 if (
Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
161 InGlobalIndexBlock =
true;
162 }
else if (
Cursor.SkipBlock()) {
179 unsigned ID = Record[Idx++];
182 if (ID == Modules.size())
183 Modules.push_back(ModuleInfo());
185 Modules.resize(ID + 1);
189 Modules[
ID].Size = Record[Idx++];
190 Modules[
ID].ModTime = Record[Idx++];
193 unsigned NameLen = Record[Idx++];
194 Modules[
ID].FileName.assign(Record.begin() + Idx,
195 Record.begin() + Idx + NameLen);
199 unsigned NumDeps = Record[Idx++];
200 Modules[
ID].Dependencies.insert(Modules[ID].Dependencies.end(),
201 Record.begin() + Idx,
202 Record.begin() + Idx + NumDeps);
206 assert(Idx == Record.size() &&
"More module info?");
211 StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName);
213 ModuleName = ModuleName.rsplit(
'-').first;
214 UnresolvedModules[ModuleName] =
ID;
218 case IDENTIFIER_INDEX:
222 (
const unsigned char *)Blob.data() + Record[0],
223 (
const unsigned char *)Blob.data() +
sizeof(uint32_t),
224 (
const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
232 delete static_cast<IdentifierIndexTable *
>(IdentifierIndex);
235 std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode>
242 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
243 llvm::MemoryBuffer::getFile(IndexPath.c_str());
245 return std::make_pair(
nullptr, EC_NotFound);
246 std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
249 llvm::BitstreamCursor
Cursor(*Buffer);
252 if (Cursor.Read(8) !=
'B' ||
253 Cursor.Read(8) !=
'C' ||
254 Cursor.Read(8) !=
'G' ||
255 Cursor.Read(8) !=
'I') {
256 return std::make_pair(
nullptr, EC_IOError);
266 for (
unsigned I = 0, N = Modules.size(); I != N; ++I) {
268 ModuleFiles.push_back(MF);
276 llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
277 = ModulesByFile.find(File);
278 if (Known == ModulesByFile.end())
282 Dependencies.clear();
284 for (
unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
286 Dependencies.push_back(MF);
294 if (!IdentifierIndex)
298 ++NumIdentifierLookups;
299 IdentifierIndexTable &Table
300 = *
static_cast<IdentifierIndexTable *
>(IdentifierIndex);
301 IdentifierIndexTable::iterator Known = Table.find(Name);
302 if (Known == Table.end()) {
307 for (
unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
308 if (
ModuleFile *MF = Modules[ModuleIDs[I]].File)
312 ++NumIdentifierLookupHits;
319 llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
320 if (Known == UnresolvedModules.end()) {
325 ModuleInfo &Info = Modules[Known->second];
333 ModulesByFile[File] = Known->second;
339 UnresolvedModules.erase(Known);
344 std::fprintf(stderr,
"*** Global Module Index Statistics:\n");
345 if (NumIdentifierLookups) {
346 fprintf(stderr,
" %u / %u identifier lookups succeeded (%f%%)\n",
347 NumIdentifierLookupHits, NumIdentifierLookups,
348 (
double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
350 std::fprintf(stderr,
"\n");
354 llvm::errs() <<
"*** Global Module Index Dump:\n";
355 llvm::errs() <<
"Module files:\n";
356 for (
auto &MI : Modules) {
357 llvm::errs() <<
"** " << MI.FileName <<
"\n";
361 llvm::errs() <<
"\n";
363 llvm::errs() <<
"\n";
382 struct ImportedModuleFileInfo {
384 time_t StoredModTime;
387 : StoredSize(Size), StoredModTime(ModTime), StoredSignature(Sig) {}
391 class GlobalModuleIndexBuilder {
396 typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
399 ModuleFilesMap ModuleFiles;
403 typedef std::multimap<const FileEntry *, ImportedModuleFileInfo>
404 ImportedModuleFilesMap;
407 ImportedModuleFilesMap ImportedModuleFiles;
411 typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
415 InterestingIdentifierMap InterestingIdentifiers;
418 void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
422 llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
423 = ModuleFiles.find(File);
424 if (Known != ModuleFiles.end())
425 return Known->second;
427 unsigned NewID = ModuleFiles.size();
434 explicit GlobalModuleIndexBuilder(
436 : FileMgr(FileMgr), PCHContainerRdr(PCHContainerRdr) {}
441 bool loadModuleFile(
const FileEntry *File);
445 bool writeIndex(llvm::BitstreamWriter &Stream);
450 llvm::BitstreamWriter &Stream,
453 Record.push_back(ID);
454 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
457 if (!Name || Name[0] == 0)
return;
460 Record.push_back(*Name++);
461 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
465 llvm::BitstreamWriter &Stream,
468 Record.push_back(ID);
470 Record.push_back(*Name++);
471 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
475 GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
477 Stream.EnterBlockInfoBlock();
479 #define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record) 480 #define RECORD(X) emitRecordID(X, #X, Stream, Record) 481 BLOCK(GLOBAL_INDEX_BLOCK);
492 class InterestingASTIdentifierLookupTrait
497 typedef std::pair<StringRef, bool> data_type;
499 data_type ReadData(
const internal_key_type& k,
500 const unsigned char* d,
504 using namespace llvm::support;
505 unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
506 bool IsInteresting = RawID & 0x01;
507 return std::make_pair(k, IsInteresting);
512 bool GlobalModuleIndexBuilder::loadModuleFile(
const FileEntry *File) {
515 auto Buffer = FileMgr.getBufferForFile(File,
true);
521 llvm::BitstreamCursor InStream(PCHContainerRdr.ExtractPCH(**Buffer));
524 if (InStream.Read(8) !=
'C' ||
525 InStream.Read(8) !=
'P' ||
526 InStream.Read(8) !=
'C' ||
527 InStream.Read(8) !=
'H') {
533 unsigned ID = getModuleFileInfo(File).ID;
536 enum { Other, ControlBlock, ASTBlock, DiagnosticOptionsBlock }
State = Other;
539 llvm::BitstreamEntry Entry = InStream.advance();
540 switch (Entry.Kind) {
545 case llvm::BitstreamEntry::Record:
547 if (State == Other) {
548 InStream.skipRecord(Entry.ID);
555 case llvm::BitstreamEntry::SubBlock:
561 State = ControlBlock;
579 State = DiagnosticOptionsBlock;
583 if (InStream.SkipBlock())
588 case llvm::BitstreamEntry::EndBlock:
596 unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob);
599 if (State == ControlBlock && Code ==
IMPORTS) {
601 unsigned Idx = 0, N = Record.size();
612 off_t StoredSize = (off_t)Record[Idx++];
613 time_t StoredModTime = (time_t)Record[Idx++];
618 {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
619 (uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
620 (uint32_t)Record[Idx++]}}};
624 Idx += Record[Idx] + 1;
627 unsigned Length = Record[Idx++];
629 Record.begin() + Idx + Length);
634 = FileMgr.getFile(ImportedFile,
false,
642 ImportedModuleFiles.insert(std::make_pair(
643 DependsOnFile, ImportedModuleFileInfo(StoredSize, StoredModTime,
647 unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
648 getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
656 typedef llvm::OnDiskIterableChainedHashTable<
657 InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
658 std::unique_ptr<InterestingIdentifierTable> Table(
660 (
const unsigned char *)Blob.data() + Record[0],
661 (
const unsigned char *)Blob.data() +
sizeof(uint32_t),
662 (
const unsigned char *)Blob.data()));
663 for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
664 DEnd = Table->data_end();
666 std::pair<StringRef, bool> Ident = *D;
668 InterestingIdentifiers[Ident.first].push_back(ID);
670 (
void)InterestingIdentifiers[Ident.first];
675 if (State == DiagnosticOptionsBlock && Code ==
SIGNATURE)
676 getModuleFileInfo(File).Signature = {
677 {{(uint32_t)Record[0], (uint32_t)Record[1], (uint32_t)Record[2],
678 (uint32_t)Record[3], (uint32_t)Record[4]}}};
690 class IdentifierIndexWriterTrait {
692 typedef StringRef key_type;
693 typedef StringRef key_type_ref;
696 typedef unsigned hash_value_type;
697 typedef unsigned offset_type;
699 static hash_value_type
ComputeHash(key_type_ref Key) {
700 return llvm::djbHash(Key);
703 std::pair<unsigned,unsigned>
704 EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
705 using namespace llvm::support;
706 endian::Writer LE(Out, little);
707 unsigned KeyLen = Key.size();
708 unsigned DataLen = Data.size() * 4;
709 LE.write<uint16_t>(KeyLen);
710 LE.write<uint16_t>(DataLen);
711 return std::make_pair(KeyLen, DataLen);
714 void EmitKey(raw_ostream& Out, key_type_ref Key,
unsigned KeyLen) {
715 Out.write(Key.data(), KeyLen);
718 void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
720 using namespace llvm::support;
721 for (
unsigned I = 0, N = Data.size(); I != N; ++I)
722 endian::write<uint32_t>(Out, Data[I], little);
728 bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
729 for (
auto MapEntry : ImportedModuleFiles) {
730 auto *File = MapEntry.first;
731 ImportedModuleFileInfo &Info = MapEntry.second;
732 if (getModuleFileInfo(File).Signature) {
733 if (getModuleFileInfo(File).Signature != Info.StoredSignature)
736 }
else if (Info.StoredSize != File->
getSize() ||
742 using namespace llvm;
745 Stream.Emit((
unsigned)
'B', 8);
746 Stream.Emit((
unsigned)
'C', 8);
747 Stream.Emit((
unsigned)
'G', 8);
748 Stream.Emit((
unsigned)
'I', 8);
752 emitBlockInfoBlock(Stream);
754 Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
759 Stream.EmitRecord(INDEX_METADATA, Record);
762 for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
763 MEnd = ModuleFiles.end();
766 Record.push_back(M->second.ID);
767 Record.push_back(M->first->getSize());
768 Record.push_back(M->first->getModificationTime());
771 StringRef Name(M->first->getName());
772 Record.push_back(Name.size());
773 Record.append(Name.begin(), Name.end());
776 Record.push_back(M->second.Dependencies.size());
777 Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
778 Stream.EmitRecord(MODULE, Record);
783 llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
784 IdentifierIndexWriterTrait Trait;
787 for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
788 IEnd = InterestingIdentifiers.end();
790 Generator.insert(I->first(), I->second, Trait);
795 uint32_t BucketOffset;
797 using namespace llvm::support;
798 llvm::raw_svector_ostream Out(IdentifierTable);
800 endian::write<uint32_t>(Out, 0, little);
801 BucketOffset = Generator.Emit(Out, Trait);
805 auto Abbrev = std::make_shared<BitCodeAbbrev>();
806 Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
807 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
808 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
809 unsigned IDTableAbbrev = Stream.EmitAbbrev(std::move(Abbrev));
812 uint64_t Record[] = {IDENTIFIER_INDEX, BucketOffset};
813 Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable);
830 llvm::LockFileManager Locked(IndexPath);
832 case llvm::LockFileManager::LFS_Error:
835 case llvm::LockFileManager::LFS_Owned:
839 case llvm::LockFileManager::LFS_Shared:
846 GlobalModuleIndexBuilder Builder(FileMgr, PCHContainerRdr);
850 for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
854 if (llvm::sys::path::extension(D->path()) !=
".pcm") {
858 if (llvm::sys::path::extension(D->path()) ==
".pcm.lock")
870 if (Builder.loadModuleFile(ModuleFile))
877 llvm::BitstreamWriter OutputStream(OutputBuffer);
878 if (Builder.writeIndex(OutputStream))
885 if (llvm::sys::fs::createUniqueFile(IndexPath +
"-%%%%%%%%", TmpFD,
890 llvm::raw_fd_ostream Out(TmpFD,
true);
895 Out.write(OutputBuffer.data(), OutputBuffer.size());
901 llvm::sys::fs::remove(IndexPath);
904 if (llvm::sys::fs::rename(IndexTmpPath, IndexPath)) {
906 llvm::sys::fs::remove(IndexTmpPath);
917 IdentifierIndexTable::key_iterator Current;
920 IdentifierIndexTable::key_iterator
End;
923 explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
924 Current = Idx.key_begin();
928 StringRef Next()
override {
932 StringRef Result = *Current;
940 IdentifierIndexTable &Table =
941 *
static_cast<IdentifierIndexTable *
>(IdentifierIndex);
942 return new GlobalIndexIdentifierIterator(Table);
Implements support for file system lookup, file system caching, and directory search management...
Defines the clang::FileManager interface and associated types.
time_t getModificationTime() const
DominatorTree GraphTraits specialization so the DominatorTree can be iterable by generic graph iterat...
std::string ModuleName
The name of the module.
static const unsigned CurrentVersion
The global index file version.
A block with unhashed content.
void printStats()
Print statistics to standard error.
static void emitRecordID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
void getModuleDependencies(ModuleFile *File, SmallVectorImpl< ModuleFile *> &Dependencies)
Retrieve the set of module files on which the given module file directly depends. ...
llvm::SmallPtrSet< ModuleFile *, 4 > HitSet
A set of module files in which we found a result.
The signature of a module, which is a hash of the AST content.
ErrorCode
An error code returned when trying to read an index.
This abstract interface provides operations for unwrapping containers for serialized ASTs (precompile...
static ErrorCode writeIndex(FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr, StringRef Path)
Write a global index into the given.
const FileEntry * getFile(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Lookup, cache, and verify the specified file (real or virtual).
void getKnownModules(SmallVectorImpl< ModuleFile *> &ModuleFiles)
Retrieve the set of modules that have up-to-date indexes.
void dump()
Print debugging view to standard error.
static void emitBlockID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
IdentifierIterator * createIdentifierIterator() const
Returns an iterator for identifiers stored in the index table.
Implements an efficient mapping from strings to IdentifierInfo nodes.
Base class for the trait describing the on-disk hash table for the identifiers in an AST file...
Information about a module that has been loaded by the ASTReader.
An iterator that walks over all of the known identifiers in the lookup table.
static const char *const IndexFileName
The name of the global index file.
Record code for the identifier table.
IndexRecordTypes
Describes the record types in the index.
Cached information about one file (either on disk or in the virtual file system). ...
The AST block, which acts as a container around the full AST block.
A global index for a set of module files, providing information about the identifiers within those mo...
unsigned ComputeHash(Selector Sel)
Record code for the signature that identifiers this AST file.
Record code for the list of other AST files imported by this AST file.
Dataflow Directional Tag Classes.
static std::pair< GlobalModuleIndex *, ErrorCode > readIndex(StringRef Path)
Read a global index file for the given directory.
Dump information about a module file.
The control block, which contains all of the information that needs to be validated prior to committi...
const FileEntry * File
The file entry for the module file.
bool lookupIdentifier(StringRef Name, HitSet &Hits)
Look for all of the module files with information about the given identifier, e.g., a global function, variable, or type with that name.
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr *> VL, ArrayRef< Expr *> PL, ArrayRef< Expr *> IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
bool loadedModuleFile(ModuleFile *File)
Note that the given module file has been loaded.