clang  5.0.0
GlobalModuleIndex.cpp
Go to the documentation of this file.
1 //===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the GlobalModuleIndex class.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "ASTReaderInternals.h"
17 #include "clang/Lex/HeaderSearch.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/ADT/MapVector.h"
23 #include "llvm/ADT/SmallString.h"
24 #include "llvm/ADT/StringExtras.h"
25 #include "llvm/Bitcode/BitstreamReader.h"
26 #include "llvm/Bitcode/BitstreamWriter.h"
27 #include "llvm/Support/FileSystem.h"
28 #include "llvm/Support/LockFileManager.h"
29 #include "llvm/Support/MemoryBuffer.h"
30 #include "llvm/Support/OnDiskHashTable.h"
31 #include "llvm/Support/Path.h"
32 #include <cstdio>
33 using namespace clang;
34 using namespace serialization;
35 
36 //----------------------------------------------------------------------------//
37 // Shared constants
38 //----------------------------------------------------------------------------//
39 namespace {
40  enum {
41  /// \brief The block containing the index.
42  GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
43  };
44 
45  /// \brief Describes the record types in the index.
47  /// \brief Contains version information and potentially other metadata,
48  /// used to determine if we can read this global index file.
49  INDEX_METADATA,
50  /// \brief Describes a module, including its file name and dependencies.
51  MODULE,
52  /// \brief The index for identifiers.
53  IDENTIFIER_INDEX
54  };
55 }
56 
57 /// \brief The name of the global index file.
58 static const char * const IndexFileName = "modules.idx";
59 
60 /// \brief The global index file version.
61 static const unsigned CurrentVersion = 1;
62 
63 //----------------------------------------------------------------------------//
64 // Global module index reader.
65 //----------------------------------------------------------------------------//
66 
67 namespace {
68 
69 /// \brief Trait used to read the identifier index from the on-disk hash
70 /// table.
71 class IdentifierIndexReaderTrait {
72 public:
73  typedef StringRef external_key_type;
74  typedef StringRef internal_key_type;
75  typedef SmallVector<unsigned, 2> data_type;
76  typedef unsigned hash_value_type;
77  typedef unsigned offset_type;
78 
79  static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
80  return a == b;
81  }
82 
83  static hash_value_type ComputeHash(const internal_key_type& a) {
84  return llvm::HashString(a);
85  }
86 
87  static std::pair<unsigned, unsigned>
88  ReadKeyDataLength(const unsigned char*& d) {
89  using namespace llvm::support;
90  unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
91  unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
92  return std::make_pair(KeyLen, DataLen);
93  }
94 
95  static const internal_key_type&
96  GetInternalKey(const external_key_type& x) { return x; }
97 
98  static const external_key_type&
99  GetExternalKey(const internal_key_type& x) { return x; }
100 
101  static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
102  return StringRef((const char *)d, n);
103  }
104 
105  static data_type ReadData(const internal_key_type& k,
106  const unsigned char* d,
107  unsigned DataLen) {
108  using namespace llvm::support;
109 
110  data_type Result;
111  while (DataLen > 0) {
112  unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
113  Result.push_back(ID);
114  DataLen -= 4;
115  }
116 
117  return Result;
118  }
119 };
120 
122  IdentifierIndexTable;
123 
124 }
125 
126 GlobalModuleIndex::GlobalModuleIndex(std::unique_ptr<llvm::MemoryBuffer> Buffer,
127  llvm::BitstreamCursor Cursor)
128  : Buffer(std::move(Buffer)), IdentifierIndex(), NumIdentifierLookups(),
129  NumIdentifierLookupHits() {
130  // Read the global index.
131  bool InGlobalIndexBlock = false;
132  bool Done = false;
133  while (!Done) {
134  llvm::BitstreamEntry Entry = Cursor.advance();
135 
136  switch (Entry.Kind) {
138  return;
139 
140  case llvm::BitstreamEntry::EndBlock:
141  if (InGlobalIndexBlock) {
142  InGlobalIndexBlock = false;
143  Done = true;
144  continue;
145  }
146  return;
147 
148 
149  case llvm::BitstreamEntry::Record:
150  // Entries in the global index block are handled below.
151  if (InGlobalIndexBlock)
152  break;
153 
154  return;
155 
156  case llvm::BitstreamEntry::SubBlock:
157  if (!InGlobalIndexBlock && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
158  if (Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
159  return;
160 
161  InGlobalIndexBlock = true;
162  } else if (Cursor.SkipBlock()) {
163  return;
164  }
165  continue;
166  }
167 
169  StringRef Blob;
170  switch ((IndexRecordTypes)Cursor.readRecord(Entry.ID, Record, &Blob)) {
171  case INDEX_METADATA:
172  // Make sure that the version matches.
173  if (Record.size() < 1 || Record[0] != CurrentVersion)
174  return;
175  break;
176 
177  case MODULE: {
178  unsigned Idx = 0;
179  unsigned ID = Record[Idx++];
180 
181  // Make room for this module's information.
182  if (ID == Modules.size())
183  Modules.push_back(ModuleInfo());
184  else
185  Modules.resize(ID + 1);
186 
187  // Size/modification time for this module file at the time the
188  // global index was built.
189  Modules[ID].Size = Record[Idx++];
190  Modules[ID].ModTime = Record[Idx++];
191 
192  // File name.
193  unsigned NameLen = Record[Idx++];
194  Modules[ID].FileName.assign(Record.begin() + Idx,
195  Record.begin() + Idx + NameLen);
196  Idx += NameLen;
197 
198  // Dependencies
199  unsigned NumDeps = Record[Idx++];
200  Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(),
201  Record.begin() + Idx,
202  Record.begin() + Idx + NumDeps);
203  Idx += NumDeps;
204 
205  // Make sure we're at the end of the record.
206  assert(Idx == Record.size() && "More module info?");
207 
208  // Record this module as an unresolved module.
209  // FIXME: this doesn't work correctly for module names containing path
210  // separators.
211  StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName);
212  // Remove the -<hash of ModuleMapPath>
213  ModuleName = ModuleName.rsplit('-').first;
214  UnresolvedModules[ModuleName] = ID;
215  break;
216  }
217 
218  case IDENTIFIER_INDEX:
219  // Wire up the identifier index.
220  if (Record[0]) {
221  IdentifierIndex = IdentifierIndexTable::Create(
222  (const unsigned char *)Blob.data() + Record[0],
223  (const unsigned char *)Blob.data() + sizeof(uint32_t),
224  (const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
225  }
226  break;
227  }
228  }
229 }
230 
231 GlobalModuleIndex::~GlobalModuleIndex() {
232  delete static_cast<IdentifierIndexTable *>(IdentifierIndex);
233 }
234 
235 std::pair<GlobalModuleIndex *, GlobalModuleIndex::ErrorCode>
237  // Load the index file, if it's there.
238  llvm::SmallString<128> IndexPath;
239  IndexPath += Path;
240  llvm::sys::path::append(IndexPath, IndexFileName);
241 
242  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
243  llvm::MemoryBuffer::getFile(IndexPath.c_str());
244  if (!BufferOrErr)
245  return std::make_pair(nullptr, EC_NotFound);
246  std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
247 
248  /// \brief The main bitstream cursor for the main block.
249  llvm::BitstreamCursor Cursor(*Buffer);
250 
251  // Sniff for the signature.
252  if (Cursor.Read(8) != 'B' ||
253  Cursor.Read(8) != 'C' ||
254  Cursor.Read(8) != 'G' ||
255  Cursor.Read(8) != 'I') {
256  return std::make_pair(nullptr, EC_IOError);
257  }
258 
259  return std::make_pair(new GlobalModuleIndex(std::move(Buffer), Cursor),
260  EC_None);
261 }
262 
263 void
265  ModuleFiles.clear();
266  for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
267  if (ModuleFile *MF = Modules[I].File)
268  ModuleFiles.push_back(MF);
269  }
270 }
271 
273  ModuleFile *File,
274  SmallVectorImpl<ModuleFile *> &Dependencies) {
275  // Look for information about this module file.
276  llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
277  = ModulesByFile.find(File);
278  if (Known == ModulesByFile.end())
279  return;
280 
281  // Record dependencies.
282  Dependencies.clear();
283  ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
284  for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
285  if (ModuleFile *MF = Modules[I].File)
286  Dependencies.push_back(MF);
287  }
288 }
289 
291  Hits.clear();
292 
293  // If there's no identifier index, there is nothing we can do.
294  if (!IdentifierIndex)
295  return false;
296 
297  // Look into the identifier index.
298  ++NumIdentifierLookups;
299  IdentifierIndexTable &Table
300  = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
301  IdentifierIndexTable::iterator Known = Table.find(Name);
302  if (Known == Table.end()) {
303  return true;
304  }
305 
306  SmallVector<unsigned, 2> ModuleIDs = *Known;
307  for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
308  if (ModuleFile *MF = Modules[ModuleIDs[I]].File)
309  Hits.insert(MF);
310  }
311 
312  ++NumIdentifierLookupHits;
313  return true;
314 }
315 
316 bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) {
317  // Look for the module in the global module index based on the module name.
318  StringRef Name = File->ModuleName;
319  llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
320  if (Known == UnresolvedModules.end()) {
321  return true;
322  }
323 
324  // Rectify this module with the global module index.
325  ModuleInfo &Info = Modules[Known->second];
326 
327  // If the size and modification time match what we expected, record this
328  // module file.
329  bool Failed = true;
330  if (File->File->getSize() == Info.Size &&
331  File->File->getModificationTime() == Info.ModTime) {
332  Info.File = File;
333  ModulesByFile[File] = Known->second;
334 
335  Failed = false;
336  }
337 
338  // One way or another, we have resolved this module file.
339  UnresolvedModules.erase(Known);
340  return Failed;
341 }
342 
344  std::fprintf(stderr, "*** Global Module Index Statistics:\n");
345  if (NumIdentifierLookups) {
346  fprintf(stderr, " %u / %u identifier lookups succeeded (%f%%)\n",
347  NumIdentifierLookupHits, NumIdentifierLookups,
348  (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
349  }
350  std::fprintf(stderr, "\n");
351 }
352 
353 LLVM_DUMP_METHOD void GlobalModuleIndex::dump() {
354  llvm::errs() << "*** Global Module Index Dump:\n";
355  llvm::errs() << "Module files:\n";
356  for (auto &MI : Modules) {
357  llvm::errs() << "** " << MI.FileName << "\n";
358  if (MI.File)
359  MI.File->dump();
360  else
361  llvm::errs() << "\n";
362  }
363  llvm::errs() << "\n";
364 }
365 
366 //----------------------------------------------------------------------------//
367 // Global module index writer.
368 //----------------------------------------------------------------------------//
369 
370 namespace {
371  /// \brief Provides information about a specific module file.
372  struct ModuleFileInfo {
373  /// \brief The numberic ID for this module file.
374  unsigned ID;
375 
376  /// \brief The set of modules on which this module depends. Each entry is
377  /// a module ID.
378  SmallVector<unsigned, 4> Dependencies;
379  ASTFileSignature Signature;
380  };
381 
382  struct ImportedModuleFileInfo {
383  off_t StoredSize;
384  time_t StoredModTime;
385  ASTFileSignature StoredSignature;
386  ImportedModuleFileInfo(off_t Size, time_t ModTime, ASTFileSignature Sig)
387  : StoredSize(Size), StoredModTime(ModTime), StoredSignature(Sig) {}
388  };
389 
390  /// \brief Builder that generates the global module index file.
391  class GlobalModuleIndexBuilder {
392  FileManager &FileMgr;
393  const PCHContainerReader &PCHContainerRdr;
394 
395  /// Mapping from files to module file information.
396  typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
397 
398  /// Information about each of the known module files.
399  ModuleFilesMap ModuleFiles;
400 
401  /// \brief Mapping from the imported module file to the imported
402  /// information.
403  typedef std::multimap<const FileEntry *, ImportedModuleFileInfo>
404  ImportedModuleFilesMap;
405 
406  /// \brief Information about each importing of a module file.
407  ImportedModuleFilesMap ImportedModuleFiles;
408 
409  /// \brief Mapping from identifiers to the list of module file IDs that
410  /// consider this identifier to be interesting.
411  typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
412 
413  /// \brief A mapping from all interesting identifiers to the set of module
414  /// files in which those identifiers are considered interesting.
415  InterestingIdentifierMap InterestingIdentifiers;
416 
417  /// \brief Write the block-info block for the global module index file.
418  void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
419 
420  /// \brief Retrieve the module file information for the given file.
421  ModuleFileInfo &getModuleFileInfo(const FileEntry *File) {
422  llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
423  = ModuleFiles.find(File);
424  if (Known != ModuleFiles.end())
425  return Known->second;
426 
427  unsigned NewID = ModuleFiles.size();
428  ModuleFileInfo &Info = ModuleFiles[File];
429  Info.ID = NewID;
430  return Info;
431  }
432 
433  public:
434  explicit GlobalModuleIndexBuilder(
435  FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr)
436  : FileMgr(FileMgr), PCHContainerRdr(PCHContainerRdr) {}
437 
438  /// \brief Load the contents of the given module file into the builder.
439  ///
440  /// \returns true if an error occurred, false otherwise.
441  bool loadModuleFile(const FileEntry *File);
442 
443  /// \brief Write the index to the given bitstream.
444  /// \returns true if an error occurred, false otherwise.
445  bool writeIndex(llvm::BitstreamWriter &Stream);
446  };
447 }
448 
449 static void emitBlockID(unsigned ID, const char *Name,
450  llvm::BitstreamWriter &Stream,
451  SmallVectorImpl<uint64_t> &Record) {
452  Record.clear();
453  Record.push_back(ID);
454  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
455 
456  // Emit the block name if present.
457  if (!Name || Name[0] == 0) return;
458  Record.clear();
459  while (*Name)
460  Record.push_back(*Name++);
461  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
462 }
463 
464 static void emitRecordID(unsigned ID, const char *Name,
465  llvm::BitstreamWriter &Stream,
466  SmallVectorImpl<uint64_t> &Record) {
467  Record.clear();
468  Record.push_back(ID);
469  while (*Name)
470  Record.push_back(*Name++);
471  Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
472 }
473 
474 void
475 GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
477  Stream.EnterBlockInfoBlock();
478 
479 #define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
480 #define RECORD(X) emitRecordID(X, #X, Stream, Record)
481  BLOCK(GLOBAL_INDEX_BLOCK);
482  RECORD(INDEX_METADATA);
483  RECORD(MODULE);
484  RECORD(IDENTIFIER_INDEX);
485 #undef RECORD
486 #undef BLOCK
487 
488  Stream.ExitBlock();
489 }
490 
491 namespace {
492  class InterestingASTIdentifierLookupTrait
494 
495  public:
496  /// \brief The identifier and whether it is "interesting".
497  typedef std::pair<StringRef, bool> data_type;
498 
499  data_type ReadData(const internal_key_type& k,
500  const unsigned char* d,
501  unsigned DataLen) {
502  // The first bit indicates whether this identifier is interesting.
503  // That's all we care about.
504  using namespace llvm::support;
505  unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
506  bool IsInteresting = RawID & 0x01;
507  return std::make_pair(k, IsInteresting);
508  }
509  };
510 }
511 
512 bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
513  // Open the module file.
514 
515  auto Buffer = FileMgr.getBufferForFile(File, /*isVolatile=*/true);
516  if (!Buffer) {
517  return true;
518  }
519 
520  // Initialize the input stream
521  llvm::BitstreamCursor InStream(PCHContainerRdr.ExtractPCH(**Buffer));
522 
523  // Sniff for the signature.
524  if (InStream.Read(8) != 'C' ||
525  InStream.Read(8) != 'P' ||
526  InStream.Read(8) != 'C' ||
527  InStream.Read(8) != 'H') {
528  return true;
529  }
530 
531  // Record this module file and assign it a unique ID (if it doesn't have
532  // one already).
533  unsigned ID = getModuleFileInfo(File).ID;
534 
535  // Search for the blocks and records we care about.
536  enum { Other, ControlBlock, ASTBlock, DiagnosticOptionsBlock } State = Other;
537  bool Done = false;
538  while (!Done) {
539  llvm::BitstreamEntry Entry = InStream.advance();
540  switch (Entry.Kind) {
542  Done = true;
543  continue;
544 
545  case llvm::BitstreamEntry::Record:
546  // In the 'other' state, just skip the record. We don't care.
547  if (State == Other) {
548  InStream.skipRecord(Entry.ID);
549  continue;
550  }
551 
552  // Handle potentially-interesting records below.
553  break;
554 
555  case llvm::BitstreamEntry::SubBlock:
556  if (Entry.ID == CONTROL_BLOCK_ID) {
557  if (InStream.EnterSubBlock(CONTROL_BLOCK_ID))
558  return true;
559 
560  // Found the control block.
561  State = ControlBlock;
562  continue;
563  }
564 
565  if (Entry.ID == AST_BLOCK_ID) {
566  if (InStream.EnterSubBlock(AST_BLOCK_ID))
567  return true;
568 
569  // Found the AST block.
570  State = ASTBlock;
571  continue;
572  }
573 
574  if (Entry.ID == UNHASHED_CONTROL_BLOCK_ID) {
575  if (InStream.EnterSubBlock(UNHASHED_CONTROL_BLOCK_ID))
576  return true;
577 
578  // Found the Diagnostic Options block.
579  State = DiagnosticOptionsBlock;
580  continue;
581  }
582 
583  if (InStream.SkipBlock())
584  return true;
585 
586  continue;
587 
588  case llvm::BitstreamEntry::EndBlock:
589  State = Other;
590  continue;
591  }
592 
593  // Read the given record.
595  StringRef Blob;
596  unsigned Code = InStream.readRecord(Entry.ID, Record, &Blob);
597 
598  // Handle module dependencies.
599  if (State == ControlBlock && Code == IMPORTS) {
600  // Load each of the imported PCH files.
601  unsigned Idx = 0, N = Record.size();
602  while (Idx < N) {
603  // Read information about the AST file.
604 
605  // Skip the imported kind
606  ++Idx;
607 
608  // Skip the import location
609  ++Idx;
610 
611  // Load stored size/modification time.
612  off_t StoredSize = (off_t)Record[Idx++];
613  time_t StoredModTime = (time_t)Record[Idx++];
614 
615  // Skip the stored signature.
616  // FIXME: we could read the signature out of the import and validate it.
617  ASTFileSignature StoredSignature = {
618  {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
619  (uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
620  (uint32_t)Record[Idx++]}}};
621 
622  // Retrieve the imported file name.
623  unsigned Length = Record[Idx++];
624  SmallString<128> ImportedFile(Record.begin() + Idx,
625  Record.begin() + Idx + Length);
626  Idx += Length;
627 
628  // Find the imported module file.
629  const FileEntry *DependsOnFile
630  = FileMgr.getFile(ImportedFile, /*openFile=*/false,
631  /*cacheFailure=*/false);
632 
633  if (!DependsOnFile)
634  return true;
635 
636  // Save the information in ImportedModuleFileInfo so we can verify after
637  // loading all pcms.
638  ImportedModuleFiles.insert(std::make_pair(
639  DependsOnFile, ImportedModuleFileInfo(StoredSize, StoredModTime,
640  StoredSignature)));
641 
642  // Record the dependency.
643  unsigned DependsOnID = getModuleFileInfo(DependsOnFile).ID;
644  getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
645  }
646 
647  continue;
648  }
649 
650  // Handle the identifier table
651  if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) {
653  InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
654  std::unique_ptr<InterestingIdentifierTable> Table(
656  (const unsigned char *)Blob.data() + Record[0],
657  (const unsigned char *)Blob.data() + sizeof(uint32_t),
658  (const unsigned char *)Blob.data()));
659  for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
660  DEnd = Table->data_end();
661  D != DEnd; ++D) {
662  std::pair<StringRef, bool> Ident = *D;
663  if (Ident.second)
664  InterestingIdentifiers[Ident.first].push_back(ID);
665  else
666  (void)InterestingIdentifiers[Ident.first];
667  }
668  }
669 
670  // Get Signature.
671  if (State == DiagnosticOptionsBlock && Code == SIGNATURE)
672  getModuleFileInfo(File).Signature = {
673  {{(uint32_t)Record[0], (uint32_t)Record[1], (uint32_t)Record[2],
674  (uint32_t)Record[3], (uint32_t)Record[4]}}};
675 
676  // We don't care about this record.
677  }
678 
679  return false;
680 }
681 
682 namespace {
683 
684 /// \brief Trait used to generate the identifier index as an on-disk hash
685 /// table.
686 class IdentifierIndexWriterTrait {
687 public:
688  typedef StringRef key_type;
689  typedef StringRef key_type_ref;
690  typedef SmallVector<unsigned, 2> data_type;
691  typedef const SmallVector<unsigned, 2> &data_type_ref;
692  typedef unsigned hash_value_type;
693  typedef unsigned offset_type;
694 
695  static hash_value_type ComputeHash(key_type_ref Key) {
696  return llvm::HashString(Key);
697  }
698 
699  std::pair<unsigned,unsigned>
700  EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
701  using namespace llvm::support;
702  endian::Writer<little> LE(Out);
703  unsigned KeyLen = Key.size();
704  unsigned DataLen = Data.size() * 4;
705  LE.write<uint16_t>(KeyLen);
706  LE.write<uint16_t>(DataLen);
707  return std::make_pair(KeyLen, DataLen);
708  }
709 
710  void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
711  Out.write(Key.data(), KeyLen);
712  }
713 
714  void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
715  unsigned DataLen) {
716  using namespace llvm::support;
717  for (unsigned I = 0, N = Data.size(); I != N; ++I)
718  endian::Writer<little>(Out).write<uint32_t>(Data[I]);
719  }
720 };
721 
722 }
723 
724 bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
725  for (auto MapEntry : ImportedModuleFiles) {
726  auto *File = MapEntry.first;
727  ImportedModuleFileInfo &Info = MapEntry.second;
728  if (getModuleFileInfo(File).Signature) {
729  if (getModuleFileInfo(File).Signature != Info.StoredSignature)
730  // Verify Signature.
731  return true;
732  } else if (Info.StoredSize != File->getSize() ||
733  Info.StoredModTime != File->getModificationTime())
734  // Verify Size and ModTime.
735  return true;
736  }
737 
738  using namespace llvm;
739 
740  // Emit the file header.
741  Stream.Emit((unsigned)'B', 8);
742  Stream.Emit((unsigned)'C', 8);
743  Stream.Emit((unsigned)'G', 8);
744  Stream.Emit((unsigned)'I', 8);
745 
746  // Write the block-info block, which describes the records in this bitcode
747  // file.
748  emitBlockInfoBlock(Stream);
749 
750  Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
751 
752  // Write the metadata.
754  Record.push_back(CurrentVersion);
755  Stream.EmitRecord(INDEX_METADATA, Record);
756 
757  // Write the set of known module files.
758  for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
759  MEnd = ModuleFiles.end();
760  M != MEnd; ++M) {
761  Record.clear();
762  Record.push_back(M->second.ID);
763  Record.push_back(M->first->getSize());
764  Record.push_back(M->first->getModificationTime());
765 
766  // File name
767  StringRef Name(M->first->getName());
768  Record.push_back(Name.size());
769  Record.append(Name.begin(), Name.end());
770 
771  // Dependencies
772  Record.push_back(M->second.Dependencies.size());
773  Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
774  Stream.EmitRecord(MODULE, Record);
775  }
776 
777  // Write the identifier -> module file mapping.
778  {
779  llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
780  IdentifierIndexWriterTrait Trait;
781 
782  // Populate the hash table.
783  for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
784  IEnd = InterestingIdentifiers.end();
785  I != IEnd; ++I) {
786  Generator.insert(I->first(), I->second, Trait);
787  }
788 
789  // Create the on-disk hash table in a buffer.
791  uint32_t BucketOffset;
792  {
793  using namespace llvm::support;
794  llvm::raw_svector_ostream Out(IdentifierTable);
795  // Make sure that no bucket is at offset 0
796  endian::Writer<little>(Out).write<uint32_t>(0);
797  BucketOffset = Generator.Emit(Out, Trait);
798  }
799 
800  // Create a blob abbreviation
801  auto Abbrev = std::make_shared<BitCodeAbbrev>();
802  Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
803  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
804  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
805  unsigned IDTableAbbrev = Stream.EmitAbbrev(std::move(Abbrev));
806 
807  // Write the identifier table
808  uint64_t Record[] = {IDENTIFIER_INDEX, BucketOffset};
809  Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable);
810  }
811 
812  Stream.ExitBlock();
813  return false;
814 }
815 
818  const PCHContainerReader &PCHContainerRdr,
819  StringRef Path) {
820  llvm::SmallString<128> IndexPath;
821  IndexPath += Path;
822  llvm::sys::path::append(IndexPath, IndexFileName);
823 
824  // Coordinate building the global index file with other processes that might
825  // try to do the same.
826  llvm::LockFileManager Locked(IndexPath);
827  switch (Locked) {
828  case llvm::LockFileManager::LFS_Error:
829  return EC_IOError;
830 
831  case llvm::LockFileManager::LFS_Owned:
832  // We're responsible for building the index ourselves. Do so below.
833  break;
834 
835  case llvm::LockFileManager::LFS_Shared:
836  // Someone else is responsible for building the index. We don't care
837  // when they finish, so we're done.
838  return EC_Building;
839  }
840 
841  // The module index builder.
842  GlobalModuleIndexBuilder Builder(FileMgr, PCHContainerRdr);
843 
844  // Load each of the module files.
845  std::error_code EC;
846  for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
847  D != DEnd && !EC;
848  D.increment(EC)) {
849  // If this isn't a module file, we don't care.
850  if (llvm::sys::path::extension(D->path()) != ".pcm") {
851  // ... unless it's a .pcm.lock file, which indicates that someone is
852  // in the process of rebuilding a module. They'll rebuild the index
853  // at the end of that translation unit, so we don't have to.
854  if (llvm::sys::path::extension(D->path()) == ".pcm.lock")
855  return EC_Building;
856 
857  continue;
858  }
859 
860  // If we can't find the module file, skip it.
861  const FileEntry *ModuleFile = FileMgr.getFile(D->path());
862  if (!ModuleFile)
863  continue;
864 
865  // Load this module file.
866  if (Builder.loadModuleFile(ModuleFile))
867  return EC_IOError;
868  }
869 
870  // The output buffer, into which the global index will be written.
871  SmallVector<char, 16> OutputBuffer;
872  {
873  llvm::BitstreamWriter OutputStream(OutputBuffer);
874  if (Builder.writeIndex(OutputStream))
875  return EC_IOError;
876  }
877 
878  // Write the global index file to a temporary file.
879  llvm::SmallString<128> IndexTmpPath;
880  int TmpFD;
881  if (llvm::sys::fs::createUniqueFile(IndexPath + "-%%%%%%%%", TmpFD,
882  IndexTmpPath))
883  return EC_IOError;
884 
885  // Open the temporary global index file for output.
886  llvm::raw_fd_ostream Out(TmpFD, true);
887  if (Out.has_error())
888  return EC_IOError;
889 
890  // Write the index.
891  Out.write(OutputBuffer.data(), OutputBuffer.size());
892  Out.close();
893  if (Out.has_error())
894  return EC_IOError;
895 
896  // Remove the old index file. It isn't relevant any more.
897  llvm::sys::fs::remove(IndexPath);
898 
899  // Rename the newly-written index file to the proper name.
900  if (llvm::sys::fs::rename(IndexTmpPath, IndexPath)) {
901  // Rename failed; just remove the
902  llvm::sys::fs::remove(IndexTmpPath);
903  return EC_IOError;
904  }
905 
906  // We're done.
907  return EC_None;
908 }
909 
910 namespace {
911  class GlobalIndexIdentifierIterator : public IdentifierIterator {
912  /// \brief The current position within the identifier lookup table.
913  IdentifierIndexTable::key_iterator Current;
914 
915  /// \brief The end position within the identifier lookup table.
916  IdentifierIndexTable::key_iterator End;
917 
918  public:
919  explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
920  Current = Idx.key_begin();
921  End = Idx.key_end();
922  }
923 
924  StringRef Next() override {
925  if (Current == End)
926  return StringRef();
927 
928  StringRef Result = *Current;
929  ++Current;
930  return Result;
931  }
932  };
933 }
934 
936  IdentifierIndexTable &Table =
937  *static_cast<IdentifierIndexTable *>(IdentifierIndex);
938  return new GlobalIndexIdentifierIterator(Table);
939 }
unsigned Length
void getKnownModules(SmallVectorImpl< ModuleFile * > &ModuleFiles)
Retrieve the set of modules that have up-to-date indexes.
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:116
Defines the clang::FileManager interface and associated types.
std::unique_ptr< llvm::MemoryBuffer > Buffer
void getModuleDependencies(ModuleFile *File, SmallVectorImpl< ModuleFile * > &Dependencies)
Retrieve the set of module files on which the given module file directly depends. ...
static const unsigned CurrentVersion
The global index file version.
A block with unhashed content.
Definition: ASTBitCodes.h:261
off_t getSize() const
Definition: FileManager.h:87
return(__x >> __y)|(__x<< (32-__y))
void printStats()
Print statistics to standard error.
static void emitRecordID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
LineState State
llvm::SmallPtrSet< ModuleFile *, 4 > HitSet
A set of module files in which we found a result.
The signature of a module, which is a hash of the AST content.
Definition: Module.h:47
IdentifierIterator * createIdentifierIterator() const
Returns an iterator for identifiers stored in the index table.
ErrorCode
An error code returned when trying to read an index.
This abstract interface provides operations for unwrapping containers for serialized ASTs (precompile...
detail::InMemoryDirectory::const_iterator I
static ErrorCode writeIndex(FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr, StringRef Path)
Write a global index into the given.
const FileEntry * getFile(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Lookup, cache, and verify the specified file (real or virtual).
const SmallVectorImpl< AnnotatedLine * >::const_iterator End
void dump()
Print debugging view to standard error.
static void emitBlockID(unsigned ID, const char *Name, llvm::BitstreamWriter &Stream, SmallVectorImpl< uint64_t > &Record)
Implements an efficient mapping from strings to IdentifierInfo nodes.
Base class for the trait describing the on-disk hash table for the identifiers in an AST file...
An iterator that walks over all of the known identifiers in the lookup table.
static const char *const IndexFileName
The name of the global index file.
The result type of a method or function.
Record code for the identifier table.
Definition: ASTBitCodes.h:406
IndexRecordTypes
Describes the record types in the index.
#define false
Definition: stdbool.h:33
StringRef FileName
Definition: Format.cpp:1465
SmallVectorImpl< AnnotatedLine * >::const_iterator Next
const std::string ID
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr * > VL, ArrayRef< Expr * > PL, ArrayRef< Expr * > IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
The AST block, which acts as a container around the full AST block.
Definition: ASTBitCodes.h:213
A global index for a set of module files, providing information about the identifiers within those mo...
unsigned ComputeHash(Selector Sel)
Definition: ASTCommon.cpp:164
Record code for the signature that identifiers this AST file.
Definition: ASTBitCodes.h:328
if(T->getSizeExpr()) TRY_TO(TraverseStmt(T-> getSizeExpr()))
Record code for the list of other AST files imported by this AST file.
Definition: ASTBitCodes.h:272
There was an unspecified I/O error reading or writing the index.
StringRef Name
Definition: USRFinder.cpp:123
static std::pair< GlobalModuleIndex *, ErrorCode > readIndex(StringRef Path)
Read a global index file for the given directory.
#define BLOCK(X)
time_t getModificationTime() const
Definition: FileManager.h:91
FormatToken * Current
BoundNodesTreeBuilder *const Builder
Dump information about a module file.
#define RECORD(X)
#define true
Definition: stdbool.h:32
The control block, which contains all of the information that needs to be validated prior to committi...
Definition: ASTBitCodes.h:239
const NamedDecl * Result
Definition: USRFinder.cpp:70
bool lookupIdentifier(StringRef Name, HitSet &Hits)
Look for all of the module files with information about the given identifier, e.g., a global function, variable, or type with that name.
bool loadedModuleFile(ModuleFile *File)
Note that the given module file has been loaded.