File: | clang/lib/Serialization/GlobalModuleIndex.cpp |
Warning: | line 133, column 43 Dereference of null smart pointer 'Buffer' of type 'std::unique_ptr' |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | // This file implements the GlobalModuleIndex class. | |||
10 | // | |||
11 | //===----------------------------------------------------------------------===// | |||
12 | ||||
13 | #include "clang/Serialization/GlobalModuleIndex.h" | |||
14 | #include "ASTReaderInternals.h" | |||
15 | #include "clang/Basic/FileManager.h" | |||
16 | #include "clang/Lex/HeaderSearch.h" | |||
17 | #include "clang/Serialization/ASTBitCodes.h" | |||
18 | #include "clang/Serialization/ModuleFile.h" | |||
19 | #include "clang/Serialization/PCHContainerOperations.h" | |||
20 | #include "llvm/ADT/DenseMap.h" | |||
21 | #include "llvm/ADT/MapVector.h" | |||
22 | #include "llvm/ADT/SmallString.h" | |||
23 | #include "llvm/ADT/StringRef.h" | |||
24 | #include "llvm/Bitstream/BitstreamReader.h" | |||
25 | #include "llvm/Bitstream/BitstreamWriter.h" | |||
26 | #include "llvm/Support/DJB.h" | |||
27 | #include "llvm/Support/FileSystem.h" | |||
28 | #include "llvm/Support/FileUtilities.h" | |||
29 | #include "llvm/Support/LockFileManager.h" | |||
30 | #include "llvm/Support/MemoryBuffer.h" | |||
31 | #include "llvm/Support/OnDiskHashTable.h" | |||
32 | #include "llvm/Support/Path.h" | |||
33 | #include "llvm/Support/TimeProfiler.h" | |||
34 | #include <cstdio> | |||
35 | using namespace clang; | |||
36 | using namespace serialization; | |||
37 | ||||
38 | //----------------------------------------------------------------------------// | |||
39 | // Shared constants | |||
40 | //----------------------------------------------------------------------------// | |||
41 | namespace { | |||
42 | enum { | |||
43 | /// The block containing the index. | |||
44 | GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID | |||
45 | }; | |||
46 | ||||
47 | /// Describes the record types in the index. | |||
48 | enum IndexRecordTypes { | |||
49 | /// Contains version information and potentially other metadata, | |||
50 | /// used to determine if we can read this global index file. | |||
51 | INDEX_METADATA, | |||
52 | /// Describes a module, including its file name and dependencies. | |||
53 | MODULE, | |||
54 | /// The index for identifiers. | |||
55 | IDENTIFIER_INDEX | |||
56 | }; | |||
57 | } | |||
58 | ||||
59 | /// The name of the global index file. | |||
60 | static const char * const IndexFileName = "modules.idx"; | |||
61 | ||||
62 | /// The global index file version. | |||
63 | static const unsigned CurrentVersion = 1; | |||
64 | ||||
65 | //----------------------------------------------------------------------------// | |||
66 | // Global module index reader. | |||
67 | //----------------------------------------------------------------------------// | |||
68 | ||||
69 | namespace { | |||
70 | ||||
71 | /// Trait used to read the identifier index from the on-disk hash | |||
72 | /// table. | |||
73 | class IdentifierIndexReaderTrait { | |||
74 | public: | |||
75 | typedef StringRef external_key_type; | |||
76 | typedef StringRef internal_key_type; | |||
77 | typedef SmallVector<unsigned, 2> data_type; | |||
78 | typedef unsigned hash_value_type; | |||
79 | typedef unsigned offset_type; | |||
80 | ||||
81 | static bool EqualKey(const internal_key_type& a, const internal_key_type& b) { | |||
82 | return a == b; | |||
83 | } | |||
84 | ||||
85 | static hash_value_type ComputeHash(const internal_key_type& a) { | |||
86 | return llvm::djbHash(a); | |||
87 | } | |||
88 | ||||
89 | static std::pair<unsigned, unsigned> | |||
90 | ReadKeyDataLength(const unsigned char*& d) { | |||
91 | using namespace llvm::support; | |||
92 | unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d); | |||
93 | unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d); | |||
94 | return std::make_pair(KeyLen, DataLen); | |||
95 | } | |||
96 | ||||
97 | static const internal_key_type& | |||
98 | GetInternalKey(const external_key_type& x) { return x; } | |||
99 | ||||
100 | static const external_key_type& | |||
101 | GetExternalKey(const internal_key_type& x) { return x; } | |||
102 | ||||
103 | static internal_key_type ReadKey(const unsigned char* d, unsigned n) { | |||
104 | return StringRef((const char *)d, n); | |||
105 | } | |||
106 | ||||
107 | static data_type ReadData(const internal_key_type& k, | |||
108 | const unsigned char* d, | |||
109 | unsigned DataLen) { | |||
110 | using namespace llvm::support; | |||
111 | ||||
112 | data_type Result; | |||
113 | while (DataLen > 0) { | |||
114 | unsigned ID = endian::readNext<uint32_t, little, unaligned>(d); | |||
115 | Result.push_back(ID); | |||
116 | DataLen -= 4; | |||
117 | } | |||
118 | ||||
119 | return Result; | |||
120 | } | |||
121 | }; | |||
122 | ||||
123 | typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait> | |||
124 | IdentifierIndexTable; | |||
125 | ||||
126 | } | |||
127 | ||||
128 | GlobalModuleIndex::GlobalModuleIndex(std::unique_ptr<llvm::MemoryBuffer> Buffer, | |||
129 | llvm::BitstreamCursor Cursor) | |||
130 | : Buffer(std::move(Buffer)), IdentifierIndex(), NumIdentifierLookups(), | |||
131 | NumIdentifierLookupHits() { | |||
132 | auto Fail = [&Buffer](llvm::Error &&Err) { | |||
133 | report_fatal_error("Module index '" + Buffer->getBufferIdentifier() + | |||
| ||||
134 | "' failed: " + toString(std::move(Err))); | |||
135 | }; | |||
136 | ||||
137 | llvm::TimeTraceScope TimeScope("Module LoadIndex"); | |||
138 | // Read the global index. | |||
139 | bool InGlobalIndexBlock = false; | |||
140 | bool Done = false; | |||
141 | while (!Done) { | |||
142 | llvm::BitstreamEntry Entry; | |||
143 | if (Expected<llvm::BitstreamEntry> Res = Cursor.advance()) | |||
144 | Entry = Res.get(); | |||
145 | else | |||
146 | Fail(Res.takeError()); | |||
147 | ||||
148 | switch (Entry.Kind) { | |||
149 | case llvm::BitstreamEntry::Error: | |||
150 | return; | |||
151 | ||||
152 | case llvm::BitstreamEntry::EndBlock: | |||
153 | if (InGlobalIndexBlock) { | |||
154 | InGlobalIndexBlock = false; | |||
155 | Done = true; | |||
156 | continue; | |||
157 | } | |||
158 | return; | |||
159 | ||||
160 | ||||
161 | case llvm::BitstreamEntry::Record: | |||
162 | // Entries in the global index block are handled below. | |||
163 | if (InGlobalIndexBlock) | |||
164 | break; | |||
165 | ||||
166 | return; | |||
167 | ||||
168 | case llvm::BitstreamEntry::SubBlock: | |||
169 | if (!InGlobalIndexBlock
| |||
170 | if (llvm::Error Err = Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID)) | |||
171 | Fail(std::move(Err)); | |||
172 | InGlobalIndexBlock = true; | |||
173 | } else if (llvm::Error Err = Cursor.SkipBlock()) | |||
174 | Fail(std::move(Err)); | |||
175 | continue; | |||
176 | } | |||
177 | ||||
178 | SmallVector<uint64_t, 64> Record; | |||
179 | StringRef Blob; | |||
180 | Expected<unsigned> MaybeIndexRecord = | |||
181 | Cursor.readRecord(Entry.ID, Record, &Blob); | |||
182 | if (!MaybeIndexRecord) | |||
183 | Fail(MaybeIndexRecord.takeError()); | |||
184 | IndexRecordTypes IndexRecord = | |||
185 | static_cast<IndexRecordTypes>(MaybeIndexRecord.get()); | |||
186 | switch (IndexRecord) { | |||
187 | case INDEX_METADATA: | |||
188 | // Make sure that the version matches. | |||
189 | if (Record.size() < 1 || Record[0] != CurrentVersion) | |||
190 | return; | |||
191 | break; | |||
192 | ||||
193 | case MODULE: { | |||
194 | unsigned Idx = 0; | |||
195 | unsigned ID = Record[Idx++]; | |||
196 | ||||
197 | // Make room for this module's information. | |||
198 | if (ID == Modules.size()) | |||
199 | Modules.push_back(ModuleInfo()); | |||
200 | else | |||
201 | Modules.resize(ID + 1); | |||
202 | ||||
203 | // Size/modification time for this module file at the time the | |||
204 | // global index was built. | |||
205 | Modules[ID].Size = Record[Idx++]; | |||
206 | Modules[ID].ModTime = Record[Idx++]; | |||
207 | ||||
208 | // File name. | |||
209 | unsigned NameLen = Record[Idx++]; | |||
210 | Modules[ID].FileName.assign(Record.begin() + Idx, | |||
211 | Record.begin() + Idx + NameLen); | |||
212 | Idx += NameLen; | |||
213 | ||||
214 | // Dependencies | |||
215 | unsigned NumDeps = Record[Idx++]; | |||
216 | Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(), | |||
217 | Record.begin() + Idx, | |||
218 | Record.begin() + Idx + NumDeps); | |||
219 | Idx += NumDeps; | |||
220 | ||||
221 | // Make sure we're at the end of the record. | |||
222 | assert(Idx == Record.size() && "More module info?")((Idx == Record.size() && "More module info?") ? static_cast <void> (0) : __assert_fail ("Idx == Record.size() && \"More module info?\"" , "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/Serialization/GlobalModuleIndex.cpp" , 222, __PRETTY_FUNCTION__)); | |||
223 | ||||
224 | // Record this module as an unresolved module. | |||
225 | // FIXME: this doesn't work correctly for module names containing path | |||
226 | // separators. | |||
227 | StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName); | |||
228 | // Remove the -<hash of ModuleMapPath> | |||
229 | ModuleName = ModuleName.rsplit('-').first; | |||
230 | UnresolvedModules[ModuleName] = ID; | |||
231 | break; | |||
232 | } | |||
233 | ||||
234 | case IDENTIFIER_INDEX: | |||
235 | // Wire up the identifier index. | |||
236 | if (Record[0]) { | |||
237 | IdentifierIndex = IdentifierIndexTable::Create( | |||
238 | (const unsigned char *)Blob.data() + Record[0], | |||
239 | (const unsigned char *)Blob.data() + sizeof(uint32_t), | |||
240 | (const unsigned char *)Blob.data(), IdentifierIndexReaderTrait()); | |||
241 | } | |||
242 | break; | |||
243 | } | |||
244 | } | |||
245 | } | |||
246 | ||||
247 | GlobalModuleIndex::~GlobalModuleIndex() { | |||
248 | delete static_cast<IdentifierIndexTable *>(IdentifierIndex); | |||
249 | } | |||
250 | ||||
251 | std::pair<GlobalModuleIndex *, llvm::Error> | |||
252 | GlobalModuleIndex::readIndex(StringRef Path) { | |||
253 | // Load the index file, if it's there. | |||
254 | llvm::SmallString<128> IndexPath; | |||
255 | IndexPath += Path; | |||
256 | llvm::sys::path::append(IndexPath, IndexFileName); | |||
257 | ||||
258 | llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr = | |||
259 | llvm::MemoryBuffer::getFile(IndexPath.c_str()); | |||
260 | if (!BufferOrErr) | |||
| ||||
261 | return std::make_pair(nullptr, | |||
262 | llvm::errorCodeToError(BufferOrErr.getError())); | |||
263 | std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get()); | |||
264 | ||||
265 | /// The main bitstream cursor for the main block. | |||
266 | llvm::BitstreamCursor Cursor(*Buffer); | |||
267 | ||||
268 | // Sniff for the signature. | |||
269 | for (unsigned char C : {'B', 'C', 'G', 'I'}) { | |||
270 | if (Expected<llvm::SimpleBitstreamCursor::word_t> Res = Cursor.Read(8)) { | |||
271 | if (Res.get() != C) | |||
272 | return std::make_pair( | |||
273 | nullptr, llvm::createStringError(std::errc::illegal_byte_sequence, | |||
274 | "expected signature BCGI")); | |||
275 | } else | |||
276 | return std::make_pair(nullptr, Res.takeError()); | |||
277 | } | |||
278 | ||||
279 | return std::make_pair(new GlobalModuleIndex(std::move(Buffer), Cursor), | |||
280 | llvm::Error::success()); | |||
281 | } | |||
282 | ||||
283 | void | |||
284 | GlobalModuleIndex::getKnownModules(SmallVectorImpl<ModuleFile *> &ModuleFiles) { | |||
285 | ModuleFiles.clear(); | |||
286 | for (unsigned I = 0, N = Modules.size(); I != N; ++I) { | |||
287 | if (ModuleFile *MF = Modules[I].File) | |||
288 | ModuleFiles.push_back(MF); | |||
289 | } | |||
290 | } | |||
291 | ||||
292 | void GlobalModuleIndex::getModuleDependencies( | |||
293 | ModuleFile *File, | |||
294 | SmallVectorImpl<ModuleFile *> &Dependencies) { | |||
295 | // Look for information about this module file. | |||
296 | llvm::DenseMap<ModuleFile *, unsigned>::iterator Known | |||
297 | = ModulesByFile.find(File); | |||
298 | if (Known == ModulesByFile.end()) | |||
299 | return; | |||
300 | ||||
301 | // Record dependencies. | |||
302 | Dependencies.clear(); | |||
303 | ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies; | |||
304 | for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) { | |||
305 | if (ModuleFile *MF = Modules[I].File) | |||
306 | Dependencies.push_back(MF); | |||
307 | } | |||
308 | } | |||
309 | ||||
310 | bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) { | |||
311 | Hits.clear(); | |||
312 | ||||
313 | // If there's no identifier index, there is nothing we can do. | |||
314 | if (!IdentifierIndex) | |||
315 | return false; | |||
316 | ||||
317 | // Look into the identifier index. | |||
318 | ++NumIdentifierLookups; | |||
319 | IdentifierIndexTable &Table | |||
320 | = *static_cast<IdentifierIndexTable *>(IdentifierIndex); | |||
321 | IdentifierIndexTable::iterator Known = Table.find(Name); | |||
322 | if (Known == Table.end()) { | |||
323 | return true; | |||
324 | } | |||
325 | ||||
326 | SmallVector<unsigned, 2> ModuleIDs = *Known; | |||
327 | for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) { | |||
328 | if (ModuleFile *MF = Modules[ModuleIDs[I]].File) | |||
329 | Hits.insert(MF); | |||
330 | } | |||
331 | ||||
332 | ++NumIdentifierLookupHits; | |||
333 | return true; | |||
334 | } | |||
335 | ||||
336 | bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) { | |||
337 | // Look for the module in the global module index based on the module name. | |||
338 | StringRef Name = File->ModuleName; | |||
339 | llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name); | |||
340 | if (Known == UnresolvedModules.end()) { | |||
341 | return true; | |||
342 | } | |||
343 | ||||
344 | // Rectify this module with the global module index. | |||
345 | ModuleInfo &Info = Modules[Known->second]; | |||
346 | ||||
347 | // If the size and modification time match what we expected, record this | |||
348 | // module file. | |||
349 | bool Failed = true; | |||
350 | if (File->File->getSize() == Info.Size && | |||
351 | File->File->getModificationTime() == Info.ModTime) { | |||
352 | Info.File = File; | |||
353 | ModulesByFile[File] = Known->second; | |||
354 | ||||
355 | Failed = false; | |||
356 | } | |||
357 | ||||
358 | // One way or another, we have resolved this module file. | |||
359 | UnresolvedModules.erase(Known); | |||
360 | return Failed; | |||
361 | } | |||
362 | ||||
363 | void GlobalModuleIndex::printStats() { | |||
364 | std::fprintf(stderrstderr, "*** Global Module Index Statistics:\n"); | |||
365 | if (NumIdentifierLookups) { | |||
366 | fprintf(stderrstderr, " %u / %u identifier lookups succeeded (%f%%)\n", | |||
367 | NumIdentifierLookupHits, NumIdentifierLookups, | |||
368 | (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups); | |||
369 | } | |||
370 | std::fprintf(stderrstderr, "\n"); | |||
371 | } | |||
372 | ||||
373 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void GlobalModuleIndex::dump() { | |||
374 | llvm::errs() << "*** Global Module Index Dump:\n"; | |||
375 | llvm::errs() << "Module files:\n"; | |||
376 | for (auto &MI : Modules) { | |||
377 | llvm::errs() << "** " << MI.FileName << "\n"; | |||
378 | if (MI.File) | |||
379 | MI.File->dump(); | |||
380 | else | |||
381 | llvm::errs() << "\n"; | |||
382 | } | |||
383 | llvm::errs() << "\n"; | |||
384 | } | |||
385 | ||||
386 | //----------------------------------------------------------------------------// | |||
387 | // Global module index writer. | |||
388 | //----------------------------------------------------------------------------// | |||
389 | ||||
390 | namespace { | |||
391 | /// Provides information about a specific module file. | |||
392 | struct ModuleFileInfo { | |||
393 | /// The numberic ID for this module file. | |||
394 | unsigned ID; | |||
395 | ||||
396 | /// The set of modules on which this module depends. Each entry is | |||
397 | /// a module ID. | |||
398 | SmallVector<unsigned, 4> Dependencies; | |||
399 | ASTFileSignature Signature; | |||
400 | }; | |||
401 | ||||
402 | struct ImportedModuleFileInfo { | |||
403 | off_t StoredSize; | |||
404 | time_t StoredModTime; | |||
405 | ASTFileSignature StoredSignature; | |||
406 | ImportedModuleFileInfo(off_t Size, time_t ModTime, ASTFileSignature Sig) | |||
407 | : StoredSize(Size), StoredModTime(ModTime), StoredSignature(Sig) {} | |||
408 | }; | |||
409 | ||||
410 | /// Builder that generates the global module index file. | |||
411 | class GlobalModuleIndexBuilder { | |||
412 | FileManager &FileMgr; | |||
413 | const PCHContainerReader &PCHContainerRdr; | |||
414 | ||||
415 | /// Mapping from files to module file information. | |||
416 | typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap; | |||
417 | ||||
418 | /// Information about each of the known module files. | |||
419 | ModuleFilesMap ModuleFiles; | |||
420 | ||||
421 | /// Mapping from the imported module file to the imported | |||
422 | /// information. | |||
423 | typedef std::multimap<const FileEntry *, ImportedModuleFileInfo> | |||
424 | ImportedModuleFilesMap; | |||
425 | ||||
426 | /// Information about each importing of a module file. | |||
427 | ImportedModuleFilesMap ImportedModuleFiles; | |||
428 | ||||
429 | /// Mapping from identifiers to the list of module file IDs that | |||
430 | /// consider this identifier to be interesting. | |||
431 | typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap; | |||
432 | ||||
433 | /// A mapping from all interesting identifiers to the set of module | |||
434 | /// files in which those identifiers are considered interesting. | |||
435 | InterestingIdentifierMap InterestingIdentifiers; | |||
436 | ||||
437 | /// Write the block-info block for the global module index file. | |||
438 | void emitBlockInfoBlock(llvm::BitstreamWriter &Stream); | |||
439 | ||||
440 | /// Retrieve the module file information for the given file. | |||
441 | ModuleFileInfo &getModuleFileInfo(const FileEntry *File) { | |||
442 | llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known | |||
443 | = ModuleFiles.find(File); | |||
444 | if (Known != ModuleFiles.end()) | |||
445 | return Known->second; | |||
446 | ||||
447 | unsigned NewID = ModuleFiles.size(); | |||
448 | ModuleFileInfo &Info = ModuleFiles[File]; | |||
449 | Info.ID = NewID; | |||
450 | return Info; | |||
451 | } | |||
452 | ||||
453 | public: | |||
454 | explicit GlobalModuleIndexBuilder( | |||
455 | FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr) | |||
456 | : FileMgr(FileMgr), PCHContainerRdr(PCHContainerRdr) {} | |||
457 | ||||
458 | /// Load the contents of the given module file into the builder. | |||
459 | llvm::Error loadModuleFile(const FileEntry *File); | |||
460 | ||||
461 | /// Write the index to the given bitstream. | |||
462 | /// \returns true if an error occurred, false otherwise. | |||
463 | bool writeIndex(llvm::BitstreamWriter &Stream); | |||
464 | }; | |||
465 | } | |||
466 | ||||
467 | static void emitBlockID(unsigned ID, const char *Name, | |||
468 | llvm::BitstreamWriter &Stream, | |||
469 | SmallVectorImpl<uint64_t> &Record) { | |||
470 | Record.clear(); | |||
471 | Record.push_back(ID); | |||
472 | Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record); | |||
473 | ||||
474 | // Emit the block name if present. | |||
475 | if (!Name || Name[0] == 0) return; | |||
476 | Record.clear(); | |||
477 | while (*Name) | |||
478 | Record.push_back(*Name++); | |||
479 | Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record); | |||
480 | } | |||
481 | ||||
482 | static void emitRecordID(unsigned ID, const char *Name, | |||
483 | llvm::BitstreamWriter &Stream, | |||
484 | SmallVectorImpl<uint64_t> &Record) { | |||
485 | Record.clear(); | |||
486 | Record.push_back(ID); | |||
487 | while (*Name) | |||
488 | Record.push_back(*Name++); | |||
489 | Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record); | |||
490 | } | |||
491 | ||||
492 | void | |||
493 | GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) { | |||
494 | SmallVector<uint64_t, 64> Record; | |||
495 | Stream.EnterBlockInfoBlock(); | |||
496 | ||||
497 | #define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record) | |||
498 | #define RECORD(X) emitRecordID(X, #X, Stream, Record) | |||
499 | BLOCK(GLOBAL_INDEX_BLOCK); | |||
500 | RECORD(INDEX_METADATA); | |||
501 | RECORD(MODULE); | |||
502 | RECORD(IDENTIFIER_INDEX); | |||
503 | #undef RECORD | |||
504 | #undef BLOCK | |||
505 | ||||
506 | Stream.ExitBlock(); | |||
507 | } | |||
508 | ||||
509 | namespace { | |||
510 | class InterestingASTIdentifierLookupTrait | |||
511 | : public serialization::reader::ASTIdentifierLookupTraitBase { | |||
512 | ||||
513 | public: | |||
514 | /// The identifier and whether it is "interesting". | |||
515 | typedef std::pair<StringRef, bool> data_type; | |||
516 | ||||
517 | data_type ReadData(const internal_key_type& k, | |||
518 | const unsigned char* d, | |||
519 | unsigned DataLen) { | |||
520 | // The first bit indicates whether this identifier is interesting. | |||
521 | // That's all we care about. | |||
522 | using namespace llvm::support; | |||
523 | unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d); | |||
524 | bool IsInteresting = RawID & 0x01; | |||
525 | return std::make_pair(k, IsInteresting); | |||
526 | } | |||
527 | }; | |||
528 | } | |||
529 | ||||
530 | llvm::Error GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) { | |||
531 | // Open the module file. | |||
532 | ||||
533 | auto Buffer = FileMgr.getBufferForFile(File, /*isVolatile=*/true); | |||
534 | if (!Buffer) | |||
535 | return llvm::createStringError(Buffer.getError(), | |||
536 | "failed getting buffer for module file"); | |||
537 | ||||
538 | // Initialize the input stream | |||
539 | llvm::BitstreamCursor InStream(PCHContainerRdr.ExtractPCH(**Buffer)); | |||
540 | ||||
541 | // Sniff for the signature. | |||
542 | for (unsigned char C : {'C', 'P', 'C', 'H'}) | |||
543 | if (Expected<llvm::SimpleBitstreamCursor::word_t> Res = InStream.Read(8)) { | |||
544 | if (Res.get() != C) | |||
545 | return llvm::createStringError(std::errc::illegal_byte_sequence, | |||
546 | "expected signature CPCH"); | |||
547 | } else | |||
548 | return Res.takeError(); | |||
549 | ||||
550 | // Record this module file and assign it a unique ID (if it doesn't have | |||
551 | // one already). | |||
552 | unsigned ID = getModuleFileInfo(File).ID; | |||
553 | ||||
554 | // Search for the blocks and records we care about. | |||
555 | enum { Other, ControlBlock, ASTBlock, DiagnosticOptionsBlock } State = Other; | |||
556 | bool Done = false; | |||
557 | while (!Done) { | |||
558 | Expected<llvm::BitstreamEntry> MaybeEntry = InStream.advance(); | |||
559 | if (!MaybeEntry) | |||
560 | return MaybeEntry.takeError(); | |||
561 | llvm::BitstreamEntry Entry = MaybeEntry.get(); | |||
562 | ||||
563 | switch (Entry.Kind) { | |||
564 | case llvm::BitstreamEntry::Error: | |||
565 | Done = true; | |||
566 | continue; | |||
567 | ||||
568 | case llvm::BitstreamEntry::Record: | |||
569 | // In the 'other' state, just skip the record. We don't care. | |||
570 | if (State == Other) { | |||
571 | if (llvm::Expected<unsigned> Skipped = InStream.skipRecord(Entry.ID)) | |||
572 | continue; | |||
573 | else | |||
574 | return Skipped.takeError(); | |||
575 | } | |||
576 | ||||
577 | // Handle potentially-interesting records below. | |||
578 | break; | |||
579 | ||||
580 | case llvm::BitstreamEntry::SubBlock: | |||
581 | if (Entry.ID == CONTROL_BLOCK_ID) { | |||
582 | if (llvm::Error Err = InStream.EnterSubBlock(CONTROL_BLOCK_ID)) | |||
583 | return Err; | |||
584 | ||||
585 | // Found the control block. | |||
586 | State = ControlBlock; | |||
587 | continue; | |||
588 | } | |||
589 | ||||
590 | if (Entry.ID == AST_BLOCK_ID) { | |||
591 | if (llvm::Error Err = InStream.EnterSubBlock(AST_BLOCK_ID)) | |||
592 | return Err; | |||
593 | ||||
594 | // Found the AST block. | |||
595 | State = ASTBlock; | |||
596 | continue; | |||
597 | } | |||
598 | ||||
599 | if (Entry.ID == UNHASHED_CONTROL_BLOCK_ID) { | |||
600 | if (llvm::Error Err = InStream.EnterSubBlock(UNHASHED_CONTROL_BLOCK_ID)) | |||
601 | return Err; | |||
602 | ||||
603 | // Found the Diagnostic Options block. | |||
604 | State = DiagnosticOptionsBlock; | |||
605 | continue; | |||
606 | } | |||
607 | ||||
608 | if (llvm::Error Err = InStream.SkipBlock()) | |||
609 | return Err; | |||
610 | ||||
611 | continue; | |||
612 | ||||
613 | case llvm::BitstreamEntry::EndBlock: | |||
614 | State = Other; | |||
615 | continue; | |||
616 | } | |||
617 | ||||
618 | // Read the given record. | |||
619 | SmallVector<uint64_t, 64> Record; | |||
620 | StringRef Blob; | |||
621 | Expected<unsigned> MaybeCode = InStream.readRecord(Entry.ID, Record, &Blob); | |||
622 | if (!MaybeCode) | |||
623 | return MaybeCode.takeError(); | |||
624 | unsigned Code = MaybeCode.get(); | |||
625 | ||||
626 | // Handle module dependencies. | |||
627 | if (State == ControlBlock && Code == IMPORTS) { | |||
628 | // Load each of the imported PCH files. | |||
629 | unsigned Idx = 0, N = Record.size(); | |||
630 | while (Idx < N) { | |||
631 | // Read information about the AST file. | |||
632 | ||||
633 | // Skip the imported kind | |||
634 | ++Idx; | |||
635 | ||||
636 | // Skip the import location | |||
637 | ++Idx; | |||
638 | ||||
639 | // Load stored size/modification time. | |||
640 | off_t StoredSize = (off_t)Record[Idx++]; | |||
641 | time_t StoredModTime = (time_t)Record[Idx++]; | |||
642 | ||||
643 | // Skip the stored signature. | |||
644 | // FIXME: we could read the signature out of the import and validate it. | |||
645 | ASTFileSignature StoredSignature = { | |||
646 | {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++], | |||
647 | (uint32_t)Record[Idx++], (uint32_t)Record[Idx++], | |||
648 | (uint32_t)Record[Idx++]}}}; | |||
649 | ||||
650 | // Skip the module name (currently this is only used for prebuilt | |||
651 | // modules while here we are only dealing with cached). | |||
652 | Idx += Record[Idx] + 1; | |||
653 | ||||
654 | // Retrieve the imported file name. | |||
655 | unsigned Length = Record[Idx++]; | |||
656 | SmallString<128> ImportedFile(Record.begin() + Idx, | |||
657 | Record.begin() + Idx + Length); | |||
658 | Idx += Length; | |||
659 | ||||
660 | // Find the imported module file. | |||
661 | auto DependsOnFile | |||
662 | = FileMgr.getFile(ImportedFile, /*OpenFile=*/false, | |||
663 | /*CacheFailure=*/false); | |||
664 | ||||
665 | if (!DependsOnFile) | |||
666 | return llvm::createStringError(std::errc::bad_file_descriptor, | |||
667 | "imported file \"%s\" not found", | |||
668 | ImportedFile.c_str()); | |||
669 | ||||
670 | // Save the information in ImportedModuleFileInfo so we can verify after | |||
671 | // loading all pcms. | |||
672 | ImportedModuleFiles.insert(std::make_pair( | |||
673 | *DependsOnFile, ImportedModuleFileInfo(StoredSize, StoredModTime, | |||
674 | StoredSignature))); | |||
675 | ||||
676 | // Record the dependency. | |||
677 | unsigned DependsOnID = getModuleFileInfo(*DependsOnFile).ID; | |||
678 | getModuleFileInfo(File).Dependencies.push_back(DependsOnID); | |||
679 | } | |||
680 | ||||
681 | continue; | |||
682 | } | |||
683 | ||||
684 | // Handle the identifier table | |||
685 | if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) { | |||
686 | typedef llvm::OnDiskIterableChainedHashTable< | |||
687 | InterestingASTIdentifierLookupTrait> InterestingIdentifierTable; | |||
688 | std::unique_ptr<InterestingIdentifierTable> Table( | |||
689 | InterestingIdentifierTable::Create( | |||
690 | (const unsigned char *)Blob.data() + Record[0], | |||
691 | (const unsigned char *)Blob.data() + sizeof(uint32_t), | |||
692 | (const unsigned char *)Blob.data())); | |||
693 | for (InterestingIdentifierTable::data_iterator D = Table->data_begin(), | |||
694 | DEnd = Table->data_end(); | |||
695 | D != DEnd; ++D) { | |||
696 | std::pair<StringRef, bool> Ident = *D; | |||
697 | if (Ident.second) | |||
698 | InterestingIdentifiers[Ident.first].push_back(ID); | |||
699 | else | |||
700 | (void)InterestingIdentifiers[Ident.first]; | |||
701 | } | |||
702 | } | |||
703 | ||||
704 | // Get Signature. | |||
705 | if (State == DiagnosticOptionsBlock && Code == SIGNATURE) | |||
706 | getModuleFileInfo(File).Signature = { | |||
707 | {{(uint32_t)Record[0], (uint32_t)Record[1], (uint32_t)Record[2], | |||
708 | (uint32_t)Record[3], (uint32_t)Record[4]}}}; | |||
709 | ||||
710 | // We don't care about this record. | |||
711 | } | |||
712 | ||||
713 | return llvm::Error::success(); | |||
714 | } | |||
715 | ||||
716 | namespace { | |||
717 | ||||
718 | /// Trait used to generate the identifier index as an on-disk hash | |||
719 | /// table. | |||
720 | class IdentifierIndexWriterTrait { | |||
721 | public: | |||
722 | typedef StringRef key_type; | |||
723 | typedef StringRef key_type_ref; | |||
724 | typedef SmallVector<unsigned, 2> data_type; | |||
725 | typedef const SmallVector<unsigned, 2> &data_type_ref; | |||
726 | typedef unsigned hash_value_type; | |||
727 | typedef unsigned offset_type; | |||
728 | ||||
729 | static hash_value_type ComputeHash(key_type_ref Key) { | |||
730 | return llvm::djbHash(Key); | |||
731 | } | |||
732 | ||||
733 | std::pair<unsigned,unsigned> | |||
734 | EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) { | |||
735 | using namespace llvm::support; | |||
736 | endian::Writer LE(Out, little); | |||
737 | unsigned KeyLen = Key.size(); | |||
738 | unsigned DataLen = Data.size() * 4; | |||
739 | LE.write<uint16_t>(KeyLen); | |||
740 | LE.write<uint16_t>(DataLen); | |||
741 | return std::make_pair(KeyLen, DataLen); | |||
742 | } | |||
743 | ||||
744 | void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) { | |||
745 | Out.write(Key.data(), KeyLen); | |||
746 | } | |||
747 | ||||
748 | void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data, | |||
749 | unsigned DataLen) { | |||
750 | using namespace llvm::support; | |||
751 | for (unsigned I = 0, N = Data.size(); I != N; ++I) | |||
752 | endian::write<uint32_t>(Out, Data[I], little); | |||
753 | } | |||
754 | }; | |||
755 | ||||
756 | } | |||
757 | ||||
758 | bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) { | |||
759 | for (auto MapEntry : ImportedModuleFiles) { | |||
760 | auto *File = MapEntry.first; | |||
761 | ImportedModuleFileInfo &Info = MapEntry.second; | |||
762 | if (getModuleFileInfo(File).Signature) { | |||
763 | if (getModuleFileInfo(File).Signature != Info.StoredSignature) | |||
764 | // Verify Signature. | |||
765 | return true; | |||
766 | } else if (Info.StoredSize != File->getSize() || | |||
767 | Info.StoredModTime != File->getModificationTime()) | |||
768 | // Verify Size and ModTime. | |||
769 | return true; | |||
770 | } | |||
771 | ||||
772 | using namespace llvm; | |||
773 | llvm::TimeTraceScope TimeScope("Module WriteIndex"); | |||
774 | ||||
775 | // Emit the file header. | |||
776 | Stream.Emit((unsigned)'B', 8); | |||
777 | Stream.Emit((unsigned)'C', 8); | |||
778 | Stream.Emit((unsigned)'G', 8); | |||
779 | Stream.Emit((unsigned)'I', 8); | |||
780 | ||||
781 | // Write the block-info block, which describes the records in this bitcode | |||
782 | // file. | |||
783 | emitBlockInfoBlock(Stream); | |||
784 | ||||
785 | Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3); | |||
786 | ||||
787 | // Write the metadata. | |||
788 | SmallVector<uint64_t, 2> Record; | |||
789 | Record.push_back(CurrentVersion); | |||
790 | Stream.EmitRecord(INDEX_METADATA, Record); | |||
791 | ||||
792 | // Write the set of known module files. | |||
793 | for (ModuleFilesMap::iterator M = ModuleFiles.begin(), | |||
794 | MEnd = ModuleFiles.end(); | |||
795 | M != MEnd; ++M) { | |||
796 | Record.clear(); | |||
797 | Record.push_back(M->second.ID); | |||
798 | Record.push_back(M->first->getSize()); | |||
799 | Record.push_back(M->first->getModificationTime()); | |||
800 | ||||
801 | // File name | |||
802 | StringRef Name(M->first->getName()); | |||
803 | Record.push_back(Name.size()); | |||
804 | Record.append(Name.begin(), Name.end()); | |||
805 | ||||
806 | // Dependencies | |||
807 | Record.push_back(M->second.Dependencies.size()); | |||
808 | Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end()); | |||
809 | Stream.EmitRecord(MODULE, Record); | |||
810 | } | |||
811 | ||||
812 | // Write the identifier -> module file mapping. | |||
813 | { | |||
814 | llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator; | |||
815 | IdentifierIndexWriterTrait Trait; | |||
816 | ||||
817 | // Populate the hash table. | |||
818 | for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(), | |||
819 | IEnd = InterestingIdentifiers.end(); | |||
820 | I != IEnd; ++I) { | |||
821 | Generator.insert(I->first(), I->second, Trait); | |||
822 | } | |||
823 | ||||
824 | // Create the on-disk hash table in a buffer. | |||
825 | SmallString<4096> IdentifierTable; | |||
826 | uint32_t BucketOffset; | |||
827 | { | |||
828 | using namespace llvm::support; | |||
829 | llvm::raw_svector_ostream Out(IdentifierTable); | |||
830 | // Make sure that no bucket is at offset 0 | |||
831 | endian::write<uint32_t>(Out, 0, little); | |||
832 | BucketOffset = Generator.Emit(Out, Trait); | |||
833 | } | |||
834 | ||||
835 | // Create a blob abbreviation | |||
836 | auto Abbrev = std::make_shared<BitCodeAbbrev>(); | |||
837 | Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX)); | |||
838 | Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); | |||
839 | Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); | |||
840 | unsigned IDTableAbbrev = Stream.EmitAbbrev(std::move(Abbrev)); | |||
841 | ||||
842 | // Write the identifier table | |||
843 | uint64_t Record[] = {IDENTIFIER_INDEX, BucketOffset}; | |||
844 | Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable); | |||
845 | } | |||
846 | ||||
847 | Stream.ExitBlock(); | |||
848 | return false; | |||
849 | } | |||
850 | ||||
851 | llvm::Error | |||
852 | GlobalModuleIndex::writeIndex(FileManager &FileMgr, | |||
853 | const PCHContainerReader &PCHContainerRdr, | |||
854 | StringRef Path) { | |||
855 | llvm::SmallString<128> IndexPath; | |||
856 | IndexPath += Path; | |||
857 | llvm::sys::path::append(IndexPath, IndexFileName); | |||
858 | ||||
859 | // Coordinate building the global index file with other processes that might | |||
860 | // try to do the same. | |||
861 | llvm::LockFileManager Locked(IndexPath); | |||
862 | switch (Locked) { | |||
863 | case llvm::LockFileManager::LFS_Error: | |||
864 | return llvm::createStringError(std::errc::io_error, "LFS error"); | |||
865 | ||||
866 | case llvm::LockFileManager::LFS_Owned: | |||
867 | // We're responsible for building the index ourselves. Do so below. | |||
868 | break; | |||
869 | ||||
870 | case llvm::LockFileManager::LFS_Shared: | |||
871 | // Someone else is responsible for building the index. We don't care | |||
872 | // when they finish, so we're done. | |||
873 | return llvm::createStringError(std::errc::device_or_resource_busy, | |||
874 | "someone else is building the index"); | |||
875 | } | |||
876 | ||||
877 | // The module index builder. | |||
878 | GlobalModuleIndexBuilder Builder(FileMgr, PCHContainerRdr); | |||
879 | ||||
880 | // Load each of the module files. | |||
881 | std::error_code EC; | |||
882 | for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd; | |||
883 | D != DEnd && !EC; | |||
884 | D.increment(EC)) { | |||
885 | // If this isn't a module file, we don't care. | |||
886 | if (llvm::sys::path::extension(D->path()) != ".pcm") { | |||
887 | // ... unless it's a .pcm.lock file, which indicates that someone is | |||
888 | // in the process of rebuilding a module. They'll rebuild the index | |||
889 | // at the end of that translation unit, so we don't have to. | |||
890 | if (llvm::sys::path::extension(D->path()) == ".pcm.lock") | |||
891 | return llvm::createStringError(std::errc::device_or_resource_busy, | |||
892 | "someone else is building the index"); | |||
893 | ||||
894 | continue; | |||
895 | } | |||
896 | ||||
897 | // If we can't find the module file, skip it. | |||
898 | auto ModuleFile = FileMgr.getFile(D->path()); | |||
899 | if (!ModuleFile) | |||
900 | continue; | |||
901 | ||||
902 | // Load this module file. | |||
903 | if (llvm::Error Err = Builder.loadModuleFile(*ModuleFile)) | |||
904 | return Err; | |||
905 | } | |||
906 | ||||
907 | // The output buffer, into which the global index will be written. | |||
908 | SmallVector<char, 16> OutputBuffer; | |||
909 | { | |||
910 | llvm::BitstreamWriter OutputStream(OutputBuffer); | |||
911 | if (Builder.writeIndex(OutputStream)) | |||
912 | return llvm::createStringError(std::errc::io_error, | |||
913 | "failed writing index"); | |||
914 | } | |||
915 | ||||
916 | return llvm::writeFileAtomically( | |||
917 | (IndexPath + "-%%%%%%%%").str(), IndexPath, | |||
918 | llvm::StringRef(OutputBuffer.data(), OutputBuffer.size())); | |||
919 | } | |||
920 | ||||
921 | namespace { | |||
922 | class GlobalIndexIdentifierIterator : public IdentifierIterator { | |||
923 | /// The current position within the identifier lookup table. | |||
924 | IdentifierIndexTable::key_iterator Current; | |||
925 | ||||
926 | /// The end position within the identifier lookup table. | |||
927 | IdentifierIndexTable::key_iterator End; | |||
928 | ||||
929 | public: | |||
930 | explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) { | |||
931 | Current = Idx.key_begin(); | |||
932 | End = Idx.key_end(); | |||
933 | } | |||
934 | ||||
935 | StringRef Next() override { | |||
936 | if (Current == End) | |||
937 | return StringRef(); | |||
938 | ||||
939 | StringRef Result = *Current; | |||
940 | ++Current; | |||
941 | return Result; | |||
942 | } | |||
943 | }; | |||
944 | } | |||
945 | ||||
946 | IdentifierIterator *GlobalModuleIndex::createIdentifierIterator() const { | |||
947 | IdentifierIndexTable &Table = | |||
948 | *static_cast<IdentifierIndexTable *>(IdentifierIndex); | |||
949 | return new GlobalIndexIdentifierIterator(Table); | |||
950 | } |