Bug Summary

File:clang/lib/Serialization/GlobalModuleIndex.cpp
Warning:line 133, column 43
Dereference of null smart pointer 'Buffer' of type 'std::unique_ptr'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name GlobalModuleIndex.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -relaxed-aliasing -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -target-cpu x86-64 -dwarf-column-info -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D CLANG_VENDOR="Debian " -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/build-llvm/tools/clang/lib/Serialization -I /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/Serialization -I /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/include -I /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/build-llvm/tools/clang/include -I /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/build-llvm/include -I /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/build-llvm/tools/clang/lib/Serialization -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fno-common -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-01-11-115256-23437-1 -x c++ /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/Serialization/GlobalModuleIndex.cpp
1//===--- GlobalModuleIndex.cpp - Global Module Index ------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the GlobalModuleIndex class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "clang/Serialization/GlobalModuleIndex.h"
14#include "ASTReaderInternals.h"
15#include "clang/Basic/FileManager.h"
16#include "clang/Lex/HeaderSearch.h"
17#include "clang/Serialization/ASTBitCodes.h"
18#include "clang/Serialization/ModuleFile.h"
19#include "clang/Serialization/PCHContainerOperations.h"
20#include "llvm/ADT/DenseMap.h"
21#include "llvm/ADT/MapVector.h"
22#include "llvm/ADT/SmallString.h"
23#include "llvm/ADT/StringRef.h"
24#include "llvm/Bitstream/BitstreamReader.h"
25#include "llvm/Bitstream/BitstreamWriter.h"
26#include "llvm/Support/DJB.h"
27#include "llvm/Support/FileSystem.h"
28#include "llvm/Support/FileUtilities.h"
29#include "llvm/Support/LockFileManager.h"
30#include "llvm/Support/MemoryBuffer.h"
31#include "llvm/Support/OnDiskHashTable.h"
32#include "llvm/Support/Path.h"
33#include "llvm/Support/TimeProfiler.h"
34#include <cstdio>
35using namespace clang;
36using namespace serialization;
37
38//----------------------------------------------------------------------------//
39// Shared constants
40//----------------------------------------------------------------------------//
41namespace {
42 enum {
43 /// The block containing the index.
44 GLOBAL_INDEX_BLOCK_ID = llvm::bitc::FIRST_APPLICATION_BLOCKID
45 };
46
47 /// Describes the record types in the index.
48 enum IndexRecordTypes {
49 /// Contains version information and potentially other metadata,
50 /// used to determine if we can read this global index file.
51 INDEX_METADATA,
52 /// Describes a module, including its file name and dependencies.
53 MODULE,
54 /// The index for identifiers.
55 IDENTIFIER_INDEX
56 };
57}
58
59/// The name of the global index file.
60static const char * const IndexFileName = "modules.idx";
61
62/// The global index file version.
63static const unsigned CurrentVersion = 1;
64
65//----------------------------------------------------------------------------//
66// Global module index reader.
67//----------------------------------------------------------------------------//
68
69namespace {
70
71/// Trait used to read the identifier index from the on-disk hash
72/// table.
73class IdentifierIndexReaderTrait {
74public:
75 typedef StringRef external_key_type;
76 typedef StringRef internal_key_type;
77 typedef SmallVector<unsigned, 2> data_type;
78 typedef unsigned hash_value_type;
79 typedef unsigned offset_type;
80
81 static bool EqualKey(const internal_key_type& a, const internal_key_type& b) {
82 return a == b;
83 }
84
85 static hash_value_type ComputeHash(const internal_key_type& a) {
86 return llvm::djbHash(a);
87 }
88
89 static std::pair<unsigned, unsigned>
90 ReadKeyDataLength(const unsigned char*& d) {
91 using namespace llvm::support;
92 unsigned KeyLen = endian::readNext<uint16_t, little, unaligned>(d);
93 unsigned DataLen = endian::readNext<uint16_t, little, unaligned>(d);
94 return std::make_pair(KeyLen, DataLen);
95 }
96
97 static const internal_key_type&
98 GetInternalKey(const external_key_type& x) { return x; }
99
100 static const external_key_type&
101 GetExternalKey(const internal_key_type& x) { return x; }
102
103 static internal_key_type ReadKey(const unsigned char* d, unsigned n) {
104 return StringRef((const char *)d, n);
105 }
106
107 static data_type ReadData(const internal_key_type& k,
108 const unsigned char* d,
109 unsigned DataLen) {
110 using namespace llvm::support;
111
112 data_type Result;
113 while (DataLen > 0) {
114 unsigned ID = endian::readNext<uint32_t, little, unaligned>(d);
115 Result.push_back(ID);
116 DataLen -= 4;
117 }
118
119 return Result;
120 }
121};
122
123typedef llvm::OnDiskIterableChainedHashTable<IdentifierIndexReaderTrait>
124 IdentifierIndexTable;
125
126}
127
128GlobalModuleIndex::GlobalModuleIndex(std::unique_ptr<llvm::MemoryBuffer> Buffer,
129 llvm::BitstreamCursor Cursor)
130 : Buffer(std::move(Buffer)), IdentifierIndex(), NumIdentifierLookups(),
4
←
Smart pointer 'Buffer' of type 'std::unique_ptr' is reset to null when moved from
→
131 NumIdentifierLookupHits() {
132 auto Fail = [&Buffer](llvm::Error &&Err) {
133 report_fatal_error("Module index '" + Buffer->getBufferIdentifier() +
13
←
Dereference of null smart pointer 'Buffer' of type 'std::unique_ptr'
134 "' failed: " + toString(std::move(Err)));
135 };
136
137 llvm::TimeTraceScope TimeScope("Module LoadIndex");
138 // Read the global index.
139 bool InGlobalIndexBlock = false;
140 bool Done = false;
141 while (!Done) {
5
←
Loop condition is true. Entering loop body
→
142 llvm::BitstreamEntry Entry;
143 if (Expected<llvm::BitstreamEntry> Res = Cursor.advance())
6
←
Taking true branch
→
144 Entry = Res.get();
145 else
146 Fail(Res.takeError());
147
148 switch (Entry.Kind) {
7
←
Control jumps to 'case SubBlock:' at line 168
→
149 case llvm::BitstreamEntry::Error:
150 return;
151
152 case llvm::BitstreamEntry::EndBlock:
153 if (InGlobalIndexBlock) {
154 InGlobalIndexBlock = false;
155 Done = true;
156 continue;
157 }
158 return;
159
160
161 case llvm::BitstreamEntry::Record:
162 // Entries in the global index block are handled below.
163 if (InGlobalIndexBlock)
164 break;
165
166 return;
167
168 case llvm::BitstreamEntry::SubBlock:
169 if (!InGlobalIndexBlock
7.1
'InGlobalIndexBlock' is false
 && Entry.ID == GLOBAL_INDEX_BLOCK_ID) {
8
←
Assuming field 'ID' is equal to GLOBAL_INDEX_BLOCK_ID
→
9
←
Taking true branch
→
170 if (llvm::Error Err = Cursor.EnterSubBlock(GLOBAL_INDEX_BLOCK_ID))
10
←
Assuming the condition is true
→
11
←
Taking true branch
→
171 Fail(std::move(Err));
12
←
Calling 'operator()'
→
172 InGlobalIndexBlock = true;
173 } else if (llvm::Error Err = Cursor.SkipBlock())
174 Fail(std::move(Err));
175 continue;
176 }
177
178 SmallVector<uint64_t, 64> Record;
179 StringRef Blob;
180 Expected<unsigned> MaybeIndexRecord =
181 Cursor.readRecord(Entry.ID, Record, &Blob);
182 if (!MaybeIndexRecord)
183 Fail(MaybeIndexRecord.takeError());
184 IndexRecordTypes IndexRecord =
185 static_cast<IndexRecordTypes>(MaybeIndexRecord.get());
186 switch (IndexRecord) {
187 case INDEX_METADATA:
188 // Make sure that the version matches.
189 if (Record.size() < 1 || Record[0] != CurrentVersion)
190 return;
191 break;
192
193 case MODULE: {
194 unsigned Idx = 0;
195 unsigned ID = Record[Idx++];
196
197 // Make room for this module's information.
198 if (ID == Modules.size())
199 Modules.push_back(ModuleInfo());
200 else
201 Modules.resize(ID + 1);
202
203 // Size/modification time for this module file at the time the
204 // global index was built.
205 Modules[ID].Size = Record[Idx++];
206 Modules[ID].ModTime = Record[Idx++];
207
208 // File name.
209 unsigned NameLen = Record[Idx++];
210 Modules[ID].FileName.assign(Record.begin() + Idx,
211 Record.begin() + Idx + NameLen);
212 Idx += NameLen;
213
214 // Dependencies
215 unsigned NumDeps = Record[Idx++];
216 Modules[ID].Dependencies.insert(Modules[ID].Dependencies.end(),
217 Record.begin() + Idx,
218 Record.begin() + Idx + NumDeps);
219 Idx += NumDeps;
220
221 // Make sure we're at the end of the record.
222 assert(Idx == Record.size() && "More module info?")((Idx == Record.size() && "More module info?") ? static_cast
<void> (0) : __assert_fail ("Idx == Record.size() && \"More module info?\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/clang/lib/Serialization/GlobalModuleIndex.cpp"
, 222, __PRETTY_FUNCTION__));
223
224 // Record this module as an unresolved module.
225 // FIXME: this doesn't work correctly for module names containing path
226 // separators.
227 StringRef ModuleName = llvm::sys::path::stem(Modules[ID].FileName);
228 // Remove the -<hash of ModuleMapPath>
229 ModuleName = ModuleName.rsplit('-').first;
230 UnresolvedModules[ModuleName] = ID;
231 break;
232 }
233
234 case IDENTIFIER_INDEX:
235 // Wire up the identifier index.
236 if (Record[0]) {
237 IdentifierIndex = IdentifierIndexTable::Create(
238 (const unsigned char *)Blob.data() + Record[0],
239 (const unsigned char *)Blob.data() + sizeof(uint32_t),
240 (const unsigned char *)Blob.data(), IdentifierIndexReaderTrait());
241 }
242 break;
243 }
244 }
245}
246
247GlobalModuleIndex::~GlobalModuleIndex() {
248 delete static_cast<IdentifierIndexTable *>(IdentifierIndex);
249}
250
251std::pair<GlobalModuleIndex *, llvm::Error>
252GlobalModuleIndex::readIndex(StringRef Path) {
253 // Load the index file, if it's there.
254 llvm::SmallString<128> IndexPath;
255 IndexPath += Path;
256 llvm::sys::path::append(IndexPath, IndexFileName);
257
258 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferOrErr =
259 llvm::MemoryBuffer::getFile(IndexPath.c_str());
260 if (!BufferOrErr)
1
Taking false branch
→
261 return std::make_pair(nullptr,
262 llvm::errorCodeToError(BufferOrErr.getError()));
263 std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
264
265 /// The main bitstream cursor for the main block.
266 llvm::BitstreamCursor Cursor(*Buffer);
267
268 // Sniff for the signature.
269 for (unsigned char C : {'B', 'C', 'G', 'I'}) {
2
←
Assuming '__begin1' is equal to '__end1'
→
270 if (Expected<llvm::SimpleBitstreamCursor::word_t> Res = Cursor.Read(8)) {
271 if (Res.get() != C)
272 return std::make_pair(
273 nullptr, llvm::createStringError(std::errc::illegal_byte_sequence,
274 "expected signature BCGI"));
275 } else
276 return std::make_pair(nullptr, Res.takeError());
277 }
278
279 return std::make_pair(new GlobalModuleIndex(std::move(Buffer), Cursor),
3
←
Calling constructor for 'GlobalModuleIndex'
→
280 llvm::Error::success());
281}
282
283void
284GlobalModuleIndex::getKnownModules(SmallVectorImpl<ModuleFile *> &ModuleFiles) {
285 ModuleFiles.clear();
286 for (unsigned I = 0, N = Modules.size(); I != N; ++I) {
287 if (ModuleFile *MF = Modules[I].File)
288 ModuleFiles.push_back(MF);
289 }
290}
291
292void GlobalModuleIndex::getModuleDependencies(
293 ModuleFile *File,
294 SmallVectorImpl<ModuleFile *> &Dependencies) {
295 // Look for information about this module file.
296 llvm::DenseMap<ModuleFile *, unsigned>::iterator Known
297 = ModulesByFile.find(File);
298 if (Known == ModulesByFile.end())
299 return;
300
301 // Record dependencies.
302 Dependencies.clear();
303 ArrayRef<unsigned> StoredDependencies = Modules[Known->second].Dependencies;
304 for (unsigned I = 0, N = StoredDependencies.size(); I != N; ++I) {
305 if (ModuleFile *MF = Modules[I].File)
306 Dependencies.push_back(MF);
307 }
308}
309
310bool GlobalModuleIndex::lookupIdentifier(StringRef Name, HitSet &Hits) {
311 Hits.clear();
312
313 // If there's no identifier index, there is nothing we can do.
314 if (!IdentifierIndex)
315 return false;
316
317 // Look into the identifier index.
318 ++NumIdentifierLookups;
319 IdentifierIndexTable &Table
320 = *static_cast<IdentifierIndexTable *>(IdentifierIndex);
321 IdentifierIndexTable::iterator Known = Table.find(Name);
322 if (Known == Table.end()) {
323 return true;
324 }
325
326 SmallVector<unsigned, 2> ModuleIDs = *Known;
327 for (unsigned I = 0, N = ModuleIDs.size(); I != N; ++I) {
328 if (ModuleFile *MF = Modules[ModuleIDs[I]].File)
329 Hits.insert(MF);
330 }
331
332 ++NumIdentifierLookupHits;
333 return true;
334}
335
336bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) {
337 // Look for the module in the global module index based on the module name.
338 StringRef Name = File->ModuleName;
339 llvm::StringMap<unsigned>::iterator Known = UnresolvedModules.find(Name);
340 if (Known == UnresolvedModules.end()) {
341 return true;
342 }
343
344 // Rectify this module with the global module index.
345 ModuleInfo &Info = Modules[Known->second];
346
347 // If the size and modification time match what we expected, record this
348 // module file.
349 bool Failed = true;
350 if (File->File->getSize() == Info.Size &&
351 File->File->getModificationTime() == Info.ModTime) {
352 Info.File = File;
353 ModulesByFile[File] = Known->second;
354
355 Failed = false;
356 }
357
358 // One way or another, we have resolved this module file.
359 UnresolvedModules.erase(Known);
360 return Failed;
361}
362
363void GlobalModuleIndex::printStats() {
364 std::fprintf(stderrstderr, "*** Global Module Index Statistics:\n");
365 if (NumIdentifierLookups) {
366 fprintf(stderrstderr, " %u / %u identifier lookups succeeded (%f%%)\n",
367 NumIdentifierLookupHits, NumIdentifierLookups,
368 (double)NumIdentifierLookupHits*100.0/NumIdentifierLookups);
369 }
370 std::fprintf(stderrstderr, "\n");
371}
372
373LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void GlobalModuleIndex::dump() {
374 llvm::errs() << "*** Global Module Index Dump:\n";
375 llvm::errs() << "Module files:\n";
376 for (auto &MI : Modules) {
377 llvm::errs() << "** " << MI.FileName << "\n";
378 if (MI.File)
379 MI.File->dump();
380 else
381 llvm::errs() << "\n";
382 }
383 llvm::errs() << "\n";
384}
385
386//----------------------------------------------------------------------------//
387// Global module index writer.
388//----------------------------------------------------------------------------//
389
390namespace {
391 /// Provides information about a specific module file.
392 struct ModuleFileInfo {
393 /// The numberic ID for this module file.
394 unsigned ID;
395
396 /// The set of modules on which this module depends. Each entry is
397 /// a module ID.
398 SmallVector<unsigned, 4> Dependencies;
399 ASTFileSignature Signature;
400 };
401
402 struct ImportedModuleFileInfo {
403 off_t StoredSize;
404 time_t StoredModTime;
405 ASTFileSignature StoredSignature;
406 ImportedModuleFileInfo(off_t Size, time_t ModTime, ASTFileSignature Sig)
407 : StoredSize(Size), StoredModTime(ModTime), StoredSignature(Sig) {}
408 };
409
410 /// Builder that generates the global module index file.
411 class GlobalModuleIndexBuilder {
412 FileManager &FileMgr;
413 const PCHContainerReader &PCHContainerRdr;
414
415 /// Mapping from files to module file information.
416 typedef llvm::MapVector<const FileEntry *, ModuleFileInfo> ModuleFilesMap;
417
418 /// Information about each of the known module files.
419 ModuleFilesMap ModuleFiles;
420
421 /// Mapping from the imported module file to the imported
422 /// information.
423 typedef std::multimap<const FileEntry *, ImportedModuleFileInfo>
424 ImportedModuleFilesMap;
425
426 /// Information about each importing of a module file.
427 ImportedModuleFilesMap ImportedModuleFiles;
428
429 /// Mapping from identifiers to the list of module file IDs that
430 /// consider this identifier to be interesting.
431 typedef llvm::StringMap<SmallVector<unsigned, 2> > InterestingIdentifierMap;
432
433 /// A mapping from all interesting identifiers to the set of module
434 /// files in which those identifiers are considered interesting.
435 InterestingIdentifierMap InterestingIdentifiers;
436
437 /// Write the block-info block for the global module index file.
438 void emitBlockInfoBlock(llvm::BitstreamWriter &Stream);
439
440 /// Retrieve the module file information for the given file.
441 ModuleFileInfo &getModuleFileInfo(const FileEntry *File) {
442 llvm::MapVector<const FileEntry *, ModuleFileInfo>::iterator Known
443 = ModuleFiles.find(File);
444 if (Known != ModuleFiles.end())
445 return Known->second;
446
447 unsigned NewID = ModuleFiles.size();
448 ModuleFileInfo &Info = ModuleFiles[File];
449 Info.ID = NewID;
450 return Info;
451 }
452
453 public:
454 explicit GlobalModuleIndexBuilder(
455 FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr)
456 : FileMgr(FileMgr), PCHContainerRdr(PCHContainerRdr) {}
457
458 /// Load the contents of the given module file into the builder.
459 llvm::Error loadModuleFile(const FileEntry *File);
460
461 /// Write the index to the given bitstream.
462 /// \returns true if an error occurred, false otherwise.
463 bool writeIndex(llvm::BitstreamWriter &Stream);
464 };
465}
466
467static void emitBlockID(unsigned ID, const char *Name,
468 llvm::BitstreamWriter &Stream,
469 SmallVectorImpl<uint64_t> &Record) {
470 Record.clear();
471 Record.push_back(ID);
472 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETBID, Record);
473
474 // Emit the block name if present.
475 if (!Name || Name[0] == 0) return;
476 Record.clear();
477 while (*Name)
478 Record.push_back(*Name++);
479 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_BLOCKNAME, Record);
480}
481
482static void emitRecordID(unsigned ID, const char *Name,
483 llvm::BitstreamWriter &Stream,
484 SmallVectorImpl<uint64_t> &Record) {
485 Record.clear();
486 Record.push_back(ID);
487 while (*Name)
488 Record.push_back(*Name++);
489 Stream.EmitRecord(llvm::bitc::BLOCKINFO_CODE_SETRECORDNAME, Record);
490}
491
492void
493GlobalModuleIndexBuilder::emitBlockInfoBlock(llvm::BitstreamWriter &Stream) {
494 SmallVector<uint64_t, 64> Record;
495 Stream.EnterBlockInfoBlock();
496
497#define BLOCK(X) emitBlockID(X ## _ID, #X, Stream, Record)
498#define RECORD(X) emitRecordID(X, #X, Stream, Record)
499 BLOCK(GLOBAL_INDEX_BLOCK);
500 RECORD(INDEX_METADATA);
501 RECORD(MODULE);
502 RECORD(IDENTIFIER_INDEX);
503#undef RECORD
504#undef BLOCK
505
506 Stream.ExitBlock();
507}
508
509namespace {
510 class InterestingASTIdentifierLookupTrait
511 : public serialization::reader::ASTIdentifierLookupTraitBase {
512
513 public:
514 /// The identifier and whether it is "interesting".
515 typedef std::pair<StringRef, bool> data_type;
516
517 data_type ReadData(const internal_key_type& k,
518 const unsigned char* d,
519 unsigned DataLen) {
520 // The first bit indicates whether this identifier is interesting.
521 // That's all we care about.
522 using namespace llvm::support;
523 unsigned RawID = endian::readNext<uint32_t, little, unaligned>(d);
524 bool IsInteresting = RawID & 0x01;
525 return std::make_pair(k, IsInteresting);
526 }
527 };
528}
529
530llvm::Error GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) {
531 // Open the module file.
532
533 auto Buffer = FileMgr.getBufferForFile(File, /*isVolatile=*/true);
534 if (!Buffer)
535 return llvm::createStringError(Buffer.getError(),
536 "failed getting buffer for module file");
537
538 // Initialize the input stream
539 llvm::BitstreamCursor InStream(PCHContainerRdr.ExtractPCH(**Buffer));
540
541 // Sniff for the signature.
542 for (unsigned char C : {'C', 'P', 'C', 'H'})
543 if (Expected<llvm::SimpleBitstreamCursor::word_t> Res = InStream.Read(8)) {
544 if (Res.get() != C)
545 return llvm::createStringError(std::errc::illegal_byte_sequence,
546 "expected signature CPCH");
547 } else
548 return Res.takeError();
549
550 // Record this module file and assign it a unique ID (if it doesn't have
551 // one already).
552 unsigned ID = getModuleFileInfo(File).ID;
553
554 // Search for the blocks and records we care about.
555 enum { Other, ControlBlock, ASTBlock, DiagnosticOptionsBlock } State = Other;
556 bool Done = false;
557 while (!Done) {
558 Expected<llvm::BitstreamEntry> MaybeEntry = InStream.advance();
559 if (!MaybeEntry)
560 return MaybeEntry.takeError();
561 llvm::BitstreamEntry Entry = MaybeEntry.get();
562
563 switch (Entry.Kind) {
564 case llvm::BitstreamEntry::Error:
565 Done = true;
566 continue;
567
568 case llvm::BitstreamEntry::Record:
569 // In the 'other' state, just skip the record. We don't care.
570 if (State == Other) {
571 if (llvm::Expected<unsigned> Skipped = InStream.skipRecord(Entry.ID))
572 continue;
573 else
574 return Skipped.takeError();
575 }
576
577 // Handle potentially-interesting records below.
578 break;
579
580 case llvm::BitstreamEntry::SubBlock:
581 if (Entry.ID == CONTROL_BLOCK_ID) {
582 if (llvm::Error Err = InStream.EnterSubBlock(CONTROL_BLOCK_ID))
583 return Err;
584
585 // Found the control block.
586 State = ControlBlock;
587 continue;
588 }
589
590 if (Entry.ID == AST_BLOCK_ID) {
591 if (llvm::Error Err = InStream.EnterSubBlock(AST_BLOCK_ID))
592 return Err;
593
594 // Found the AST block.
595 State = ASTBlock;
596 continue;
597 }
598
599 if (Entry.ID == UNHASHED_CONTROL_BLOCK_ID) {
600 if (llvm::Error Err = InStream.EnterSubBlock(UNHASHED_CONTROL_BLOCK_ID))
601 return Err;
602
603 // Found the Diagnostic Options block.
604 State = DiagnosticOptionsBlock;
605 continue;
606 }
607
608 if (llvm::Error Err = InStream.SkipBlock())
609 return Err;
610
611 continue;
612
613 case llvm::BitstreamEntry::EndBlock:
614 State = Other;
615 continue;
616 }
617
618 // Read the given record.
619 SmallVector<uint64_t, 64> Record;
620 StringRef Blob;
621 Expected<unsigned> MaybeCode = InStream.readRecord(Entry.ID, Record, &Blob);
622 if (!MaybeCode)
623 return MaybeCode.takeError();
624 unsigned Code = MaybeCode.get();
625
626 // Handle module dependencies.
627 if (State == ControlBlock && Code == IMPORTS) {
628 // Load each of the imported PCH files.
629 unsigned Idx = 0, N = Record.size();
630 while (Idx < N) {
631 // Read information about the AST file.
632
633 // Skip the imported kind
634 ++Idx;
635
636 // Skip the import location
637 ++Idx;
638
639 // Load stored size/modification time.
640 off_t StoredSize = (off_t)Record[Idx++];
641 time_t StoredModTime = (time_t)Record[Idx++];
642
643 // Skip the stored signature.
644 // FIXME: we could read the signature out of the import and validate it.
645 ASTFileSignature StoredSignature = {
646 {{(uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
647 (uint32_t)Record[Idx++], (uint32_t)Record[Idx++],
648 (uint32_t)Record[Idx++]}}};
649
650 // Skip the module name (currently this is only used for prebuilt
651 // modules while here we are only dealing with cached).
652 Idx += Record[Idx] + 1;
653
654 // Retrieve the imported file name.
655 unsigned Length = Record[Idx++];
656 SmallString<128> ImportedFile(Record.begin() + Idx,
657 Record.begin() + Idx + Length);
658 Idx += Length;
659
660 // Find the imported module file.
661 auto DependsOnFile
662 = FileMgr.getFile(ImportedFile, /*OpenFile=*/false,
663 /*CacheFailure=*/false);
664
665 if (!DependsOnFile)
666 return llvm::createStringError(std::errc::bad_file_descriptor,
667 "imported file \"%s\" not found",
668 ImportedFile.c_str());
669
670 // Save the information in ImportedModuleFileInfo so we can verify after
671 // loading all pcms.
672 ImportedModuleFiles.insert(std::make_pair(
673 *DependsOnFile, ImportedModuleFileInfo(StoredSize, StoredModTime,
674 StoredSignature)));
675
676 // Record the dependency.
677 unsigned DependsOnID = getModuleFileInfo(*DependsOnFile).ID;
678 getModuleFileInfo(File).Dependencies.push_back(DependsOnID);
679 }
680
681 continue;
682 }
683
684 // Handle the identifier table
685 if (State == ASTBlock && Code == IDENTIFIER_TABLE && Record[0] > 0) {
686 typedef llvm::OnDiskIterableChainedHashTable<
687 InterestingASTIdentifierLookupTrait> InterestingIdentifierTable;
688 std::unique_ptr<InterestingIdentifierTable> Table(
689 InterestingIdentifierTable::Create(
690 (const unsigned char *)Blob.data() + Record[0],
691 (const unsigned char *)Blob.data() + sizeof(uint32_t),
692 (const unsigned char *)Blob.data()));
693 for (InterestingIdentifierTable::data_iterator D = Table->data_begin(),
694 DEnd = Table->data_end();
695 D != DEnd; ++D) {
696 std::pair<StringRef, bool> Ident = *D;
697 if (Ident.second)
698 InterestingIdentifiers[Ident.first].push_back(ID);
699 else
700 (void)InterestingIdentifiers[Ident.first];
701 }
702 }
703
704 // Get Signature.
705 if (State == DiagnosticOptionsBlock && Code == SIGNATURE)
706 getModuleFileInfo(File).Signature = {
707 {{(uint32_t)Record[0], (uint32_t)Record[1], (uint32_t)Record[2],
708 (uint32_t)Record[3], (uint32_t)Record[4]}}};
709
710 // We don't care about this record.
711 }
712
713 return llvm::Error::success();
714}
715
716namespace {
717
718/// Trait used to generate the identifier index as an on-disk hash
719/// table.
720class IdentifierIndexWriterTrait {
721public:
722 typedef StringRef key_type;
723 typedef StringRef key_type_ref;
724 typedef SmallVector<unsigned, 2> data_type;
725 typedef const SmallVector<unsigned, 2> &data_type_ref;
726 typedef unsigned hash_value_type;
727 typedef unsigned offset_type;
728
729 static hash_value_type ComputeHash(key_type_ref Key) {
730 return llvm::djbHash(Key);
731 }
732
733 std::pair<unsigned,unsigned>
734 EmitKeyDataLength(raw_ostream& Out, key_type_ref Key, data_type_ref Data) {
735 using namespace llvm::support;
736 endian::Writer LE(Out, little);
737 unsigned KeyLen = Key.size();
738 unsigned DataLen = Data.size() * 4;
739 LE.write<uint16_t>(KeyLen);
740 LE.write<uint16_t>(DataLen);
741 return std::make_pair(KeyLen, DataLen);
742 }
743
744 void EmitKey(raw_ostream& Out, key_type_ref Key, unsigned KeyLen) {
745 Out.write(Key.data(), KeyLen);
746 }
747
748 void EmitData(raw_ostream& Out, key_type_ref Key, data_type_ref Data,
749 unsigned DataLen) {
750 using namespace llvm::support;
751 for (unsigned I = 0, N = Data.size(); I != N; ++I)
752 endian::write<uint32_t>(Out, Data[I], little);
753 }
754};
755
756}
757
758bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) {
759 for (auto MapEntry : ImportedModuleFiles) {
760 auto *File = MapEntry.first;
761 ImportedModuleFileInfo &Info = MapEntry.second;
762 if (getModuleFileInfo(File).Signature) {
763 if (getModuleFileInfo(File).Signature != Info.StoredSignature)
764 // Verify Signature.
765 return true;
766 } else if (Info.StoredSize != File->getSize() ||
767 Info.StoredModTime != File->getModificationTime())
768 // Verify Size and ModTime.
769 return true;
770 }
771
772 using namespace llvm;
773 llvm::TimeTraceScope TimeScope("Module WriteIndex");
774
775 // Emit the file header.
776 Stream.Emit((unsigned)'B', 8);
777 Stream.Emit((unsigned)'C', 8);
778 Stream.Emit((unsigned)'G', 8);
779 Stream.Emit((unsigned)'I', 8);
780
781 // Write the block-info block, which describes the records in this bitcode
782 // file.
783 emitBlockInfoBlock(Stream);
784
785 Stream.EnterSubblock(GLOBAL_INDEX_BLOCK_ID, 3);
786
787 // Write the metadata.
788 SmallVector<uint64_t, 2> Record;
789 Record.push_back(CurrentVersion);
790 Stream.EmitRecord(INDEX_METADATA, Record);
791
792 // Write the set of known module files.
793 for (ModuleFilesMap::iterator M = ModuleFiles.begin(),
794 MEnd = ModuleFiles.end();
795 M != MEnd; ++M) {
796 Record.clear();
797 Record.push_back(M->second.ID);
798 Record.push_back(M->first->getSize());
799 Record.push_back(M->first->getModificationTime());
800
801 // File name
802 StringRef Name(M->first->getName());
803 Record.push_back(Name.size());
804 Record.append(Name.begin(), Name.end());
805
806 // Dependencies
807 Record.push_back(M->second.Dependencies.size());
808 Record.append(M->second.Dependencies.begin(), M->second.Dependencies.end());
809 Stream.EmitRecord(MODULE, Record);
810 }
811
812 // Write the identifier -> module file mapping.
813 {
814 llvm::OnDiskChainedHashTableGenerator<IdentifierIndexWriterTrait> Generator;
815 IdentifierIndexWriterTrait Trait;
816
817 // Populate the hash table.
818 for (InterestingIdentifierMap::iterator I = InterestingIdentifiers.begin(),
819 IEnd = InterestingIdentifiers.end();
820 I != IEnd; ++I) {
821 Generator.insert(I->first(), I->second, Trait);
822 }
823
824 // Create the on-disk hash table in a buffer.
825 SmallString<4096> IdentifierTable;
826 uint32_t BucketOffset;
827 {
828 using namespace llvm::support;
829 llvm::raw_svector_ostream Out(IdentifierTable);
830 // Make sure that no bucket is at offset 0
831 endian::write<uint32_t>(Out, 0, little);
832 BucketOffset = Generator.Emit(Out, Trait);
833 }
834
835 // Create a blob abbreviation
836 auto Abbrev = std::make_shared<BitCodeAbbrev>();
837 Abbrev->Add(BitCodeAbbrevOp(IDENTIFIER_INDEX));
838 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
839 Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
840 unsigned IDTableAbbrev = Stream.EmitAbbrev(std::move(Abbrev));
841
842 // Write the identifier table
843 uint64_t Record[] = {IDENTIFIER_INDEX, BucketOffset};
844 Stream.EmitRecordWithBlob(IDTableAbbrev, Record, IdentifierTable);
845 }
846
847 Stream.ExitBlock();
848 return false;
849}
850
851llvm::Error
852GlobalModuleIndex::writeIndex(FileManager &FileMgr,
853 const PCHContainerReader &PCHContainerRdr,
854 StringRef Path) {
855 llvm::SmallString<128> IndexPath;
856 IndexPath += Path;
857 llvm::sys::path::append(IndexPath, IndexFileName);
858
859 // Coordinate building the global index file with other processes that might
860 // try to do the same.
861 llvm::LockFileManager Locked(IndexPath);
862 switch (Locked) {
863 case llvm::LockFileManager::LFS_Error:
864 return llvm::createStringError(std::errc::io_error, "LFS error");
865
866 case llvm::LockFileManager::LFS_Owned:
867 // We're responsible for building the index ourselves. Do so below.
868 break;
869
870 case llvm::LockFileManager::LFS_Shared:
871 // Someone else is responsible for building the index. We don't care
872 // when they finish, so we're done.
873 return llvm::createStringError(std::errc::device_or_resource_busy,
874 "someone else is building the index");
875 }
876
877 // The module index builder.
878 GlobalModuleIndexBuilder Builder(FileMgr, PCHContainerRdr);
879
880 // Load each of the module files.
881 std::error_code EC;
882 for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd;
883 D != DEnd && !EC;
884 D.increment(EC)) {
885 // If this isn't a module file, we don't care.
886 if (llvm::sys::path::extension(D->path()) != ".pcm") {
887 // ... unless it's a .pcm.lock file, which indicates that someone is
888 // in the process of rebuilding a module. They'll rebuild the index
889 // at the end of that translation unit, so we don't have to.
890 if (llvm::sys::path::extension(D->path()) == ".pcm.lock")
891 return llvm::createStringError(std::errc::device_or_resource_busy,
892 "someone else is building the index");
893
894 continue;
895 }
896
897 // If we can't find the module file, skip it.
898 auto ModuleFile = FileMgr.getFile(D->path());
899 if (!ModuleFile)
900 continue;
901
902 // Load this module file.
903 if (llvm::Error Err = Builder.loadModuleFile(*ModuleFile))
904 return Err;
905 }
906
907 // The output buffer, into which the global index will be written.
908 SmallVector<char, 16> OutputBuffer;
909 {
910 llvm::BitstreamWriter OutputStream(OutputBuffer);
911 if (Builder.writeIndex(OutputStream))
912 return llvm::createStringError(std::errc::io_error,
913 "failed writing index");
914 }
915
916 return llvm::writeFileAtomically(
917 (IndexPath + "-%%%%%%%%").str(), IndexPath,
918 llvm::StringRef(OutputBuffer.data(), OutputBuffer.size()));
919}
920
921namespace {
922 class GlobalIndexIdentifierIterator : public IdentifierIterator {
923 /// The current position within the identifier lookup table.
924 IdentifierIndexTable::key_iterator Current;
925
926 /// The end position within the identifier lookup table.
927 IdentifierIndexTable::key_iterator End;
928
929 public:
930 explicit GlobalIndexIdentifierIterator(IdentifierIndexTable &Idx) {
931 Current = Idx.key_begin();
932 End = Idx.key_end();
933 }
934
935 StringRef Next() override {
936 if (Current == End)
937 return StringRef();
938
939 StringRef Result = *Current;
940 ++Current;
941 return Result;
942 }
943 };
944}
945
946IdentifierIterator *GlobalModuleIndex::createIdentifierIterator() const {
947 IdentifierIndexTable &Table =
948 *static_cast<IdentifierIndexTable *>(IdentifierIndex);
949 return new GlobalIndexIdentifierIterator(Table);
950}