clang  5.0.0
ModuleManager.cpp
Go to the documentation of this file.
1 //===--- ModuleManager.cpp - Module Manager ---------------------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines the ModuleManager class, which manages a set of loaded
11 // modules for the ASTReader.
12 //
13 //===----------------------------------------------------------------------===//
17 #include "clang/Lex/HeaderSearch.h"
18 #include "clang/Lex/ModuleMap.h"
20 #include "llvm/Support/MemoryBuffer.h"
21 #include "llvm/Support/Path.h"
22 #include <system_error>
23 
24 #ifndef NDEBUG
25 #include "llvm/Support/GraphWriter.h"
26 #endif
27 
28 using namespace clang;
29 using namespace serialization;
30 
32  const FileEntry *Entry = FileMgr.getFile(Name, /*openFile=*/false,
33  /*cacheFailure=*/false);
34  if (Entry)
35  return lookup(Entry);
36 
37  return nullptr;
38 }
39 
41  auto Known = Modules.find(File);
42  if (Known == Modules.end())
43  return nullptr;
44 
45  return Known->second;
46 }
47 
48 std::unique_ptr<llvm::MemoryBuffer>
50  const FileEntry *Entry = FileMgr.getFile(Name, /*openFile=*/false,
51  /*cacheFailure=*/false);
52  return std::move(InMemoryBuffers[Entry]);
53 }
54 
55 static bool checkSignature(ASTFileSignature Signature,
56  ASTFileSignature ExpectedSignature,
57  std::string &ErrorStr) {
58  if (!ExpectedSignature || Signature == ExpectedSignature)
59  return false;
60 
61  ErrorStr =
62  Signature ? "signature mismatch" : "could not read module signature";
63  return true;
64 }
65 
66 static void updateModuleImports(ModuleFile &MF, ModuleFile *ImportedBy,
67  SourceLocation ImportLoc) {
68  if (ImportedBy) {
69  MF.ImportedBy.insert(ImportedBy);
70  ImportedBy->Imports.insert(&MF);
71  } else {
72  if (!MF.DirectlyImported)
73  MF.ImportLoc = ImportLoc;
74 
75  MF.DirectlyImported = true;
76  }
77 }
78 
81  SourceLocation ImportLoc, ModuleFile *ImportedBy,
82  unsigned Generation,
83  off_t ExpectedSize, time_t ExpectedModTime,
84  ASTFileSignature ExpectedSignature,
85  ASTFileSignatureReader ReadSignature,
87  std::string &ErrorStr) {
88  Module = nullptr;
89 
90  // Look for the file entry. This only fails if the expected size or
91  // modification time differ.
92  const FileEntry *Entry;
93  if (Type == MK_ExplicitModule || Type == MK_PrebuiltModule) {
94  // If we're not expecting to pull this file out of the module cache, it
95  // might have a different mtime due to being moved across filesystems in
96  // a distributed build. The size must still match, though. (As must the
97  // contents, but we can't check that.)
98  ExpectedModTime = 0;
99  }
100  if (lookupModuleFile(FileName, ExpectedSize, ExpectedModTime, Entry)) {
101  ErrorStr = "module file out of date";
102  return OutOfDate;
103  }
104 
105  if (!Entry && FileName != "-") {
106  ErrorStr = "module file not found";
107  return Missing;
108  }
109 
110  // Check whether we already loaded this module, before
111  if (ModuleFile *ModuleEntry = Modules.lookup(Entry)) {
112  // Check the stored signature.
113  if (checkSignature(ModuleEntry->Signature, ExpectedSignature, ErrorStr))
114  return OutOfDate;
115 
116  Module = ModuleEntry;
117  updateModuleImports(*ModuleEntry, ImportedBy, ImportLoc);
118  return AlreadyLoaded;
119  }
120 
121  // Allocate a new module.
122  auto NewModule = llvm::make_unique<ModuleFile>(Type, Generation);
123  NewModule->Index = Chain.size();
124  NewModule->FileName = FileName.str();
125  NewModule->File = Entry;
126  NewModule->ImportLoc = ImportLoc;
127  NewModule->InputFilesValidationTimestamp = 0;
128 
129  if (NewModule->Kind == MK_ImplicitModule) {
130  std::string TimestampFilename = NewModule->getTimestampFilename();
132  // A cached stat value would be fine as well.
133  if (!FileMgr.getNoncachedStatValue(TimestampFilename, Status))
134  NewModule->InputFilesValidationTimestamp =
135  llvm::sys::toTimeT(Status.getLastModificationTime());
136  }
137 
138  // Load the contents of the module
139  if (std::unique_ptr<llvm::MemoryBuffer> Buffer = lookupBuffer(FileName)) {
140  // The buffer was already provided for us.
141  NewModule->Buffer = &PCMCache->addBuffer(FileName, std::move(Buffer));
142  } else if (llvm::MemoryBuffer *Buffer = PCMCache->lookupBuffer(FileName)) {
143  NewModule->Buffer = Buffer;
144  } else {
145  // Open the AST file.
146  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buf((std::error_code()));
147  if (FileName == "-") {
148  Buf = llvm::MemoryBuffer::getSTDIN();
149  } else {
150  // Leave the FileEntry open so if it gets read again by another
151  // ModuleManager it must be the same underlying file.
152  // FIXME: Because FileManager::getFile() doesn't guarantee that it will
153  // give us an open file, this may not be 100% reliable.
154  Buf = FileMgr.getBufferForFile(NewModule->File,
155  /*IsVolatile=*/false,
156  /*ShouldClose=*/false);
157  }
158 
159  if (!Buf) {
160  ErrorStr = Buf.getError().message();
161  return Missing;
162  }
163 
164  NewModule->Buffer = &PCMCache->addBuffer(FileName, std::move(*Buf));
165  }
166 
167  // Initialize the stream.
168  NewModule->Data = PCHContainerRdr.ExtractPCH(*NewModule->Buffer);
169 
170  // Read the signature eagerly now so that we can check it. Avoid calling
171  // ReadSignature unless there's something to check though.
172  if (ExpectedSignature && checkSignature(ReadSignature(NewModule->Data),
173  ExpectedSignature, ErrorStr)) {
174  // Try to remove the buffer. If it can't be removed, then it was already
175  // validated by this process.
176  if (!PCMCache->tryToRemoveBuffer(NewModule->FileName))
177  FileMgr.invalidateCache(NewModule->File);
178  return OutOfDate;
179  }
180 
181  // We're keeping this module. Store it everywhere.
182  Module = Modules[Entry] = NewModule.get();
183 
184  updateModuleImports(*NewModule, ImportedBy, ImportLoc);
185 
186  if (!NewModule->isModule())
187  PCHChain.push_back(NewModule.get());
188  if (!ImportedBy)
189  Roots.push_back(NewModule.get());
190 
191  Chain.push_back(std::move(NewModule));
192  return NewlyLoaded;
193 }
194 
196  ModuleIterator First,
197  llvm::SmallPtrSetImpl<ModuleFile *> &LoadedSuccessfully,
198  ModuleMap *modMap) {
199  auto Last = end();
200  if (First == Last)
201  return;
202 
203 
204  // Explicitly clear VisitOrder since we might not notice it is stale.
205  VisitOrder.clear();
206 
207  // Collect the set of module file pointers that we'll be removing.
208  llvm::SmallPtrSet<ModuleFile *, 4> victimSet(
209  (llvm::pointer_iterator<ModuleIterator>(First)),
210  (llvm::pointer_iterator<ModuleIterator>(Last)));
211 
212  auto IsVictim = [&](ModuleFile *MF) {
213  return victimSet.count(MF);
214  };
215  // Remove any references to the now-destroyed modules.
216  for (auto I = begin(); I != First; ++I) {
217  I->Imports.remove_if(IsVictim);
218  I->ImportedBy.remove_if(IsVictim);
219  }
220  Roots.erase(std::remove_if(Roots.begin(), Roots.end(), IsVictim),
221  Roots.end());
222 
223  // Remove the modules from the PCH chain.
224  for (auto I = First; I != Last; ++I) {
225  if (!I->isModule()) {
226  PCHChain.erase(std::find(PCHChain.begin(), PCHChain.end(), &*I),
227  PCHChain.end());
228  break;
229  }
230  }
231 
232  // Delete the modules and erase them from the various structures.
233  for (ModuleIterator victim = First; victim != Last; ++victim) {
234  Modules.erase(victim->File);
235 
236  if (modMap) {
237  StringRef ModuleName = victim->ModuleName;
238  if (Module *mod = modMap->findModule(ModuleName)) {
239  mod->setASTFile(nullptr);
240  }
241  }
242 
243  // Files that didn't make it through ReadASTCore successfully will be
244  // rebuilt (or there was an error). Invalidate them so that we can load the
245  // new files that will be renamed over the old ones.
246  //
247  // The PCMCache tracks whether the module was successfully loaded in another
248  // thread/context; in that case, it won't need to be rebuilt (and we can't
249  // safely invalidate it anyway).
250  if (LoadedSuccessfully.count(&*victim) == 0 &&
251  !PCMCache->tryToRemoveBuffer(victim->FileName))
252  FileMgr.invalidateCache(victim->File);
253  }
254 
255  // Delete the modules.
256  Chain.erase(Chain.begin() + (First - begin()), Chain.end());
257 }
258 
259 void
261  std::unique_ptr<llvm::MemoryBuffer> Buffer) {
262 
263  const FileEntry *Entry =
264  FileMgr.getVirtualFile(FileName, Buffer->getBufferSize(), 0);
265  InMemoryBuffers[Entry] = std::move(Buffer);
266 }
267 
268 ModuleManager::VisitState *ModuleManager::allocateVisitState() {
269  // Fast path: if we have a cached state, use it.
270  if (FirstVisitState) {
271  VisitState *Result = FirstVisitState;
272  FirstVisitState = FirstVisitState->NextState;
273  Result->NextState = nullptr;
274  return Result;
275  }
276 
277  // Allocate and return a new state.
278  return new VisitState(size());
279 }
280 
281 void ModuleManager::returnVisitState(VisitState *State) {
282  assert(State->NextState == nullptr && "Visited state is in list?");
283  State->NextState = FirstVisitState;
284  FirstVisitState = State;
285 }
286 
288  GlobalIndex = Index;
289  if (!GlobalIndex) {
290  ModulesInCommonWithGlobalIndex.clear();
291  return;
292  }
293 
294  // Notify the global module index about all of the modules we've already
295  // loaded.
296  for (ModuleFile &M : *this)
297  if (!GlobalIndex->loadedModuleFile(&M))
298  ModulesInCommonWithGlobalIndex.push_back(&M);
299 }
300 
302  if (!GlobalIndex || GlobalIndex->loadedModuleFile(MF))
303  return;
304 
305  ModulesInCommonWithGlobalIndex.push_back(MF);
306 }
307 
309  const PCHContainerReader &PCHContainerRdr)
310  : FileMgr(FileMgr), PCMCache(&PCMCache), PCHContainerRdr(PCHContainerRdr),
311  GlobalIndex(), FirstVisitState(nullptr) {}
312 
313 ModuleManager::~ModuleManager() { delete FirstVisitState; }
314 
315 void ModuleManager::visit(llvm::function_ref<bool(ModuleFile &M)> Visitor,
316  llvm::SmallPtrSetImpl<ModuleFile *> *ModuleFilesHit) {
317  // If the visitation order vector is the wrong size, recompute the order.
318  if (VisitOrder.size() != Chain.size()) {
319  unsigned N = size();
320  VisitOrder.clear();
321  VisitOrder.reserve(N);
322 
323  // Record the number of incoming edges for each module. When we
324  // encounter a module with no incoming edges, push it into the queue
325  // to seed the queue.
327  Queue.reserve(N);
328  llvm::SmallVector<unsigned, 4> UnusedIncomingEdges;
329  UnusedIncomingEdges.resize(size());
330  for (ModuleFile &M : llvm::reverse(*this)) {
331  unsigned Size = M.ImportedBy.size();
332  UnusedIncomingEdges[M.Index] = Size;
333  if (!Size)
334  Queue.push_back(&M);
335  }
336 
337  // Traverse the graph, making sure to visit a module before visiting any
338  // of its dependencies.
339  while (!Queue.empty()) {
340  ModuleFile *CurrentModule = Queue.pop_back_val();
341  VisitOrder.push_back(CurrentModule);
342 
343  // For any module that this module depends on, push it on the
344  // stack (if it hasn't already been marked as visited).
345  for (auto M = CurrentModule->Imports.rbegin(),
346  MEnd = CurrentModule->Imports.rend();
347  M != MEnd; ++M) {
348  // Remove our current module as an impediment to visiting the
349  // module we depend on. If we were the last unvisited module
350  // that depends on this particular module, push it into the
351  // queue to be visited.
352  unsigned &NumUnusedEdges = UnusedIncomingEdges[(*M)->Index];
353  if (NumUnusedEdges && (--NumUnusedEdges == 0))
354  Queue.push_back(*M);
355  }
356  }
357 
358  assert(VisitOrder.size() == N && "Visitation order is wrong?");
359 
360  delete FirstVisitState;
361  FirstVisitState = nullptr;
362  }
363 
364  VisitState *State = allocateVisitState();
365  unsigned VisitNumber = State->NextVisitNumber++;
366 
367  // If the caller has provided us with a hit-set that came from the global
368  // module index, mark every module file in common with the global module
369  // index that is *not* in that set as 'visited'.
370  if (ModuleFilesHit && !ModulesInCommonWithGlobalIndex.empty()) {
371  for (unsigned I = 0, N = ModulesInCommonWithGlobalIndex.size(); I != N; ++I)
372  {
373  ModuleFile *M = ModulesInCommonWithGlobalIndex[I];
374  if (!ModuleFilesHit->count(M))
375  State->VisitNumber[M->Index] = VisitNumber;
376  }
377  }
378 
379  for (unsigned I = 0, N = VisitOrder.size(); I != N; ++I) {
380  ModuleFile *CurrentModule = VisitOrder[I];
381  // Should we skip this module file?
382  if (State->VisitNumber[CurrentModule->Index] == VisitNumber)
383  continue;
384 
385  // Visit the module.
386  assert(State->VisitNumber[CurrentModule->Index] == VisitNumber - 1);
387  State->VisitNumber[CurrentModule->Index] = VisitNumber;
388  if (!Visitor(*CurrentModule))
389  continue;
390 
391  // The visitor has requested that cut off visitation of any
392  // module that the current module depends on. To indicate this
393  // behavior, we mark all of the reachable modules as having been visited.
394  ModuleFile *NextModule = CurrentModule;
395  do {
396  // For any module that this module depends on, push it on the
397  // stack (if it hasn't already been marked as visited).
398  for (llvm::SetVector<ModuleFile *>::iterator
399  M = NextModule->Imports.begin(),
400  MEnd = NextModule->Imports.end();
401  M != MEnd; ++M) {
402  if (State->VisitNumber[(*M)->Index] != VisitNumber) {
403  State->Stack.push_back(*M);
404  State->VisitNumber[(*M)->Index] = VisitNumber;
405  }
406  }
407 
408  if (State->Stack.empty())
409  break;
410 
411  // Pop the next module off the stack.
412  NextModule = State->Stack.pop_back_val();
413  } while (true);
414  }
415 
416  returnVisitState(State);
417 }
418 
420  off_t ExpectedSize,
421  time_t ExpectedModTime,
422  const FileEntry *&File) {
423  if (FileName == "-") {
424  File = nullptr;
425  return false;
426  }
427 
428  // Open the file immediately to ensure there is no race between stat'ing and
429  // opening the file.
430  File = FileMgr.getFile(FileName, /*openFile=*/true, /*cacheFailure=*/false);
431  if (!File)
432  return false;
433 
434  if ((ExpectedSize && ExpectedSize != File->getSize()) ||
435  (ExpectedModTime && ExpectedModTime != File->getModificationTime()))
436  // Do not destroy File, as it may be referenced. If we need to rebuild it,
437  // it will be destroyed by removeModules.
438  return true;
439 
440  return false;
441 }
442 
443 #ifndef NDEBUG
444 namespace llvm {
445  template<>
446  struct GraphTraits<ModuleManager> {
447  typedef ModuleFile *NodeRef;
448  typedef llvm::SetVector<ModuleFile *>::const_iterator ChildIteratorType;
449  typedef pointer_iterator<ModuleManager::ModuleConstIterator> nodes_iterator;
450 
452  return Node->Imports.begin();
453  }
454 
456  return Node->Imports.end();
457  }
458 
459  static nodes_iterator nodes_begin(const ModuleManager &Manager) {
460  return nodes_iterator(Manager.begin());
461  }
462 
463  static nodes_iterator nodes_end(const ModuleManager &Manager) {
464  return nodes_iterator(Manager.end());
465  }
466  };
467 
468  template<>
469  struct DOTGraphTraits<ModuleManager> : public DefaultDOTGraphTraits {
470  explicit DOTGraphTraits(bool IsSimple = false)
471  : DefaultDOTGraphTraits(IsSimple) { }
472 
473  static bool renderGraphFromBottomUp() {
474  return true;
475  }
476 
477  std::string getNodeLabel(ModuleFile *M, const ModuleManager&) {
478  return M->ModuleName;
479  }
480  };
481 }
482 
484  llvm::ViewGraph(*this, "Modules");
485 }
486 #endif
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:116
virtual StringRef ExtractPCH(llvm::MemoryBufferRef Buffer) const =0
Returns the serialized AST inside the PCH container Buffer.
Module * findModule(StringRef Name) const
Retrieve a module with the given name.
Definition: ModuleMap.cpp:705
unsigned size() const
Number of modules loaded.
std::string ModuleName
The name of the module.
Definition: Module.h:118
The module file is out-of-date.
The base class of the type hierarchy.
Definition: Type.h:1303
unsigned Index
The index of this module in the list of modules.
Definition: Module.h:109
std::unique_ptr< llvm::MemoryBuffer > Buffer
void visit(llvm::function_ref< bool(ModuleFile &M)> Visitor, llvm::SmallPtrSetImpl< ModuleFile * > *ModuleFilesHit=nullptr)
Visit each of the modules.
Manages the set of modules loaded by an AST reader.
Definition: ModuleManager.h:34
Manage memory buffers across multiple users.
off_t getSize() const
Definition: FileManager.h:87
SourceLocation ImportLoc
The source location where this module was first imported.
Definition: Module.h:194
AddModuleResult
The result of attempting to add a new module.
ModuleIterator begin()
Forward iterator to traverse all loaded modules.
LineState State
void viewGraph()
View the graphviz representation of the module graph.
The signature of a module, which is a hash of the AST content.
Definition: Module.h:47
void setGlobalIndex(GlobalModuleIndex *Index)
Set the global module index.
Describes a module or submodule.
Definition: Module.h:57
bool lookupModuleFile(StringRef FileName, off_t ExpectedSize, time_t ExpectedModTime, const FileEntry *&File)
Attempt to resolve the given module file name to a file entry.
ModuleIterator end()
Forward iterator end-point to traverse all loaded modules.
llvm::sys::TimePoint getLastModificationTime() const
The result of a status operation.
This abstract interface provides operations for unwrapping containers for serialized ASTs (precompile...
detail::InMemoryDirectory::const_iterator I
const FileEntry * getFile(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Lookup, cache, and verify the specified file (real or virtual).
ModuleKind
Specifies the kind of module that has been loaded.
Definition: Module.h:47
File is from a prebuilt module path.
Definition: Module.h:53
llvm::pointee_iterator< SmallVectorImpl< std::unique_ptr< ModuleFile > >::iterator > ModuleIterator
static nodes_iterator nodes_end(const ModuleManager &Manager)
Information about a module that has been loaded by the ASTReader.
Definition: Module.h:100
AddModuleResult addModule(StringRef FileName, ModuleKind Type, SourceLocation ImportLoc, ModuleFile *ImportedBy, unsigned Generation, off_t ExpectedSize, time_t ExpectedModTime, ASTFileSignature ExpectedSignature, ASTFileSignatureReader ReadSignature, ModuleFile *&Module, std::string &ErrorStr)
Attempts to create a new module and add it to the list of known modules.
void addInMemoryBuffer(StringRef FileName, std::unique_ptr< llvm::MemoryBuffer > Buffer)
Add an in-memory buffer the list of known buffers.
static nodes_iterator nodes_begin(const ModuleManager &Manager)
StringRef FileName
Definition: Format.cpp:1465
Encodes a location in the source.
File is an implicitly-loaded module.
Definition: Module.h:48
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
static ChildIteratorType child_begin(NodeRef Node)
ModuleManager(FileManager &FileMgr, MemoryBufferCache &PCMCache, const PCHContainerReader &PCHContainerRdr)
llvm::SetVector< ModuleFile * >::const_iterator ChildIteratorType
const FileEntry * getVirtualFile(StringRef Filename, off_t Size, time_t ModificationTime)
Retrieve a file entry for a "virtual" file that acts as if there were a file with the given name on d...
llvm::SetVector< ModuleFile * > ImportedBy
List of modules which depend on this module.
Definition: Module.h:448
A global index for a set of module files, providing information about the identifiers within those mo...
The module file was just loaded in response to this call.
ast_type_traits::DynTypedNode Node
void removeModules(ModuleIterator First, llvm::SmallPtrSetImpl< ModuleFile * > &LoadedSuccessfully, ModuleMap *modMap)
Remove the modules starting from First (to the end).
pointer_iterator< ModuleManager::ModuleConstIterator > nodes_iterator
StringRef Name
Definition: USRFinder.cpp:123
File is an explicitly-loaded module.
Definition: Module.h:49
llvm::ErrorOr< std::unique_ptr< llvm::MemoryBuffer > > getBufferForFile(const FileEntry *Entry, bool isVolatile=false, bool ShouldCloseOpenFile=true)
Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful, otherwise returning null.
static void updateModuleImports(ModuleFile &MF, ModuleFile *ImportedBy, SourceLocation ImportLoc)
void moduleFileAccepted(ModuleFile *MF)
Notification from the AST reader that the given module file has been "accepted", and will not (can no...
time_t getModificationTime() const
Definition: FileManager.h:91
std::string getNodeLabel(ModuleFile *M, const ModuleManager &)
ModuleFile * lookup(StringRef Name) const
Returns the module associated with the given name.
bool getNoncachedStatValue(StringRef Path, vfs::Status &Result)
Get the 'stat' information for the given Path.
The module file had already been loaded.
void invalidateCache(const FileEntry *Entry)
Remove the real file Entry from the cache.
FormattingAttemptStatus * Status
Definition: Format.cpp:1073
static ChildIteratorType child_end(NodeRef Node)
bool DirectlyImported
Whether this module has been directly imported by the user.
Definition: Module.h:161
llvm::SetVector< ModuleFile * > Imports
List of modules which this module depends on.
Definition: Module.h:451
const NamedDecl * Result
Definition: USRFinder.cpp:70
std::unique_ptr< llvm::MemoryBuffer > lookupBuffer(StringRef Name)
Returns the in-memory (virtual file) buffer with the given name.
bool loadedModuleFile(ModuleFile *File)
Note that the given module file has been loaded.
static bool checkSignature(ASTFileSignature Signature, ASTFileSignature ExpectedSignature, std::string &ErrorStr)