clang  5.0.0
PPLexerChange.cpp
Go to the documentation of this file.
1 //===--- PPLexerChange.cpp - Handle changing lexers in the preprocessor ---===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements pieces of the Preprocessor interface that manage the
11 // current lexer stack.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "clang/Lex/Preprocessor.h"
18 #include "clang/Lex/HeaderSearch.h"
20 #include "clang/Lex/MacroInfo.h"
21 #include "clang/Lex/PTHManager.h"
22 #include "llvm/ADT/StringSwitch.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/MemoryBuffer.h"
25 #include "llvm/Support/Path.h"
26 using namespace clang;
27 
29 
30 //===----------------------------------------------------------------------===//
31 // Miscellaneous Methods.
32 //===----------------------------------------------------------------------===//
33 
34 /// isInPrimaryFile - Return true if we're in the top-level file, not in a
35 /// \#include. This looks through macro expansions and active _Pragma lexers.
37  if (IsFileLexer())
38  return IncludeMacroStack.empty();
39 
40  // If there are any stacked lexers, we're in a #include.
41  assert(IsFileLexer(IncludeMacroStack[0]) &&
42  "Top level include stack isn't our primary lexer?");
43  return std::none_of(IncludeMacroStack.begin() + 1, IncludeMacroStack.end(),
44  [this](const IncludeStackInfo &ISI) -> bool {
45  return IsFileLexer(ISI);
46  });
47 }
48 
49 /// getCurrentLexer - Return the current file lexer being lexed from. Note
50 /// that this ignores any potentially active macro expansions and _Pragma
51 /// expansions going on at the time.
53  if (IsFileLexer())
54  return CurPPLexer;
55 
56  // Look for a stacked lexer.
57  for (const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
58  if (IsFileLexer(ISI))
59  return ISI.ThePPLexer;
60  }
61  return nullptr;
62 }
63 
64 
65 //===----------------------------------------------------------------------===//
66 // Methods for Entering and Callbacks for leaving various contexts
67 //===----------------------------------------------------------------------===//
68 
69 /// EnterSourceFile - Add a source file to the top of the include stack and
70 /// start lexing tokens from it instead of the current buffer.
72  SourceLocation Loc) {
73  assert(!CurTokenLexer && "Cannot #include a file inside a macro!");
74  ++NumEnteredSourceFiles;
75 
76  if (MaxIncludeStackDepth < IncludeMacroStack.size())
77  MaxIncludeStackDepth = IncludeMacroStack.size();
78 
79  if (PTH) {
80  if (PTHLexer *PL = PTH->CreateLexer(FID)) {
81  EnterSourceFileWithPTH(PL, CurDir);
82  return false;
83  }
84  }
85 
86  // Get the MemoryBuffer for this FID, if it fails, we fail.
87  bool Invalid = false;
88  const llvm::MemoryBuffer *InputFile =
89  getSourceManager().getBuffer(FID, Loc, &Invalid);
90  if (Invalid) {
91  SourceLocation FileStart = SourceMgr.getLocForStartOfFile(FID);
92  Diag(Loc, diag::err_pp_error_opening_file)
93  << std::string(SourceMgr.getBufferName(FileStart)) << "";
94  return true;
95  }
96 
98  SourceMgr.getFileEntryForID(FID) == CodeCompletionFile) {
99  CodeCompletionFileLoc = SourceMgr.getLocForStartOfFile(FID);
100  CodeCompletionLoc =
101  CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset);
102  }
103 
104  EnterSourceFileWithLexer(new Lexer(FID, InputFile, *this), CurDir);
105  return false;
106 }
107 
108 /// EnterSourceFileWithLexer - Add a source file to the top of the include stack
109 /// and start lexing tokens from it instead of the current buffer.
110 void Preprocessor::EnterSourceFileWithLexer(Lexer *TheLexer,
111  const DirectoryLookup *CurDir) {
112 
113  // Add the current lexer to the include stack.
114  if (CurPPLexer || CurTokenLexer)
115  PushIncludeMacroStack();
116 
117  CurLexer.reset(TheLexer);
118  CurPPLexer = TheLexer;
119  CurDirLookup = CurDir;
120  CurLexerSubmodule = nullptr;
121  if (CurLexerKind != CLK_LexAfterModuleImport)
122  CurLexerKind = CLK_Lexer;
123 
124  // Notify the client, if desired, that we are in a new source file.
125  if (Callbacks && !CurLexer->Is_PragmaLexer) {
126  SrcMgr::CharacteristicKind FileType =
127  SourceMgr.getFileCharacteristic(CurLexer->getFileLoc());
128 
129  Callbacks->FileChanged(CurLexer->getFileLoc(),
130  PPCallbacks::EnterFile, FileType);
131  }
132 }
133 
134 /// EnterSourceFileWithPTH - Add a source file to the top of the include stack
135 /// and start getting tokens from it using the PTH cache.
136 void Preprocessor::EnterSourceFileWithPTH(PTHLexer *PL,
137  const DirectoryLookup *CurDir) {
138 
139  if (CurPPLexer || CurTokenLexer)
140  PushIncludeMacroStack();
141 
142  CurDirLookup = CurDir;
143  CurPTHLexer.reset(PL);
144  CurPPLexer = CurPTHLexer.get();
145  CurLexerSubmodule = nullptr;
146  if (CurLexerKind != CLK_LexAfterModuleImport)
147  CurLexerKind = CLK_PTHLexer;
148 
149  // Notify the client, if desired, that we are in a new source file.
150  if (Callbacks) {
151  FileID FID = CurPPLexer->getFileID();
152  SourceLocation EnterLoc = SourceMgr.getLocForStartOfFile(FID);
153  SrcMgr::CharacteristicKind FileType =
154  SourceMgr.getFileCharacteristic(EnterLoc);
155  Callbacks->FileChanged(EnterLoc, PPCallbacks::EnterFile, FileType);
156  }
157 }
158 
159 /// EnterMacro - Add a Macro to the top of the include stack and start lexing
160 /// tokens from it instead of the current buffer.
162  MacroInfo *Macro, MacroArgs *Args) {
163  std::unique_ptr<TokenLexer> TokLexer;
164  if (NumCachedTokenLexers == 0) {
165  TokLexer = llvm::make_unique<TokenLexer>(Tok, ILEnd, Macro, Args, *this);
166  } else {
167  TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
168  TokLexer->Init(Tok, ILEnd, Macro, Args);
169  }
170 
171  PushIncludeMacroStack();
172  CurDirLookup = nullptr;
173  CurTokenLexer = std::move(TokLexer);
174  if (CurLexerKind != CLK_LexAfterModuleImport)
175  CurLexerKind = CLK_TokenLexer;
176 }
177 
178 /// EnterTokenStream - Add a "macro" context to the top of the include stack,
179 /// which will cause the lexer to start returning the specified tokens.
180 ///
181 /// If DisableMacroExpansion is true, tokens lexed from the token stream will
182 /// not be subject to further macro expansion. Otherwise, these tokens will
183 /// be re-macro-expanded when/if expansion is enabled.
184 ///
185 /// If OwnsTokens is false, this method assumes that the specified stream of
186 /// tokens has a permanent owner somewhere, so they do not need to be copied.
187 /// If it is true, it assumes the array of tokens is allocated with new[] and
188 /// must be freed.
189 ///
190 void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
191  bool DisableMacroExpansion,
192  bool OwnsTokens) {
193  if (CurLexerKind == CLK_CachingLexer) {
194  if (CachedLexPos < CachedTokens.size()) {
195  // We're entering tokens into the middle of our cached token stream. We
196  // can't represent that, so just insert the tokens into the buffer.
197  CachedTokens.insert(CachedTokens.begin() + CachedLexPos,
198  Toks, Toks + NumToks);
199  if (OwnsTokens)
200  delete [] Toks;
201  return;
202  }
203 
204  // New tokens are at the end of the cached token sequnece; insert the
205  // token stream underneath the caching lexer.
206  ExitCachingLexMode();
207  EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
208  EnterCachingLexMode();
209  return;
210  }
211 
212  // Create a macro expander to expand from the specified token stream.
213  std::unique_ptr<TokenLexer> TokLexer;
214  if (NumCachedTokenLexers == 0) {
215  TokLexer = llvm::make_unique<TokenLexer>(
216  Toks, NumToks, DisableMacroExpansion, OwnsTokens, *this);
217  } else {
218  TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
219  TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
220  }
221 
222  // Save our current state.
223  PushIncludeMacroStack();
224  CurDirLookup = nullptr;
225  CurTokenLexer = std::move(TokLexer);
226  if (CurLexerKind != CLK_LexAfterModuleImport)
227  CurLexerKind = CLK_TokenLexer;
228 }
229 
230 /// \brief Compute the relative path that names the given file relative to
231 /// the given directory.
232 static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir,
233  const FileEntry *File,
234  SmallString<128> &Result) {
235  Result.clear();
236 
237  StringRef FilePath = File->getDir()->getName();
238  StringRef Path = FilePath;
239  while (!Path.empty()) {
240  if (const DirectoryEntry *CurDir = FM.getDirectory(Path)) {
241  if (CurDir == Dir) {
242  Result = FilePath.substr(Path.size());
243  llvm::sys::path::append(Result,
244  llvm::sys::path::filename(File->getName()));
245  return;
246  }
247  }
248 
249  Path = llvm::sys::path::parent_path(Path);
250  }
251 
252  Result = File->getName();
253 }
254 
255 void Preprocessor::PropagateLineStartLeadingSpaceInfo(Token &Result) {
256  if (CurTokenLexer) {
257  CurTokenLexer->PropagateLineStartLeadingSpaceInfo(Result);
258  return;
259  }
260  if (CurLexer) {
261  CurLexer->PropagateLineStartLeadingSpaceInfo(Result);
262  return;
263  }
264  // FIXME: Handle other kinds of lexers? It generally shouldn't matter,
265  // but it might if they're empty?
266 }
267 
268 /// \brief Determine the location to use as the end of the buffer for a lexer.
269 ///
270 /// If the file ends with a newline, form the EOF token on the newline itself,
271 /// rather than "on the line following it", which doesn't exist. This makes
272 /// diagnostics relating to the end of file include the last file that the user
273 /// actually typed, which is goodness.
274 const char *Preprocessor::getCurLexerEndPos() {
275  const char *EndPos = CurLexer->BufferEnd;
276  if (EndPos != CurLexer->BufferStart &&
277  (EndPos[-1] == '\n' || EndPos[-1] == '\r')) {
278  --EndPos;
279 
280  // Handle \n\r and \r\n:
281  if (EndPos != CurLexer->BufferStart &&
282  (EndPos[-1] == '\n' || EndPos[-1] == '\r') &&
283  EndPos[-1] != EndPos[0])
284  --EndPos;
285  }
286 
287  return EndPos;
288 }
289 
291  const Module &Mod, SmallVectorImpl<const Module *> &SubMods) {
292  if (Mod.getUmbrellaHeader())
293  SubMods.push_back(&Mod);
294  for (auto *M : Mod.submodules())
296 }
297 
298 void Preprocessor::diagnoseMissingHeaderInUmbrellaDir(const Module &Mod) {
299  assert(Mod.getUmbrellaHeader() && "Module must use umbrella header");
300  SourceLocation StartLoc =
301  SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID());
302  if (getDiagnostics().isIgnored(diag::warn_uncovered_module_header, StartLoc))
303  return;
304 
306  const DirectoryEntry *Dir = Mod.getUmbrellaDir().Entry;
307  vfs::FileSystem &FS = *FileMgr.getVirtualFileSystem();
308  std::error_code EC;
309  for (vfs::recursive_directory_iterator Entry(FS, Dir->getName(), EC), End;
310  Entry != End && !EC; Entry.increment(EC)) {
311  using llvm::StringSwitch;
312 
313  // Check whether this entry has an extension typically associated with
314  // headers.
315  if (!StringSwitch<bool>(llvm::sys::path::extension(Entry->getName()))
316  .Cases(".h", ".H", ".hh", ".hpp", true)
317  .Default(false))
318  continue;
319 
320  if (const FileEntry *Header = getFileManager().getFile(Entry->getName()))
321  if (!getSourceManager().hasFileInfo(Header)) {
322  if (!ModMap.isHeaderInUnavailableModule(Header)) {
323  // Find the relative path that would access this header.
324  SmallString<128> RelativePath;
325  computeRelativePath(FileMgr, Dir, Header, RelativePath);
326  Diag(StartLoc, diag::warn_uncovered_module_header)
327  << Mod.getFullModuleName() << RelativePath;
328  }
329  }
330  }
331 }
332 
333 /// HandleEndOfFile - This callback is invoked when the lexer hits the end of
334 /// the current file. This either returns the EOF token or pops a level off
335 /// the include stack and keeps going.
336 bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
337  assert(!CurTokenLexer &&
338  "Ending a file when currently in a macro!");
339 
340  // If we have an unclosed module region from a pragma at the end of a
341  // module, complain and close it now.
342  // FIXME: This is not correct if we are building a module from PTH.
343  const bool LeavingSubmodule = CurLexer && CurLexerSubmodule;
344  if ((LeavingSubmodule || IncludeMacroStack.empty()) &&
345  !BuildingSubmoduleStack.empty() &&
346  BuildingSubmoduleStack.back().IsPragma) {
347  Diag(BuildingSubmoduleStack.back().ImportLoc,
348  diag::err_pp_module_begin_without_module_end);
349  Module *M = LeaveSubmodule(/*ForPragma*/true);
350 
351  Result.startToken();
352  const char *EndPos = getCurLexerEndPos();
353  CurLexer->BufferPtr = EndPos;
354  CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
355  Result.setAnnotationEndLoc(Result.getLocation());
356  Result.setAnnotationValue(M);
357  return true;
358  }
359 
360  // See if this file had a controlling macro.
361  if (CurPPLexer) { // Not ending a macro, ignore it.
362  if (const IdentifierInfo *ControllingMacro =
363  CurPPLexer->MIOpt.GetControllingMacroAtEndOfFile()) {
364  // Okay, this has a controlling macro, remember in HeaderFileInfo.
365  if (const FileEntry *FE = CurPPLexer->getFileEntry()) {
366  HeaderInfo.SetFileControllingMacro(FE, ControllingMacro);
367  if (MacroInfo *MI =
368  getMacroInfo(const_cast<IdentifierInfo*>(ControllingMacro)))
369  MI->setUsedForHeaderGuard(true);
370  if (const IdentifierInfo *DefinedMacro =
371  CurPPLexer->MIOpt.GetDefinedMacro()) {
372  if (!isMacroDefined(ControllingMacro) &&
373  DefinedMacro != ControllingMacro &&
374  HeaderInfo.FirstTimeLexingFile(FE)) {
375 
376  // If the edit distance between the two macros is more than 50%,
377  // DefinedMacro may not be header guard, or can be header guard of
378  // another header file. Therefore, it maybe defining something
379  // completely different. This can be observed in the wild when
380  // handling feature macros or header guards in different files.
381 
382  const StringRef ControllingMacroName = ControllingMacro->getName();
383  const StringRef DefinedMacroName = DefinedMacro->getName();
384  const size_t MaxHalfLength = std::max(ControllingMacroName.size(),
385  DefinedMacroName.size()) / 2;
386  const unsigned ED = ControllingMacroName.edit_distance(
387  DefinedMacroName, true, MaxHalfLength);
388  if (ED <= MaxHalfLength) {
389  // Emit a warning for a bad header guard.
390  Diag(CurPPLexer->MIOpt.GetMacroLocation(),
391  diag::warn_header_guard)
392  << CurPPLexer->MIOpt.GetMacroLocation() << ControllingMacro;
393  Diag(CurPPLexer->MIOpt.GetDefinedLocation(),
394  diag::note_header_guard)
395  << CurPPLexer->MIOpt.GetDefinedLocation() << DefinedMacro
396  << ControllingMacro
398  CurPPLexer->MIOpt.GetDefinedLocation(),
399  ControllingMacro->getName());
400  }
401  }
402  }
403  }
404  }
405  }
406 
407  // Complain about reaching a true EOF within arc_cf_code_audited.
408  // We don't want to complain about reaching the end of a macro
409  // instantiation or a _Pragma.
410  if (PragmaARCCFCodeAuditedLoc.isValid() &&
411  !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
412  Diag(PragmaARCCFCodeAuditedLoc, diag::err_pp_eof_in_arc_cf_code_audited);
413 
414  // Recover by leaving immediately.
415  PragmaARCCFCodeAuditedLoc = SourceLocation();
416  }
417 
418  // Complain about reaching a true EOF within assume_nonnull.
419  // We don't want to complain about reaching the end of a macro
420  // instantiation or a _Pragma.
421  if (PragmaAssumeNonNullLoc.isValid() &&
422  !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
423  Diag(PragmaAssumeNonNullLoc, diag::err_pp_eof_in_assume_nonnull);
424 
425  // Recover by leaving immediately.
426  PragmaAssumeNonNullLoc = SourceLocation();
427  }
428 
429  // If this is a #include'd file, pop it off the include stack and continue
430  // lexing the #includer file.
431  if (!IncludeMacroStack.empty()) {
432 
433  // If we lexed the code-completion file, act as if we reached EOF.
434  if (isCodeCompletionEnabled() && CurPPLexer &&
435  SourceMgr.getLocForStartOfFile(CurPPLexer->getFileID()) ==
436  CodeCompletionFileLoc) {
437  if (CurLexer) {
438  Result.startToken();
439  CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd, tok::eof);
440  CurLexer.reset();
441  } else {
442  assert(CurPTHLexer && "Got EOF but no current lexer set!");
443  CurPTHLexer->getEOF(Result);
444  CurPTHLexer.reset();
445  }
446 
447  CurPPLexer = nullptr;
448  return true;
449  }
450 
451  if (!isEndOfMacro && CurPPLexer &&
452  SourceMgr.getIncludeLoc(CurPPLexer->getFileID()).isValid()) {
453  // Notify SourceManager to record the number of FileIDs that were created
454  // during lexing of the #include'd file.
455  unsigned NumFIDs =
456  SourceMgr.local_sloc_entry_size() -
457  CurPPLexer->getInitialNumSLocEntries() + 1/*#include'd file*/;
458  SourceMgr.setNumCreatedFIDsForFileID(CurPPLexer->getFileID(), NumFIDs);
459  }
460 
461  bool ExitedFromPredefinesFile = false;
462  FileID ExitedFID;
463  if (!isEndOfMacro && CurPPLexer) {
464  ExitedFID = CurPPLexer->getFileID();
465 
466  assert(PredefinesFileID.isValid() &&
467  "HandleEndOfFile is called before PredefinesFileId is set");
468  ExitedFromPredefinesFile = (PredefinesFileID == ExitedFID);
469  }
470 
471  if (LeavingSubmodule) {
472  // We're done with this submodule.
473  Module *M = LeaveSubmodule(/*ForPragma*/false);
474 
475  // Notify the parser that we've left the module.
476  const char *EndPos = getCurLexerEndPos();
477  Result.startToken();
478  CurLexer->BufferPtr = EndPos;
479  CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
480  Result.setAnnotationEndLoc(Result.getLocation());
481  Result.setAnnotationValue(M);
482  }
483 
484  // We're done with the #included file.
486 
487  // Propagate info about start-of-line/leading white-space/etc.
488  PropagateLineStartLeadingSpaceInfo(Result);
489 
490  // Notify the client, if desired, that we are in a new source file.
491  if (Callbacks && !isEndOfMacro && CurPPLexer) {
492  SrcMgr::CharacteristicKind FileType =
493  SourceMgr.getFileCharacteristic(CurPPLexer->getSourceLocation());
494  Callbacks->FileChanged(CurPPLexer->getSourceLocation(),
495  PPCallbacks::ExitFile, FileType, ExitedFID);
496  }
497 
498  // Restore conditional stack from the preamble right after exiting from the
499  // predefines file.
500  if (ExitedFromPredefinesFile)
501  replayPreambleConditionalStack();
502 
503  // Client should lex another token unless we generated an EOM.
504  return LeavingSubmodule;
505  }
506 
507  // If this is the end of the main file, form an EOF token.
508  if (CurLexer) {
509  const char *EndPos = getCurLexerEndPos();
510  Result.startToken();
511  CurLexer->BufferPtr = EndPos;
512  CurLexer->FormTokenWithChars(Result, EndPos, tok::eof);
513 
514  if (isCodeCompletionEnabled()) {
515  // Inserting the code-completion point increases the source buffer by 1,
516  // but the main FileID was created before inserting the point.
517  // Compensate by reducing the EOF location by 1, otherwise the location
518  // will point to the next FileID.
519  // FIXME: This is hacky, the code-completion point should probably be
520  // inserted before the main FileID is created.
521  if (CurLexer->getFileLoc() == CodeCompletionFileLoc)
522  Result.setLocation(Result.getLocation().getLocWithOffset(-1));
523  }
524 
526  // We're done with lexing.
527  CurLexer.reset();
528  } else {
529  assert(CurPTHLexer && "Got EOF but no current lexer set!");
530  CurPTHLexer->getEOF(Result);
531  CurPTHLexer.reset();
532  }
533 
535  CurPPLexer = nullptr;
536 
537  if (TUKind == TU_Complete) {
538  // This is the end of the top-level file. 'WarnUnusedMacroLocs' has
539  // collected all macro locations that we need to warn because they are not
540  // used.
541  for (WarnUnusedMacroLocsTy::iterator
542  I=WarnUnusedMacroLocs.begin(), E=WarnUnusedMacroLocs.end();
543  I!=E; ++I)
544  Diag(*I, diag::pp_macro_not_used);
545  }
546 
547  // If we are building a module that has an umbrella header, make sure that
548  // each of the headers within the directory, including all submodules, is
549  // covered by the umbrella header was actually included by the umbrella
550  // header.
551  if (Module *Mod = getCurrentModule()) {
554  for (auto *M : AllMods)
555  diagnoseMissingHeaderInUmbrellaDir(*M);
556  }
557 
558  return true;
559 }
560 
561 /// HandleEndOfTokenLexer - This callback is invoked when the current TokenLexer
562 /// hits the end of its token stream.
564  assert(CurTokenLexer && !CurPPLexer &&
565  "Ending a macro when currently in a #include file!");
566 
567  if (!MacroExpandingLexersStack.empty() &&
568  MacroExpandingLexersStack.back().first == CurTokenLexer.get())
569  removeCachedMacroExpandedTokensOfLastLexer();
570 
571  // Delete or cache the now-dead macro expander.
572  if (NumCachedTokenLexers == TokenLexerCacheSize)
573  CurTokenLexer.reset();
574  else
575  TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
576 
577  // Handle this like a #include file being popped off the stack.
578  return HandleEndOfFile(Result, true);
579 }
580 
581 /// RemoveTopOfLexerStack - Pop the current lexer/macro exp off the top of the
582 /// lexer stack. This should only be used in situations where the current
583 /// state of the top-of-stack lexer is unknown.
585  assert(!IncludeMacroStack.empty() && "Ran out of stack entries to load");
586 
587  if (CurTokenLexer) {
588  // Delete or cache the now-dead macro expander.
589  if (NumCachedTokenLexers == TokenLexerCacheSize)
590  CurTokenLexer.reset();
591  else
592  TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
593  }
594 
595  PopIncludeMacroStack();
596 }
597 
598 /// HandleMicrosoftCommentPaste - When the macro expander pastes together a
599 /// comment (/##/) in microsoft mode, this method handles updating the current
600 /// state, returning the token on the next source line.
602  assert(CurTokenLexer && !CurPPLexer &&
603  "Pasted comment can only be formed from macro");
604  // We handle this by scanning for the closest real lexer, switching it to
605  // raw mode and preprocessor mode. This will cause it to return \n as an
606  // explicit EOD token.
607  PreprocessorLexer *FoundLexer = nullptr;
608  bool LexerWasInPPMode = false;
609  for (const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
610  if (ISI.ThePPLexer == nullptr) continue; // Scan for a real lexer.
611 
612  // Once we find a real lexer, mark it as raw mode (disabling macro
613  // expansions) and preprocessor mode (return EOD). We know that the lexer
614  // was *not* in raw mode before, because the macro that the comment came
615  // from was expanded. However, it could have already been in preprocessor
616  // mode (#if COMMENT) in which case we have to return it to that mode and
617  // return EOD.
618  FoundLexer = ISI.ThePPLexer;
619  FoundLexer->LexingRawMode = true;
620  LexerWasInPPMode = FoundLexer->ParsingPreprocessorDirective;
621  FoundLexer->ParsingPreprocessorDirective = true;
622  break;
623  }
624 
625  // Okay, we either found and switched over the lexer, or we didn't find a
626  // lexer. In either case, finish off the macro the comment came from, getting
627  // the next token.
628  if (!HandleEndOfTokenLexer(Tok)) Lex(Tok);
629 
630  // Discarding comments as long as we don't have EOF or EOD. This 'comments
631  // out' the rest of the line, including any tokens that came from other macros
632  // that were active, as in:
633  // #define submacro a COMMENT b
634  // submacro c
635  // which should lex to 'a' only: 'b' and 'c' should be removed.
636  while (Tok.isNot(tok::eod) && Tok.isNot(tok::eof))
637  Lex(Tok);
638 
639  // If we got an eod token, then we successfully found the end of the line.
640  if (Tok.is(tok::eod)) {
641  assert(FoundLexer && "Can't get end of line without an active lexer");
642  // Restore the lexer back to normal mode instead of raw mode.
643  FoundLexer->LexingRawMode = false;
644 
645  // If the lexer was already in preprocessor mode, just return the EOD token
646  // to finish the preprocessor line.
647  if (LexerWasInPPMode) return;
648 
649  // Otherwise, switch out of PP mode and return the next lexed token.
650  FoundLexer->ParsingPreprocessorDirective = false;
651  return Lex(Tok);
652  }
653 
654  // If we got an EOF token, then we reached the end of the token stream but
655  // didn't find an explicit \n. This can only happen if there was no lexer
656  // active (an active lexer would return EOD at EOF if there was no \n in
657  // preprocessor directive mode), so just return EOF as our token.
658  assert(!FoundLexer && "Lexer should return EOD before EOF in PP mode");
659 }
660 
662  bool ForPragma) {
663  if (!getLangOpts().ModulesLocalVisibility) {
664  // Just track that we entered this submodule.
665  BuildingSubmoduleStack.push_back(
666  BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
667  PendingModuleMacroNames.size()));
668  return;
669  }
670 
671  // Resolve as much of the module definition as we can now, before we enter
672  // one of its headers.
673  // FIXME: Can we enable Complain here?
674  // FIXME: Can we do this when local visibility is disabled?
676  ModMap.resolveExports(M, /*Complain=*/false);
677  ModMap.resolveUses(M, /*Complain=*/false);
678  ModMap.resolveConflicts(M, /*Complain=*/false);
679 
680  // If this is the first time we've entered this module, set up its state.
681  auto R = Submodules.insert(std::make_pair(M, SubmoduleState()));
682  auto &State = R.first->second;
683  bool FirstTime = R.second;
684  if (FirstTime) {
685  // Determine the set of starting macros for this submodule; take these
686  // from the "null" module (the predefines buffer).
687  //
688  // FIXME: If we have local visibility but not modules enabled, the
689  // NullSubmoduleState is polluted by #defines in the top-level source
690  // file.
691  auto &StartingMacros = NullSubmoduleState.Macros;
692 
693  // Restore to the starting state.
694  // FIXME: Do this lazily, when each macro name is first referenced.
695  for (auto &Macro : StartingMacros) {
696  // Skip uninteresting macros.
697  if (!Macro.second.getLatest() &&
698  Macro.second.getOverriddenMacros().empty())
699  continue;
700 
701  MacroState MS(Macro.second.getLatest());
702  MS.setOverriddenMacros(*this, Macro.second.getOverriddenMacros());
703  State.Macros.insert(std::make_pair(Macro.first, std::move(MS)));
704  }
705  }
706 
707  // Track that we entered this module.
708  BuildingSubmoduleStack.push_back(
709  BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
710  PendingModuleMacroNames.size()));
711 
712  // Switch to this submodule as the current submodule.
713  CurSubmoduleState = &State;
714 
715  // This module is visible to itself.
716  if (FirstTime)
717  makeModuleVisible(M, ImportLoc);
718 }
719 
720 bool Preprocessor::needModuleMacros() const {
721  // If we're not within a submodule, we never need to create ModuleMacros.
722  if (BuildingSubmoduleStack.empty())
723  return false;
724  // If we are tracking module macro visibility even for textually-included
725  // headers, we need ModuleMacros.
726  if (getLangOpts().ModulesLocalVisibility)
727  return true;
728  // Otherwise, we only need module macros if we're actually compiling a module
729  // interface.
730  return getLangOpts().isCompilingModule();
731 }
732 
734  if (BuildingSubmoduleStack.empty() ||
735  BuildingSubmoduleStack.back().IsPragma != ForPragma) {
736  assert(ForPragma && "non-pragma module enter/leave mismatch");
737  return nullptr;
738  }
739 
740  auto &Info = BuildingSubmoduleStack.back();
741 
742  Module *LeavingMod = Info.M;
743  SourceLocation ImportLoc = Info.ImportLoc;
744 
745  if (!needModuleMacros() ||
746  (!getLangOpts().ModulesLocalVisibility &&
747  LeavingMod->getTopLevelModuleName() != getLangOpts().CurrentModule)) {
748  // If we don't need module macros, or this is not a module for which we
749  // are tracking macro visibility, don't build any, and preserve the list
750  // of pending names for the surrounding submodule.
751  BuildingSubmoduleStack.pop_back();
752  makeModuleVisible(LeavingMod, ImportLoc);
753  return LeavingMod;
754  }
755 
756  // Create ModuleMacros for any macros defined in this submodule.
757  llvm::SmallPtrSet<const IdentifierInfo*, 8> VisitedMacros;
758  for (unsigned I = Info.OuterPendingModuleMacroNames;
759  I != PendingModuleMacroNames.size(); ++I) {
760  auto *II = const_cast<IdentifierInfo*>(PendingModuleMacroNames[I]);
761  if (!VisitedMacros.insert(II).second)
762  continue;
763 
764  auto MacroIt = CurSubmoduleState->Macros.find(II);
765  if (MacroIt == CurSubmoduleState->Macros.end())
766  continue;
767  auto &Macro = MacroIt->second;
768 
769  // Find the starting point for the MacroDirective chain in this submodule.
770  MacroDirective *OldMD = nullptr;
771  auto *OldState = Info.OuterSubmoduleState;
772  if (getLangOpts().ModulesLocalVisibility)
773  OldState = &NullSubmoduleState;
774  if (OldState && OldState != CurSubmoduleState) {
775  // FIXME: It'd be better to start at the state from when we most recently
776  // entered this submodule, but it doesn't really matter.
777  auto &OldMacros = OldState->Macros;
778  auto OldMacroIt = OldMacros.find(II);
779  if (OldMacroIt == OldMacros.end())
780  OldMD = nullptr;
781  else
782  OldMD = OldMacroIt->second.getLatest();
783  }
784 
785  // This module may have exported a new macro. If so, create a ModuleMacro
786  // representing that fact.
787  bool ExplicitlyPublic = false;
788  for (auto *MD = Macro.getLatest(); MD != OldMD; MD = MD->getPrevious()) {
789  assert(MD && "broken macro directive chain");
790 
791  if (auto *VisMD = dyn_cast<VisibilityMacroDirective>(MD)) {
792  // The latest visibility directive for a name in a submodule affects
793  // all the directives that come before it.
794  if (VisMD->isPublic())
795  ExplicitlyPublic = true;
796  else if (!ExplicitlyPublic)
797  // Private with no following public directive: not exported.
798  break;
799  } else {
800  MacroInfo *Def = nullptr;
801  if (DefMacroDirective *DefMD = dyn_cast<DefMacroDirective>(MD))
802  Def = DefMD->getInfo();
803 
804  // FIXME: Issue a warning if multiple headers for the same submodule
805  // define a macro, rather than silently ignoring all but the first.
806  bool IsNew;
807  // Don't bother creating a module macro if it would represent a #undef
808  // that doesn't override anything.
809  if (Def || !Macro.getOverriddenMacros().empty())
810  addModuleMacro(LeavingMod, II, Def,
811  Macro.getOverriddenMacros(), IsNew);
812 
813  if (!getLangOpts().ModulesLocalVisibility) {
814  // This macro is exposed to the rest of this compilation as a
815  // ModuleMacro; we don't need to track its MacroDirective any more.
816  Macro.setLatest(nullptr);
817  Macro.setOverriddenMacros(*this, {});
818  }
819  break;
820  }
821  }
822  }
823  PendingModuleMacroNames.resize(Info.OuterPendingModuleMacroNames);
824 
825  // FIXME: Before we leave this submodule, we should parse all the other
826  // headers within it. Otherwise, we're left with an inconsistent state
827  // where we've made the module visible but don't yet have its complete
828  // contents.
829 
830  // Put back the outer module's state, if we're tracking it.
831  if (getLangOpts().ModulesLocalVisibility)
832  CurSubmoduleState = Info.OuterSubmoduleState;
833 
834  BuildingSubmoduleStack.pop_back();
835 
836  // A nested #include makes the included submodule visible.
837  makeModuleVisible(LeavingMod, ImportLoc);
838  return LeavingMod;
839 }
SourceManager & getSourceManager() const
Definition: Preprocessor.h:729
Header getUmbrellaHeader() const
Retrieve the header that serves as the umbrella header for this module.
Definition: Module.h:441
unsigned getInitialNumSLocEntries() const
Number of SLocEntries before lexing the file.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Definition: Lexer.h:46
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:116
unsigned local_sloc_entry_size() const
Get the number of local SLocEntries we have.
Defines the clang::FileManager interface and associated types.
bool isHeaderInUnavailableModule(const FileEntry *Header) const
Determine whether the given header is part of a module marked 'unavailable'.
Definition: ModuleMap.cpp:608
Defines the SourceManager interface.
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
llvm::MemoryBuffer * getBuffer(FileID FID, SourceLocation Loc, bool *Invalid=nullptr) const
Return the buffer for the specified FileID.
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool resolveUses(Module *Mod, bool Complain)
Resolve all of the unresolved uses in the given module.
Definition: ModuleMap.cpp:1143
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
Definition: SourceManager.h:82
ModuleMap & getModuleMap()
Retrieve the module map.
Definition: HeaderSearch.h:624
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
SourceLocation GetDefinedLocation() const
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
bool hasFileInfo(const FileEntry *File) const
The virtual file system interface.
const StringRef FilePath
One of these records is kept for each identifier that is lexed.
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:397
bool ParsingPreprocessorDirective
True when parsing #XXX; turns '\n' into a tok::eod token.
bool resolveConflicts(Module *Mod, bool Complain)
Resolve all of the unresolved conflicts in the given module.
Definition: ModuleMap.cpp:1156
LineState State
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
Module * LeaveSubmodule(bool ForPragma)
StringRef getBufferName(SourceLocation Loc, bool *Invalid=nullptr) const
Return the filename or buffer identifier of the buffer the location is in.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
Definition: Preprocessor.h:895
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:725
std::string getFullModuleName(bool AllowStringLiterals=false) const
Retrieve the full name of this module, including the path from its top-level module.
Definition: Module.cpp:158
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
An input iterator over the recursive contents of a virtual path, similar to llvm::sys::fs::recursive_...
Describes a module or submodule.
Definition: Module.h:57
FileManager & getFileManager() const
Definition: Preprocessor.h:728
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
IntrusiveRefCntPtr< vfs::FileSystem > getVirtualFileSystem() const
Definition: FileManager.h:225
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:731
static void collectAllSubModulesWithUmbrellaHeader(const Module &Mod, SmallVectorImpl< const Module * > &SubMods)
StringRef getName() const
Definition: FileManager.h:84
const FileEntry * getFileEntry() const
getFileEntry - Return the FileEntry corresponding to this FileID.
detail::InMemoryDirectory::const_iterator I
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
const FileEntry * getFile(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Lookup, cache, and verify the specified file (real or virtual).
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
std::string CurrentModule
The name of the current module, of which the main source file is a part.
Definition: LangOptions.h:130
const DirectoryEntry * getDirectory(StringRef DirName, bool CacheFailure=true)
Lookup, cache, and verify the specified directory (real or virtual).
const SmallVectorImpl< AnnotatedLine * >::const_iterator End
const DirectoryEntry * Entry
Definition: Module.h:146
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
void setAnnotationValue(void *val)
Definition: Token.h:228
bool LexingRawMode
True if in raw mode.
void makeModuleVisible(Module *M, SourceLocation Loc)
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
Defines the clang::Preprocessor interface.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
bool resolveExports(Module *Mod, bool Complain)
Resolve all of the unresolved exports in the given module.
Definition: ModuleMap.cpp:1130
MultipleIncludeOpt MIOpt
A state machine that detects the #ifndef-wrapping a file idiom for the multiple-include optimization...
static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir, const FileEntry *File, SmallString< 128 > &Result)
Compute the relative path that names the given file relative to the given directory.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
bool isNot(tok::TokenKind K) const
Definition: Token.h:96
SourceLocation getIncludeLoc(FileID FID) const
Returns the include location if FID is a #include'd file otherwise it returns an invalid location...
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
DirectoryName getUmbrellaDir() const
Retrieve the directory for which this module serves as the umbrella.
Definition: Module.cpp:183
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
StringRef getTopLevelModuleName() const
Retrieve the name of the top-level module.
Definition: Module.h:419
virtual SourceLocation getSourceLocation()=0
Return the source location for the next observable location.
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Definition: MacroInfo.h:286
Encodes a location in the source.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
bool isValid() const
Return true if this is a valid SourceLocation object.
void setAnnotationEndLoc(SourceLocation L)
Definition: Token.h:142
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
void Lex(Token &Result)
Lex the next token for this preprocessor.
void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
FileID getMainFileID() const
Returns the FileID of the main source file.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:95
bool isIgnored(unsigned DiagID, SourceLocation Loc) const
Determine whether the diagnostic is known to be ignored.
Definition: Diagnostic.h:732
DiagnosticsEngine & getDiagnostics() const
Definition: Preprocessor.h:722
bool isMacroDefined(StringRef Id)
Definition: Preprocessor.h:830
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
llvm::iterator_range< submodule_iterator > submodules()
Definition: Module.h:519
SrcMgr::CharacteristicKind getFileCharacteristic(SourceLocation Loc) const
Return the file characteristic of the specified source location, indicating whether this is a normal ...
const IdentifierInfo * GetDefinedMacro() const
If the ControllingMacro is followed by a macro definition, return the macro that was defined...
StringRef getName() const
Definition: FileManager.h:51
detail::InMemoryDirectory::const_iterator E
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:34
const IdentifierInfo * GetControllingMacroAtEndOfFile() const
Once the entire file has been lexed, if there is a controlling macro, return it.
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
Cached information about one directory (either on disk or in the virtual file system).
Definition: FileManager.h:45
char __ovld __cnfn max(char x, char y)
Returns y if x < y, otherwise it returns x.
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
static FixItHint CreateReplacement(CharSourceRange RemoveRange, StringRef Code)
Create a code modification hint that replaces the given source range with the given code string...
Definition: Diagnostic.h:127
void setNumCreatedFIDsForFileID(FileID FID, unsigned NumFIDs) const
Set the number of FileIDs (files and macros) that were created during preprocessing of FID...
The translation unit is a complete translation unit.
Definition: LangOptions.h:239
SourceLocation GetMacroLocation() const
bool isCompilingModule() const
Are we compiling a module interface (.cppm or module map)?
Definition: LangOptions.h:166
bool FirstTimeLexingFile(const FileEntry *File)
Return true if this is the first time encountering this header.
Definition: HeaderSearch.h:456
SourceLocation getLocForStartOfFile(FileID FID) const
Return the source location corresponding to the first byte of the specified file. ...
void setLocation(SourceLocation L)
Definition: Token.h:132
bool isValid() const
const DirectoryEntry * getDir() const
Return the directory the file lives in.
Definition: FileManager.h:94
void startToken()
Reset all flags to cleared.
Definition: Token.h:169
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
Definition: MacroInfo.h:321
void SetFileControllingMacro(const FileEntry *File, const IdentifierInfo *ControllingMacro)
Mark the specified file as having a controlling macro.
Definition: HeaderSearch.h:450
bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...