clang  5.0.0
Preprocessor.h
Go to the documentation of this file.
1 //===--- Preprocessor.h - C Language Family Preprocessor --------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 ///
10 /// \file
11 /// \brief Defines the clang::Preprocessor interface.
12 ///
13 //===----------------------------------------------------------------------===//
14 
15 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
16 #define LLVM_CLANG_LEX_PREPROCESSOR_H
17 
18 #include "clang/Basic/Builtins.h"
19 #include "clang/Basic/Diagnostic.h"
22 #include "clang/Lex/Lexer.h"
23 #include "clang/Lex/MacroInfo.h"
24 #include "clang/Lex/ModuleMap.h"
25 #include "clang/Lex/PPCallbacks.h"
26 #include "clang/Lex/PTHLexer.h"
27 #include "clang/Lex/TokenLexer.h"
28 #include "llvm/ADT/ArrayRef.h"
29 #include "llvm/ADT/DenseMap.h"
30 #include "llvm/ADT/IntrusiveRefCntPtr.h"
31 #include "llvm/ADT/SmallPtrSet.h"
32 #include "llvm/ADT/SmallVector.h"
33 #include "llvm/ADT/TinyPtrVector.h"
34 #include "llvm/Support/Allocator.h"
35 #include "llvm/Support/Registry.h"
36 #include <memory>
37 #include <vector>
38 
39 namespace llvm {
40  template<unsigned InternalLen> class SmallString;
41 }
42 
43 namespace clang {
44 
45 class SourceManager;
46 class ExternalPreprocessorSource;
47 class FileManager;
48 class FileEntry;
49 class HeaderSearch;
50 class MemoryBufferCache;
51 class PragmaNamespace;
52 class PragmaHandler;
53 class CommentHandler;
54 class ScratchBuffer;
55 class TargetInfo;
56 class PPCallbacks;
57 class CodeCompletionHandler;
58 class DirectoryLookup;
59 class PreprocessingRecord;
60 class ModuleLoader;
61 class PTHManager;
62 class PreprocessorOptions;
63 
64 /// \brief Stores token information for comparing actual tokens with
65 /// predefined values. Only handles simple tokens and identifiers.
66 class TokenValue {
68  IdentifierInfo *II;
69 
70 public:
71  TokenValue(tok::TokenKind Kind) : Kind(Kind), II(nullptr) {
72  assert(Kind != tok::raw_identifier && "Raw identifiers are not supported.");
73  assert(Kind != tok::identifier &&
74  "Identifiers should be created by TokenValue(IdentifierInfo *)");
75  assert(!tok::isLiteral(Kind) && "Literals are not supported.");
76  assert(!tok::isAnnotation(Kind) && "Annotations are not supported.");
77  }
78  TokenValue(IdentifierInfo *II) : Kind(tok::identifier), II(II) {}
79  bool operator==(const Token &Tok) const {
80  return Tok.getKind() == Kind &&
81  (!II || II == Tok.getIdentifierInfo());
82  }
83 };
84 
85 /// \brief Context in which macro name is used.
86 enum MacroUse {
87  MU_Other = 0, // other than #define or #undef
88  MU_Define = 1, // macro name specified in #define
89  MU_Undef = 2 // macro name specified in #undef
90 };
91 
92 /// \brief Engages in a tight little dance with the lexer to efficiently
93 /// preprocess tokens.
94 ///
95 /// Lexers know only about tokens within a single source file, and don't
96 /// know anything about preprocessor-level issues like the \#include stack,
97 /// token expansion, etc.
98 class Preprocessor {
99  std::shared_ptr<PreprocessorOptions> PPOpts;
100  DiagnosticsEngine *Diags;
101  LangOptions &LangOpts;
102  const TargetInfo *Target;
103  const TargetInfo *AuxTarget;
104  FileManager &FileMgr;
105  SourceManager &SourceMgr;
106  MemoryBufferCache &PCMCache;
107  std::unique_ptr<ScratchBuffer> ScratchBuf;
108  HeaderSearch &HeaderInfo;
109  ModuleLoader &TheModuleLoader;
110 
111  /// \brief External source of macros.
112  ExternalPreprocessorSource *ExternalSource;
113 
114 
115  /// An optional PTHManager object used for getting tokens from
116  /// a token cache rather than lexing the original source file.
117  std::unique_ptr<PTHManager> PTH;
118 
119  /// A BumpPtrAllocator object used to quickly allocate and release
120  /// objects internal to the Preprocessor.
121  llvm::BumpPtrAllocator BP;
122 
123  /// Identifiers for builtin macros and other builtins.
124  IdentifierInfo *Ident__LINE__, *Ident__FILE__; // __LINE__, __FILE__
125  IdentifierInfo *Ident__DATE__, *Ident__TIME__; // __DATE__, __TIME__
126  IdentifierInfo *Ident__INCLUDE_LEVEL__; // __INCLUDE_LEVEL__
127  IdentifierInfo *Ident__BASE_FILE__; // __BASE_FILE__
128  IdentifierInfo *Ident__TIMESTAMP__; // __TIMESTAMP__
129  IdentifierInfo *Ident__COUNTER__; // __COUNTER__
130  IdentifierInfo *Ident_Pragma, *Ident__pragma; // _Pragma, __pragma
131  IdentifierInfo *Ident__identifier; // __identifier
132  IdentifierInfo *Ident__VA_ARGS__; // __VA_ARGS__
133  IdentifierInfo *Ident__has_feature; // __has_feature
134  IdentifierInfo *Ident__has_extension; // __has_extension
135  IdentifierInfo *Ident__has_builtin; // __has_builtin
136  IdentifierInfo *Ident__has_attribute; // __has_attribute
137  IdentifierInfo *Ident__has_include; // __has_include
138  IdentifierInfo *Ident__has_include_next; // __has_include_next
139  IdentifierInfo *Ident__has_warning; // __has_warning
140  IdentifierInfo *Ident__is_identifier; // __is_identifier
141  IdentifierInfo *Ident__building_module; // __building_module
142  IdentifierInfo *Ident__MODULE__; // __MODULE__
143  IdentifierInfo *Ident__has_cpp_attribute; // __has_cpp_attribute
144  IdentifierInfo *Ident__has_declspec; // __has_declspec_attribute
145 
146  SourceLocation DATELoc, TIMELoc;
147  unsigned CounterValue; // Next __COUNTER__ value.
148 
149  enum {
150  /// \brief Maximum depth of \#includes.
151  MaxAllowedIncludeStackDepth = 200
152  };
153 
154  // State that is set before the preprocessor begins.
155  bool KeepComments : 1;
156  bool KeepMacroComments : 1;
157  bool SuppressIncludeNotFoundError : 1;
158 
159  // State that changes while the preprocessor runs:
160  bool InMacroArgs : 1; // True if parsing fn macro invocation args.
161 
162  /// Whether the preprocessor owns the header search object.
163  bool OwnsHeaderSearch : 1;
164 
165  /// True if macro expansion is disabled.
166  bool DisableMacroExpansion : 1;
167 
168  /// Temporarily disables DisableMacroExpansion (i.e. enables expansion)
169  /// when parsing preprocessor directives.
170  bool MacroExpansionInDirectivesOverride : 1;
171 
173 
174  /// \brief Whether we have already loaded macros from the external source.
175  mutable bool ReadMacrosFromExternalSource : 1;
176 
177  /// \brief True if pragmas are enabled.
178  bool PragmasEnabled : 1;
179 
180  /// \brief True if the current build action is a preprocessing action.
181  bool PreprocessedOutput : 1;
182 
183  /// \brief True if we are currently preprocessing a #if or #elif directive
184  bool ParsingIfOrElifDirective;
185 
186  /// \brief True if we are pre-expanding macro arguments.
187  bool InMacroArgPreExpansion;
188 
189  /// \brief Mapping/lookup information for all identifiers in
190  /// the program, including program keywords.
191  mutable IdentifierTable Identifiers;
192 
193  /// \brief This table contains all the selectors in the program.
194  ///
195  /// Unlike IdentifierTable above, this table *isn't* populated by the
196  /// preprocessor. It is declared/expanded here because its role/lifetime is
197  /// conceptually similar to the IdentifierTable. In addition, the current
198  /// control flow (in clang::ParseAST()), make it convenient to put here.
199  ///
200  /// FIXME: Make sure the lifetime of Identifiers/Selectors *isn't* tied to
201  /// the lifetime of the preprocessor.
202  SelectorTable Selectors;
203 
204  /// \brief Information about builtins.
206 
207  /// \brief Tracks all of the pragmas that the client registered
208  /// with this preprocessor.
209  std::unique_ptr<PragmaNamespace> PragmaHandlers;
210 
211  /// \brief Pragma handlers of the original source is stored here during the
212  /// parsing of a model file.
213  std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
214 
215  /// \brief Tracks all of the comment handlers that the client registered
216  /// with this preprocessor.
217  std::vector<CommentHandler *> CommentHandlers;
218 
219  /// \brief True if we want to ignore EOF token and continue later on (thus
220  /// avoid tearing the Lexer and etc. down).
221  bool IncrementalProcessing;
222 
223  /// The kind of translation unit we are processing.
224  TranslationUnitKind TUKind;
225 
226  /// \brief The code-completion handler.
227  CodeCompletionHandler *CodeComplete;
228 
229  /// \brief The file that we're performing code-completion for, if any.
230  const FileEntry *CodeCompletionFile;
231 
232  /// \brief The offset in file for the code-completion point.
233  unsigned CodeCompletionOffset;
234 
235  /// \brief The location for the code-completion point. This gets instantiated
236  /// when the CodeCompletionFile gets \#include'ed for preprocessing.
237  SourceLocation CodeCompletionLoc;
238 
239  /// \brief The start location for the file of the code-completion point.
240  ///
241  /// This gets instantiated when the CodeCompletionFile gets \#include'ed
242  /// for preprocessing.
243  SourceLocation CodeCompletionFileLoc;
244 
245  /// \brief The source location of the \c import contextual keyword we just
246  /// lexed, if any.
247  SourceLocation ModuleImportLoc;
248 
249  /// \brief The module import path that we're currently processing.
251 
252  /// \brief Whether the last token we lexed was an '@'.
253  bool LastTokenWasAt;
254 
255  /// \brief Whether the module import expects an identifier next. Otherwise,
256  /// it expects a '.' or ';'.
257  bool ModuleImportExpectsIdentifier;
258 
259  /// \brief The source location of the currently-active
260  /// \#pragma clang arc_cf_code_audited begin.
261  SourceLocation PragmaARCCFCodeAuditedLoc;
262 
263  /// \brief The source location of the currently-active
264  /// \#pragma clang assume_nonnull begin.
265  SourceLocation PragmaAssumeNonNullLoc;
266 
267  /// \brief True if we hit the code-completion point.
268  bool CodeCompletionReached;
269 
270  /// \brief The code completion token containing the information
271  /// on the stem that is to be code completed.
272  IdentifierInfo *CodeCompletionII;
273 
274  /// \brief The directory that the main file should be considered to occupy,
275  /// if it does not correspond to a real file (as happens when building a
276  /// module).
277  const DirectoryEntry *MainFileDir;
278 
279  /// \brief The number of bytes that we will initially skip when entering the
280  /// main file, along with a flag that indicates whether skipping this number
281  /// of bytes will place the lexer at the start of a line.
282  ///
283  /// This is used when loading a precompiled preamble.
284  std::pair<int, bool> SkipMainFilePreamble;
285 
286  class PreambleConditionalStackStore {
287  enum State {
288  Off = 0,
289  Recording = 1,
290  Replaying = 2,
291  };
292 
293  public:
294  PreambleConditionalStackStore() : ConditionalStackState(Off) {}
295 
296  void startRecording() { ConditionalStackState = Recording; }
297  void startReplaying() { ConditionalStackState = Replaying; }
298  bool isRecording() const { return ConditionalStackState == Recording; }
299  bool isReplaying() const { return ConditionalStackState == Replaying; }
300 
301  ArrayRef<PPConditionalInfo> getStack() const {
302  return ConditionalStack;
303  }
304 
305  void doneReplaying() {
306  ConditionalStack.clear();
307  ConditionalStackState = Off;
308  }
309 
310  void setStack(ArrayRef<PPConditionalInfo> s) {
311  if (!isRecording() && !isReplaying())
312  return;
313  ConditionalStack.clear();
314  ConditionalStack.append(s.begin(), s.end());
315  }
316 
317  bool hasRecordedPreamble() const { return !ConditionalStack.empty(); }
318 
319  private:
320  SmallVector<PPConditionalInfo, 4> ConditionalStack;
321  State ConditionalStackState;
322  } PreambleConditionalStack;
323 
324  /// \brief The current top of the stack that we're lexing from if
325  /// not expanding a macro and we are lexing directly from source code.
326  ///
327  /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
328  std::unique_ptr<Lexer> CurLexer;
329 
330  /// \brief The current top of stack that we're lexing from if
331  /// not expanding from a macro and we are lexing from a PTH cache.
332  ///
333  /// Only one of CurLexer, CurPTHLexer, or CurTokenLexer will be non-null.
334  std::unique_ptr<PTHLexer> CurPTHLexer;
335 
336  /// \brief The current top of the stack what we're lexing from
337  /// if not expanding a macro.
338  ///
339  /// This is an alias for either CurLexer or CurPTHLexer.
340  PreprocessorLexer *CurPPLexer;
341 
342  /// \brief Used to find the current FileEntry, if CurLexer is non-null
343  /// and if applicable.
344  ///
345  /// This allows us to implement \#include_next and find directory-specific
346  /// properties.
347  const DirectoryLookup *CurDirLookup;
348 
349  /// \brief The current macro we are expanding, if we are expanding a macro.
350  ///
351  /// One of CurLexer and CurTokenLexer must be null.
352  std::unique_ptr<TokenLexer> CurTokenLexer;
353 
354  /// \brief The kind of lexer we're currently working with.
355  enum CurLexerKind {
356  CLK_Lexer,
357  CLK_PTHLexer,
358  CLK_TokenLexer,
359  CLK_CachingLexer,
360  CLK_LexAfterModuleImport
361  } CurLexerKind;
362 
363  /// \brief If the current lexer is for a submodule that is being built, this
364  /// is that submodule.
365  Module *CurLexerSubmodule;
366 
367  /// \brief Keeps track of the stack of files currently
368  /// \#included, and macros currently being expanded from, not counting
369  /// CurLexer/CurTokenLexer.
370  struct IncludeStackInfo {
371  enum CurLexerKind CurLexerKind;
372  Module *TheSubmodule;
373  std::unique_ptr<Lexer> TheLexer;
374  std::unique_ptr<PTHLexer> ThePTHLexer;
375  PreprocessorLexer *ThePPLexer;
376  std::unique_ptr<TokenLexer> TheTokenLexer;
377  const DirectoryLookup *TheDirLookup;
378 
379  // The following constructors are completely useless copies of the default
380  // versions, only needed to pacify MSVC.
381  IncludeStackInfo(enum CurLexerKind CurLexerKind, Module *TheSubmodule,
382  std::unique_ptr<Lexer> &&TheLexer,
383  std::unique_ptr<PTHLexer> &&ThePTHLexer,
384  PreprocessorLexer *ThePPLexer,
385  std::unique_ptr<TokenLexer> &&TheTokenLexer,
386  const DirectoryLookup *TheDirLookup)
387  : CurLexerKind(std::move(CurLexerKind)),
388  TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
389  ThePTHLexer(std::move(ThePTHLexer)),
390  ThePPLexer(std::move(ThePPLexer)),
391  TheTokenLexer(std::move(TheTokenLexer)),
392  TheDirLookup(std::move(TheDirLookup)) {}
393  };
394  std::vector<IncludeStackInfo> IncludeMacroStack;
395 
396  /// \brief Actions invoked when some preprocessor activity is
397  /// encountered (e.g. a file is \#included, etc).
398  std::unique_ptr<PPCallbacks> Callbacks;
399 
400  struct MacroExpandsInfo {
401  Token Tok;
402  MacroDefinition MD;
403  SourceRange Range;
404  MacroExpandsInfo(Token Tok, MacroDefinition MD, SourceRange Range)
405  : Tok(Tok), MD(MD), Range(Range) { }
406  };
407  SmallVector<MacroExpandsInfo, 2> DelayedMacroExpandsCallbacks;
408 
409  /// Information about a name that has been used to define a module macro.
410  struct ModuleMacroInfo {
411  ModuleMacroInfo(MacroDirective *MD)
412  : MD(MD), ActiveModuleMacrosGeneration(0), IsAmbiguous(false) {}
413 
414  /// The most recent macro directive for this identifier.
415  MacroDirective *MD;
416  /// The active module macros for this identifier.
417  llvm::TinyPtrVector<ModuleMacro*> ActiveModuleMacros;
418  /// The generation number at which we last updated ActiveModuleMacros.
419  /// \see Preprocessor::VisibleModules.
420  unsigned ActiveModuleMacrosGeneration;
421  /// Whether this macro name is ambiguous.
422  bool IsAmbiguous;
423  /// The module macros that are overridden by this macro.
424  llvm::TinyPtrVector<ModuleMacro*> OverriddenMacros;
425  };
426 
427  /// The state of a macro for an identifier.
428  class MacroState {
429  mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *> State;
430 
431  ModuleMacroInfo *getModuleInfo(Preprocessor &PP,
432  const IdentifierInfo *II) const {
433  if (II->isOutOfDate())
434  PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
435  // FIXME: Find a spare bit on IdentifierInfo and store a
436  // HasModuleMacros flag.
437  if (!II->hasMacroDefinition() ||
438  (!PP.getLangOpts().Modules &&
439  !PP.getLangOpts().ModulesLocalVisibility) ||
440  !PP.CurSubmoduleState->VisibleModules.getGeneration())
441  return nullptr;
442 
443  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
444  if (!Info) {
445  Info = new (PP.getPreprocessorAllocator())
446  ModuleMacroInfo(State.get<MacroDirective *>());
447  State = Info;
448  }
449 
450  if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
451  Info->ActiveModuleMacrosGeneration)
452  PP.updateModuleMacroInfo(II, *Info);
453  return Info;
454  }
455 
456  public:
457  MacroState() : MacroState(nullptr) {}
458  MacroState(MacroDirective *MD) : State(MD) {}
459  MacroState(MacroState &&O) noexcept : State(O.State) {
460  O.State = (MacroDirective *)nullptr;
461  }
462  MacroState &operator=(MacroState &&O) noexcept {
463  auto S = O.State;
464  O.State = (MacroDirective *)nullptr;
465  State = S;
466  return *this;
467  }
468  ~MacroState() {
469  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
470  Info->~ModuleMacroInfo();
471  }
472 
473  MacroDirective *getLatest() const {
474  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
475  return Info->MD;
476  return State.get<MacroDirective*>();
477  }
478  void setLatest(MacroDirective *MD) {
479  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
480  Info->MD = MD;
481  else
482  State = MD;
483  }
484 
485  bool isAmbiguous(Preprocessor &PP, const IdentifierInfo *II) const {
486  auto *Info = getModuleInfo(PP, II);
487  return Info ? Info->IsAmbiguous : false;
488  }
490  getActiveModuleMacros(Preprocessor &PP, const IdentifierInfo *II) const {
491  if (auto *Info = getModuleInfo(PP, II))
492  return Info->ActiveModuleMacros;
493  return None;
494  }
495 
496  MacroDirective::DefInfo findDirectiveAtLoc(SourceLocation Loc,
497  SourceManager &SourceMgr) const {
498  // FIXME: Incorporate module macros into the result of this.
499  if (auto *Latest = getLatest())
500  return Latest->findDirectiveAtLoc(Loc, SourceMgr);
501  return MacroDirective::DefInfo();
502  }
503 
504  void overrideActiveModuleMacros(Preprocessor &PP, IdentifierInfo *II) {
505  if (auto *Info = getModuleInfo(PP, II)) {
506  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
507  Info->ActiveModuleMacros.begin(),
508  Info->ActiveModuleMacros.end());
509  Info->ActiveModuleMacros.clear();
510  Info->IsAmbiguous = false;
511  }
512  }
513  ArrayRef<ModuleMacro*> getOverriddenMacros() const {
514  if (auto *Info = State.dyn_cast<ModuleMacroInfo*>())
515  return Info->OverriddenMacros;
516  return None;
517  }
518  void setOverriddenMacros(Preprocessor &PP,
519  ArrayRef<ModuleMacro *> Overrides) {
520  auto *Info = State.dyn_cast<ModuleMacroInfo*>();
521  if (!Info) {
522  if (Overrides.empty())
523  return;
524  Info = new (PP.getPreprocessorAllocator())
525  ModuleMacroInfo(State.get<MacroDirective *>());
526  State = Info;
527  }
528  Info->OverriddenMacros.clear();
529  Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
530  Overrides.begin(), Overrides.end());
531  Info->ActiveModuleMacrosGeneration = 0;
532  }
533  };
534 
535  /// For each IdentifierInfo that was associated with a macro, we
536  /// keep a mapping to the history of all macro definitions and #undefs in
537  /// the reverse order (the latest one is in the head of the list).
538  ///
539  /// This mapping lives within the \p CurSubmoduleState.
540  typedef llvm::DenseMap<const IdentifierInfo *, MacroState> MacroMap;
541 
542  friend class ASTReader;
543 
544  struct SubmoduleState;
545 
546  /// \brief Information about a submodule that we're currently building.
547  struct BuildingSubmoduleInfo {
548  BuildingSubmoduleInfo(Module *M, SourceLocation ImportLoc, bool IsPragma,
549  SubmoduleState *OuterSubmoduleState,
550  unsigned OuterPendingModuleMacroNames)
551  : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
552  OuterSubmoduleState(OuterSubmoduleState),
553  OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
554 
555  /// The module that we are building.
556  Module *M;
557  /// The location at which the module was included.
558  SourceLocation ImportLoc;
559  /// Whether we entered this submodule via a pragma.
560  bool IsPragma;
561  /// The previous SubmoduleState.
562  SubmoduleState *OuterSubmoduleState;
563  /// The number of pending module macro names when we started building this.
564  unsigned OuterPendingModuleMacroNames;
565  };
566  SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
567 
568  /// \brief Information about a submodule's preprocessor state.
569  struct SubmoduleState {
570  /// The macros for the submodule.
571  MacroMap Macros;
572  /// The set of modules that are visible within the submodule.
573  VisibleModuleSet VisibleModules;
574  // FIXME: CounterValue?
575  // FIXME: PragmaPushMacroInfo?
576  };
577  std::map<Module*, SubmoduleState> Submodules;
578 
579  /// The preprocessor state for preprocessing outside of any submodule.
580  SubmoduleState NullSubmoduleState;
581 
582  /// The current submodule state. Will be \p NullSubmoduleState if we're not
583  /// in a submodule.
584  SubmoduleState *CurSubmoduleState;
585 
586  /// The set of known macros exported from modules.
587  llvm::FoldingSet<ModuleMacro> ModuleMacros;
588 
589  /// The names of potential module macros that we've not yet processed.
590  llvm::SmallVector<const IdentifierInfo*, 32> PendingModuleMacroNames;
591 
592  /// The list of module macros, for each identifier, that are not overridden by
593  /// any other module macro.
594  llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro*>>
595  LeafModuleMacros;
596 
597  /// \brief Macros that we want to warn because they are not used at the end
598  /// of the translation unit.
599  ///
600  /// We store just their SourceLocations instead of
601  /// something like MacroInfo*. The benefit of this is that when we are
602  /// deserializing from PCH, we don't need to deserialize identifier & macros
603  /// just so that we can report that they are unused, we just warn using
604  /// the SourceLocations of this set (that will be filled by the ASTReader).
605  /// We are using SmallPtrSet instead of a vector for faster removal.
606  typedef llvm::SmallPtrSet<SourceLocation, 32> WarnUnusedMacroLocsTy;
607  WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
608 
609  /// \brief A "freelist" of MacroArg objects that can be
610  /// reused for quick allocation.
611  MacroArgs *MacroArgCache;
612  friend class MacroArgs;
613 
614  /// For each IdentifierInfo used in a \#pragma push_macro directive,
615  /// we keep a MacroInfo stack used to restore the previous macro value.
616  llvm::DenseMap<IdentifierInfo*, std::vector<MacroInfo*> > PragmaPushMacroInfo;
617 
618  // Various statistics we track for performance analysis.
619  unsigned NumDirectives, NumDefined, NumUndefined, NumPragma;
620  unsigned NumIf, NumElse, NumEndif;
621  unsigned NumEnteredSourceFiles, MaxIncludeStackDepth;
622  unsigned NumMacroExpanded, NumFnMacroExpanded, NumBuiltinMacroExpanded;
623  unsigned NumFastMacroExpanded, NumTokenPaste, NumFastTokenPaste;
624  unsigned NumSkipped;
625 
626  /// \brief The predefined macros that preprocessor should use from the
627  /// command line etc.
628  std::string Predefines;
629 
630  /// \brief The file ID for the preprocessor predefines.
631  FileID PredefinesFileID;
632 
633  /// \{
634  /// \brief Cache of macro expanders to reduce malloc traffic.
635  enum { TokenLexerCacheSize = 8 };
636  unsigned NumCachedTokenLexers;
637  std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
638  /// \}
639 
640  /// \brief Keeps macro expanded tokens for TokenLexers.
641  //
642  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
643  /// going to lex in the cache and when it finishes the tokens are removed
644  /// from the end of the cache.
645  SmallVector<Token, 16> MacroExpandedTokens;
646  std::vector<std::pair<TokenLexer *, size_t> > MacroExpandingLexersStack;
647 
648  /// \brief A record of the macro definitions and expansions that
649  /// occurred during preprocessing.
650  ///
651  /// This is an optional side structure that can be enabled with
652  /// \c createPreprocessingRecord() prior to preprocessing.
653  PreprocessingRecord *Record;
654 
655  /// Cached tokens state.
656  typedef SmallVector<Token, 1> CachedTokensTy;
657 
658  /// \brief Cached tokens are stored here when we do backtracking or
659  /// lookahead. They are "lexed" by the CachingLex() method.
660  CachedTokensTy CachedTokens;
661 
662  /// \brief The position of the cached token that CachingLex() should
663  /// "lex" next.
664  ///
665  /// If it points beyond the CachedTokens vector, it means that a normal
666  /// Lex() should be invoked.
667  CachedTokensTy::size_type CachedLexPos;
668 
669  /// \brief Stack of backtrack positions, allowing nested backtracks.
670  ///
671  /// The EnableBacktrackAtThisPos() method pushes a position to
672  /// indicate where CachedLexPos should be set when the BackTrack() method is
673  /// invoked (at which point the last position is popped).
674  std::vector<CachedTokensTy::size_type> BacktrackPositions;
675 
676  struct MacroInfoChain {
677  MacroInfo MI;
678  MacroInfoChain *Next;
679  };
680 
681  /// MacroInfos are managed as a chain for easy disposal. This is the head
682  /// of that list.
683  MacroInfoChain *MIChainHead;
684 
685  void updateOutOfDateIdentifier(IdentifierInfo &II) const;
686 
687 public:
688  Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
689  DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM,
690  MemoryBufferCache &PCMCache,
691  HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
692  IdentifierInfoLookup *IILookup = nullptr,
693  bool OwnsHeaderSearch = false,
695 
696  ~Preprocessor();
697 
698  /// \brief Initialize the preprocessor using information about the target.
699  ///
700  /// \param Target is owned by the caller and must remain valid for the
701  /// lifetime of the preprocessor.
702  /// \param AuxTarget is owned by the caller and must remain valid for
703  /// the lifetime of the preprocessor.
704  void Initialize(const TargetInfo &Target,
705  const TargetInfo *AuxTarget = nullptr);
706 
707  /// \brief Initialize the preprocessor to parse a model file
708  ///
709  /// To parse model files the preprocessor of the original source is reused to
710  /// preserver the identifier table. However to avoid some duplicate
711  /// information in the preprocessor some cleanup is needed before it is used
712  /// to parse model files. This method does that cleanup.
713  void InitializeForModelFile();
714 
715  /// \brief Cleanup after model file parsing
716  void FinalizeForModelFile();
717 
718  /// \brief Retrieve the preprocessor options used to initialize this
719  /// preprocessor.
720  PreprocessorOptions &getPreprocessorOpts() const { return *PPOpts; }
721 
722  DiagnosticsEngine &getDiagnostics() const { return *Diags; }
723  void setDiagnostics(DiagnosticsEngine &D) { Diags = &D; }
724 
725  const LangOptions &getLangOpts() const { return LangOpts; }
726  const TargetInfo &getTargetInfo() const { return *Target; }
727  const TargetInfo *getAuxTargetInfo() const { return AuxTarget; }
728  FileManager &getFileManager() const { return FileMgr; }
729  SourceManager &getSourceManager() const { return SourceMgr; }
730  MemoryBufferCache &getPCMCache() const { return PCMCache; }
731  HeaderSearch &getHeaderSearchInfo() const { return HeaderInfo; }
732 
733  IdentifierTable &getIdentifierTable() { return Identifiers; }
734  const IdentifierTable &getIdentifierTable() const { return Identifiers; }
735  SelectorTable &getSelectorTable() { return Selectors; }
736  Builtin::Context &getBuiltinInfo() { return BuiltinInfo; }
737  llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
738 
739  void setPTHManager(PTHManager* pm);
740 
741  PTHManager *getPTHManager() { return PTH.get(); }
742 
744  ExternalSource = Source;
745  }
746 
748  return ExternalSource;
749  }
750 
751  /// \brief Retrieve the module loader associated with this preprocessor.
752  ModuleLoader &getModuleLoader() const { return TheModuleLoader; }
753 
755  return TheModuleLoader.HadFatalFailure;
756  }
757 
758  /// \brief True if we are currently preprocessing a #if or #elif directive
760  return ParsingIfOrElifDirective;
761  }
762 
763  /// \brief Control whether the preprocessor retains comments in output.
764  void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments) {
765  this->KeepComments = KeepComments | KeepMacroComments;
766  this->KeepMacroComments = KeepMacroComments;
767  }
768 
769  bool getCommentRetentionState() const { return KeepComments; }
770 
771  void setPragmasEnabled(bool Enabled) { PragmasEnabled = Enabled; }
772  bool getPragmasEnabled() const { return PragmasEnabled; }
773 
774  void SetSuppressIncludeNotFoundError(bool Suppress) {
775  SuppressIncludeNotFoundError = Suppress;
776  }
777 
779  return SuppressIncludeNotFoundError;
780  }
781 
782  /// Sets whether the preprocessor is responsible for producing output or if
783  /// it is producing tokens to be consumed by Parse and Sema.
784  void setPreprocessedOutput(bool IsPreprocessedOutput) {
785  PreprocessedOutput = IsPreprocessedOutput;
786  }
787 
788  /// Returns true if the preprocessor is responsible for generating output,
789  /// false if it is producing tokens to be consumed by Parse and Sema.
790  bool isPreprocessedOutput() const { return PreprocessedOutput; }
791 
792  /// \brief Return true if we are lexing directly from the specified lexer.
793  bool isCurrentLexer(const PreprocessorLexer *L) const {
794  return CurPPLexer == L;
795  }
796 
797  /// \brief Return the current lexer being lexed from.
798  ///
799  /// Note that this ignores any potentially active macro expansions and _Pragma
800  /// expansions going on at the time.
801  PreprocessorLexer *getCurrentLexer() const { return CurPPLexer; }
802 
803  /// \brief Return the current file lexer being lexed from.
804  ///
805  /// Note that this ignores any potentially active macro expansions and _Pragma
806  /// expansions going on at the time.
808 
809  /// \brief Return the submodule owning the file being lexed. This may not be
810  /// the current module if we have changed modules since entering the file.
811  Module *getCurrentLexerSubmodule() const { return CurLexerSubmodule; }
812 
813  /// \brief Returns the FileID for the preprocessor predefines.
814  FileID getPredefinesFileID() const { return PredefinesFileID; }
815 
816  /// \{
817  /// \brief Accessors for preprocessor callbacks.
818  ///
819  /// Note that this class takes ownership of any PPCallbacks object given to
820  /// it.
821  PPCallbacks *getPPCallbacks() const { return Callbacks.get(); }
822  void addPPCallbacks(std::unique_ptr<PPCallbacks> C) {
823  if (Callbacks)
824  C = llvm::make_unique<PPChainedCallbacks>(std::move(C),
825  std::move(Callbacks));
826  Callbacks = std::move(C);
827  }
828  /// \}
829 
830  bool isMacroDefined(StringRef Id) {
831  return isMacroDefined(&Identifiers.get(Id));
832  }
833  bool isMacroDefined(const IdentifierInfo *II) {
834  return II->hasMacroDefinition() &&
835  (!getLangOpts().Modules || (bool)getMacroDefinition(II));
836  }
837 
838  /// \brief Determine whether II is defined as a macro within the module M,
839  /// if that is a module that we've already preprocessed. Does not check for
840  /// macros imported into M.
842  if (!II->hasMacroDefinition())
843  return false;
844  auto I = Submodules.find(M);
845  if (I == Submodules.end())
846  return false;
847  auto J = I->second.Macros.find(II);
848  if (J == I->second.Macros.end())
849  return false;
850  auto *MD = J->second.getLatest();
851  return MD && MD->isDefined();
852  }
853 
855  if (!II->hasMacroDefinition())
856  return MacroDefinition();
857 
858  MacroState &S = CurSubmoduleState->Macros[II];
859  auto *MD = S.getLatest();
860  while (MD && isa<VisibilityMacroDirective>(MD))
861  MD = MD->getPrevious();
862  return MacroDefinition(dyn_cast_or_null<DefMacroDirective>(MD),
863  S.getActiveModuleMacros(*this, II),
864  S.isAmbiguous(*this, II));
865  }
866 
868  SourceLocation Loc) {
869  if (!II->hadMacroDefinition())
870  return MacroDefinition();
871 
872  MacroState &S = CurSubmoduleState->Macros[II];
874  if (auto *MD = S.getLatest())
875  DI = MD->findDirectiveAtLoc(Loc, getSourceManager());
876  // FIXME: Compute the set of active module macros at the specified location.
877  return MacroDefinition(DI.getDirective(),
878  S.getActiveModuleMacros(*this, II),
879  S.isAmbiguous(*this, II));
880  }
881 
882  /// \brief Given an identifier, return its latest non-imported MacroDirective
883  /// if it is \#define'd and not \#undef'd, or null if it isn't \#define'd.
885  if (!II->hasMacroDefinition())
886  return nullptr;
887 
888  auto *MD = getLocalMacroDirectiveHistory(II);
889  if (!MD || MD->getDefinition().isUndefined())
890  return nullptr;
891 
892  return MD;
893  }
894 
895  const MacroInfo *getMacroInfo(const IdentifierInfo *II) const {
896  return const_cast<Preprocessor*>(this)->getMacroInfo(II);
897  }
898 
900  if (!II->hasMacroDefinition())
901  return nullptr;
902  if (auto MD = getMacroDefinition(II))
903  return MD.getMacroInfo();
904  return nullptr;
905  }
906 
907  /// \brief Given an identifier, return the latest non-imported macro
908  /// directive for that identifier.
909  ///
910  /// One can iterate over all previous macro directives from the most recent
911  /// one.
913 
914  /// \brief Add a directive to the macro directive history for this identifier.
917  SourceLocation Loc) {
918  DefMacroDirective *MD = AllocateDefMacroDirective(MI, Loc);
919  appendMacroDirective(II, MD);
920  return MD;
921  }
923  MacroInfo *MI) {
924  return appendDefMacroDirective(II, MI, MI->getDefinitionLoc());
925  }
926  /// \brief Set a MacroDirective that was loaded from a PCH file.
928  MacroDirective *MD);
929 
930  /// \brief Register an exported macro for a module and identifier.
932  ArrayRef<ModuleMacro *> Overrides, bool &IsNew);
934 
935  /// \brief Get the list of leaf (non-overridden) module macros for a name.
937  if (II->isOutOfDate())
938  updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
939  auto I = LeafModuleMacros.find(II);
940  if (I != LeafModuleMacros.end())
941  return I->second;
942  return None;
943  }
944 
945  /// \{
946  /// Iterators for the macro history table. Currently defined macros have
947  /// IdentifierInfo::hasMacroDefinition() set and an empty
948  /// MacroInfo::getUndefLoc() at the head of the list.
949  typedef MacroMap::const_iterator macro_iterator;
950  macro_iterator macro_begin(bool IncludeExternalMacros = true) const;
951  macro_iterator macro_end(bool IncludeExternalMacros = true) const;
952  llvm::iterator_range<macro_iterator>
953  macros(bool IncludeExternalMacros = true) const {
954  return llvm::make_range(macro_begin(IncludeExternalMacros),
955  macro_end(IncludeExternalMacros));
956  }
957  /// \}
958 
959  /// \brief Return the name of the macro defined before \p Loc that has
960  /// spelling \p Tokens. If there are multiple macros with same spelling,
961  /// return the last one defined.
964 
965  const std::string &getPredefines() const { return Predefines; }
966  /// \brief Set the predefines for this Preprocessor.
967  ///
968  /// These predefines are automatically injected when parsing the main file.
969  void setPredefines(const char *P) { Predefines = P; }
970  void setPredefines(StringRef P) { Predefines = P; }
971 
972  /// Return information about the specified preprocessor
973  /// identifier token.
975  return &Identifiers.get(Name);
976  }
977 
978  /// \brief Add the specified pragma handler to this preprocessor.
979  ///
980  /// If \p Namespace is non-null, then it is a token required to exist on the
981  /// pragma line before the pragma string starts, e.g. "STDC" or "GCC".
982  void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler);
984  AddPragmaHandler(StringRef(), Handler);
985  }
986 
987  /// \brief Remove the specific pragma handler from this preprocessor.
988  ///
989  /// If \p Namespace is non-null, then it should be the namespace that
990  /// \p Handler was added to. It is an error to remove a handler that
991  /// has not been registered.
992  void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler);
994  RemovePragmaHandler(StringRef(), Handler);
995  }
996 
997  /// Install empty handlers for all pragmas (making them ignored).
998  void IgnorePragmas();
999 
1000  /// \brief Add the specified comment handler to the preprocessor.
1001  void addCommentHandler(CommentHandler *Handler);
1002 
1003  /// \brief Remove the specified comment handler.
1004  ///
1005  /// It is an error to remove a handler that has not been registered.
1006  void removeCommentHandler(CommentHandler *Handler);
1007 
1008  /// \brief Set the code completion handler to the given object.
1010  CodeComplete = &Handler;
1011  }
1012 
1013  /// \brief Retrieve the current code-completion handler.
1015  return CodeComplete;
1016  }
1017 
1018  /// \brief Clear out the code completion handler.
1020  CodeComplete = nullptr;
1021  }
1022 
1023  /// \brief Hook used by the lexer to invoke the "natural language" code
1024  /// completion point.
1026 
1027  /// \brief Set the code completion token for filtering purposes.
1029  CodeCompletionII = Filter;
1030  }
1031 
1032  /// \brief Get the code completion token for filtering purposes.
1034  if (CodeCompletionII)
1035  return CodeCompletionII->getName();
1036  return {};
1037  }
1038 
1039  /// \brief Retrieve the preprocessing record, or NULL if there is no
1040  /// preprocessing record.
1041  PreprocessingRecord *getPreprocessingRecord() const { return Record; }
1042 
1043  /// \brief Create a new preprocessing record, which will keep track of
1044  /// all macro expansions, macro definitions, etc.
1046 
1047  /// \brief Enter the specified FileID as the main source file,
1048  /// which implicitly adds the builtin defines etc.
1049  void EnterMainSourceFile();
1050 
1051  /// \brief Inform the preprocessor callbacks that processing is complete.
1052  void EndSourceFile();
1053 
1054  /// \brief Add a source file to the top of the include stack and
1055  /// start lexing tokens from it instead of the current buffer.
1056  ///
1057  /// Emits a diagnostic, doesn't enter the file, and returns true on error.
1058  bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir,
1059  SourceLocation Loc);
1060 
1061  /// \brief Add a Macro to the top of the include stack and start lexing
1062  /// tokens from it instead of the current buffer.
1063  ///
1064  /// \param Args specifies the tokens input to a function-like macro.
1065  /// \param ILEnd specifies the location of the ')' for a function-like macro
1066  /// or the identifier for an object-like macro.
1067  void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro,
1068  MacroArgs *Args);
1069 
1070  /// \brief Add a "macro" context to the top of the include stack,
1071  /// which will cause the lexer to start returning the specified tokens.
1072  ///
1073  /// If \p DisableMacroExpansion is true, tokens lexed from the token stream
1074  /// will not be subject to further macro expansion. Otherwise, these tokens
1075  /// will be re-macro-expanded when/if expansion is enabled.
1076  ///
1077  /// If \p OwnsTokens is false, this method assumes that the specified stream
1078  /// of tokens has a permanent owner somewhere, so they do not need to be
1079  /// copied. If it is true, it assumes the array of tokens is allocated with
1080  /// \c new[] and the Preprocessor will delete[] it.
1081 private:
1082  void EnterTokenStream(const Token *Toks, unsigned NumToks,
1083  bool DisableMacroExpansion, bool OwnsTokens);
1084 
1085 public:
1086  void EnterTokenStream(std::unique_ptr<Token[]> Toks, unsigned NumToks,
1087  bool DisableMacroExpansion) {
1088  EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion, true);
1089  }
1090  void EnterTokenStream(ArrayRef<Token> Toks, bool DisableMacroExpansion) {
1091  EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion, false);
1092  }
1093 
1094  /// \brief Pop the current lexer/macro exp off the top of the lexer stack.
1095  ///
1096  /// This should only be used in situations where the current state of the
1097  /// top-of-stack lexer is known.
1098  void RemoveTopOfLexerStack();
1099 
1100  /// From the point that this method is called, and until
1101  /// CommitBacktrackedTokens() or Backtrack() is called, the Preprocessor
1102  /// keeps track of the lexed tokens so that a subsequent Backtrack() call will
1103  /// make the Preprocessor re-lex the same tokens.
1104  ///
1105  /// Nested backtracks are allowed, meaning that EnableBacktrackAtThisPos can
1106  /// be called multiple times and CommitBacktrackedTokens/Backtrack calls will
1107  /// be combined with the EnableBacktrackAtThisPos calls in reverse order.
1108  ///
1109  /// NOTE: *DO NOT* forget to call either CommitBacktrackedTokens or Backtrack
1110  /// at some point after EnableBacktrackAtThisPos. If you don't, caching of
1111  /// tokens will continue indefinitely.
1112  ///
1113  void EnableBacktrackAtThisPos();
1114 
1115  /// \brief Disable the last EnableBacktrackAtThisPos call.
1116  void CommitBacktrackedTokens();
1117 
1119  CachedTokensTy::size_type Begin, End;
1120  };
1121 
1122 private:
1123  /// \brief A range of cached tokens that should be erased after lexing
1124  /// when backtracking requires the erasure of such cached tokens.
1125  Optional<CachedTokensRange> CachedTokenRangeToErase;
1126 
1127 public:
1128  /// \brief Returns the range of cached tokens that were lexed since
1129  /// EnableBacktrackAtThisPos() was previously called.
1131 
1132  /// \brief Erase the range of cached tokens that were lexed since
1133  /// EnableBacktrackAtThisPos() was previously called.
1134  void EraseCachedTokens(CachedTokensRange TokenRange);
1135 
1136  /// \brief Make Preprocessor re-lex the tokens that were lexed since
1137  /// EnableBacktrackAtThisPos() was previously called.
1138  void Backtrack();
1139 
1140  /// \brief True if EnableBacktrackAtThisPos() was called and
1141  /// caching of tokens is on.
1142  bool isBacktrackEnabled() const { return !BacktrackPositions.empty(); }
1143 
1144  /// \brief Lex the next token for this preprocessor.
1145  void Lex(Token &Result);
1146 
1148 
1149  void makeModuleVisible(Module *M, SourceLocation Loc);
1150 
1152  return CurSubmoduleState->VisibleModules.getImportLoc(M);
1153  }
1154 
1155  /// \brief Lex a string literal, which may be the concatenation of multiple
1156  /// string literals and may even come from macro expansion.
1157  /// \returns true on success, false if a error diagnostic has been generated.
1158  bool LexStringLiteral(Token &Result, std::string &String,
1159  const char *DiagnosticTag, bool AllowMacroExpansion) {
1160  if (AllowMacroExpansion)
1161  Lex(Result);
1162  else
1163  LexUnexpandedToken(Result);
1164  return FinishLexStringLiteral(Result, String, DiagnosticTag,
1165  AllowMacroExpansion);
1166  }
1167 
1168  /// \brief Complete the lexing of a string literal where the first token has
1169  /// already been lexed (see LexStringLiteral).
1170  bool FinishLexStringLiteral(Token &Result, std::string &String,
1171  const char *DiagnosticTag,
1172  bool AllowMacroExpansion);
1173 
1174  /// \brief Lex a token. If it's a comment, keep lexing until we get
1175  /// something not a comment.
1176  ///
1177  /// This is useful in -E -C mode where comments would foul up preprocessor
1178  /// directive handling.
1180  do
1181  Lex(Result);
1182  while (Result.getKind() == tok::comment);
1183  }
1184 
1185  /// \brief Just like Lex, but disables macro expansion of identifier tokens.
1187  // Disable macro expansion.
1188  bool OldVal = DisableMacroExpansion;
1189  DisableMacroExpansion = true;
1190  // Lex the token.
1191  Lex(Result);
1192 
1193  // Reenable it.
1194  DisableMacroExpansion = OldVal;
1195  }
1196 
1197  /// \brief Like LexNonComment, but this disables macro expansion of
1198  /// identifier tokens.
1200  do
1201  LexUnexpandedToken(Result);
1202  while (Result.getKind() == tok::comment);
1203  }
1204 
1205  /// \brief Parses a simple integer literal to get its numeric value. Floating
1206  /// point literals and user defined literals are rejected. Used primarily to
1207  /// handle pragmas that accept integer arguments.
1208  bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value);
1209 
1210  /// Disables macro expansion everywhere except for preprocessor directives.
1212  DisableMacroExpansion = true;
1213  MacroExpansionInDirectivesOverride = true;
1214  }
1215 
1216  /// \brief Peeks ahead N tokens and returns that token without consuming any
1217  /// tokens.
1218  ///
1219  /// LookAhead(0) returns the next token that would be returned by Lex(),
1220  /// LookAhead(1) returns the token after it, etc. This returns normal
1221  /// tokens after phase 5. As such, it is equivalent to using
1222  /// 'Lex', not 'LexUnexpandedToken'.
1223  const Token &LookAhead(unsigned N) {
1224  if (CachedLexPos + N < CachedTokens.size())
1225  return CachedTokens[CachedLexPos+N];
1226  else
1227  return PeekAhead(N+1);
1228  }
1229 
1230  /// \brief When backtracking is enabled and tokens are cached,
1231  /// this allows to revert a specific number of tokens.
1232  ///
1233  /// Note that the number of tokens being reverted should be up to the last
1234  /// backtrack position, not more.
1235  void RevertCachedTokens(unsigned N) {
1236  assert(isBacktrackEnabled() &&
1237  "Should only be called when tokens are cached for backtracking");
1238  assert(signed(CachedLexPos) - signed(N) >= signed(BacktrackPositions.back())
1239  && "Should revert tokens up to the last backtrack position, not more");
1240  assert(signed(CachedLexPos) - signed(N) >= 0 &&
1241  "Corrupted backtrack positions ?");
1242  CachedLexPos -= N;
1243  }
1244 
1245  /// \brief Enters a token in the token stream to be lexed next.
1246  ///
1247  /// If BackTrack() is called afterwards, the token will remain at the
1248  /// insertion point.
1249  void EnterToken(const Token &Tok) {
1250  EnterCachingLexMode();
1251  CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
1252  }
1253 
1254  /// We notify the Preprocessor that if it is caching tokens (because
1255  /// backtrack is enabled) it should replace the most recent cached tokens
1256  /// with the given annotation token. This function has no effect if
1257  /// backtracking is not enabled.
1258  ///
1259  /// Note that the use of this function is just for optimization, so that the
1260  /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
1261  /// invoked.
1262  void AnnotateCachedTokens(const Token &Tok) {
1263  assert(Tok.isAnnotation() && "Expected annotation token");
1264  if (CachedLexPos != 0 && isBacktrackEnabled())
1265  AnnotatePreviousCachedTokens(Tok);
1266  }
1267 
1268  /// Get the location of the last cached token, suitable for setting the end
1269  /// location of an annotation token.
1271  assert(CachedLexPos != 0);
1272  return CachedTokens[CachedLexPos-1].getLastLoc();
1273  }
1274 
1275  /// \brief Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
1276  /// CachedTokens.
1277  bool IsPreviousCachedToken(const Token &Tok) const;
1278 
1279  /// \brief Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
1280  /// in \p NewToks.
1281  ///
1282  /// Useful when a token needs to be split in smaller ones and CachedTokens
1283  /// most recent token must to be updated to reflect that.
1285 
1286  /// \brief Replace the last token with an annotation token.
1287  ///
1288  /// Like AnnotateCachedTokens(), this routine replaces an
1289  /// already-parsed (and resolved) token with an annotation
1290  /// token. However, this routine only replaces the last token with
1291  /// the annotation token; it does not affect any other cached
1292  /// tokens. This function has no effect if backtracking is not
1293  /// enabled.
1295  assert(Tok.isAnnotation() && "Expected annotation token");
1296  if (CachedLexPos != 0 && isBacktrackEnabled())
1297  CachedTokens[CachedLexPos-1] = Tok;
1298  }
1299 
1300  /// Enter an annotation token into the token stream.
1302  void *AnnotationVal);
1303 
1304  /// Update the current token to represent the provided
1305  /// identifier, in order to cache an action performed by typo correction.
1306  void TypoCorrectToken(const Token &Tok) {
1307  assert(Tok.getIdentifierInfo() && "Expected identifier token");
1308  if (CachedLexPos != 0 && isBacktrackEnabled())
1309  CachedTokens[CachedLexPos-1] = Tok;
1310  }
1311 
1312  /// \brief Recompute the current lexer kind based on the CurLexer/CurPTHLexer/
1313  /// CurTokenLexer pointers.
1314  void recomputeCurLexerKind();
1315 
1316  /// \brief Returns true if incremental processing is enabled
1317  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
1318 
1319  /// \brief Enables the incremental processing
1320  void enableIncrementalProcessing(bool value = true) {
1321  IncrementalProcessing = value;
1322  }
1323 
1324  /// \brief Specify the point at which code-completion will be performed.
1325  ///
1326  /// \param File the file in which code completion should occur. If
1327  /// this file is included multiple times, code-completion will
1328  /// perform completion the first time it is included. If NULL, this
1329  /// function clears out the code-completion point.
1330  ///
1331  /// \param Line the line at which code completion should occur
1332  /// (1-based).
1333  ///
1334  /// \param Column the column at which code completion should occur
1335  /// (1-based).
1336  ///
1337  /// \returns true if an error occurred, false otherwise.
1338  bool SetCodeCompletionPoint(const FileEntry *File,
1339  unsigned Line, unsigned Column);
1340 
1341  /// \brief Determine if we are performing code completion.
1342  bool isCodeCompletionEnabled() const { return CodeCompletionFile != nullptr; }
1343 
1344  /// \brief Returns the location of the code-completion point.
1345  ///
1346  /// Returns an invalid location if code-completion is not enabled or the file
1347  /// containing the code-completion point has not been lexed yet.
1348  SourceLocation getCodeCompletionLoc() const { return CodeCompletionLoc; }
1349 
1350  /// \brief Returns the start location of the file of code-completion point.
1351  ///
1352  /// Returns an invalid location if code-completion is not enabled or the file
1353  /// containing the code-completion point has not been lexed yet.
1355  return CodeCompletionFileLoc;
1356  }
1357 
1358  /// \brief Returns true if code-completion is enabled and we have hit the
1359  /// code-completion point.
1360  bool isCodeCompletionReached() const { return CodeCompletionReached; }
1361 
1362  /// \brief Note that we hit the code-completion point.
1364  assert(isCodeCompletionEnabled() && "Code-completion not enabled!");
1365  CodeCompletionReached = true;
1366  // Silence any diagnostics that occur after we hit the code-completion.
1368  }
1369 
1370  /// \brief The location of the currently-active \#pragma clang
1371  /// arc_cf_code_audited begin.
1372  ///
1373  /// Returns an invalid location if there is no such pragma active.
1375  return PragmaARCCFCodeAuditedLoc;
1376  }
1377 
1378  /// \brief Set the location of the currently-active \#pragma clang
1379  /// arc_cf_code_audited begin. An invalid location ends the pragma.
1381  PragmaARCCFCodeAuditedLoc = Loc;
1382  }
1383 
1384  /// \brief The location of the currently-active \#pragma clang
1385  /// assume_nonnull begin.
1386  ///
1387  /// Returns an invalid location if there is no such pragma active.
1389  return PragmaAssumeNonNullLoc;
1390  }
1391 
1392  /// \brief Set the location of the currently-active \#pragma clang
1393  /// assume_nonnull begin. An invalid location ends the pragma.
1395  PragmaAssumeNonNullLoc = Loc;
1396  }
1397 
1398  /// \brief Set the directory in which the main file should be considered
1399  /// to have been found, if it is not a real file.
1400  void setMainFileDir(const DirectoryEntry *Dir) {
1401  MainFileDir = Dir;
1402  }
1403 
1404  /// \brief Instruct the preprocessor to skip part of the main source file.
1405  ///
1406  /// \param Bytes The number of bytes in the preamble to skip.
1407  ///
1408  /// \param StartOfLine Whether skipping these bytes puts the lexer at the
1409  /// start of a line.
1410  void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine) {
1411  SkipMainFilePreamble.first = Bytes;
1412  SkipMainFilePreamble.second = StartOfLine;
1413  }
1414 
1415  /// Forwarding function for diagnostics. This emits a diagnostic at
1416  /// the specified Token's location, translating the token's start
1417  /// position in the current buffer into a SourcePosition object for rendering.
1418  DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const {
1419  return Diags->Report(Loc, DiagID);
1420  }
1421 
1422  DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const {
1423  return Diags->Report(Tok.getLocation(), DiagID);
1424  }
1425 
1426  /// Return the 'spelling' of the token at the given
1427  /// location; does not go up to the spelling location or down to the
1428  /// expansion location.
1429  ///
1430  /// \param buffer A buffer which will be used only if the token requires
1431  /// "cleaning", e.g. if it contains trigraphs or escaped newlines
1432  /// \param invalid If non-null, will be set \c true if an error occurs.
1434  SmallVectorImpl<char> &buffer,
1435  bool *invalid = nullptr) const {
1436  return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
1437  }
1438 
1439  /// \brief Return the 'spelling' of the Tok token.
1440  ///
1441  /// The spelling of a token is the characters used to represent the token in
1442  /// the source file after trigraph expansion and escaped-newline folding. In
1443  /// particular, this wants to get the true, uncanonicalized, spelling of
1444  /// things like digraphs, UCNs, etc.
1445  ///
1446  /// \param Invalid If non-null, will be set \c true if an error occurs.
1447  std::string getSpelling(const Token &Tok, bool *Invalid = nullptr) const {
1448  return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
1449  }
1450 
1451  /// \brief Get the spelling of a token into a preallocated buffer, instead
1452  /// of as an std::string.
1453  ///
1454  /// The caller is required to allocate enough space for the token, which is
1455  /// guaranteed to be at least Tok.getLength() bytes long. The length of the
1456  /// actual result is returned.
1457  ///
1458  /// Note that this method may do two possible things: it may either fill in
1459  /// the buffer specified with characters, or it may *change the input pointer*
1460  /// to point to a constant buffer with the data already in it (avoiding a
1461  /// copy). The caller is not allowed to modify the returned buffer pointer
1462  /// if an internal buffer is returned.
1463  unsigned getSpelling(const Token &Tok, const char *&Buffer,
1464  bool *Invalid = nullptr) const {
1465  return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
1466  }
1467 
1468  /// \brief Get the spelling of a token into a SmallVector.
1469  ///
1470  /// Note that the returned StringRef may not point to the
1471  /// supplied buffer if a copy can be avoided.
1472  StringRef getSpelling(const Token &Tok,
1474  bool *Invalid = nullptr) const;
1475 
1476  /// \brief Relex the token at the specified location.
1477  /// \returns true if there was a failure, false on success.
1479  bool IgnoreWhiteSpace = false) {
1480  return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
1481  }
1482 
1483  /// \brief Given a Token \p Tok that is a numeric constant with length 1,
1484  /// return the character.
1485  char
1487  bool *Invalid = nullptr) const {
1488  assert(Tok.is(tok::numeric_constant) &&
1489  Tok.getLength() == 1 && "Called on unsupported token");
1490  assert(!Tok.needsCleaning() && "Token can't need cleaning with length 1");
1491 
1492  // If the token is carrying a literal data pointer, just use it.
1493  if (const char *D = Tok.getLiteralData())
1494  return *D;
1495 
1496  // Otherwise, fall back on getCharacterData, which is slower, but always
1497  // works.
1498  return *SourceMgr.getCharacterData(Tok.getLocation(), Invalid);
1499  }
1500 
1501  /// \brief Retrieve the name of the immediate macro expansion.
1502  ///
1503  /// This routine starts from a source location, and finds the name of the
1504  /// macro responsible for its immediate expansion. It looks through any
1505  /// intervening macro argument expansions to compute this. It returns a
1506  /// StringRef that refers to the SourceManager-owned buffer of the source
1507  /// where that macro name is spelled. Thus, the result shouldn't out-live
1508  /// the SourceManager.
1510  return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
1511  }
1512 
1513  /// \brief Plop the specified string into a scratch buffer and set the
1514  /// specified token's location and length to it.
1515  ///
1516  /// If specified, the source location provides a location of the expansion
1517  /// point of the token.
1518  void CreateString(StringRef Str, Token &Tok,
1519  SourceLocation ExpansionLocStart = SourceLocation(),
1520  SourceLocation ExpansionLocEnd = SourceLocation());
1521 
1522  /// \brief Computes the source location just past the end of the
1523  /// token at this source location.
1524  ///
1525  /// This routine can be used to produce a source location that
1526  /// points just past the end of the token referenced by \p Loc, and
1527  /// is generally used when a diagnostic needs to point just after a
1528  /// token where it expected something different that it received. If
1529  /// the returned source location would not be meaningful (e.g., if
1530  /// it points into a macro), this routine returns an invalid
1531  /// source location.
1532  ///
1533  /// \param Offset an offset from the end of the token, where the source
1534  /// location should refer to. The default offset (0) produces a source
1535  /// location pointing just past the end of the token; an offset of 1 produces
1536  /// a source location pointing to the last character in the token, etc.
1538  return Lexer::getLocForEndOfToken(Loc, Offset, SourceMgr, LangOpts);
1539  }
1540 
1541  /// \brief Returns true if the given MacroID location points at the first
1542  /// token of the macro expansion.
1543  ///
1544  /// \param MacroBegin If non-null and function returns true, it is set to
1545  /// begin location of the macro.
1547  SourceLocation *MacroBegin = nullptr) const {
1548  return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
1549  MacroBegin);
1550  }
1551 
1552  /// \brief Returns true if the given MacroID location points at the last
1553  /// token of the macro expansion.
1554  ///
1555  /// \param MacroEnd If non-null and function returns true, it is set to
1556  /// end location of the macro.
1558  SourceLocation *MacroEnd = nullptr) const {
1559  return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
1560  }
1561 
1562  /// \brief Print the token to stderr, used for debugging.
1563  void DumpToken(const Token &Tok, bool DumpFlags = false) const;
1564  void DumpLocation(SourceLocation Loc) const;
1565  void DumpMacro(const MacroInfo &MI) const;
1566  void dumpMacroInfo(const IdentifierInfo *II);
1567 
1568  /// \brief Given a location that specifies the start of a
1569  /// token, return a new location that specifies a character within the token.
1571  unsigned Char) const {
1572  return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
1573  }
1574 
1575  /// \brief Increment the counters for the number of token paste operations
1576  /// performed.
1577  ///
1578  /// If fast was specified, this is a 'fast paste' case we handled.
1579  void IncrementPasteCounter(bool isFast) {
1580  if (isFast)
1581  ++NumFastTokenPaste;
1582  else
1583  ++NumTokenPaste;
1584  }
1585 
1586  void PrintStats();
1587 
1588  size_t getTotalMemory() const;
1589 
1590  /// When the macro expander pastes together a comment (/##/) in Microsoft
1591  /// mode, this method handles updating the current state, returning the
1592  /// token on the next source line.
1594 
1595  //===--------------------------------------------------------------------===//
1596  // Preprocessor callback methods. These are invoked by a lexer as various
1597  // directives and events are found.
1598 
1599  /// Given a tok::raw_identifier token, look up the
1600  /// identifier information for the token and install it into the token,
1601  /// updating the token kind accordingly.
1602  IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
1603 
1604 private:
1605  llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1606 
1607 public:
1608 
1609  /// \brief Specifies the reason for poisoning an identifier.
1610  ///
1611  /// If that identifier is accessed while poisoned, then this reason will be
1612  /// used instead of the default "poisoned" diagnostic.
1613  void SetPoisonReason(IdentifierInfo *II, unsigned DiagID);
1614 
1615  /// \brief Display reason for poisoned identifier.
1616  void HandlePoisonedIdentifier(Token & Tok);
1617 
1619  if(IdentifierInfo * II = Identifier.getIdentifierInfo()) {
1620  if(II->isPoisoned()) {
1621  HandlePoisonedIdentifier(Identifier);
1622  }
1623  }
1624  }
1625 
1626 private:
1627  /// Identifiers used for SEH handling in Borland. These are only
1628  /// allowed in particular circumstances
1629  // __except block
1630  IdentifierInfo *Ident__exception_code,
1631  *Ident___exception_code,
1632  *Ident_GetExceptionCode;
1633  // __except filter expression
1634  IdentifierInfo *Ident__exception_info,
1635  *Ident___exception_info,
1636  *Ident_GetExceptionInfo;
1637  // __finally
1638  IdentifierInfo *Ident__abnormal_termination,
1639  *Ident___abnormal_termination,
1640  *Ident_AbnormalTermination;
1641 
1642  const char *getCurLexerEndPos();
1643  void diagnoseMissingHeaderInUmbrellaDir(const Module &Mod);
1644 
1645 public:
1646  void PoisonSEHIdentifiers(bool Poison = true); // Borland
1647 
1648  /// \brief Callback invoked when the lexer reads an identifier and has
1649  /// filled in the tokens IdentifierInfo member.
1650  ///
1651  /// This callback potentially macro expands it or turns it into a named
1652  /// token (like 'for').
1653  ///
1654  /// \returns true if we actually computed a token, false if we need to
1655  /// lex again.
1656  bool HandleIdentifier(Token &Identifier);
1657 
1658 
1659  /// \brief Callback invoked when the lexer hits the end of the current file.
1660  ///
1661  /// This either returns the EOF token and returns true, or
1662  /// pops a level off the include stack and returns false, at which point the
1663  /// client should call lex again.
1664  bool HandleEndOfFile(Token &Result, bool isEndOfMacro = false);
1665 
1666  /// \brief Callback invoked when the current TokenLexer hits the end of its
1667  /// token stream.
1669 
1670  /// \brief Callback invoked when the lexer sees a # token at the start of a
1671  /// line.
1672  ///
1673  /// This consumes the directive, modifies the lexer/preprocessor state, and
1674  /// advances the lexer(s) so that the next token read is the correct one.
1675  void HandleDirective(Token &Result);
1676 
1677  /// \brief Ensure that the next token is a tok::eod token.
1678  ///
1679  /// If not, emit a diagnostic and consume up until the eod.
1680  /// If \p EnableMacros is true, then we consider macros that expand to zero
1681  /// tokens as being ok.
1682  void CheckEndOfDirective(const char *Directive, bool EnableMacros = false);
1683 
1684  /// \brief Read and discard all tokens remaining on the current line until
1685  /// the tok::eod token is found.
1687 
1688  /// \brief Returns true if the preprocessor has seen a use of
1689  /// __DATE__ or __TIME__ in the file so far.
1690  bool SawDateOrTime() const {
1691  return DATELoc != SourceLocation() || TIMELoc != SourceLocation();
1692  }
1693  unsigned getCounterValue() const { return CounterValue; }
1694  void setCounterValue(unsigned V) { CounterValue = V; }
1695 
1696  /// \brief Retrieves the module that we're currently building, if any.
1698 
1699  /// \brief Allocate a new MacroInfo object with the provided SourceLocation.
1701 
1702  /// \brief Turn the specified lexer token into a fully checked and spelled
1703  /// filename, e.g. as an operand of \#include.
1704  ///
1705  /// The caller is expected to provide a buffer that is large enough to hold
1706  /// the spelling of the filename, but is also expected to handle the case
1707  /// when this method decides to use a different buffer.
1708  ///
1709  /// \returns true if the input filename was in <>'s or false if it was
1710  /// in ""'s.
1711  bool GetIncludeFilenameSpelling(SourceLocation Loc,StringRef &Filename);
1712 
1713  /// \brief Given a "foo" or <foo> reference, look up the indicated file.
1714  ///
1715  /// Returns null on failure. \p isAngled indicates whether the file
1716  /// reference is for system \#include's or not (i.e. using <> instead of "").
1717  const FileEntry *LookupFile(SourceLocation FilenameLoc, StringRef Filename,
1718  bool isAngled, const DirectoryLookup *FromDir,
1719  const FileEntry *FromFile,
1720  const DirectoryLookup *&CurDir,
1721  SmallVectorImpl<char> *SearchPath,
1722  SmallVectorImpl<char> *RelativePath,
1723  ModuleMap::KnownHeader *SuggestedModule,
1724  bool *IsMapped, bool SkipCache = false);
1725 
1726  /// \brief Get the DirectoryLookup structure used to find the current
1727  /// FileEntry, if CurLexer is non-null and if applicable.
1728  ///
1729  /// This allows us to implement \#include_next and find directory-specific
1730  /// properties.
1731  const DirectoryLookup *GetCurDirLookup() { return CurDirLookup; }
1732 
1733  /// \brief Return true if we're in the top-level file, not in a \#include.
1734  bool isInPrimaryFile() const;
1735 
1736  /// \brief Handle cases where the \#include name is expanded
1737  /// from a macro as multiple tokens, which need to be glued together.
1738  ///
1739  /// This occurs for code like:
1740  /// \code
1741  /// \#define FOO <x/y.h>
1742  /// \#include FOO
1743  /// \endcode
1744  /// because in this case, "<x/y.h>" is returned as 7 tokens, not one.
1745  ///
1746  /// This code concatenates and consumes tokens up to the '>' token. It
1747  /// returns false if the > was found, otherwise it returns true if it finds
1748  /// and consumes the EOD marker.
1749  bool ConcatenateIncludeName(SmallString<128> &FilenameBuffer,
1750  SourceLocation &End);
1751 
1752  /// \brief Lex an on-off-switch (C99 6.10.6p2) and verify that it is
1753  /// followed by EOD. Return true if the token is not a valid on-off-switch.
1754  bool LexOnOffSwitch(tok::OnOffSwitch &OOS);
1755 
1756  bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef,
1757  bool *ShadowFlag = nullptr);
1758 
1759  void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma);
1760  Module *LeaveSubmodule(bool ForPragma);
1761 
1762 private:
1763  void PushIncludeMacroStack() {
1764  assert(CurLexerKind != CLK_CachingLexer && "cannot push a caching lexer");
1765  IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
1766  std::move(CurLexer), std::move(CurPTHLexer),
1767  CurPPLexer, std::move(CurTokenLexer),
1768  CurDirLookup);
1769  CurPPLexer = nullptr;
1770  }
1771 
1772  void PopIncludeMacroStack() {
1773  CurLexer = std::move(IncludeMacroStack.back().TheLexer);
1774  CurPTHLexer = std::move(IncludeMacroStack.back().ThePTHLexer);
1775  CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1776  CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
1777  CurDirLookup = IncludeMacroStack.back().TheDirLookup;
1778  CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
1779  CurLexerKind = IncludeMacroStack.back().CurLexerKind;
1780  IncludeMacroStack.pop_back();
1781  }
1782 
1783  void PropagateLineStartLeadingSpaceInfo(Token &Result);
1784 
1785  /// Determine whether we need to create module macros for #defines in the
1786  /// current context.
1787  bool needModuleMacros() const;
1788 
1789  /// Update the set of active module macros and ambiguity flag for a module
1790  /// macro name.
1791  void updateModuleMacroInfo(const IdentifierInfo *II, ModuleMacroInfo &Info);
1792 
1793  DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
1794  SourceLocation Loc);
1795  UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
1796  VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
1797  bool isPublic);
1798 
1799  /// \brief Lex and validate a macro name, which occurs after a
1800  /// \#define or \#undef.
1801  ///
1802  /// \param MacroNameTok Token that represents the name defined or undefined.
1803  /// \param IsDefineUndef Kind if preprocessor directive.
1804  /// \param ShadowFlag Points to flag that is set if macro name shadows
1805  /// a keyword.
1806  ///
1807  /// This emits a diagnostic, sets the token kind to eod,
1808  /// and discards the rest of the macro line if the macro name is invalid.
1809  void ReadMacroName(Token &MacroNameTok, MacroUse IsDefineUndef = MU_Other,
1810  bool *ShadowFlag = nullptr);
1811 
1812  /// ReadOptionalMacroParameterListAndBody - This consumes all (i.e. the
1813  /// entire line) of the macro's tokens and adds them to MacroInfo, and while
1814  /// doing so performs certain validity checks including (but not limited to):
1815  /// - # (stringization) is followed by a macro parameter
1816  /// \param MacroNameTok - Token that represents the macro name
1817  /// \param ImmediatelyAfterHeaderGuard - Macro follows an #ifdef header guard
1818  ///
1819  /// Either returns a pointer to a MacroInfo object OR emits a diagnostic and
1820  /// returns a nullptr if an invalid sequence of tokens is encountered.
1821 
1822  MacroInfo *ReadOptionalMacroParameterListAndBody(
1823  const Token &MacroNameTok, bool ImmediatelyAfterHeaderGuard);
1824 
1825  /// The ( starting an argument list of a macro definition has just been read.
1826  /// Lex the rest of the parameters and the closing ), updating \p MI with
1827  /// what we learn and saving in \p LastTok the last token read.
1828  /// Return true if an error occurs parsing the arg list.
1829  bool ReadMacroParameterList(MacroInfo *MI, Token& LastTok);
1830 
1831  /// We just read a \#if or related directive and decided that the
1832  /// subsequent tokens are in the \#if'd out portion of the
1833  /// file. Lex the rest of the file, until we see an \#endif. If \p
1834  /// FoundNonSkipPortion is true, then we have already emitted code for part of
1835  /// this \#if directive, so \#else/\#elif blocks should never be entered. If
1836  /// \p FoundElse is false, then \#else directives are ok, if not, then we have
1837  /// already seen one so a \#else directive is a duplicate. When this returns,
1838  /// the caller can lex the first valid token.
1839  void SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
1840  bool FoundNonSkipPortion, bool FoundElse,
1841  SourceLocation ElseLoc = SourceLocation());
1842 
1843  /// \brief A fast PTH version of SkipExcludedConditionalBlock.
1844  void PTHSkipExcludedConditionalBlock();
1845 
1846  /// Information about the result for evaluating an expression for a
1847  /// preprocessor directive.
1848  struct DirectiveEvalResult {
1849  /// Whether the expression was evaluated as true or not.
1850  bool Conditional;
1851  /// True if the expression contained identifiers that were undefined.
1852  bool IncludedUndefinedIds;
1853  };
1854 
1855  /// \brief Evaluate an integer constant expression that may occur after a
1856  /// \#if or \#elif directive and return a \p DirectiveEvalResult object.
1857  ///
1858  /// If the expression is equivalent to "!defined(X)" return X in IfNDefMacro.
1859  DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
1860 
1861  /// \brief Install the standard preprocessor pragmas:
1862  /// \#pragma GCC poison/system_header/dependency and \#pragma once.
1863  void RegisterBuiltinPragmas();
1864 
1865  /// \brief Register builtin macros such as __LINE__ with the identifier table.
1866  void RegisterBuiltinMacros();
1867 
1868  /// If an identifier token is read that is to be expanded as a macro, handle
1869  /// it and return the next token as 'Tok'. If we lexed a token, return true;
1870  /// otherwise the caller should lex again.
1871  bool HandleMacroExpandedIdentifier(Token &Tok, const MacroDefinition &MD);
1872 
1873  /// \brief Cache macro expanded tokens for TokenLexers.
1874  //
1875  /// Works like a stack; a TokenLexer adds the macro expanded tokens that is
1876  /// going to lex in the cache and when it finishes the tokens are removed
1877  /// from the end of the cache.
1878  Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
1879  ArrayRef<Token> tokens);
1880  void removeCachedMacroExpandedTokensOfLastLexer();
1882 
1883  /// Determine whether the next preprocessor token to be
1884  /// lexed is a '('. If so, consume the token and return true, if not, this
1885  /// method should have no observable side-effect on the lexed tokens.
1886  bool isNextPPTokenLParen();
1887 
1888  /// After reading "MACRO(", this method is invoked to read all of the formal
1889  /// arguments specified for the macro invocation. Returns null on error.
1890  MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI,
1891  SourceLocation &ExpansionEnd);
1892 
1893  /// \brief If an identifier token is read that is to be expanded
1894  /// as a builtin macro, handle it and return the next token as 'Tok'.
1895  void ExpandBuiltinMacro(Token &Tok);
1896 
1897  /// \brief Read a \c _Pragma directive, slice it up, process it, then
1898  /// return the first token after the directive.
1899  /// This assumes that the \c _Pragma token has just been read into \p Tok.
1900  void Handle_Pragma(Token &Tok);
1901 
1902  /// \brief Like Handle_Pragma except the pragma text is not enclosed within
1903  /// a string literal.
1904  void HandleMicrosoft__pragma(Token &Tok);
1905 
1906  /// \brief Add a lexer to the top of the include stack and
1907  /// start lexing tokens from it instead of the current buffer.
1908  void EnterSourceFileWithLexer(Lexer *TheLexer, const DirectoryLookup *Dir);
1909 
1910  /// \brief Add a lexer to the top of the include stack and
1911  /// start getting tokens from it using the PTH cache.
1912  void EnterSourceFileWithPTH(PTHLexer *PL, const DirectoryLookup *Dir);
1913 
1914  /// \brief Set the FileID for the preprocessor predefines.
1915  void setPredefinesFileID(FileID FID) {
1916  assert(PredefinesFileID.isInvalid() && "PredefinesFileID already set!");
1917  PredefinesFileID = FID;
1918  }
1919 
1920  /// \brief Returns true if we are lexing from a file and not a
1921  /// pragma or a macro.
1922  static bool IsFileLexer(const Lexer* L, const PreprocessorLexer* P) {
1923  return L ? !L->isPragmaLexer() : P != nullptr;
1924  }
1925 
1926  static bool IsFileLexer(const IncludeStackInfo& I) {
1927  return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
1928  }
1929 
1930  bool IsFileLexer() const {
1931  return IsFileLexer(CurLexer.get(), CurPPLexer);
1932  }
1933 
1934  //===--------------------------------------------------------------------===//
1935  // Caching stuff.
1936  void CachingLex(Token &Result);
1937  bool InCachingLexMode() const {
1938  // If the Lexer pointers are 0 and IncludeMacroStack is empty, it means
1939  // that we are past EOF, not that we are in CachingLex mode.
1940  return !CurPPLexer && !CurTokenLexer && !CurPTHLexer &&
1941  !IncludeMacroStack.empty();
1942  }
1943  void EnterCachingLexMode();
1944  void ExitCachingLexMode() {
1945  if (InCachingLexMode())
1947  }
1948  const Token &PeekAhead(unsigned N);
1949  void AnnotatePreviousCachedTokens(const Token &Tok);
1950 
1951  //===--------------------------------------------------------------------===//
1952  /// Handle*Directive - implement the various preprocessor directives. These
1953  /// should side-effect the current preprocessor object so that the next call
1954  /// to Lex() will return the appropriate token next.
1955  void HandleLineDirective();
1956  void HandleDigitDirective(Token &Tok);
1957  void HandleUserDiagnosticDirective(Token &Tok, bool isWarning);
1958  void HandleIdentSCCSDirective(Token &Tok);
1959  void HandleMacroPublicDirective(Token &Tok);
1960  void HandleMacroPrivateDirective();
1961 
1962  // File inclusion.
1963  void HandleIncludeDirective(SourceLocation HashLoc,
1964  Token &Tok,
1965  const DirectoryLookup *LookupFrom = nullptr,
1966  const FileEntry *LookupFromFile = nullptr,
1967  bool isImport = false);
1968  void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok);
1969  void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok);
1970  void HandleImportDirective(SourceLocation HashLoc, Token &Tok);
1971  void HandleMicrosoftImportDirective(Token &Tok);
1972 
1973 public:
1974  /// Check that the given module is available, producing a diagnostic if not.
1975  /// \return \c true if the check failed (because the module is not available).
1976  /// \c false if the module appears to be usable.
1977  static bool checkModuleIsAvailable(const LangOptions &LangOpts,
1978  const TargetInfo &TargetInfo,
1979  DiagnosticsEngine &Diags, Module *M);
1980 
1981  // Module inclusion testing.
1982  /// \brief Find the module that owns the source or header file that
1983  /// \p Loc points to. If the location is in a file that was included
1984  /// into a module, or is outside any module, returns nullptr.
1985  Module *getModuleForLocation(SourceLocation Loc);
1986 
1987  /// \brief We want to produce a diagnostic at location IncLoc concerning a
1988  /// missing module import.
1989  ///
1990  /// \param IncLoc The location at which the missing import was detected.
1991  /// \param M The desired module.
1992  /// \param MLoc A location within the desired module at which some desired
1993  /// effect occurred (eg, where a desired entity was declared).
1994  ///
1995  /// \return A file that can be #included to import a module containing MLoc.
1996  /// Null if no such file could be determined or if a #include is not
1997  /// appropriate.
1998  const FileEntry *getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc,
1999  Module *M,
2000  SourceLocation MLoc);
2001 
2002  bool isRecordingPreamble() const {
2003  return PreambleConditionalStack.isRecording();
2004  }
2005 
2006  bool hasRecordedPreamble() const {
2007  return PreambleConditionalStack.hasRecordedPreamble();
2008  }
2009 
2011  return PreambleConditionalStack.getStack();
2012  }
2013 
2015  PreambleConditionalStack.setStack(s);
2016  }
2017 
2019  PreambleConditionalStack.startReplaying();
2020  PreambleConditionalStack.setStack(s);
2021  }
2022 
2023 private:
2024  /// \brief After processing predefined file, initialize the conditional stack from
2025  /// the preamble.
2026  void replayPreambleConditionalStack();
2027 
2028  // Macro handling.
2029  void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterTopLevelIfndef);
2030  void HandleUndefDirective();
2031 
2032  // Conditional Inclusion.
2033  void HandleIfdefDirective(Token &Tok, bool isIfndef,
2034  bool ReadAnyTokensBeforeDirective);
2035  void HandleIfDirective(Token &Tok, bool ReadAnyTokensBeforeDirective);
2036  void HandleEndifDirective(Token &Tok);
2037  void HandleElseDirective(Token &Tok);
2038  void HandleElifDirective(Token &Tok);
2039 
2040  // Pragmas.
2041  void HandlePragmaDirective(SourceLocation IntroducerLoc,
2042  PragmaIntroducerKind Introducer);
2043 public:
2044  void HandlePragmaOnce(Token &OnceTok);
2045  void HandlePragmaMark();
2046  void HandlePragmaPoison();
2047  void HandlePragmaSystemHeader(Token &SysHeaderTok);
2048  void HandlePragmaDependency(Token &DependencyTok);
2049  void HandlePragmaPushMacro(Token &Tok);
2050  void HandlePragmaPopMacro(Token &Tok);
2051  void HandlePragmaIncludeAlias(Token &Tok);
2052  void HandlePragmaModuleBuild(Token &Tok);
2054 
2055  // Return true and store the first token only if any CommentHandler
2056  // has inserted some tokens and getCommentRetentionState() is false.
2057  bool HandleComment(Token &Token, SourceRange Comment);
2058 
2059  /// \brief A macro is used, update information about macros that need unused
2060  /// warnings.
2061  void markMacroAsUsed(MacroInfo *MI);
2062 };
2063 
2064 /// \brief Abstract base class that describes a handler that will receive
2065 /// source ranges for each of the comments encountered in the source file.
2067 public:
2068  virtual ~CommentHandler();
2069 
2070  // The handler shall return true if it has pushed any tokens
2071  // to be read using e.g. EnterToken or EnterTokenStream.
2072  virtual bool HandleComment(Preprocessor &PP, SourceRange Comment) = 0;
2073 };
2074 
2075 /// \brief Registry of pragma handlers added by plugins
2076 typedef llvm::Registry<PragmaHandler> PragmaHandlerRegistry;
2077 
2078 } // end namespace clang
2079 
2080 #endif
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character. ...
SourceManager & getSourceManager() const
Definition: Preprocessor.h:729
void ReplacePreviousCachedToken(ArrayRef< Token > NewToks)
Replace token in CachedLexPos - 1 in CachedTokens by the tokens in NewToks.
Definition: PPCaching.cpp:168
static unsigned getSpelling(const Token &Tok, const char *&Buffer, const SourceManager &SourceMgr, const LangOptions &LangOpts, bool *Invalid=nullptr)
getSpelling - This method is used to get the spelling of a token into a preallocated buffer...
Definition: Lexer.cpp:370
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
Definition: Preprocessor.h:953
Module * getModuleForLocation(SourceLocation Loc)
Find the module that owns the source or header file that Loc points to.
llvm::BumpPtrAllocator & getPreprocessorAllocator()
Definition: Preprocessor.h:737
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:747
void FinalizeForModelFile()
Cleanup after model file parsing.
SelectorTable & getSelectorTable()
Definition: Preprocessor.h:735
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Definition: Builtins.h:65
MacroInfo * AllocateMacroInfo(SourceLocation L)
Allocate a new MacroInfo object with the provided SourceLocation.
bool ConcatenateIncludeName(SmallString< 128 > &FilenameBuffer, SourceLocation &End)
Handle cases where the #include name is expanded from a macro as multiple tokens, which need to be gl...
void markMacroAsUsed(MacroInfo *MI)
A macro is used, update information about macros that need unused warnings.
Implements support for file system lookup, file system caching, and directory search management...
Definition: FileManager.h:116
void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Add the specified pragma handler to this preprocessor.
Definition: Pragma.cpp:880
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
friend void TokenLexer::ExpandFunctionArguments()
void MaybeHandlePoisonedIdentifier(Token &Identifier)
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
Definition: Preprocessor.h:974
static const Builtin::Info BuiltinInfo[]
Definition: Builtins.cpp:21
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
MemoryBufferCache & getPCMCache() const
Definition: Preprocessor.h:730
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
void dumpMacroInfo(const IdentifierInfo *II)
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
const std::string & getPredefines() const
Definition: Preprocessor.h:965
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
Definition: Preprocessor.h:811
StringRef P
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
Definition: MacroInfo.h:542
void LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
std::unique_ptr< llvm::MemoryBuffer > Buffer
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
Definition: Diagnostic.h:1205
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:283
MacroMap::const_iterator macro_iterator
Definition: Preprocessor.h:949
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions...
unsigned getCounterValue() const
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
void setPTHManager(PTHManager *pm)
static bool isAtStartOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroBegin=nullptr)
Returns true if the given MacroID location points at the first token of the macro expansion...
Definition: Lexer.cpp:784
void EnterToken(const Token &Tok)
Enters a token in the token stream to be lexed next.
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
void IgnorePragmas()
Install empty handlers for all pragmas (making them ignored).
Definition: Pragma.cpp:1832
Manage memory buffers across multiple users.
const FileEntry * LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, const DirectoryLookup *FromDir, const FileEntry *FromFile, const DirectoryLookup *&CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool SkipCache=false)
Given a "foo" or <foo> reference, look up the indicated file.
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
Definition: MacroInfo.h:117
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
virtual bool HandleComment(Preprocessor &PP, SourceRange Comment)=0
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
Builtin::Context & getBuiltinInfo()
Definition: Preprocessor.h:736
void setPredefines(const char *P)
Set the predefines for this Preprocessor.
Definition: Preprocessor.h:969
This interface provides a way to observe the actions of the preprocessor as it does its thing...
Definition: PPCallbacks.h:36
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool getPragmasEnabled() const
Definition: Preprocessor.h:772
TokenValue(tok::TokenKind Kind)
Definition: Preprocessor.h:71
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
Definition: Preprocessor.h:916
One of these records is kept for each identifier that is lexed.
Represents a macro directive exported by a module.
Definition: MacroInfo.h:476
A directive for a defined macro or a macro imported from a module.
Definition: MacroInfo.h:397
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
This table allows us to fully hide how we implement multi-keyword caching.
LineState State
bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef, bool *ShadowFlag=nullptr)
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
Module * LeaveSubmodule(bool ForPragma)
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, MemoryBufferCache &PCMCache, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
void HandlePragmaPushMacro(Token &Tok)
Handle #pragma push_macro.
Definition: Pragma.cpp:595
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
Definition: Preprocessor.h:895
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:725
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
Definition: Preprocessor.h:764
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
CachedTokensTy::size_type Begin
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:48
Describes a module or submodule.
Definition: Module.h:57
FileManager & getFileManager() const
Definition: Preprocessor.h:728
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
Definition: Preprocessor.h:759
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
void SetSuppressIncludeNotFoundError(bool Suppress)
Definition: Preprocessor.h:774
void CheckEndOfDirective(const char *Directive, bool EnableMacros=false)
Ensure that the next token is a tok::eod token.
MacroUse
Context in which macro name is used.
Definition: Preprocessor.h:86
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
uint32_t Offset
Definition: CacheTokens.cpp:43
bool getCommentRetentionState() const
Definition: Preprocessor.h:769
const IdentifierTable & getIdentifierTable() const
Definition: Preprocessor.h:734
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
void LexNonComment(Token &Result)
Lex a token.
static bool getRawToken(SourceLocation Loc, Token &Result, const SourceManager &SM, const LangOptions &LangOpts, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
Definition: Lexer.cpp:428
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:147
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:731
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
bool isMacroDefined(const IdentifierInfo *II)
Definition: Preprocessor.h:833
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we've alread...
Definition: Preprocessor.h:841
Defines the Diagnostic-related interfaces.
bool hadModuleLoaderFatalFailure() const
Definition: Preprocessor.h:754
void CommitBacktrackedTokens()
Disable the last EnableBacktrackAtThisPos call.
Definition: PPCaching.cpp:32
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
tok::TokenKind getKind() const
Definition: Token.h:90
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
const TargetInfo & getTargetInfo() const
Definition: Preprocessor.h:726
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
Definition: Preprocessor.h:784
static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Character, const SourceManager &SM, const LangOptions &LangOpts)
AdvanceToTokenCharacter - If the current SourceLocation specifies a location at the start of a token...
Definition: Lexer.cpp:701
detail::InMemoryDirectory::const_iterator I
PragmaIntroducerKind
Describes how the pragma was introduced, e.g., with #pragma, _Pragma, or __pragma.
Definition: Pragma.h:32
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Add a directive to the macro directive history for this identifier.
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/CurPTHLexer/ CurTokenLexer pointers...
Encapsulates the information needed to find the file referenced by a #include or #include_next, (sub-)framework lookup, etc.
Definition: HeaderSearch.h:137
void HandlePragmaPoison()
HandlePragmaPoison - Handle #pragma GCC poison.
Definition: Pragma.cpp:410
friend class MacroArgs
Definition: Preprocessor.h:612
A little helper class used to produce diagnostics.
Definition: Diagnostic.h:953
void HandlePragmaIncludeAlias(Token &Tok)
Definition: Pragma.cpp:653
void clearCodeCompletionHandler()
Clear out the code completion handler.
StringRef Filename
Definition: Format.cpp:1301
void Backtrack()
Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...
Definition: PPCaching.cpp:63
const SmallVectorImpl< AnnotatedLine * >::const_iterator End
void HandlePragmaOnce(Token &OnceTok)
HandlePragmaOnce - Handle #pragma once.
Definition: Pragma.cpp:385
bool isRecordingPreamble() const
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
Exposes information about the current target.
Definition: TargetInfo.h:54
static bool isAtEndOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroEnd=nullptr)
Returns true if the given MacroID location points at the last token of the macro expansion.
Definition: Lexer.cpp:806
Abstract interface for external sources of preprocessor information.
StringRef getName() const
Return the actual identifier string.
static SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset, const SourceManager &SM, const LangOptions &LangOpts)
Computes the source location just past the end of the token at this source location.
Definition: Lexer.cpp:762
void makeModuleVisible(Module *M, SourceLocation Loc)
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
void EnableBacktrackAtThisPos()
From the point that this method is called, and until CommitBacktrackedTokens() or Backtrack() is call...
Definition: PPCaching.cpp:26
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
Implements an efficient mapping from strings to IdentifierInfo nodes.
bool LexOnOffSwitch(tok::OnOffSwitch &OOS)
Lex an on-off-switch (C99 6.10.6p2) and verify that it is followed by EOD.
Definition: Pragma.cpp:933
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
bool operator==(const Token &Tok) const
Definition: Preprocessor.h:79
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
#define bool
Definition: stdbool.h:31
void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Remove the specific pragma handler from this preprocessor.
Definition: Pragma.cpp:911
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
Definition: Preprocessor.h:752
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
FormatToken * Token
Stores token information for comparing actual tokens with predefined values.
Definition: Preprocessor.h:66
static bool checkModuleIsAvailable(const LangOptions &LangOpts, const TargetInfo &TargetInfo, DiagnosticsEngine &Diags, Module *M)
Check that the given module is available, producing a diagnostic if not.
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
void HandlePragmaDependency(Token &DependencyTok)
HandlePragmaDependency - Handle #pragma GCC dependency "foo" blah.
Definition: Pragma.cpp:486
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
TokenValue(IdentifierInfo *II)
Definition: Preprocessor.h:78
SourceLocation getModuleImportLoc(Module *M) const
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
The result type of a method or function.
const SourceManager & SM
Definition: Format.cpp:1293
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
Definition: Token.h:215
size_t getTotalMemory() const
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Definition: MacroInfo.h:286
Kind
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
SmallVectorImpl< AnnotatedLine * >::const_iterator Next
void setExternalSource(ExternalPreprocessorSource *Source)
Definition: Preprocessor.h:743
Encodes a location in the source.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
Definition: Preprocessor.h:801
AnnotatedLine & Line
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
Definition: TokenKinds.h:95
void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin. ...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
Definition: Preprocessor.h:854
macro_iterator macro_end(bool IncludeExternalMacros=true) const
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define'd and not #undef'...
Definition: Preprocessor.h:884
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
IdentifierTable & getIdentifierTable()
Definition: Preprocessor.h:733
void setPragmasEnabled(bool Enabled)
Definition: Preprocessor.h:771
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
const TargetInfo * getAuxTargetInfo() const
Definition: Preprocessor.h:727
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the 'spelling' of the Tok token.
void Lex(Token &Result)
Lex the next token for this preprocessor.
ArrayRef< FormatToken * > Tokens
void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED, MacroDirective *MD)
Set a MacroDirective that was loaded from a PCH file.
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
Definition: TokenKinds.h:87
void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
Definition: TokenKinds.h:25
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
Definition: Preprocessor.h:867
PPCallbacks * getPPCallbacks() const
Accessors for preprocessor callbacks.
Definition: Preprocessor.h:821
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:95
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
DiagnosticsEngine & getDiagnostics() const
Definition: Preprocessor.h:722
static StringRef getImmediateMacroName(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts)
Retrieve the name of the immediate macro expansion.
Definition: Lexer.cpp:959
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
Definition: Preprocessor.h:814
bool isMacroDefined(StringRef Id)
Definition: Preprocessor.h:830
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion)
SmallVector< Token, 4 > CachedTokens
A set of tokens that has been cached for later parsing.
Definition: DeclSpec.h:1127
StringRef Name
Definition: USRFinder.cpp:123
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion)
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
void HandlePragmaPopMacro(Token &Tok)
Handle #pragma pop_macro.
Definition: Pragma.cpp:618
Reads an AST files chain containing the contents of a translation unit.
Definition: ASTReader.h:328
bool hasRecordedPreamble() const
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
Definition: Pragma.h:59
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
Abstract interface for a module loader.
Definition: ModuleLoader.h:69
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
void PoisonSEHIdentifiers(bool Poison=true)
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:34
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
Definition: TokenKinds.h:49
bool GetIncludeFilenameSpelling(SourceLocation Loc, StringRef &Filename)
Turn the specified lexer token into a fully checked and spelled filename, e.g.
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
bool isInvalid() const
MacroDirective * getLocalMacroDirectiveHistory(const IdentifierInfo *II) const
Given an identifier, return the latest non-imported macro directive for that identifier.
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
IdentifierInfo * ParsePragmaPushOrPopMacro(Token &Tok)
ParsePragmaPushOrPopMacro - Handle parsing of pragma push_macro/pop_macro.
Definition: Pragma.cpp:540
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
Cached information about one directory (either on disk or in the virtual file system).
Definition: FileManager.h:45
Defines the PPCallbacks interface.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
Definition: Preprocessor.h:922
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
Definition: Preprocessor.h:793
void HandlePoisonedIdentifier(Token &Tok)
Display reason for poisoned identifier.
void EraseCachedTokens(CachedTokensRange TokenRange)
Erase the range of cached tokens that were lexed since EnableBacktrackAtThisPos() was previously call...
Definition: PPCaching.cpp:44
bool IsPreviousCachedToken(const Token &Tok) const
Whether Tok is the most recent token (CachedLexPos - 1) in CachedTokens.
Definition: PPCaching.cpp:150
bool HandleComment(Token &Token, SourceRange Comment)
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
Definition: Preprocessor.h:720
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
void setSuppressAllDiagnostics(bool Val=true)
Suppress all diagnostics, to silence the front end when we know that we don't want any more diagnosti...
Definition: Diagnostic.h:548
Defines the clang::SourceLocation class and associated facilities.
void RemovePragmaHandler(PragmaHandler *Handler)
Definition: Preprocessor.h:993
void DumpMacro(const MacroInfo &MI) const
ModuleMacro * getModuleMacro(Module *Mod, IdentifierInfo *II)
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
CachedTokensRange LastCachedTokenRange()
Returns the range of cached tokens that were lexed since EnableBacktrackAtThisPos() was previously ca...
Definition: PPCaching.cpp:38
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:237
const FileEntry * getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc, Module *M, SourceLocation MLoc)
We want to produce a diagnostic at location IncLoc concerning a missing module import.
void setPredefines(StringRef P)
Definition: Preprocessor.h:970
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
Definition: Preprocessor.h:936
The translation unit is a complete translation unit.
Definition: LangOptions.h:239
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
void HandlePragmaModuleBuild(Token &Tok)
Definition: Pragma.cpp:797
Abstract base class that describes a handler that will receive source ranges for each of the comments...
void HandlePragmaSystemHeader(Token &SysHeaderTok)
HandlePragmaSystemHeader - Implement #pragma GCC system_header.
Definition: Pragma.cpp:452
void DiscardUntilEndOfDirective()
Read and discard all tokens remaining on the current line until the tok::eod token is found...
unsigned getLength() const
Definition: Token.h:127
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
Definition: Preprocessor.h:790
void DumpLocation(SourceLocation Loc) const
void setCounterValue(unsigned V)
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
A trivial tuple used to represent a source range.
void HandlePragmaMark()
Definition: Pragma.cpp:400
bool GetSuppressIncludeNotFoundError()
Definition: Preprocessor.h:778
Directive - Abstract class representing a parsed verify directive.
MacroInfo * getMacroInfo(const IdentifierInfo *II)
Definition: Preprocessor.h:899
void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind, void *AnnotationVal)
Enter an annotation token into the token stream.
Callback handler that receives notifications when performing code completion within the preprocessor...
A header that is known to reside within a given module, whether it was included or excluded...
Definition: ModuleMap.h:127
PTHManager * getPTHManager()
Definition: Preprocessor.h:741
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
Definition: Token.h:118
SourceLocation getPragmaARCCFCodeAuditedLoc() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:822
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
This class handles loading and caching of source files into memory.
Defines enum values for all the target-independent builtin functions.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
void AddPragmaHandler(PragmaHandler *Handler)
Definition: Preprocessor.h:983
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
Definition: Preprocessor.h:98
const DefMacroDirective * getDirective() const
Definition: MacroInfo.h:343
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
void setDiagnostics(DiagnosticsEngine &D)
Definition: Preprocessor.h:723
bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:177