clang  5.0.0
Preprocessor.cpp
Go to the documentation of this file.
1 //===--- Preprocess.cpp - C Language Family Preprocessor Implementation ---===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the Preprocessor interface.
11 //
12 //===----------------------------------------------------------------------===//
13 //
14 // Options to support:
15 // -H - Print the name of each header file used.
16 // -d[DNI] - Dump various things.
17 // -fworking-directory - #line's with preprocessor's working dir.
18 // -fpreprocessed
19 // -dependency-file,-M,-MM,-MF,-MG,-MP,-MT,-MQ,-MD,-MMD
20 // -W*
21 // -w
22 //
23 // Messages to emit:
24 // "Multiple include guards may be useful for:\n"
25 //
26 //===----------------------------------------------------------------------===//
27 
28 #include "clang/Lex/Preprocessor.h"
32 #include "clang/Basic/TargetInfo.h"
35 #include "clang/Lex/HeaderSearch.h"
38 #include "clang/Lex/MacroArgs.h"
39 #include "clang/Lex/MacroInfo.h"
40 #include "clang/Lex/ModuleLoader.h"
41 #include "clang/Lex/PTHManager.h"
42 #include "clang/Lex/Pragma.h"
46 #include "llvm/ADT/APInt.h"
47 #include "llvm/ADT/DenseMap.h"
48 #include "llvm/ADT/SmallString.h"
49 #include "llvm/ADT/SmallVector.h"
50 #include "llvm/ADT/STLExtras.h"
51 #include "llvm/ADT/StringRef.h"
52 #include "llvm/ADT/StringSwitch.h"
53 #include "llvm/Support/Capacity.h"
54 #include "llvm/Support/ErrorHandling.h"
55 #include "llvm/Support/MemoryBuffer.h"
56 #include "llvm/Support/raw_ostream.h"
57 #include <algorithm>
58 #include <cassert>
59 #include <memory>
60 #include <string>
61 #include <utility>
62 #include <vector>
63 
64 using namespace clang;
65 
66 LLVM_INSTANTIATE_REGISTRY(PragmaHandlerRegistry)
67 
68 //===----------------------------------------------------------------------===//
70 
71 Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
72  DiagnosticsEngine &diags, LangOptions &opts,
74  HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
75  IdentifierInfoLookup *IILookup, bool OwnsHeaders,
76  TranslationUnitKind TUKind)
77  : PPOpts(std::move(PPOpts)), Diags(&diags), LangOpts(opts), Target(nullptr),
78  AuxTarget(nullptr), FileMgr(Headers.getFileMgr()), SourceMgr(SM),
79  PCMCache(PCMCache), ScratchBuf(new ScratchBuffer(SourceMgr)),
80  HeaderInfo(Headers), TheModuleLoader(TheModuleLoader),
81  ExternalSource(nullptr), Identifiers(opts, IILookup),
82  PragmaHandlers(new PragmaNamespace(StringRef())),
83  IncrementalProcessing(false), TUKind(TUKind), CodeComplete(nullptr),
84  CodeCompletionFile(nullptr), CodeCompletionOffset(0),
85  LastTokenWasAt(false), ModuleImportExpectsIdentifier(false),
86  CodeCompletionReached(false), CodeCompletionII(nullptr),
87  MainFileDir(nullptr), SkipMainFilePreamble(0, true), CurPPLexer(nullptr),
88  CurDirLookup(nullptr), CurLexerKind(CLK_Lexer),
89  CurLexerSubmodule(nullptr), Callbacks(nullptr),
90  CurSubmoduleState(&NullSubmoduleState), MacroArgCache(nullptr),
91  Record(nullptr), MIChainHead(nullptr) {
92  OwnsHeaderSearch = OwnsHeaders;
93 
94  CounterValue = 0; // __COUNTER__ starts at 0.
95 
96  // Clear stats.
97  NumDirectives = NumDefined = NumUndefined = NumPragma = 0;
98  NumIf = NumElse = NumEndif = 0;
99  NumEnteredSourceFiles = 0;
100  NumMacroExpanded = NumFnMacroExpanded = NumBuiltinMacroExpanded = 0;
101  NumFastMacroExpanded = NumTokenPaste = NumFastTokenPaste = 0;
102  MaxIncludeStackDepth = 0;
103  NumSkipped = 0;
104 
105  // Default to discarding comments.
106  KeepComments = false;
107  KeepMacroComments = false;
108  SuppressIncludeNotFoundError = false;
109 
110  // Macro expansion is enabled.
111  DisableMacroExpansion = false;
112  MacroExpansionInDirectivesOverride = false;
113  InMacroArgs = false;
114  InMacroArgPreExpansion = false;
115  NumCachedTokenLexers = 0;
116  PragmasEnabled = true;
117  ParsingIfOrElifDirective = false;
118  PreprocessedOutput = false;
119 
120  CachedLexPos = 0;
121 
122  // We haven't read anything from the external source.
123  ReadMacrosFromExternalSource = false;
124 
125  // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro.
126  // This gets unpoisoned where it is allowed.
127  (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
128  SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
129 
130  // Initialize the pragma handlers.
131  RegisterBuiltinPragmas();
132 
133  // Initialize builtin macros like __LINE__ and friends.
134  RegisterBuiltinMacros();
135 
136  if(LangOpts.Borland) {
137  Ident__exception_info = getIdentifierInfo("_exception_info");
138  Ident___exception_info = getIdentifierInfo("__exception_info");
139  Ident_GetExceptionInfo = getIdentifierInfo("GetExceptionInformation");
140  Ident__exception_code = getIdentifierInfo("_exception_code");
141  Ident___exception_code = getIdentifierInfo("__exception_code");
142  Ident_GetExceptionCode = getIdentifierInfo("GetExceptionCode");
143  Ident__abnormal_termination = getIdentifierInfo("_abnormal_termination");
144  Ident___abnormal_termination = getIdentifierInfo("__abnormal_termination");
145  Ident_AbnormalTermination = getIdentifierInfo("AbnormalTermination");
146  } else {
147  Ident__exception_info = Ident__exception_code = nullptr;
148  Ident__abnormal_termination = Ident___exception_info = nullptr;
149  Ident___exception_code = Ident___abnormal_termination = nullptr;
150  Ident_GetExceptionInfo = Ident_GetExceptionCode = nullptr;
151  Ident_AbnormalTermination = nullptr;
152  }
153 
154  if (this->PPOpts->GeneratePreamble)
155  PreambleConditionalStack.startRecording();
156 }
157 
159  assert(BacktrackPositions.empty() && "EnableBacktrack/Backtrack imbalance!");
160 
161  IncludeMacroStack.clear();
162 
163  // Destroy any macro definitions.
164  while (MacroInfoChain *I = MIChainHead) {
165  MIChainHead = I->Next;
166  I->~MacroInfoChain();
167  }
168 
169  // Free any cached macro expanders.
170  // This populates MacroArgCache, so all TokenLexers need to be destroyed
171  // before the code below that frees up the MacroArgCache list.
172  std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers, nullptr);
173  CurTokenLexer.reset();
174 
175  // Free any cached MacroArgs.
176  for (MacroArgs *ArgList = MacroArgCache; ArgList;)
177  ArgList = ArgList->deallocate();
178 
179  // Delete the header search info, if we own it.
180  if (OwnsHeaderSearch)
181  delete &HeaderInfo;
182 }
183 
185  const TargetInfo *AuxTarget) {
186  assert((!this->Target || this->Target == &Target) &&
187  "Invalid override of target information");
188  this->Target = &Target;
189 
190  assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
191  "Invalid override of aux target information.");
192  this->AuxTarget = AuxTarget;
193 
194  // Initialize information about built-ins.
195  BuiltinInfo.InitializeTarget(Target, AuxTarget);
196  HeaderInfo.setTarget(Target);
197 }
198 
200  NumEnteredSourceFiles = 0;
201 
202  // Reset pragmas
203  PragmaHandlersBackup = std::move(PragmaHandlers);
204  PragmaHandlers = llvm::make_unique<PragmaNamespace>(StringRef());
205  RegisterBuiltinPragmas();
206 
207  // Reset PredefinesFileID
208  PredefinesFileID = FileID();
209 }
210 
212  NumEnteredSourceFiles = 1;
213 
214  PragmaHandlers = std::move(PragmaHandlersBackup);
215 }
216 
218  PTH.reset(pm);
219  FileMgr.addStatCache(PTH->createStatCache());
220 }
221 
222 void Preprocessor::DumpToken(const Token &Tok, bool DumpFlags) const {
223  llvm::errs() << tok::getTokenName(Tok.getKind()) << " '"
224  << getSpelling(Tok) << "'";
225 
226  if (!DumpFlags) return;
227 
228  llvm::errs() << "\t";
229  if (Tok.isAtStartOfLine())
230  llvm::errs() << " [StartOfLine]";
231  if (Tok.hasLeadingSpace())
232  llvm::errs() << " [LeadingSpace]";
233  if (Tok.isExpandDisabled())
234  llvm::errs() << " [ExpandDisabled]";
235  if (Tok.needsCleaning()) {
236  const char *Start = SourceMgr.getCharacterData(Tok.getLocation());
237  llvm::errs() << " [UnClean='" << StringRef(Start, Tok.getLength())
238  << "']";
239  }
240 
241  llvm::errs() << "\tLoc=<";
242  DumpLocation(Tok.getLocation());
243  llvm::errs() << ">";
244 }
245 
247  Loc.dump(SourceMgr);
248 }
249 
250 void Preprocessor::DumpMacro(const MacroInfo &MI) const {
251  llvm::errs() << "MACRO: ";
252  for (unsigned i = 0, e = MI.getNumTokens(); i != e; ++i) {
254  llvm::errs() << " ";
255  }
256  llvm::errs() << "\n";
257 }
258 
260  llvm::errs() << "\n*** Preprocessor Stats:\n";
261  llvm::errs() << NumDirectives << " directives found:\n";
262  llvm::errs() << " " << NumDefined << " #define.\n";
263  llvm::errs() << " " << NumUndefined << " #undef.\n";
264  llvm::errs() << " #include/#include_next/#import:\n";
265  llvm::errs() << " " << NumEnteredSourceFiles << " source files entered.\n";
266  llvm::errs() << " " << MaxIncludeStackDepth << " max include stack depth\n";
267  llvm::errs() << " " << NumIf << " #if/#ifndef/#ifdef.\n";
268  llvm::errs() << " " << NumElse << " #else/#elif.\n";
269  llvm::errs() << " " << NumEndif << " #endif.\n";
270  llvm::errs() << " " << NumPragma << " #pragma.\n";
271  llvm::errs() << NumSkipped << " #if/#ifndef#ifdef regions skipped\n";
272 
273  llvm::errs() << NumMacroExpanded << "/" << NumFnMacroExpanded << "/"
274  << NumBuiltinMacroExpanded << " obj/fn/builtin macros expanded, "
275  << NumFastMacroExpanded << " on the fast path.\n";
276  llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
277  << " token paste (##) operations performed, "
278  << NumFastTokenPaste << " on the fast path.\n";
279 
280  llvm::errs() << "\nPreprocessor Memory: " << getTotalMemory() << "B total";
281 
282  llvm::errs() << "\n BumpPtr: " << BP.getTotalMemory();
283  llvm::errs() << "\n Macro Expanded Tokens: "
284  << llvm::capacity_in_bytes(MacroExpandedTokens);
285  llvm::errs() << "\n Predefines Buffer: " << Predefines.capacity();
286  // FIXME: List information for all submodules.
287  llvm::errs() << "\n Macros: "
288  << llvm::capacity_in_bytes(CurSubmoduleState->Macros);
289  llvm::errs() << "\n #pragma push_macro Info: "
290  << llvm::capacity_in_bytes(PragmaPushMacroInfo);
291  llvm::errs() << "\n Poison Reasons: "
292  << llvm::capacity_in_bytes(PoisonReasons);
293  llvm::errs() << "\n Comment Handlers: "
294  << llvm::capacity_in_bytes(CommentHandlers) << "\n";
295 }
296 
298 Preprocessor::macro_begin(bool IncludeExternalMacros) const {
299  if (IncludeExternalMacros && ExternalSource &&
300  !ReadMacrosFromExternalSource) {
301  ReadMacrosFromExternalSource = true;
302  ExternalSource->ReadDefinedMacros();
303  }
304 
305  // Make sure we cover all macros in visible modules.
306  for (const ModuleMacro &Macro : ModuleMacros)
307  CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState()));
308 
309  return CurSubmoduleState->Macros.begin();
310 }
311 
313  return BP.getTotalMemory()
314  + llvm::capacity_in_bytes(MacroExpandedTokens)
315  + Predefines.capacity() /* Predefines buffer. */
316  // FIXME: Include sizes from all submodules, and include MacroInfo sizes,
317  // and ModuleMacros.
318  + llvm::capacity_in_bytes(CurSubmoduleState->Macros)
319  + llvm::capacity_in_bytes(PragmaPushMacroInfo)
320  + llvm::capacity_in_bytes(PoisonReasons)
321  + llvm::capacity_in_bytes(CommentHandlers);
322 }
323 
325 Preprocessor::macro_end(bool IncludeExternalMacros) const {
326  if (IncludeExternalMacros && ExternalSource &&
327  !ReadMacrosFromExternalSource) {
328  ReadMacrosFromExternalSource = true;
329  ExternalSource->ReadDefinedMacros();
330  }
331 
332  return CurSubmoduleState->Macros.end();
333 }
334 
335 /// \brief Compares macro tokens with a specified token value sequence.
336 static bool MacroDefinitionEquals(const MacroInfo *MI,
338  return Tokens.size() == MI->getNumTokens() &&
339  std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin());
340 }
341 
343  SourceLocation Loc,
345  SourceLocation BestLocation;
346  StringRef BestSpelling;
348  I != E; ++I) {
350  Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
351  if (!Def || !Def.getMacroInfo())
352  continue;
353  if (!Def.getMacroInfo()->isObjectLike())
354  continue;
356  continue;
357  SourceLocation Location = Def.getLocation();
358  // Choose the macro defined latest.
359  if (BestLocation.isInvalid() ||
360  (Location.isValid() &&
361  SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) {
362  BestLocation = Location;
363  BestSpelling = I->first->getName();
364  }
365  }
366  return BestSpelling;
367 }
368 
370  if (CurLexer)
371  CurLexerKind = CLK_Lexer;
372  else if (CurPTHLexer)
373  CurLexerKind = CLK_PTHLexer;
374  else if (CurTokenLexer)
375  CurLexerKind = CLK_TokenLexer;
376  else
377  CurLexerKind = CLK_CachingLexer;
378 }
379 
381  unsigned CompleteLine,
382  unsigned CompleteColumn) {
383  assert(File);
384  assert(CompleteLine && CompleteColumn && "Starts from 1:1");
385  assert(!CodeCompletionFile && "Already set");
386 
387  using llvm::MemoryBuffer;
388 
389  // Load the actual file's contents.
390  bool Invalid = false;
391  const MemoryBuffer *Buffer = SourceMgr.getMemoryBufferForFile(File, &Invalid);
392  if (Invalid)
393  return true;
394 
395  // Find the byte position of the truncation point.
396  const char *Position = Buffer->getBufferStart();
397  for (unsigned Line = 1; Line < CompleteLine; ++Line) {
398  for (; *Position; ++Position) {
399  if (*Position != '\r' && *Position != '\n')
400  continue;
401 
402  // Eat \r\n or \n\r as a single line.
403  if ((Position[1] == '\r' || Position[1] == '\n') &&
404  Position[0] != Position[1])
405  ++Position;
406  ++Position;
407  break;
408  }
409  }
410 
411  Position += CompleteColumn - 1;
412 
413  // If pointing inside the preamble, adjust the position at the beginning of
414  // the file after the preamble.
415  if (SkipMainFilePreamble.first &&
416  SourceMgr.getFileEntryForID(SourceMgr.getMainFileID()) == File) {
417  if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
418  Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
419  }
420 
421  if (Position > Buffer->getBufferEnd())
422  Position = Buffer->getBufferEnd();
423 
424  CodeCompletionFile = File;
425  CodeCompletionOffset = Position - Buffer->getBufferStart();
426 
427  std::unique_ptr<MemoryBuffer> NewBuffer =
428  MemoryBuffer::getNewUninitMemBuffer(Buffer->getBufferSize() + 1,
429  Buffer->getBufferIdentifier());
430  char *NewBuf = const_cast<char*>(NewBuffer->getBufferStart());
431  char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
432  *NewPos = '\0';
433  std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
434  SourceMgr.overrideFileContents(File, std::move(NewBuffer));
435 
436  return false;
437 }
438 
440  if (CodeComplete)
441  CodeComplete->CodeCompleteNaturalLanguage();
443 }
444 
445 /// getSpelling - This method is used to get the spelling of a token into a
446 /// SmallVector. Note that the returned StringRef may not point to the
447 /// supplied buffer if a copy can be avoided.
448 StringRef Preprocessor::getSpelling(const Token &Tok,
450  bool *Invalid) const {
451  // NOTE: this has to be checked *before* testing for an IdentifierInfo.
452  if (Tok.isNot(tok::raw_identifier) && !Tok.hasUCN()) {
453  // Try the fast path.
454  if (const IdentifierInfo *II = Tok.getIdentifierInfo())
455  return II->getName();
456  }
457 
458  // Resize the buffer if we need to copy into it.
459  if (Tok.needsCleaning())
460  Buffer.resize(Tok.getLength());
461 
462  const char *Ptr = Buffer.data();
463  unsigned Len = getSpelling(Tok, Ptr, Invalid);
464  return StringRef(Ptr, Len);
465 }
466 
467 /// CreateString - Plop the specified string into a scratch buffer and return a
468 /// location for it. If specified, the source location provides a source
469 /// location for the token.
470 void Preprocessor::CreateString(StringRef Str, Token &Tok,
471  SourceLocation ExpansionLocStart,
472  SourceLocation ExpansionLocEnd) {
473  Tok.setLength(Str.size());
474 
475  const char *DestPtr;
476  SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
477 
478  if (ExpansionLocStart.isValid())
479  Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
480  ExpansionLocEnd, Str.size());
481  Tok.setLocation(Loc);
482 
483  // If this is a raw identifier or a literal token, set the pointer data.
484  if (Tok.is(tok::raw_identifier))
485  Tok.setRawIdentifierData(DestPtr);
486  else if (Tok.isLiteral())
487  Tok.setLiteralData(DestPtr);
488 }
489 
491  if (!getLangOpts().isCompilingModule())
492  return nullptr;
493 
494  return getHeaderSearchInfo().lookupModule(getLangOpts().CurrentModule);
495 }
496 
497 //===----------------------------------------------------------------------===//
498 // Preprocessor Initialization Methods
499 //===----------------------------------------------------------------------===//
500 
501 /// EnterMainSourceFile - Enter the specified FileID as the main source file,
502 /// which implicitly adds the builtin defines etc.
504  // We do not allow the preprocessor to reenter the main file. Doing so will
505  // cause FileID's to accumulate information from both runs (e.g. #line
506  // information) and predefined macros aren't guaranteed to be set properly.
507  assert(NumEnteredSourceFiles == 0 && "Cannot reenter the main file!");
508  FileID MainFileID = SourceMgr.getMainFileID();
509 
510  // If MainFileID is loaded it means we loaded an AST file, no need to enter
511  // a main file.
512  if (!SourceMgr.isLoadedFileID(MainFileID)) {
513  // Enter the main file source buffer.
514  EnterSourceFile(MainFileID, nullptr, SourceLocation());
515 
516  // If we've been asked to skip bytes in the main file (e.g., as part of a
517  // precompiled preamble), do so now.
518  if (SkipMainFilePreamble.first > 0)
519  CurLexer->SkipBytes(SkipMainFilePreamble.first,
520  SkipMainFilePreamble.second);
521 
522  // Tell the header info that the main file was entered. If the file is later
523  // #imported, it won't be re-entered.
524  if (const FileEntry *FE = SourceMgr.getFileEntryForID(MainFileID))
525  HeaderInfo.IncrementIncludeCount(FE);
526  }
527 
528  // Preprocess Predefines to populate the initial preprocessor state.
529  std::unique_ptr<llvm::MemoryBuffer> SB =
530  llvm::MemoryBuffer::getMemBufferCopy(Predefines, "<built-in>");
531  assert(SB && "Cannot create predefined source buffer");
532  FileID FID = SourceMgr.createFileID(std::move(SB));
533  assert(FID.isValid() && "Could not create FileID for predefines?");
534  setPredefinesFileID(FID);
535 
536  // Start parsing the predefines.
537  EnterSourceFile(FID, nullptr, SourceLocation());
538 }
539 
540 void Preprocessor::replayPreambleConditionalStack() {
541  // Restore the conditional stack from the preamble, if there is one.
542  if (PreambleConditionalStack.isReplaying()) {
543  assert(CurPPLexer &&
544  "CurPPLexer is null when calling replayPreambleConditionalStack.");
545  CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack());
546  PreambleConditionalStack.doneReplaying();
547  }
548 }
549 
551  // Notify the client that we reached the end of the source file.
552  if (Callbacks)
553  Callbacks->EndOfMainFile();
554 }
555 
556 //===----------------------------------------------------------------------===//
557 // Lexer Event Handling.
558 //===----------------------------------------------------------------------===//
559 
560 /// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
561 /// identifier information for the token and install it into the token,
562 /// updating the token kind accordingly.
564  assert(!Identifier.getRawIdentifier().empty() && "No raw identifier data!");
565 
566  // Look up this token, see if it is a macro, or if it is a language keyword.
567  IdentifierInfo *II;
568  if (!Identifier.needsCleaning() && !Identifier.hasUCN()) {
569  // No cleaning needed, just use the characters from the lexed buffer.
570  II = getIdentifierInfo(Identifier.getRawIdentifier());
571  } else {
572  // Cleaning needed, alloca a buffer, clean into it, then use the buffer.
573  SmallString<64> IdentifierBuffer;
574  StringRef CleanedStr = getSpelling(Identifier, IdentifierBuffer);
575 
576  if (Identifier.hasUCN()) {
577  SmallString<64> UCNIdentifierBuffer;
578  expandUCNs(UCNIdentifierBuffer, CleanedStr);
579  II = getIdentifierInfo(UCNIdentifierBuffer);
580  } else {
581  II = getIdentifierInfo(CleanedStr);
582  }
583  }
584 
585  // Update the token info (identifier info and appropriate token kind).
586  Identifier.setIdentifierInfo(II);
587  if (getLangOpts().MSVCCompat && II->isCPlusPlusOperatorKeyword() &&
589  Identifier.setKind(clang::tok::identifier);
590  else
591  Identifier.setKind(II->getTokenID());
592 
593  return II;
594 }
595 
596 void Preprocessor::SetPoisonReason(IdentifierInfo *II, unsigned DiagID) {
597  PoisonReasons[II] = DiagID;
598 }
599 
601  assert(Ident__exception_code && Ident__exception_info);
602  assert(Ident___exception_code && Ident___exception_info);
603  Ident__exception_code->setIsPoisoned(Poison);
604  Ident___exception_code->setIsPoisoned(Poison);
605  Ident_GetExceptionCode->setIsPoisoned(Poison);
606  Ident__exception_info->setIsPoisoned(Poison);
607  Ident___exception_info->setIsPoisoned(Poison);
608  Ident_GetExceptionInfo->setIsPoisoned(Poison);
609  Ident__abnormal_termination->setIsPoisoned(Poison);
610  Ident___abnormal_termination->setIsPoisoned(Poison);
611  Ident_AbnormalTermination->setIsPoisoned(Poison);
612 }
613 
615  assert(Identifier.getIdentifierInfo() &&
616  "Can't handle identifiers without identifier info!");
617  llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
618  PoisonReasons.find(Identifier.getIdentifierInfo());
619  if(it == PoisonReasons.end())
620  Diag(Identifier, diag::err_pp_used_poisoned_id);
621  else
622  Diag(Identifier,it->second) << Identifier.getIdentifierInfo();
623 }
624 
625 /// \brief Returns a diagnostic message kind for reporting a future keyword as
626 /// appropriate for the identifier and specified language.
628  const LangOptions &LangOpts) {
629  assert(II.isFutureCompatKeyword() && "diagnostic should not be needed");
630 
631  if (LangOpts.CPlusPlus)
632  return llvm::StringSwitch<diag::kind>(II.getName())
633 #define CXX11_KEYWORD(NAME, FLAGS) \
634  .Case(#NAME, diag::warn_cxx11_keyword)
635 #include "clang/Basic/TokenKinds.def"
636  ;
637 
638  llvm_unreachable(
639  "Keyword not known to come from a newer Standard or proposed Standard");
640 }
641 
642 void Preprocessor::updateOutOfDateIdentifier(IdentifierInfo &II) const {
643  assert(II.isOutOfDate() && "not out of date");
645 }
646 
647 /// HandleIdentifier - This callback is invoked when the lexer reads an
648 /// identifier. This callback looks up the identifier in the map and/or
649 /// potentially macro expands it or turns it into a named token (like 'for').
650 ///
651 /// Note that callers of this method are guarded by checking the
652 /// IdentifierInfo's 'isHandleIdentifierCase' bit. If this method changes, the
653 /// IdentifierInfo methods that compute these properties will need to change to
654 /// match.
656  assert(Identifier.getIdentifierInfo() &&
657  "Can't handle identifiers without identifier info!");
658 
659  IdentifierInfo &II = *Identifier.getIdentifierInfo();
660 
661  // If the information about this identifier is out of date, update it from
662  // the external source.
663  // We have to treat __VA_ARGS__ in a special way, since it gets
664  // serialized with isPoisoned = true, but our preprocessor may have
665  // unpoisoned it if we're defining a C99 macro.
666  if (II.isOutOfDate()) {
667  bool CurrentIsPoisoned = false;
668  if (&II == Ident__VA_ARGS__)
669  CurrentIsPoisoned = Ident__VA_ARGS__->isPoisoned();
670 
671  updateOutOfDateIdentifier(II);
672  Identifier.setKind(II.getTokenID());
673 
674  if (&II == Ident__VA_ARGS__)
675  II.setIsPoisoned(CurrentIsPoisoned);
676  }
677 
678  // If this identifier was poisoned, and if it was not produced from a macro
679  // expansion, emit an error.
680  if (II.isPoisoned() && CurPPLexer) {
681  HandlePoisonedIdentifier(Identifier);
682  }
683 
684  // If this is a macro to be expanded, do it.
685  if (MacroDefinition MD = getMacroDefinition(&II)) {
686  auto *MI = MD.getMacroInfo();
687  assert(MI && "macro definition with no macro info?");
688  if (!DisableMacroExpansion) {
689  if (!Identifier.isExpandDisabled() && MI->isEnabled()) {
690  // C99 6.10.3p10: If the preprocessing token immediately after the
691  // macro name isn't a '(', this macro should not be expanded.
692  if (!MI->isFunctionLike() || isNextPPTokenLParen())
693  return HandleMacroExpandedIdentifier(Identifier, MD);
694  } else {
695  // C99 6.10.3.4p2 says that a disabled macro may never again be
696  // expanded, even if it's in a context where it could be expanded in the
697  // future.
698  Identifier.setFlag(Token::DisableExpand);
699  if (MI->isObjectLike() || isNextPPTokenLParen())
700  Diag(Identifier, diag::pp_disabled_macro_expansion);
701  }
702  }
703  }
704 
705  // If this identifier is a keyword in a newer Standard or proposed Standard,
706  // produce a warning. Don't warn if we're not considering macro expansion,
707  // since this identifier might be the name of a macro.
708  // FIXME: This warning is disabled in cases where it shouldn't be, like
709  // "#define constexpr constexpr", "int constexpr;"
710  if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
711  Diag(Identifier, getFutureCompatDiagKind(II, getLangOpts()))
712  << II.getName();
713  // Don't diagnose this keyword again in this translation unit.
714  II.setIsFutureCompatKeyword(false);
715  }
716 
717  // If this is an extension token, diagnose its use.
718  // We avoid diagnosing tokens that originate from macro definitions.
719  // FIXME: This warning is disabled in cases where it shouldn't be,
720  // like "#define TY typeof", "TY(1) x".
721  if (II.isExtensionToken() && !DisableMacroExpansion)
722  Diag(Identifier, diag::ext_token_used);
723 
724  // If this is the 'import' contextual keyword following an '@', note
725  // that the next token indicates a module name.
726  //
727  // Note that we do not treat 'import' as a contextual
728  // keyword when we're in a caching lexer, because caching lexers only get
729  // used in contexts where import declarations are disallowed.
730  //
731  // Likewise if this is the C++ Modules TS import keyword.
732  if (((LastTokenWasAt && II.isModulesImport()) ||
733  Identifier.is(tok::kw_import)) &&
734  !InMacroArgs && !DisableMacroExpansion &&
735  (getLangOpts().Modules || getLangOpts().DebuggerSupport) &&
736  CurLexerKind != CLK_CachingLexer) {
737  ModuleImportLoc = Identifier.getLocation();
738  ModuleImportPath.clear();
739  ModuleImportExpectsIdentifier = true;
740  CurLexerKind = CLK_LexAfterModuleImport;
741  }
742  return true;
743 }
744 
745 void Preprocessor::Lex(Token &Result) {
746  // We loop here until a lex function returns a token; this avoids recursion.
747  bool ReturnedToken;
748  do {
749  switch (CurLexerKind) {
750  case CLK_Lexer:
751  ReturnedToken = CurLexer->Lex(Result);
752  break;
753  case CLK_PTHLexer:
754  ReturnedToken = CurPTHLexer->Lex(Result);
755  break;
756  case CLK_TokenLexer:
757  ReturnedToken = CurTokenLexer->Lex(Result);
758  break;
759  case CLK_CachingLexer:
760  CachingLex(Result);
761  ReturnedToken = true;
762  break;
763  case CLK_LexAfterModuleImport:
764  LexAfterModuleImport(Result);
765  ReturnedToken = true;
766  break;
767  }
768  } while (!ReturnedToken);
769 
770  if (Result.is(tok::code_completion))
772 
773  LastTokenWasAt = Result.is(tok::at);
774 }
775 
776 /// \brief Lex a token following the 'import' contextual keyword.
777 ///
779  // Figure out what kind of lexer we actually have.
781 
782  // Lex the next token.
783  Lex(Result);
784 
785  // The token sequence
786  //
787  // import identifier (. identifier)*
788  //
789  // indicates a module import directive. We already saw the 'import'
790  // contextual keyword, so now we're looking for the identifiers.
791  if (ModuleImportExpectsIdentifier && Result.getKind() == tok::identifier) {
792  // We expected to see an identifier here, and we did; continue handling
793  // identifiers.
794  ModuleImportPath.push_back(std::make_pair(Result.getIdentifierInfo(),
795  Result.getLocation()));
796  ModuleImportExpectsIdentifier = false;
797  CurLexerKind = CLK_LexAfterModuleImport;
798  return;
799  }
800 
801  // If we're expecting a '.' or a ';', and we got a '.', then wait until we
802  // see the next identifier. (We can also see a '[[' that begins an
803  // attribute-specifier-seq here under the C++ Modules TS.)
804  if (!ModuleImportExpectsIdentifier && Result.getKind() == tok::period) {
805  ModuleImportExpectsIdentifier = true;
806  CurLexerKind = CLK_LexAfterModuleImport;
807  return;
808  }
809 
810  // If we have a non-empty module path, load the named module.
811  if (!ModuleImportPath.empty()) {
812  // Under the Modules TS, the dot is just part of the module name, and not
813  // a real hierarachy separator. Flatten such module names now.
814  //
815  // FIXME: Is this the right level to be performing this transformation?
816  std::string FlatModuleName;
817  if (getLangOpts().ModulesTS) {
818  for (auto &Piece : ModuleImportPath) {
819  if (!FlatModuleName.empty())
820  FlatModuleName += ".";
821  FlatModuleName += Piece.first->getName();
822  }
823  SourceLocation FirstPathLoc = ModuleImportPath[0].second;
824  ModuleImportPath.clear();
825  ModuleImportPath.push_back(
826  std::make_pair(getIdentifierInfo(FlatModuleName), FirstPathLoc));
827  }
828 
829  Module *Imported = nullptr;
830  if (getLangOpts().Modules) {
831  Imported = TheModuleLoader.loadModule(ModuleImportLoc,
832  ModuleImportPath,
834  /*IsIncludeDirective=*/false);
835  if (Imported)
836  makeModuleVisible(Imported, ModuleImportLoc);
837  }
838  if (Callbacks && (getLangOpts().Modules || getLangOpts().DebuggerSupport))
839  Callbacks->moduleImport(ModuleImportLoc, ModuleImportPath, Imported);
840  }
841 }
842 
844  CurSubmoduleState->VisibleModules.setVisible(
845  M, Loc, [](Module *) {},
846  [&](ArrayRef<Module *> Path, Module *Conflict, StringRef Message) {
847  // FIXME: Include the path in the diagnostic.
848  // FIXME: Include the import location for the conflicting module.
849  Diag(ModuleImportLoc, diag::warn_module_conflict)
850  << Path[0]->getFullModuleName()
851  << Conflict->getFullModuleName()
852  << Message;
853  });
854 
855  // Add this module to the imports list of the currently-built submodule.
856  if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
857  BuildingSubmoduleStack.back().M->Imports.insert(M);
858 }
859 
860 bool Preprocessor::FinishLexStringLiteral(Token &Result, std::string &String,
861  const char *DiagnosticTag,
862  bool AllowMacroExpansion) {
863  // We need at least one string literal.
864  if (Result.isNot(tok::string_literal)) {
865  Diag(Result, diag::err_expected_string_literal)
866  << /*Source='in...'*/0 << DiagnosticTag;
867  return false;
868  }
869 
870  // Lex string literal tokens, optionally with macro expansion.
871  SmallVector<Token, 4> StrToks;
872  do {
873  StrToks.push_back(Result);
874 
875  if (Result.hasUDSuffix())
876  Diag(Result, diag::err_invalid_string_udl);
877 
878  if (AllowMacroExpansion)
879  Lex(Result);
880  else
881  LexUnexpandedToken(Result);
882  } while (Result.is(tok::string_literal));
883 
884  // Concatenate and parse the strings.
885  StringLiteralParser Literal(StrToks, *this);
886  assert(Literal.isAscii() && "Didn't allow wide strings in");
887 
888  if (Literal.hadError)
889  return false;
890 
891  if (Literal.Pascal) {
892  Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
893  << /*Source='in...'*/0 << DiagnosticTag;
894  return false;
895  }
896 
897  String = Literal.GetString();
898  return true;
899 }
900 
902  assert(Tok.is(tok::numeric_constant));
903  SmallString<8> IntegerBuffer;
904  bool NumberInvalid = false;
905  StringRef Spelling = getSpelling(Tok, IntegerBuffer, &NumberInvalid);
906  if (NumberInvalid)
907  return false;
908  NumericLiteralParser Literal(Spelling, Tok.getLocation(), *this);
909  if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
910  return false;
911  llvm::APInt APVal(64, 0);
912  if (Literal.GetIntegerValue(APVal))
913  return false;
914  Lex(Tok);
915  Value = APVal.getLimitedValue();
916  return true;
917 }
918 
920  assert(Handler && "NULL comment handler");
921  assert(std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler) ==
922  CommentHandlers.end() && "Comment handler already registered");
923  CommentHandlers.push_back(Handler);
924 }
925 
927  std::vector<CommentHandler *>::iterator Pos
928  = std::find(CommentHandlers.begin(), CommentHandlers.end(), Handler);
929  assert(Pos != CommentHandlers.end() && "Comment handler not registered");
930  CommentHandlers.erase(Pos);
931 }
932 
934  bool AnyPendingTokens = false;
935  for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
936  HEnd = CommentHandlers.end();
937  H != HEnd; ++H) {
938  if ((*H)->HandleComment(*this, Comment))
939  AnyPendingTokens = true;
940  }
941  if (!AnyPendingTokens || getCommentRetentionState())
942  return false;
943  Lex(result);
944  return true;
945 }
946 
948 
950 
952 
954  if (Record)
955  return;
956 
957  Record = new PreprocessingRecord(getSourceManager());
958  addPPCallbacks(std::unique_ptr<PPCallbacks>(Record));
959 }
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
Definition: Token.h:266
SourceManager & getSourceManager() const
Definition: Preprocessor.h:729
bool isPoisoned() const
Return true if this token has been poisoned.
int Position
ExternalPreprocessorSource * getExternalSource() const
Definition: Preprocessor.h:747
void FinalizeForModelFile()
Cleanup after model file parsing.
bool isLoadedFileID(FileID FID) const
Returns true if FID came from a PCH/Module.
Defines the clang::FileManager interface and associated types.
Defines the SourceManager interface.
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
Definition: Preprocessor.h:974
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
Defines the FileSystemStatCache interface.
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
bool isObjectLike() const
Definition: MacroInfo.h:194
virtual void CodeCompleteNaturalLanguage()
Callback invoked when performing code completion in a part of the file where we expect natural langua...
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
Definition: Token.h:270
A description of the current definition of a macro.
Definition: MacroInfo.h:542
void LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
std::unique_ptr< llvm::MemoryBuffer > Buffer
bool hasUCN() const
Returns true if this token contains a universal character name.
Definition: Token.h:294
void setFlag(TokenFlags Flag)
Set the specified flag.
Definition: Token.h:234
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
Definition: Token.h:283
MacroMap::const_iterator macro_iterator
Definition: Preprocessor.h:949
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions...
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
void setPTHManager(PTHManager *pm)
Manage memory buffers across multiple users.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
One of these records is kept for each identifier that is lexed.
Represents a macro directive exported by a module.
Definition: MacroInfo.h:476
void setRawIdentifierData(const char *Ptr)
Definition: Token.h:207
SourceLocation getLocation() const
Definition: MacroInfo.h:450
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, MemoryBufferCache &PCMCache, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
const LangOptions & getLangOpts() const
Definition: Preprocessor.h:725
void setTarget(const TargetInfo &Target)
Set the target information for the header search, if not already known.
Token - This structure provides full information about a lexed token.
Definition: Token.h:35
void setKind(tok::TokenKind K)
Definition: Token.h:91
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Definition: LangOptions.h:48
Describes a module or submodule.
Definition: Module.h:57
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
bool getCommentRetentionState() const
Definition: Preprocessor.h:769
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
Concrete class used by the front-end to report problems and issues.
Definition: Diagnostic.h:147
HeaderSearch & getHeaderSearchInfo() const
Definition: Preprocessor.h:731
bool isFutureCompatKeyword() const
is/setIsFutureCompatKeyword - Initialize information about whether or not this language token is a ke...
tokens_iterator tokens_begin() const
Definition: MacroInfo.h:236
void dump(const SourceManager &SM) const
tok::TokenKind getKind() const
Definition: Token.h:90
detail::InMemoryDirectory::const_iterator I
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
bool isInvalid() const
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/CurPTHLexer/ CurTokenLexer pointers...
Encapsulates the information needed to find the file referenced by a #include or #include_next, (sub-)framework lookup, etc.
Definition: HeaderSearch.h:137
StringRef getRawIdentifier() const
getRawIdentifier - For a raw identifier token (i.e., an identifier lexed in raw mode), returns a reference to the text substring in the buffer if known.
Definition: Token.h:203
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
Provides lookups to, and iteration over, IdentiferInfo objects.
Exposes information about the current target.
Definition: TargetInfo.h:54
Abstract interface for external sources of preprocessor information.
StringRef getName() const
Return the actual identifier string.
void makeModuleVisible(Module *M, SourceLocation Loc)
bool isBeforeInTranslationUnit(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the translation unit.
FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, unsigned LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
virtual void ReadDefinedMacros()=0
Read the set of macros defined by this external macro source.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Definition: MacroArgs.h:29
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
Defines the clang::Preprocessor interface.
bool hasUDSuffix() const
Return true if this token is a string or character literal which has a ud-suffix. ...
Definition: Token.h:291
void setIsPoisoned(bool Value=true)
setIsPoisoned - Mark this identifier as poisoned.
bool isInSystemHeader(SourceLocation Loc) const
Returns if a SourceLocation is in a system header.
static bool MacroDefinitionEquals(const MacroInfo *MI, ArrayRef< TokenValue > Tokens)
Compares macro tokens with a specified token value sequence.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Definition: Token.h:124
bool isNot(tok::TokenKind K) const
Definition: Token.h:96
unsigned getNumTokens() const
Return the number of tokens that this macro expands to.
Definition: MacroInfo.h:228
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
void IncrementIncludeCount(const FileEntry *File)
Increment the count for the number of times the specified FileEntry has been entered.
Definition: HeaderSearch.h:442
Module * lookupModule(StringRef ModuleName, bool AllowSearch=true)
Lookup a module Search for a module with the given name.
const SourceManager & SM
Definition: Format.cpp:1293
void overrideFileContents(const FileEntry *SourceFile, llvm::MemoryBuffer *Buffer, bool DoNotFree)
Override the contents of the given source file by providing an already-allocated buffer.
size_t getTotalMemory() const
#define false
Definition: stdbool.h:33
void addStatCache(std::unique_ptr< FileSystemStatCache > statCache, bool AtBeginning=false)
Installs the provided FileSystemStatCache object within the FileManager.
Definition: FileManager.cpp:66
Encodes a location in the source.
void setLength(unsigned Len)
Definition: Token.h:133
AnnotatedLine & Line
StringRef GetString() const
bool isValid() const
Return true if this is a valid SourceLocation object.
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
Definition: Preprocessor.h:854
macro_iterator macro_end(bool IncludeExternalMacros=true) const
All of the names in this module are hidden.
Definition: Module.h:246
Cached information about one file (either on disk or in the virtual file system). ...
Definition: FileManager.h:59
void setIdentifierInfo(IdentifierInfo *II)
Definition: Token.h:186
void Lex(Token &Result)
Lex the next token for this preprocessor.
llvm::MemoryBuffer * getMemoryBufferForFile(const FileEntry *File, bool *Invalid=nullptr)
Retrieve the memory buffer associated with the given file.
ArrayRef< FormatToken * > Tokens
const Token & getReplacementToken(unsigned Tok) const
Definition: MacroInfo.h:230
FileID getMainFileID() const
Returns the FileID of the main source file.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Definition: Token.h:95
void expandUCNs(SmallVectorImpl< char > &Buf, StringRef Input)
Copy characters from Input to Buf, expanding any UCNs.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
static diag::kind getFutureCompatDiagKind(const IdentifierInfo &II, const LangOptions &LangOpts)
Returns a diagnostic message kind for reporting a future keyword as appropriate for the identifier an...
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
virtual ModuleLoadResult loadModule(SourceLocation ImportLoc, ModuleIdPath Path, Module::NameVisibilityKind Visibility, bool IsInclusionDirective)=0
Attempt to load the given module.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
detail::InMemoryDirectory::const_iterator E
Abstract interface for a module loader.
Definition: ModuleLoader.h:69
SourceMgr(SourceMgr)
void PoisonSEHIdentifiers(bool Poison=true)
void setLiteralData(const char *Ptr)
Definition: Token.h:219
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
Definition: Token.h:113
Encapsulates the data about a macro definition (e.g.
Definition: MacroInfo.h:34
PragmaNamespace - This PragmaHandler subdivides the namespace of pragmas, allowing hierarchical pragm...
Definition: Pragma.h:89
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
void setConditionalLevels(ArrayRef< PPConditionalInfo > CL)
virtual void updateOutOfDateIdentifier(IdentifierInfo &II)=0
Update an out-of-date identifier.
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
void HandlePoisonedIdentifier(Token &Tok)
Display reason for poisoned identifier.
#define CXX11_KEYWORD(NAME, FLAGS)
bool HandleComment(Token &Token, SourceRange Comment)
void DumpMacro(const MacroInfo &MI) const
TranslationUnitKind
Describes the kind of translation unit being processed.
Definition: LangOptions.h:237
StringLiteralParser - This decodes string escape characters and performs wide string analysis and Tra...
unsigned kind
All of the diagnostics that can be emitted by the frontend.
Definition: DiagnosticIDs.h:44
const char * getTokenName(TokenKind Kind) LLVM_READNONE
Determines the name of a token as used within the front end.
Definition: TokenKinds.cpp:25
Defines the clang::TargetInfo interface.
NumericLiteralParser - This performs strict semantic analysis of the content of a ppnumber...
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Abstract base class that describes a handler that will receive source ranges for each of the comments...
unsigned getLength() const
Definition: Token.h:127
void DumpLocation(SourceLocation Loc) const
void setLocation(SourceLocation L)
Definition: Token.h:132
#define true
Definition: stdbool.h:32
A trivial tuple used to represent a source range.
bool isValid() const
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens...
Definition: ScratchBuffer.h:25
bool isExpandDisabled() const
Return true if this identifier token should never be expanded in the future, due to C99 6...
Definition: Token.h:274
void InitializeTarget(const TargetInfo &Target, const TargetInfo *AuxTarget)
Perform target-specific initialization.
Definition: Builtins.cpp:43
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
Definition: Preprocessor.h:822
This class handles loading and caching of source files into memory.
bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
IdentifierInfo * getIdentifierInfo() const
Definition: Token.h:177