14 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H 15 #define LLVM_CLANG_LEX_PREPROCESSOR_H 33 #include "llvm/ADT/ArrayRef.h" 34 #include "llvm/ADT/DenseMap.h" 35 #include "llvm/ADT/FoldingSet.h" 36 #include "llvm/ADT/FunctionExtras.h" 37 #include "llvm/ADT/None.h" 38 #include "llvm/ADT/Optional.h" 39 #include "llvm/ADT/PointerUnion.h" 40 #include "llvm/ADT/STLExtras.h" 41 #include "llvm/ADT/SmallPtrSet.h" 42 #include "llvm/ADT/SmallVector.h" 43 #include "llvm/ADT/StringRef.h" 44 #include "llvm/ADT/TinyPtrVector.h" 45 #include "llvm/ADT/iterator_range.h" 46 #include "llvm/Support/Allocator.h" 47 #include "llvm/Support/Casting.h" 48 #include "llvm/Support/Registry.h" 60 template<
unsigned InternalLen>
class SmallString;
66 class CodeCompletionHandler;
69 class DirectoryLookup;
70 class ExternalPreprocessorSource;
76 class PragmaNamespace;
77 class PreprocessingRecord;
78 class PreprocessorLexer;
79 class PreprocessorOptions;
91 assert(Kind != tok::raw_identifier &&
"Raw identifiers are not supported.");
92 assert(Kind != tok::identifier &&
93 "Identifiers should be created by TokenValue(IdentifierInfo *)");
101 return Tok.
getKind() == Kind &&
128 llvm::unique_function<void(const clang::Token &)> OnToken;
129 std::shared_ptr<PreprocessorOptions> PPOpts;
136 std::unique_ptr<ScratchBuffer> ScratchBuf;
145 llvm::BumpPtrAllocator BP;
183 unsigned CounterValue = 0;
187 MaxAllowedIncludeStackDepth = 200
191 bool KeepComments : 1;
192 bool KeepMacroComments : 1;
193 bool SuppressIncludeNotFoundError : 1;
196 bool InMacroArgs : 1;
199 bool OwnsHeaderSearch : 1;
202 bool DisableMacroExpansion : 1;
206 bool MacroExpansionInDirectivesOverride : 1;
211 mutable bool ReadMacrosFromExternalSource : 1;
214 bool PragmasEnabled : 1;
217 bool PreprocessedOutput : 1;
220 bool ParsingIfOrElifDirective;
223 bool InMacroArgPreExpansion;
245 std::unique_ptr<PragmaNamespace> PragmaHandlers;
249 std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
253 std::vector<CommentHandler *> CommentHandlers;
257 bool IncrementalProcessing =
false;
266 const FileEntry *CodeCompletionFile =
nullptr;
269 unsigned CodeCompletionOffset = 0;
289 bool LastTokenWasAt =
false;
297 AfterTopLevelTokenSeq = -1,
302 ImportSeq(
State S) : S(S) {}
305 void handleOpenBracket() {
306 S =
static_cast<State>(std::max<int>(S, 0) + 1);
309 void handleCloseBracket() {
310 S =
static_cast<State>(std::max<int>(S, 1) - 1);
313 void handleCloseBrace() {
314 handleCloseBracket();
315 if (S == AtTopLevel && !AfterHeaderName)
316 S = AfterTopLevelTokenSeq;
321 S = AfterTopLevelTokenSeq;
322 AfterHeaderName =
false;
327 void handleExport() {
328 if (S == AfterTopLevelTokenSeq)
334 void handleImport() {
335 if (S == AfterTopLevelTokenSeq || S == AfterExport)
343 void handleHeaderName() {
344 if (S == AfterImportSeq)
345 AfterHeaderName =
true;
355 bool atTopLevel() {
return S <= 0; }
356 bool afterImportSeq() {
return S == AfterImportSeq; }
363 bool AfterHeaderName =
false;
367 ImportSeq ImportSeqState = ImportSeq::AfterTopLevelTokenSeq;
371 bool ModuleImportExpectsIdentifier =
false;
382 bool CodeCompletionReached =
false;
401 std::pair<int, bool> SkipMainFilePreamble;
405 bool HasReachedMaxIncludeDepth =
false;
413 unsigned LexLevel = 0;
424 bool FoundNonSkipPortion,
bool FoundElse,
426 : HashTokenLoc(HashTokenLoc), IfTokenLoc(IfTokenLoc),
427 FoundNonSkipPortion(FoundNonSkipPortion), FoundElse(FoundElse),
435 class PreambleConditionalStackStore {
443 PreambleConditionalStackStore() =
default;
445 void startRecording() { ConditionalStackState = Recording; }
446 void startReplaying() { ConditionalStackState = Replaying; }
447 bool isRecording()
const {
return ConditionalStackState == Recording; }
448 bool isReplaying()
const {
return ConditionalStackState == Replaying; }
451 return ConditionalStack;
454 void doneReplaying() {
455 ConditionalStack.clear();
456 ConditionalStackState = Off;
460 if (!isRecording() && !isReplaying())
462 ConditionalStack.clear();
463 ConditionalStack.append(s.begin(), s.end());
466 bool hasRecordedPreamble()
const {
return !ConditionalStack.empty(); }
468 bool reachedEOFWhileSkipping()
const {
return SkipInfo.hasValue(); }
470 void clearSkipInfo() { SkipInfo.reset(); }
476 State ConditionalStackState = Off;
477 } PreambleConditionalStack;
483 std::unique_ptr<Lexer> CurLexer;
501 std::unique_ptr<TokenLexer> CurTokenLexer;
508 CLK_LexAfterModuleImport
509 } CurLexerKind = CLK_Lexer;
513 Module *CurLexerSubmodule =
nullptr;
518 struct IncludeStackInfo {
519 enum CurLexerKind CurLexerKind;
521 std::unique_ptr<Lexer> TheLexer;
523 std::unique_ptr<TokenLexer> TheTokenLexer;
528 IncludeStackInfo(
enum CurLexerKind CurLexerKind,
Module *TheSubmodule,
529 std::unique_ptr<Lexer> &&TheLexer,
531 std::unique_ptr<TokenLexer> &&TheTokenLexer,
533 : CurLexerKind(
std::move(CurLexerKind)),
534 TheSubmodule(
std::move(TheSubmodule)), TheLexer(
std::move(TheLexer)),
535 ThePPLexer(
std::move(ThePPLexer)),
536 TheTokenLexer(
std::move(TheTokenLexer)),
537 TheDirLookup(
std::move(TheDirLookup)) {}
539 std::vector<IncludeStackInfo> IncludeMacroStack;
543 std::unique_ptr<PPCallbacks> Callbacks;
545 struct MacroExpandsInfo {
551 : Tok(Tok), MD(MD), Range(Range) {}
556 struct ModuleMacroInfo {
561 llvm::TinyPtrVector<ModuleMacro *> ActiveModuleMacros;
565 unsigned ActiveModuleMacrosGeneration = 0;
568 bool IsAmbiguous =
false;
571 llvm::TinyPtrVector<ModuleMacro *> OverriddenMacros;
578 mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *>
State;
583 PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
589 !PP.CurSubmoduleState->VisibleModules.getGeneration())
592 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
599 if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
600 Info->ActiveModuleMacrosGeneration)
601 PP.updateModuleMacroInfo(II, *Info);
606 MacroState() : MacroState(nullptr) {}
609 MacroState(MacroState &&O) noexcept :
State(O.State) {
613 MacroState &operator=(MacroState &&O) noexcept {
621 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
622 Info->~ModuleMacroInfo();
626 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
632 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
639 auto *Info = getModuleInfo(PP, II);
640 return Info ? Info->IsAmbiguous :
false;
645 if (
auto *Info = getModuleInfo(PP, II))
646 return Info->ActiveModuleMacros;
653 if (
auto *Latest = getLatest())
654 return Latest->findDirectiveAtLoc(Loc, SourceMgr);
659 if (
auto *Info = getModuleInfo(PP, II)) {
660 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
661 Info->ActiveModuleMacros.begin(),
662 Info->ActiveModuleMacros.end());
663 Info->ActiveModuleMacros.clear();
664 Info->IsAmbiguous =
false;
669 if (
auto *Info = State.dyn_cast<ModuleMacroInfo*>())
670 return Info->OverriddenMacros;
676 auto *Info = State.dyn_cast<ModuleMacroInfo*>();
678 if (Overrides.empty())
684 Info->OverriddenMacros.clear();
685 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
686 Overrides.begin(), Overrides.end());
687 Info->ActiveModuleMacrosGeneration = 0;
696 using MacroMap = llvm::DenseMap<const IdentifierInfo *, MacroState>;
698 struct SubmoduleState;
701 struct BuildingSubmoduleInfo {
712 SubmoduleState *OuterSubmoduleState;
715 unsigned OuterPendingModuleMacroNames;
718 SubmoduleState *OuterSubmoduleState,
719 unsigned OuterPendingModuleMacroNames)
720 : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
721 OuterSubmoduleState(OuterSubmoduleState),
722 OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
727 struct SubmoduleState {
737 std::map<Module *, SubmoduleState> Submodules;
740 SubmoduleState NullSubmoduleState;
744 SubmoduleState *CurSubmoduleState;
747 llvm::FoldingSet<ModuleMacro> ModuleMacros;
754 llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro *>>
766 using WarnUnusedMacroLocsTy = llvm::SmallPtrSet<SourceLocation, 32>;
767 WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
775 llvm::DenseMap<IdentifierInfo *, std::vector<MacroInfo *>>
779 unsigned NumDirectives = 0;
780 unsigned NumDefined = 0;
781 unsigned NumUndefined = 0;
782 unsigned NumPragma = 0;
784 unsigned NumElse = 0;
785 unsigned NumEndif = 0;
786 unsigned NumEnteredSourceFiles = 0;
787 unsigned MaxIncludeStackDepth = 0;
788 unsigned NumMacroExpanded = 0;
789 unsigned NumFnMacroExpanded = 0;
790 unsigned NumBuiltinMacroExpanded = 0;
791 unsigned NumFastMacroExpanded = 0;
792 unsigned NumTokenPaste = 0;
793 unsigned NumFastTokenPaste = 0;
794 unsigned NumSkipped = 0;
798 std::string Predefines;
804 FileID PCHThroughHeaderFileID;
807 bool SkippingUntilPragmaHdrStop =
false;
810 bool SkippingUntilPCHThroughHeader =
false;
814 enum { TokenLexerCacheSize = 8 };
815 unsigned NumCachedTokenLexers;
816 std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
825 std::vector<std::pair<TokenLexer *, size_t>> MacroExpandingLexersStack;
846 CachedTokensTy::size_type CachedLexPos = 0;
853 std::vector<CachedTokensTy::size_type> BacktrackPositions;
855 struct MacroInfoChain {
857 MacroInfoChain *Next;
862 MacroInfoChain *MIChainHead =
nullptr;
867 Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
871 bool OwnsHeaderSearch =
false,
891 void InitializeForModelFile();
894 void FinalizeForModelFile();
917 ExternalSource = Source;
921 return ExternalSource;
933 return ParsingIfOrElifDirective;
938 this->KeepComments = KeepComments | KeepMacroComments;
939 this->KeepMacroComments = KeepMacroComments;
948 SuppressIncludeNotFoundError = Suppress;
952 return SuppressIncludeNotFoundError;
958 PreprocessedOutput = IsPreprocessedOutput;
967 return CurPPLexer == L;
997 C = llvm::make_unique<PPChainedCallbacks>(std::move(C),
998 std::move(Callbacks));
999 Callbacks = std::move(C);
1007 OnToken = std::move(F);
1015 (!getLangOpts().Modules || (
bool)getMacroDefinition(II));
1024 auto I = Submodules.find(M);
1025 if (I == Submodules.end())
1027 auto J = I->second.Macros.find(II);
1028 if (J == I->second.Macros.end())
1030 auto *MD = J->second.getLatest();
1038 MacroState &S = CurSubmoduleState->Macros[II];
1039 auto *MD = S.getLatest();
1040 while (MD && isa<VisibilityMacroDirective>(MD))
1043 S.getActiveModuleMacros(*
this, II),
1044 S.isAmbiguous(*
this, II));
1052 MacroState &S = CurSubmoduleState->Macros[II];
1054 if (
auto *MD = S.getLatest())
1058 S.getActiveModuleMacros(*
this, II),
1059 S.isAmbiguous(*
this, II));
1068 auto *MD = getLocalMacroDirectiveHistory(II);
1076 return const_cast<Preprocessor*
>(
this)->getMacroInfo(II);
1082 if (
auto MD = getMacroDefinition(II))
1099 appendMacroDirective(II, MD);
1119 updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
1120 auto I = LeafModuleMacros.find(II);
1121 if (I != LeafModuleMacros.end())
1132 macro_iterator macro_begin(
bool IncludeExternalMacros =
true)
const;
1133 macro_iterator macro_end(
bool IncludeExternalMacros =
true)
const;
1135 llvm::iterator_range<macro_iterator>
1136 macros(
bool IncludeExternalMacros =
true)
const {
1139 return llvm::make_range(begin, end);
1161 return &Identifiers.
get(Name);
1168 void AddPragmaHandler(StringRef Namespace,
PragmaHandler *Handler);
1170 AddPragmaHandler(StringRef(), Handler);
1178 void RemovePragmaHandler(StringRef Namespace,
PragmaHandler *Handler);
1180 RemovePragmaHandler(StringRef(), Handler);
1184 void IgnorePragmas();
1196 CodeComplete = &Handler;
1201 return CodeComplete;
1206 CodeComplete =
nullptr;
1211 void CodeCompleteIncludedFile(llvm::StringRef Dir,
bool IsAngled);
1215 void CodeCompleteNaturalLanguage();
1219 CodeCompletionII =
Filter;
1226 CodeCompletionTokenRange = {Start, End};
1229 return CodeCompletionTokenRange;
1234 if (CodeCompletionII)
1235 return CodeCompletionII->
getName();
1245 void createPreprocessingRecord();
1248 bool isPCHThroughHeader(
const FileEntry *FE);
1251 bool creatingPCHWithThroughHeader();
1254 bool usingPCHWithThroughHeader();
1257 bool creatingPCHWithPragmaHdrStop();
1260 bool usingPCHWithPragmaHdrStop();
1264 void SkipTokensWhileUsingPCH();
1268 void HandleSkippedDirectiveWhileUsingPCH(
Token &Result,
1273 void EnterMainSourceFile();
1276 void EndSourceFile();
1309 void EnterTokenStream(
const Token *Toks,
unsigned NumToks,
1310 bool DisableMacroExpansion,
bool OwnsTokens,
1315 bool DisableMacroExpansion,
bool IsReinject) {
1316 EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion,
true,
1322 EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion,
false,
1330 void RemoveTopOfLexerStack();
1345 void EnableBacktrackAtThisPos();
1348 void CommitBacktrackedTokens();
1359 void Lex(
Token &Result);
1362 bool LexHeaderName(
Token &Result,
bool AllowMacroExpansion =
true);
1364 bool LexAfterModuleImport(
Token &Result);
1370 return CurSubmoduleState->VisibleModules.getImportLoc(M);
1377 const char *DiagnosticTag,
bool AllowMacroExpansion) {
1378 if (AllowMacroExpansion)
1381 LexUnexpandedToken(Result);
1382 return FinishLexStringLiteral(Result, String, DiagnosticTag,
1383 AllowMacroExpansion);
1388 bool FinishLexStringLiteral(
Token &Result, std::string &String,
1389 const char *DiagnosticTag,
1390 bool AllowMacroExpansion);
1400 while (Result.
getKind() == tok::comment);
1406 bool OldVal = DisableMacroExpansion;
1407 DisableMacroExpansion =
true;
1412 DisableMacroExpansion = OldVal;
1419 LexUnexpandedToken(Result);
1420 while (Result.
getKind() == tok::comment);
1426 bool parseSimpleIntegerLiteral(
Token &Tok, uint64_t &
Value);
1430 DisableMacroExpansion =
true;
1431 MacroExpansionInDirectivesOverride =
true;
1442 assert(LexLevel == 0 &&
"cannot use lookahead while lexing");
1443 if (CachedLexPos + N < CachedTokens.size())
1444 return CachedTokens[CachedLexPos+N];
1446 return PeekAhead(N+1);
1455 assert(isBacktrackEnabled() &&
1456 "Should only be called when tokens are cached for backtracking");
1457 assert(
signed(CachedLexPos) -
signed(N) >=
signed(BacktrackPositions.back())
1458 &&
"Should revert tokens up to the last backtrack position, not more");
1459 assert(
signed(CachedLexPos) -
signed(N) >= 0 &&
1460 "Corrupted backtrack positions ?");
1474 auto TokCopy = llvm::make_unique<Token[]>(1);
1476 EnterTokenStream(std::move(TokCopy), 1,
true, IsReinject);
1478 EnterCachingLexMode();
1479 assert(IsReinject &&
"new tokens in the middle of cached stream");
1480 CachedTokens.insert(CachedTokens.begin()+CachedLexPos,
Tok);
1493 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1494 if (CachedLexPos != 0 && isBacktrackEnabled())
1495 AnnotatePreviousCachedTokens(Tok);
1501 assert(CachedLexPos != 0);
1502 return CachedTokens[CachedLexPos-1].getLastLoc();
1507 bool IsPreviousCachedToken(
const Token &Tok)
const;
1525 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1526 if (CachedLexPos != 0 && isBacktrackEnabled())
1527 CachedTokens[CachedLexPos-1] = Tok;
1532 void *AnnotationVal);
1538 if (CachedLexPos != 0 && isBacktrackEnabled())
1539 CachedTokens[CachedLexPos-1] = Tok;
1544 void recomputeCurLexerKind();
1551 IncrementalProcessing = value;
1568 bool SetCodeCompletionPoint(
const FileEntry *File,
1569 unsigned Line,
unsigned Column);
1585 return CodeCompletionFileLoc;
1594 assert(isCodeCompletionEnabled() &&
"Code-completion not enabled!");
1595 CodeCompletionReached =
true;
1597 getDiagnostics().setSuppressAllDiagnostics(
true);
1605 return PragmaARCCFCodeAuditedLoc;
1611 PragmaARCCFCodeAuditedLoc = Loc;
1619 return PragmaAssumeNonNullLoc;
1625 PragmaAssumeNonNullLoc = Loc;
1641 SkipMainFilePreamble.first = Bytes;
1642 SkipMainFilePreamble.second = StartOfLine;
1649 return Diags->
Report(Loc, DiagID);
1665 bool *invalid =
nullptr)
const {
1666 return Lexer::getSpelling(loc, buffer, SourceMgr, LangOpts, invalid);
1678 return Lexer::getSpelling(Tok, SourceMgr, LangOpts, Invalid);
1694 bool *Invalid =
nullptr)
const {
1695 return Lexer::getSpelling(Tok, Buffer, SourceMgr, LangOpts, Invalid);
1702 StringRef getSpelling(
const Token &Tok,
1704 bool *Invalid =
nullptr)
const;
1709 bool IgnoreWhiteSpace =
false) {
1710 return Lexer::getRawToken(Loc, Result, SourceMgr, LangOpts, IgnoreWhiteSpace);
1717 bool *Invalid =
nullptr)
const {
1718 assert(Tok.
is(tok::numeric_constant) &&
1719 Tok.
getLength() == 1 &&
"Called on unsupported token");
1720 assert(!Tok.
needsCleaning() &&
"Token can't need cleaning with length 1");
1740 return Lexer::getImmediateMacroName(Loc, SourceMgr, getLangOpts());
1748 void CreateString(StringRef Str,
Token &Tok,
1773 return Lexer::getLocForEndOfToken(Loc,
Offset, SourceMgr, LangOpts);
1783 return Lexer::isAtStartOfMacroExpansion(loc, SourceMgr, LangOpts,
1794 return Lexer::isAtEndOfMacroExpansion(loc, SourceMgr, LangOpts, MacroEnd);
1798 void DumpToken(
const Token &Tok,
bool DumpFlags =
false)
const;
1800 void DumpMacro(
const MacroInfo &MI)
const;
1806 unsigned Char)
const {
1807 return Lexer::AdvanceToTokenCharacter(TokStart, Char, SourceMgr, LangOpts);
1816 ++NumFastTokenPaste;
1823 size_t getTotalMemory()
const;
1828 void HandleMicrosoftCommentPaste(
Token &Tok);
1840 llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1850 void HandlePoisonedIdentifier(
Token & Identifier);
1855 HandlePoisonedIdentifier(Identifier);
1865 *Ident___exception_code,
1866 *Ident_GetExceptionCode;
1869 *Ident___exception_info,
1870 *Ident_GetExceptionInfo;
1873 *Ident___abnormal_termination,
1874 *Ident_AbnormalTermination;
1876 const char *getCurLexerEndPos();
1877 void diagnoseMissingHeaderInUmbrellaDir(
const Module &Mod);
1880 void PoisonSEHIdentifiers(
bool Poison =
true);
1890 bool HandleIdentifier(
Token &Identifier);
1897 bool HandleEndOfFile(
Token &Result,
bool isEndOfMacro =
false);
1901 bool HandleEndOfTokenLexer(
Token &Result);
1908 void HandleDirective(
Token &Result);
1919 bool EnableMacros =
false);
1934 Module *getCurrentModule();
1948 bool GetIncludeFilenameSpelling(
SourceLocation Loc,StringRef &Buffer);
1961 bool *IsMapped,
bool *IsFrameworkFound,
1962 bool SkipCache =
false);
1972 bool isInPrimaryFile()
const;
1978 bool CheckMacroName(
Token &MacroNameTok,
MacroUse isDefineUndef,
1979 bool *ShadowFlag =
nullptr);
1982 Module *LeaveSubmodule(
bool ForPragma);
1985 friend void TokenLexer::ExpandFunctionArguments();
1987 void PushIncludeMacroStack() {
1988 assert(CurLexerKind != CLK_CachingLexer &&
"cannot push a caching lexer");
1989 IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
1990 std::move(CurLexer), CurPPLexer,
1991 std::move(CurTokenLexer), CurDirLookup);
1992 CurPPLexer =
nullptr;
1995 void PopIncludeMacroStack() {
1996 CurLexer = std::move(IncludeMacroStack.back().TheLexer);
1997 CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1998 CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
1999 CurDirLookup = IncludeMacroStack.back().TheDirLookup;
2000 CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
2001 CurLexerKind = IncludeMacroStack.back().CurLexerKind;
2002 IncludeMacroStack.pop_back();
2005 void PropagateLineStartLeadingSpaceInfo(
Token &Result);
2009 bool needModuleMacros()
const;
2013 void updateModuleMacroInfo(
const IdentifierInfo *II, ModuleMacroInfo &Info);
2032 bool *ShadowFlag =
nullptr);
2043 MacroInfo *ReadOptionalMacroParameterListAndBody(
2044 const Token &MacroNameTok,
bool ImmediatelyAfterHeaderGuard);
2062 bool FoundNonSkipPortion,
bool FoundElse,
2067 struct DirectiveEvalResult {
2072 bool IncludedUndefinedIds;
2082 DirectiveEvalResult EvaluateDirectiveExpression(
IdentifierInfo *&IfNDefMacro);
2086 void RegisterBuiltinPragmas();
2089 void RegisterBuiltinMacros();
2104 void removeCachedMacroExpandedTokensOfLastLexer();
2109 bool isNextPPTokenLParen();
2118 void ExpandBuiltinMacro(
Token &Tok);
2123 void Handle_Pragma(
Token &Tok);
2127 void HandleMicrosoft__pragma(
Token &Tok);
2134 void setPredefinesFileID(
FileID FID) {
2135 assert(PredefinesFileID.
isInvalid() &&
"PredefinesFileID already set!");
2136 PredefinesFileID = FID;
2140 void setPCHThroughHeaderFileID(
FileID FID);
2148 static bool IsFileLexer(
const IncludeStackInfo& I) {
2149 return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
2152 bool IsFileLexer()
const {
2153 return IsFileLexer(CurLexer.get(), CurPPLexer);
2158 void CachingLex(
Token &Result);
2160 bool InCachingLexMode()
const {
2163 return !CurPPLexer && !CurTokenLexer && !IncludeMacroStack.empty();
2166 void EnterCachingLexMode();
2167 void EnterCachingLexModeUnchecked();
2169 void ExitCachingLexMode() {
2170 if (InCachingLexMode())
2171 RemoveTopOfLexerStack();
2174 const Token &PeekAhead(
unsigned N);
2175 void AnnotatePreviousCachedTokens(
const Token &Tok);
2181 void HandleLineDirective();
2182 void HandleDigitDirective(
Token &Tok);
2183 void HandleUserDiagnosticDirective(
Token &Tok,
bool isWarning);
2184 void HandleIdentSCCSDirective(
Token &Tok);
2185 void HandleMacroPublicDirective(
Token &Tok);
2186 void HandleMacroPrivateDirective();
2190 struct ImportAction {
2195 SkippedModuleImport,
2197 Module *ModuleForHeader =
nullptr;
2200 :
Kind(AK), ModuleForHeader(Mod) {
2201 assert((AK == None || Mod) &&
"no module for module action");
2208 const FileEntry *LookupFromFile =
nullptr);
2213 const FileEntry *LookupFromFile =
nullptr);
2217 void HandleMicrosoftImportDirective(
Token &Tok);
2223 static bool checkModuleIsAvailable(
const LangOptions &LangOpts,
2249 return PreambleConditionalStack.isRecording();
2253 return PreambleConditionalStack.hasRecordedPreamble();
2257 return PreambleConditionalStack.getStack();
2261 PreambleConditionalStack.setStack(s);
2266 PreambleConditionalStack.startReplaying();
2267 PreambleConditionalStack.setStack(s);
2268 PreambleConditionalStack.SkipInfo = SkipInfo;
2272 return PreambleConditionalStack.SkipInfo;
2278 void replayPreambleConditionalStack();
2281 void HandleDefineDirective(
Token &Tok,
bool ImmediatelyAfterHeaderGuard);
2282 void HandleUndefDirective();
2285 void HandleIfdefDirective(
Token &Result,
const Token &HashToken,
2286 bool isIfndef,
bool ReadAnyTokensBeforeDirective);
2287 void HandleIfDirective(
Token &IfToken,
const Token &HashToken,
2288 bool ReadAnyTokensBeforeDirective);
2289 void HandleEndifDirective(
Token &EndifToken);
2290 void HandleElseDirective(
Token &Result,
const Token &HashToken);
2291 void HandleElifDirective(
Token &ElifToken,
const Token &HashToken);
2297 void HandlePragmaOnce(
Token &OnceTok);
2298 void HandlePragmaMark();
2299 void HandlePragmaPoison();
2300 void HandlePragmaSystemHeader(
Token &SysHeaderTok);
2301 void HandlePragmaDependency(
Token &DependencyTok);
2302 void HandlePragmaPushMacro(
Token &Tok);
2303 void HandlePragmaPopMacro(
Token &Tok);
2304 void HandlePragmaIncludeAlias(
Token &Tok);
2305 void HandlePragmaModuleBuild(
Token &Tok);
2306 void HandlePragmaHdrstop(
Token &Tok);
2334 #endif // LLVM_CLANG_LEX_PREPROCESSOR_H PreambleSkipInfo(SourceLocation HashTokenLoc, SourceLocation IfTokenLoc, bool FoundNonSkipPortion, bool FoundElse, SourceLocation ElseLoc)
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion, bool IsReinject)
A set of visible modules.
llvm::BumpPtrAllocator & getPreprocessorAllocator()
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
SelectorTable & getSelectorTable()
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
Implements support for file system lookup, file system caching, and directory search management...
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the 'spelling' of the Tok token.
Specialize PointerLikeTypeTraits to allow LazyGenerationalUpdatePtr to be placed into a PointerUnion...
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
void MaybeHandlePoisonedIdentifier(Token &Identifier)
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Defines the SourceManager interface.
static const Builtin::Info BuiltinInfo[]
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
FileManager & getFileManager() const
Defines the clang::Module class, which describes a module in the source code.
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
SourceLocation getPragmaARCCFCodeAuditedLoc() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
A directive for an undefined macro.
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
bool hadModuleLoaderFatalFailure() const
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion, bool IsReinject)
bool operator==(const Token &Tok) const
Builtin::Context & getBuiltinInfo()
void setPredefines(const char *P)
Set the predefines for this Preprocessor.
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character. ...
This interface provides a way to observe the actions of the preprocessor as it does its thing...
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
tok::TokenKind getKind() const
TokenValue(tok::TokenKind Kind)
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
One of these records is kept for each identifier that is lexed.
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
Represents a macro directive exported by a module.
A directive for a defined macro or a macro imported from a module.
This table allows us to fully hide how we implement multi-keyword caching.
const TargetInfo & getTargetInfo() const
Token - This structure provides full information about a lexed token.
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
const LangOptions & getLangOpts() const
Describes a module or submodule.
bool isPragmaLexer() const
isPragmaLexer - Returns true if this Lexer is being used to lex a pragma.
A directive for setting the module visibility of a macro.
void SetSuppressIncludeNotFoundError(bool Suppress)
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
MacroUse
Context in which macro name is used.
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
HeaderSearch & getHeaderSearchInfo() const
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
void LexNonComment(Token &Result)
Lex a token.
Concrete class used by the front-end to report problems and issues.
bool isMacroDefined(const IdentifierInfo *II)
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we've alread...
Defines the Diagnostic-related interfaces.
const TargetInfo * getAuxTargetInfo() const
TokenLexer - This implements a lexer that returns tokens from a macro body or token stream instead of...
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
void setTokenWatcher(llvm::unique_function< void(const clang::Token &)> F)
Register a function that would be called on each token in the final expanded token stream...
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
A little helper class used to produce diagnostics.
void clearCodeCompletionHandler()
Clear out the code completion handler.
Provides lookups to, and iteration over, IdentiferInfo objects.
Exposes information about the current target.
const IdentifierTable & getIdentifierTable() const
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
Abstract interface for external sources of preprocessor information.
Defines the clang::LangOptions interface.
unsigned getCounterValue() const
const DefMacroDirective * getDirective() const
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
const AnnotatedLine * Line
SourceLocation HashTokenLoc
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
Implements an efficient mapping from strings to IdentifierInfo nodes.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a variadic mac...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
DefInfo getDefinition()
Traverses the macro directives history and returns the next macro definition directive along with inf...
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Stores token information for comparing actual tokens with predefined values.
llvm::Optional< PreambleSkipInfo > getPreambleSkipInfo() const
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
bool getCommentRetentionState() const
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
TokenValue(IdentifierInfo *II)
PPCallbacks * getPPCallbacks() const
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
bool isRecordingPreamble() const
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
SourceManager & getSourceManager() const
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define'd and not #undef'...
const DefInfo findDirectiveAtLoc(SourceLocation L, const SourceManager &SM) const
Find macro definition active in the specified source location.
ExternalPreprocessorSource * getExternalSource() const
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
void setExternalSource(ExternalPreprocessorSource *Source)
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin. ...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
IdentifierInfo * getIdentifierInfo() const
IdentifierTable & getIdentifierTable()
void setPragmasEnabled(bool Enabled)
Cached information about one file (either on disk or in the virtual file system). ...
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion...
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
bool isPoisoned() const
Return true if this token has been poisoned.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
void EnterToken(const Token &Tok, bool IsReinject)
Enters a token in the token stream to be lexed next.
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
const MacroInfo * getMacroInfo() const
StringRef getName() const
Return the actual identifier string.
bool isMacroDefined(StringRef Id)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
Dataflow Directional Tag Classes.
SourceLocation IfTokenLoc
SmallVector< Token, 4 > CachedTokens
A set of tokens that has been cached for later parsing.
SourceRange getCodeCompletionTokenRange() const
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
Reads an AST files chain containing the contents of a translation unit.
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
Abstract interface for a module loader.
unsigned getLength() const
Encapsulates the data about a macro definition (e.g.
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
bool getPragmasEnabled() const
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
Cached information about one directory (either on disk or in the virtual file system).
Defines the PPCallbacks interface.
Defines the clang::TokenKind enum and support functions.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
SourceLocation getModuleImportLoc(Module *M) const
Defines the clang::SourceLocation class and associated facilities.
void RemovePragmaHandler(PragmaHandler *Handler)
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
DiagnosticsEngine & getDiagnostics() const
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
static bool isMacroDefined(const Sema &S, SourceLocation Loc, StringRef Name)
TranslationUnitKind
Describes the kind of translation unit being processed.
bool hasRecordedPreamble() const
const std::string & getPredefines() const
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
void setPredefines(StringRef P)
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
The translation unit is a complete translation unit.
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
MacroMap::const_iterator macro_iterator
Describes how and where the pragma was introduced.
void setCounterValue(unsigned V)
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
A trivial tuple used to represent a source range.
bool GetSuppressIncludeNotFoundError()
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s, llvm::Optional< PreambleSkipInfo > SkipInfo)
MacroInfo * getMacroInfo(const IdentifierInfo *II)
Callback handler that receives notifications when performing code completion within the preprocessor...
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
This class handles loading and caching of source files into memory.
Defines enum values for all the target-independent builtin functions.
An RAII class that tracks when the Preprocessor starts and stops lexing the definition of a (ISO C/C+...
void AddPragmaHandler(PragmaHandler *Handler)
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
void setDiagnostics(DiagnosticsEngine &D)