15 #ifndef LLVM_CLANG_LEX_PREPROCESSOR_H
16 #define LLVM_CLANG_LEX_PREPROCESSOR_H
28 #include "llvm/ADT/ArrayRef.h"
29 #include "llvm/ADT/DenseMap.h"
30 #include "llvm/ADT/IntrusiveRefCntPtr.h"
31 #include "llvm/ADT/SmallPtrSet.h"
32 #include "llvm/ADT/SmallVector.h"
33 #include "llvm/ADT/TinyPtrVector.h"
34 #include "llvm/Support/Allocator.h"
35 #include "llvm/Support/Registry.h"
40 template<
unsigned InternalLen>
class SmallString;
46 class ExternalPreprocessorSource;
50 class MemoryBufferCache;
51 class PragmaNamespace;
57 class CodeCompletionHandler;
58 class DirectoryLookup;
59 class PreprocessingRecord;
62 class PreprocessorOptions;
72 assert(Kind != tok::raw_identifier &&
"Raw identifiers are not supported.");
73 assert(Kind != tok::identifier &&
74 "Identifiers should be created by TokenValue(IdentifierInfo *)");
99 std::shared_ptr<PreprocessorOptions> PPOpts;
107 std::unique_ptr<ScratchBuffer> ScratchBuf;
117 std::unique_ptr<PTHManager> PTH;
121 llvm::BumpPtrAllocator BP;
147 unsigned CounterValue;
151 MaxAllowedIncludeStackDepth = 200
155 bool KeepComments : 1;
156 bool KeepMacroComments : 1;
157 bool SuppressIncludeNotFoundError : 1;
160 bool InMacroArgs : 1;
163 bool OwnsHeaderSearch : 1;
166 bool DisableMacroExpansion : 1;
170 bool MacroExpansionInDirectivesOverride : 1;
175 mutable bool ReadMacrosFromExternalSource : 1;
178 bool PragmasEnabled : 1;
181 bool PreprocessedOutput : 1;
184 bool ParsingIfOrElifDirective;
187 bool InMacroArgPreExpansion;
209 std::unique_ptr<PragmaNamespace> PragmaHandlers;
213 std::unique_ptr<PragmaNamespace> PragmaHandlersBackup;
217 std::vector<CommentHandler *> CommentHandlers;
221 bool IncrementalProcessing;
233 unsigned CodeCompletionOffset;
257 bool ModuleImportExpectsIdentifier;
268 bool CodeCompletionReached;
284 std::pair<int, bool> SkipMainFilePreamble;
286 class PreambleConditionalStackStore {
294 PreambleConditionalStackStore() : ConditionalStackState(Off) {}
296 void startRecording() { ConditionalStackState = Recording; }
297 void startReplaying() { ConditionalStackState = Replaying; }
298 bool isRecording()
const {
return ConditionalStackState == Recording; }
299 bool isReplaying()
const {
return ConditionalStackState == Replaying; }
302 return ConditionalStack;
305 void doneReplaying() {
306 ConditionalStack.clear();
307 ConditionalStackState = Off;
311 if (!isRecording() && !isReplaying())
313 ConditionalStack.clear();
314 ConditionalStack.append(s.begin(), s.end());
321 State ConditionalStackState;
322 } PreambleConditionalStack;
328 std::unique_ptr<Lexer> CurLexer;
334 std::unique_ptr<PTHLexer> CurPTHLexer;
352 std::unique_ptr<TokenLexer> CurTokenLexer;
360 CLK_LexAfterModuleImport
365 Module *CurLexerSubmodule;
370 struct IncludeStackInfo {
371 enum CurLexerKind CurLexerKind;
373 std::unique_ptr<Lexer> TheLexer;
374 std::unique_ptr<PTHLexer> ThePTHLexer;
376 std::unique_ptr<TokenLexer> TheTokenLexer;
381 IncludeStackInfo(
enum CurLexerKind CurLexerKind,
Module *TheSubmodule,
382 std::unique_ptr<Lexer> &&TheLexer,
383 std::unique_ptr<PTHLexer> &&ThePTHLexer,
385 std::unique_ptr<TokenLexer> &&TheTokenLexer,
387 : CurLexerKind(std::move(CurLexerKind)),
388 TheSubmodule(std::move(TheSubmodule)), TheLexer(std::move(TheLexer)),
389 ThePTHLexer(std::move(ThePTHLexer)),
390 ThePPLexer(std::move(ThePPLexer)),
391 TheTokenLexer(std::move(TheTokenLexer)),
392 TheDirLookup(std::move(TheDirLookup)) {}
394 std::vector<IncludeStackInfo> IncludeMacroStack;
398 std::unique_ptr<PPCallbacks> Callbacks;
400 struct MacroExpandsInfo {
405 : Tok(Tok), MD(MD), Range(Range) { }
410 struct ModuleMacroInfo {
412 : MD(MD), ActiveModuleMacrosGeneration(0), IsAmbiguous(
false) {}
417 llvm::TinyPtrVector<ModuleMacro*> ActiveModuleMacros;
420 unsigned ActiveModuleMacrosGeneration;
424 llvm::TinyPtrVector<ModuleMacro*> OverriddenMacros;
429 mutable llvm::PointerUnion<MacroDirective *, ModuleMacroInfo *>
State;
434 PP.updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
440 !PP.CurSubmoduleState->VisibleModules.getGeneration())
443 auto *Info =
State.dyn_cast<ModuleMacroInfo*>();
450 if (PP.CurSubmoduleState->VisibleModules.getGeneration() !=
451 Info->ActiveModuleMacrosGeneration)
452 PP.updateModuleMacroInfo(II, *Info);
457 MacroState() : MacroState(
nullptr) {}
459 MacroState(MacroState &&O) noexcept :
State(O.State) {
462 MacroState &operator=(MacroState &&O) noexcept {
469 if (
auto *Info =
State.dyn_cast<ModuleMacroInfo*>())
470 Info->~ModuleMacroInfo();
474 if (
auto *Info =
State.dyn_cast<ModuleMacroInfo*>())
479 if (
auto *Info =
State.dyn_cast<ModuleMacroInfo*>())
486 auto *Info = getModuleInfo(PP, II);
487 return Info ? Info->IsAmbiguous :
false;
491 if (
auto *Info = getModuleInfo(PP, II))
492 return Info->ActiveModuleMacros;
499 if (
auto *Latest = getLatest())
500 return Latest->findDirectiveAtLoc(Loc, SourceMgr);
505 if (
auto *Info = getModuleInfo(PP, II)) {
506 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
507 Info->ActiveModuleMacros.begin(),
508 Info->ActiveModuleMacros.end());
509 Info->ActiveModuleMacros.clear();
510 Info->IsAmbiguous =
false;
514 if (
auto *Info =
State.dyn_cast<ModuleMacroInfo*>())
515 return Info->OverriddenMacros;
520 auto *Info =
State.dyn_cast<ModuleMacroInfo*>();
522 if (Overrides.empty())
528 Info->OverriddenMacros.clear();
529 Info->OverriddenMacros.insert(Info->OverriddenMacros.end(),
530 Overrides.begin(), Overrides.end());
531 Info->ActiveModuleMacrosGeneration = 0;
540 typedef llvm::DenseMap<const IdentifierInfo *, MacroState> MacroMap;
544 struct SubmoduleState;
547 struct BuildingSubmoduleInfo {
549 SubmoduleState *OuterSubmoduleState,
550 unsigned OuterPendingModuleMacroNames)
551 : M(M), ImportLoc(ImportLoc), IsPragma(IsPragma),
552 OuterSubmoduleState(OuterSubmoduleState),
553 OuterPendingModuleMacroNames(OuterPendingModuleMacroNames) {}
562 SubmoduleState *OuterSubmoduleState;
564 unsigned OuterPendingModuleMacroNames;
566 SmallVector<BuildingSubmoduleInfo, 8> BuildingSubmoduleStack;
569 struct SubmoduleState {
573 VisibleModuleSet VisibleModules;
577 std::map<Module*, SubmoduleState> Submodules;
580 SubmoduleState NullSubmoduleState;
584 SubmoduleState *CurSubmoduleState;
587 llvm::FoldingSet<ModuleMacro> ModuleMacros;
594 llvm::DenseMap<const IdentifierInfo *, llvm::TinyPtrVector<ModuleMacro*>>
606 typedef llvm::SmallPtrSet<SourceLocation, 32> WarnUnusedMacroLocsTy;
607 WarnUnusedMacroLocsTy WarnUnusedMacroLocs;
616 llvm::DenseMap<IdentifierInfo*, std::vector<MacroInfo*> > PragmaPushMacroInfo;
619 unsigned NumDirectives, NumDefined, NumUndefined, NumPragma;
620 unsigned NumIf, NumElse, NumEndif;
621 unsigned NumEnteredSourceFiles, MaxIncludeStackDepth;
622 unsigned NumMacroExpanded, NumFnMacroExpanded, NumBuiltinMacroExpanded;
623 unsigned NumFastMacroExpanded, NumTokenPaste, NumFastTokenPaste;
628 std::string Predefines;
635 enum { TokenLexerCacheSize = 8 };
636 unsigned NumCachedTokenLexers;
637 std::unique_ptr<TokenLexer> TokenLexerCache[TokenLexerCacheSize];
646 std::vector<std::pair<TokenLexer *, size_t> > MacroExpandingLexersStack;
667 CachedTokensTy::size_type CachedLexPos;
674 std::vector<CachedTokensTy::size_type> BacktrackPositions;
676 struct MacroInfoChain {
678 MacroInfoChain *
Next;
683 MacroInfoChain *MIChainHead;
685 void updateOutOfDateIdentifier(IdentifierInfo &II)
const;
688 Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
689 DiagnosticsEngine &diags, LangOptions &opts, SourceManager &
SM,
690 MemoryBufferCache &PCMCache,
691 HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
692 IdentifierInfoLookup *IILookup =
nullptr,
693 bool OwnsHeaderSearch =
false,
705 const TargetInfo *AuxTarget =
nullptr);
744 ExternalSource = Source;
748 return ExternalSource;
760 return ParsingIfOrElifDirective;
765 this->KeepComments = KeepComments | KeepMacroComments;
766 this->KeepMacroComments = KeepMacroComments;
775 SuppressIncludeNotFoundError = Suppress;
779 return SuppressIncludeNotFoundError;
785 PreprocessedOutput = IsPreprocessedOutput;
794 return CurPPLexer == L;
824 C = llvm::make_unique<PPChainedCallbacks>(std::move(C),
825 std::move(Callbacks));
826 Callbacks = std::move(C);
844 auto I = Submodules.find(M);
845 if (
I == Submodules.end())
847 auto J =
I->second.Macros.find(II);
848 if (J ==
I->second.Macros.end())
850 auto *MD = J->second.getLatest();
851 return MD && MD->isDefined();
858 MacroState &
S = CurSubmoduleState->Macros[II];
859 auto *MD = S.getLatest();
860 while (MD && isa<VisibilityMacroDirective>(MD))
861 MD = MD->getPrevious();
863 S.getActiveModuleMacros(*
this, II),
864 S.isAmbiguous(*
this, II));
872 MacroState &
S = CurSubmoduleState->Macros[II];
874 if (
auto *MD = S.getLatest())
878 S.getActiveModuleMacros(*
this, II),
879 S.isAmbiguous(*
this, II));
889 if (!MD || MD->getDefinition().isUndefined())
903 return MD.getMacroInfo();
938 updateOutOfDateIdentifier(const_cast<IdentifierInfo&>(*II));
939 auto I = LeafModuleMacros.find(II);
940 if (
I != LeafModuleMacros.end())
952 llvm::iterator_range<macro_iterator>
953 macros(
bool IncludeExternalMacros =
true)
const {
954 return llvm::make_range(
macro_begin(IncludeExternalMacros),
975 return &Identifiers.
get(Name);
1010 CodeComplete = &Handler;
1015 return CodeComplete;
1020 CodeComplete =
nullptr;
1029 CodeCompletionII = Filter;
1034 if (CodeCompletionII)
1035 return CodeCompletionII->
getName();
1082 void EnterTokenStream(
const Token *Toks,
unsigned NumToks,
1083 bool DisableMacroExpansion,
bool OwnsTokens);
1087 bool DisableMacroExpansion) {
1088 EnterTokenStream(Toks.release(), NumToks, DisableMacroExpansion,
true);
1091 EnterTokenStream(Toks.data(), Toks.size(), DisableMacroExpansion,
false);
1152 return CurSubmoduleState->VisibleModules.getImportLoc(M);
1159 const char *DiagnosticTag,
bool AllowMacroExpansion) {
1160 if (AllowMacroExpansion)
1165 AllowMacroExpansion);
1171 const char *DiagnosticTag,
1172 bool AllowMacroExpansion);
1182 while (Result.
getKind() == tok::comment);
1188 bool OldVal = DisableMacroExpansion;
1189 DisableMacroExpansion =
true;
1194 DisableMacroExpansion = OldVal;
1202 while (Result.
getKind() == tok::comment);
1212 DisableMacroExpansion =
true;
1213 MacroExpansionInDirectivesOverride =
true;
1227 return PeekAhead(N+1);
1237 "Should only be called when tokens are cached for backtracking");
1238 assert(
signed(CachedLexPos) -
signed(N) >=
signed(BacktrackPositions.back())
1239 &&
"Should revert tokens up to the last backtrack position, not more");
1240 assert(
signed(CachedLexPos) -
signed(N) >= 0 &&
1241 "Corrupted backtrack positions ?");
1250 EnterCachingLexMode();
1263 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1265 AnnotatePreviousCachedTokens(Tok);
1271 assert(CachedLexPos != 0);
1295 assert(Tok.
isAnnotation() &&
"Expected annotation token");
1302 void *AnnotationVal);
1321 IncrementalProcessing = value;
1355 return CodeCompletionFileLoc;
1365 CodeCompletionReached =
true;
1375 return PragmaARCCFCodeAuditedLoc;
1381 PragmaARCCFCodeAuditedLoc = Loc;
1389 return PragmaAssumeNonNullLoc;
1395 PragmaAssumeNonNullLoc = Loc;
1411 SkipMainFilePreamble.first = Bytes;
1412 SkipMainFilePreamble.second = StartOfLine;
1419 return Diags->
Report(Loc, DiagID);
1435 bool *invalid =
nullptr)
const {
1464 bool *Invalid =
nullptr)
const {
1474 bool *Invalid =
nullptr)
const;
1479 bool IgnoreWhiteSpace =
false) {
1487 bool *Invalid =
nullptr)
const {
1488 assert(Tok.
is(tok::numeric_constant) &&
1489 Tok.
getLength() == 1 &&
"Called on unsupported token");
1490 assert(!Tok.
needsCleaning() &&
"Token can't need cleaning with length 1");
1571 unsigned Char)
const {
1581 ++NumFastTokenPaste;
1605 llvm::DenseMap<IdentifierInfo*,unsigned> PoisonReasons;
1620 if(II->isPoisoned()) {
1631 *Ident___exception_code,
1632 *Ident_GetExceptionCode;
1635 *Ident___exception_info,
1636 *Ident_GetExceptionInfo;
1639 *Ident___abnormal_termination,
1640 *Ident_AbnormalTermination;
1642 const char *getCurLexerEndPos();
1643 void diagnoseMissingHeaderInUmbrellaDir(
const Module &Mod);
1724 bool *IsMapped,
bool SkipCache =
false);
1757 bool *ShadowFlag =
nullptr);
1763 void PushIncludeMacroStack() {
1764 assert(CurLexerKind != CLK_CachingLexer &&
"cannot push a caching lexer");
1765 IncludeMacroStack.emplace_back(CurLexerKind, CurLexerSubmodule,
1766 std::move(CurLexer), std::move(CurPTHLexer),
1767 CurPPLexer, std::move(CurTokenLexer),
1769 CurPPLexer =
nullptr;
1772 void PopIncludeMacroStack() {
1773 CurLexer = std::move(IncludeMacroStack.back().TheLexer);
1774 CurPTHLexer = std::move(IncludeMacroStack.back().ThePTHLexer);
1775 CurPPLexer = IncludeMacroStack.back().ThePPLexer;
1776 CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer);
1777 CurDirLookup = IncludeMacroStack.back().TheDirLookup;
1778 CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule;
1779 CurLexerKind = IncludeMacroStack.back().CurLexerKind;
1780 IncludeMacroStack.pop_back();
1783 void PropagateLineStartLeadingSpaceInfo(
Token &
Result);
1787 bool needModuleMacros()
const;
1791 void updateModuleMacroInfo(
const IdentifierInfo *II, ModuleMacroInfo &Info);
1793 DefMacroDirective *AllocateDefMacroDirective(MacroInfo *MI,
1794 SourceLocation Loc);
1795 UndefMacroDirective *AllocateUndefMacroDirective(SourceLocation UndefLoc);
1796 VisibilityMacroDirective *AllocateVisibilityMacroDirective(SourceLocation Loc,
1810 bool *ShadowFlag =
nullptr);
1822 MacroInfo *ReadOptionalMacroParameterListAndBody(
1823 const Token &MacroNameTok,
bool ImmediatelyAfterHeaderGuard);
1829 bool ReadMacroParameterList(MacroInfo *MI,
Token& LastTok);
1839 void SkipExcludedConditionalBlock(SourceLocation IfTokenLoc,
1840 bool FoundNonSkipPortion,
bool FoundElse,
1841 SourceLocation ElseLoc = SourceLocation());
1844 void PTHSkipExcludedConditionalBlock();
1848 struct DirectiveEvalResult {
1852 bool IncludedUndefinedIds;
1859 DirectiveEvalResult EvaluateDirectiveExpression(IdentifierInfo *&IfNDefMacro);
1863 void RegisterBuiltinPragmas();
1866 void RegisterBuiltinMacros();
1871 bool HandleMacroExpandedIdentifier(
Token &Tok,
const MacroDefinition &MD);
1878 Token *cacheMacroExpandedTokens(TokenLexer *tokLexer,
1879 ArrayRef<Token> tokens);
1880 void removeCachedMacroExpandedTokensOfLastLexer();
1886 bool isNextPPTokenLParen();
1890 MacroArgs *ReadMacroCallArgumentList(
Token &MacroName, MacroInfo *MI,
1891 SourceLocation &ExpansionEnd);
1895 void ExpandBuiltinMacro(
Token &Tok);
1900 void Handle_Pragma(
Token &Tok);
1904 void HandleMicrosoft__pragma(
Token &Tok);
1908 void EnterSourceFileWithLexer(Lexer *TheLexer,
const DirectoryLookup *Dir);
1912 void EnterSourceFileWithPTH(PTHLexer *PL,
const DirectoryLookup *Dir);
1915 void setPredefinesFileID(FileID FID) {
1916 assert(PredefinesFileID.
isInvalid() &&
"PredefinesFileID already set!");
1917 PredefinesFileID = FID;
1922 static bool IsFileLexer(
const Lexer* L,
const PreprocessorLexer*
P) {
1923 return L ? !L->isPragmaLexer() : P !=
nullptr;
1926 static bool IsFileLexer(
const IncludeStackInfo&
I) {
1927 return IsFileLexer(I.TheLexer.get(), I.ThePPLexer);
1930 bool IsFileLexer()
const {
1931 return IsFileLexer(CurLexer.get(), CurPPLexer);
1937 bool InCachingLexMode()
const {
1940 return !CurPPLexer && !CurTokenLexer && !CurPTHLexer &&
1941 !IncludeMacroStack.empty();
1943 void EnterCachingLexMode();
1944 void ExitCachingLexMode() {
1945 if (InCachingLexMode())
1948 const Token &PeekAhead(
unsigned N);
1949 void AnnotatePreviousCachedTokens(
const Token &Tok);
1955 void HandleLineDirective();
1956 void HandleDigitDirective(
Token &Tok);
1957 void HandleUserDiagnosticDirective(
Token &Tok,
bool isWarning);
1958 void HandleIdentSCCSDirective(
Token &Tok);
1959 void HandleMacroPublicDirective(
Token &Tok);
1960 void HandleMacroPrivateDirective();
1963 void HandleIncludeDirective(SourceLocation HashLoc,
1965 const DirectoryLookup *LookupFrom =
nullptr,
1966 const FileEntry *LookupFromFile =
nullptr,
1967 bool isImport =
false);
1968 void HandleIncludeNextDirective(SourceLocation HashLoc,
Token &Tok);
1969 void HandleIncludeMacrosDirective(SourceLocation HashLoc,
Token &Tok);
1970 void HandleImportDirective(SourceLocation HashLoc,
Token &Tok);
1971 void HandleMicrosoftImportDirective(
Token &Tok);
1978 const TargetInfo &TargetInfo,
1979 DiagnosticsEngine &Diags, Module *M);
2000 SourceLocation MLoc);
2003 return PreambleConditionalStack.isRecording();
2007 return PreambleConditionalStack.hasRecordedPreamble();
2011 return PreambleConditionalStack.getStack();
2015 PreambleConditionalStack.setStack(s);
2019 PreambleConditionalStack.startReplaying();
2020 PreambleConditionalStack.setStack(s);
2026 void replayPreambleConditionalStack();
2029 void HandleDefineDirective(
Token &Tok,
bool ImmediatelyAfterTopLevelIfndef);
2030 void HandleUndefDirective();
2033 void HandleIfdefDirective(
Token &Tok,
bool isIfndef,
2034 bool ReadAnyTokensBeforeDirective);
2035 void HandleIfDirective(
Token &Tok,
bool ReadAnyTokensBeforeDirective);
2036 void HandleEndifDirective(
Token &Tok);
2037 void HandleElseDirective(
Token &Tok);
2038 void HandleElifDirective(
Token &Tok);
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
char getSpellingOfSingleCharacterNumericConstant(const Token &Tok, bool *Invalid=nullptr) const
Given a Token Tok that is a numeric constant with length 1, return the character. ...
SourceManager & getSourceManager() const
void ReplacePreviousCachedToken(ArrayRef< Token > NewToks)
Replace token in CachedLexPos - 1 in CachedTokens by the tokens in NewToks.
static unsigned getSpelling(const Token &Tok, const char *&Buffer, const SourceManager &SourceMgr, const LangOptions &LangOpts, bool *Invalid=nullptr)
getSpelling - This method is used to get the spelling of a token into a preallocated buffer...
CachedTokensTy::size_type End
llvm::iterator_range< macro_iterator > macros(bool IncludeExternalMacros=true) const
Module * getModuleForLocation(SourceLocation Loc)
Find the module that owns the source or header file that Loc points to.
llvm::BumpPtrAllocator & getPreprocessorAllocator()
ExternalPreprocessorSource * getExternalSource() const
void FinalizeForModelFile()
Cleanup after model file parsing.
SelectorTable & getSelectorTable()
Holds information about both target-independent and target-specific builtins, allowing easy queries b...
MacroInfo * AllocateMacroInfo(SourceLocation L)
Allocate a new MacroInfo object with the provided SourceLocation.
bool ConcatenateIncludeName(SmallString< 128 > &FilenameBuffer, SourceLocation &End)
Handle cases where the #include name is expanded from a macro as multiple tokens, which need to be gl...
void markMacroAsUsed(MacroInfo *MI)
A macro is used, update information about macros that need unused warnings.
Implements support for file system lookup, file system caching, and directory search management...
void AddPragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Add the specified pragma handler to this preprocessor.
const Token & LookAhead(unsigned N)
Peeks ahead N tokens and returns that token without consuming any tokens.
friend void TokenLexer::ExpandFunctionArguments()
void MaybeHandlePoisonedIdentifier(Token &Identifier)
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
static const Builtin::Info BuiltinInfo[]
bool LexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Lex a string literal, which may be the concatenation of multiple string literals and may even come fr...
MemoryBufferCache & getPCMCache() const
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
void dumpMacroInfo(const IdentifierInfo *II)
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
const std::string & getPredefines() const
Module * getCurrentLexerSubmodule() const
Return the submodule owning the file being lexed.
bool isAtStartOfMacroExpansion(SourceLocation loc, SourceLocation *MacroBegin=nullptr) const
Returns true if the given MacroID location points at the first token of the macro expansion...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
void LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
std::unique_ptr< llvm::MemoryBuffer > Buffer
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
PreprocessorOptions - This class is used for passing the various options used in preprocessor initial...
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
MacroMap::const_iterator macro_iterator
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions...
unsigned getCounterValue() const
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
void setPTHManager(PTHManager *pm)
static bool isAtStartOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroBegin=nullptr)
Returns true if the given MacroID location points at the first token of the macro expansion...
void EnterToken(const Token &Tok)
Enters a token in the token stream to be lexed next.
CodeCompletionHandler * getCodeCompletionHandler() const
Retrieve the current code-completion handler.
void IgnorePragmas()
Install empty handlers for all pragmas (making them ignored).
Manage memory buffers across multiple users.
const FileEntry * LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, const DirectoryLookup *FromDir, const FileEntry *FromFile, const DirectoryLookup *&CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool SkipCache=false)
Given a "foo" or <foo> reference, look up the indicated file.
SourceLocation getDefinitionLoc() const
Return the location that the macro was defined at.
bool getRawToken(SourceLocation Loc, Token &Result, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
SourceLocation getLastCachedTokenLocation() const
Get the location of the last cached token, suitable for setting the end location of an annotation tok...
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
Builtin::Context & getBuiltinInfo()
void setPredefines(const char *P)
Set the predefines for this Preprocessor.
This interface provides a way to observe the actions of the preprocessor as it does its thing...
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool getPragmasEnabled() const
TokenValue(tok::TokenKind Kind)
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
SourceLocation getPragmaAssumeNonNullLoc() const
The location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset=0)
Computes the source location just past the end of the token at this source location.
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI, SourceLocation Loc)
One of these records is kept for each identifier that is lexed.
Represents a macro directive exported by a module.
A directive for a defined macro or a macro imported from a module.
bool hasMacroDefinition() const
Return true if this identifier is #defined to some other value.
This table allows us to fully hide how we implement multi-keyword caching.
bool CheckMacroName(Token &MacroNameTok, MacroUse isDefineUndef, bool *ShadowFlag=nullptr)
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
Module * LeaveSubmodule(bool ForPragma)
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, MemoryBufferCache &PCMCache, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
void HandlePragmaPushMacro(Token &Tok)
Handle #pragma push_macro.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
const LangOptions & getLangOpts() const
Token - This structure provides full information about a lexed token.
void SetCommentRetentionState(bool KeepComments, bool KeepMacroComments)
Control whether the preprocessor retains comments in output.
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
CachedTokensTy::size_type Begin
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Describes a module or submodule.
FileManager & getFileManager() const
bool isParsingIfOrElifDirective() const
True if we are currently preprocessing a if or #elif directive.
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
void SetSuppressIncludeNotFoundError(bool Suppress)
void CheckEndOfDirective(const char *Directive, bool EnableMacros=false)
Ensure that the next token is a tok::eod token.
MacroUse
Context in which macro name is used.
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
bool getCommentRetentionState() const
const IdentifierTable & getIdentifierTable() const
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
void LexNonComment(Token &Result)
Lex a token.
static bool getRawToken(SourceLocation Loc, Token &Result, const SourceManager &SM, const LangOptions &LangOpts, bool IgnoreWhiteSpace=false)
Relex the token at the specified location.
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
Concrete class used by the front-end to report problems and issues.
HeaderSearch & getHeaderSearchInfo() const
bool isAtEndOfMacroExpansion(SourceLocation loc, SourceLocation *MacroEnd=nullptr) const
Returns true if the given MacroID location points at the last token of the macro expansion.
bool isMacroDefined(const IdentifierInfo *II)
bool isMacroDefinedInLocalModule(const IdentifierInfo *II, Module *M)
Determine whether II is defined as a macro within the module M, if that is a module that we've alread...
Defines the Diagnostic-related interfaces.
bool hadModuleLoaderFatalFailure() const
void CommitBacktrackedTokens()
Disable the last EnableBacktrackAtThisPos call.
SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Char) const
Given a location that specifies the start of a token, return a new location that specifies a characte...
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
tok::TokenKind getKind() const
void setCodeCompletionHandler(CodeCompletionHandler &Handler)
Set the code completion handler to the given object.
const TargetInfo & getTargetInfo() const
void setPreprocessedOutput(bool IsPreprocessedOutput)
Sets whether the preprocessor is responsible for producing output or if it is producing tokens to be ...
static SourceLocation AdvanceToTokenCharacter(SourceLocation TokStart, unsigned Character, const SourceManager &SM, const LangOptions &LangOpts)
AdvanceToTokenCharacter - If the current SourceLocation specifies a location at the start of a token...
detail::InMemoryDirectory::const_iterator I
PragmaIntroducerKind
Describes how the pragma was introduced, e.g., with #pragma, _Pragma, or __pragma.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void appendMacroDirective(IdentifierInfo *II, MacroDirective *MD)
Add a directive to the macro directive history for this identifier.
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/CurPTHLexer/ CurTokenLexer pointers...
void HandlePragmaPoison()
HandlePragmaPoison - Handle #pragma GCC poison.
A little helper class used to produce diagnostics.
void HandlePragmaIncludeAlias(Token &Tok)
void clearCodeCompletionHandler()
Clear out the code completion handler.
void Backtrack()
Make Preprocessor re-lex the tokens that were lexed since EnableBacktrackAtThisPos() was previously c...
void HandlePragmaOnce(Token &OnceTok)
HandlePragmaOnce - Handle #pragma once.
bool isRecordingPreamble() const
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
Exposes information about the current target.
static bool isAtEndOfMacroExpansion(SourceLocation loc, const SourceManager &SM, const LangOptions &LangOpts, SourceLocation *MacroEnd=nullptr)
Returns true if the given MacroID location points at the last token of the macro expansion.
Abstract interface for external sources of preprocessor information.
StringRef getName() const
Return the actual identifier string.
static SourceLocation getLocForEndOfToken(SourceLocation Loc, unsigned Offset, const SourceManager &SM, const LangOptions &LangOpts)
Computes the source location just past the end of the token at this source location.
void makeModuleVisible(Module *M, SourceLocation Loc)
void AnnotateCachedTokens(const Token &Tok)
We notify the Preprocessor that if it is caching tokens (because backtrack is enabled) it should repl...
void EnableBacktrackAtThisPos()
From the point that this method is called, and until CommitBacktrackedTokens() or Backtrack() is call...
void SetMacroExpansionOnlyInDirectives()
Disables macro expansion everywhere except for preprocessor directives.
Implements an efficient mapping from strings to IdentifierInfo nodes.
bool LexOnOffSwitch(tok::OnOffSwitch &OOS)
Lex an on-off-switch (C99 6.10.6p2) and verify that it is followed by EOD.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
void RevertCachedTokens(unsigned N)
When backtracking is enabled and tokens are cached, this allows to revert a specific number of tokens...
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
bool operator==(const Token &Tok) const
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
void RemovePragmaHandler(StringRef Namespace, PragmaHandler *Handler)
Remove the specific pragma handler from this preprocessor.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
ModuleLoader & getModuleLoader() const
Retrieve the module loader associated with this preprocessor.
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Stores token information for comparing actual tokens with predefined values.
static bool checkModuleIsAvailable(const LangOptions &LangOpts, const TargetInfo &TargetInfo, DiagnosticsEngine &Diags, Module *M)
Check that the given module is available, producing a diagnostic if not.
void setPragmaAssumeNonNullLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang assume_nonnull begin.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
void HandlePragmaDependency(Token &DependencyTok)
HandlePragmaDependency - Handle #pragma GCC dependency "foo" blah.
void setReplayablePreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
TokenValue(IdentifierInfo *II)
SourceLocation getModuleImportLoc(Module *M) const
SourceLocation getCodeCompletionLoc() const
Returns the location of the code-completion point.
The result type of a method or function.
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
unsigned getSpelling(const Token &Tok, const char *&Buffer, bool *Invalid=nullptr) const
Get the spelling of a token into a preallocated buffer, instead of as an std::string.
const DirectoryLookup * GetCurDirLookup()
Get the DirectoryLookup structure used to find the current FileEntry, if CurLexer is non-null and if ...
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
size_t getTotalMemory() const
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
void TypoCorrectToken(const Token &Tok)
Update the current token to represent the provided identifier, in order to cache an action performed ...
void setExternalSource(ExternalPreprocessorSource *Source)
Encodes a location in the source.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
PreprocessorLexer * getCurrentLexer() const
Return the current lexer being lexed from.
bool isAnnotation(TokenKind K)
Return true if this is any of tok::annot_* kinds.
void setPragmaARCCFCodeAuditedLoc(SourceLocation Loc)
Set the location of the currently-active #pragma clang arc_cf_code_audited begin. ...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
macro_iterator macro_end(bool IncludeExternalMacros=true) const
MacroDirective * getLocalMacroDirective(const IdentifierInfo *II) const
Given an identifier, return its latest non-imported MacroDirective if it is #define'd and not #undef'...
bool isCodeCompletionReached() const
Returns true if code-completion is enabled and we have hit the code-completion point.
IdentifierTable & getIdentifierTable()
void setPragmasEnabled(bool Enabled)
Cached information about one file (either on disk or in the virtual file system). ...
const TargetInfo * getAuxTargetInfo() const
std::string getSpelling(const Token &Tok, bool *Invalid=nullptr) const
Return the 'spelling' of the Tok token.
void Lex(Token &Result)
Lex the next token for this preprocessor.
ArrayRef< FormatToken * > Tokens
void setLoadedMacroDirective(IdentifierInfo *II, MacroDirective *ED, MacroDirective *MD)
Set a MacroDirective that was loaded from a PCH file.
bool isLiteral(TokenKind K)
Return true if this is a "literal" kind, like a numeric constant, string, etc.
void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
void enableIncrementalProcessing(bool value=true)
Enables the incremental processing.
MacroDefinition getMacroDefinitionAtLoc(const IdentifierInfo *II, SourceLocation Loc)
PPCallbacks * getPPCallbacks() const
Accessors for preprocessor callbacks.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
void ReplaceLastTokenWithAnnotation(const Token &Tok)
Replace the last token with an annotation token.
DiagnosticsEngine & getDiagnostics() const
static StringRef getImmediateMacroName(SourceLocation Loc, const SourceManager &SM, const LangOptions &LangOpts)
Retrieve the name of the immediate macro expansion.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
bool isMacroDefined(StringRef Id)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
void EnterTokenStream(ArrayRef< Token > Toks, bool DisableMacroExpansion)
SmallVector< Token, 4 > CachedTokens
A set of tokens that has been cached for later parsing.
void EnterTokenStream(std::unique_ptr< Token[]> Toks, unsigned NumToks, bool DisableMacroExpansion)
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
void HandlePragmaPopMacro(Token &Tok)
Handle #pragma pop_macro.
Reads an AST files chain containing the contents of a translation unit.
bool hasRecordedPreamble() const
PragmaHandler - Instances of this interface defined to handle the various pragmas that the language f...
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
PreprocessingRecord * getPreprocessingRecord() const
Retrieve the preprocessing record, or NULL if there is no preprocessing record.
StringRef getCodeCompletionFilter()
Get the code completion token for filtering purposes.
Abstract interface for a module loader.
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
void PoisonSEHIdentifiers(bool Poison=true)
Encapsulates the data about a macro definition (e.g.
OnOffSwitch
Defines the possible values of an on-off-switch (C99 6.10.6p2).
bool GetIncludeFilenameSpelling(SourceLocation Loc, StringRef &Filename)
Turn the specified lexer token into a fully checked and spelled filename, e.g.
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
MacroDirective * getLocalMacroDirectiveHistory(const IdentifierInfo *II) const
Given an identifier, return the latest non-imported macro directive for that identifier.
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
IdentifierInfo * ParsePragmaPushOrPopMacro(Token &Tok)
ParsePragmaPushOrPopMacro - Handle parsing of pragma push_macro/pop_macro.
void LexUnexpandedNonComment(Token &Result)
Like LexNonComment, but this disables macro expansion of identifier tokens.
Cached information about one directory (either on disk or in the virtual file system).
Defines the PPCallbacks interface.
DefMacroDirective * appendDefMacroDirective(IdentifierInfo *II, MacroInfo *MI)
bool isCurrentLexer(const PreprocessorLexer *L) const
Return true if we are lexing directly from the specified lexer.
void HandlePoisonedIdentifier(Token &Tok)
Display reason for poisoned identifier.
void EraseCachedTokens(CachedTokensRange TokenRange)
Erase the range of cached tokens that were lexed since EnableBacktrackAtThisPos() was previously call...
bool IsPreviousCachedToken(const Token &Tok) const
Whether Tok is the most recent token (CachedLexPos - 1) in CachedTokens.
bool HandleComment(Token &Token, SourceRange Comment)
PreprocessorOptions & getPreprocessorOpts() const
Retrieve the preprocessor options used to initialize this preprocessor.
DiagnosticBuilder Diag(const Token &Tok, unsigned DiagID) const
void setSuppressAllDiagnostics(bool Val=true)
Suppress all diagnostics, to silence the front end when we know that we don't want any more diagnosti...
Defines the clang::SourceLocation class and associated facilities.
void RemovePragmaHandler(PragmaHandler *Handler)
void DumpMacro(const MacroInfo &MI) const
ModuleMacro * getModuleMacro(Module *Mod, IdentifierInfo *II)
void setMainFileDir(const DirectoryEntry *Dir)
Set the directory in which the main file should be considered to have been found, if it is not a real...
CachedTokensRange LastCachedTokenRange()
Returns the range of cached tokens that were lexed since EnableBacktrackAtThisPos() was previously ca...
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
TranslationUnitKind
Describes the kind of translation unit being processed.
const FileEntry * getModuleHeaderToIncludeForDiagnostics(SourceLocation IncLoc, Module *M, SourceLocation MLoc)
We want to produce a diagnostic at location IncLoc concerning a missing module import.
void setPredefines(StringRef P)
ArrayRef< ModuleMacro * > getLeafModuleMacros(const IdentifierInfo *II) const
Get the list of leaf (non-overridden) module macros for a name.
The translation unit is a complete translation unit.
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
StringRef getImmediateMacroName(SourceLocation Loc)
Retrieve the name of the immediate macro expansion.
void HandlePragmaModuleBuild(Token &Tok)
void HandlePragmaSystemHeader(Token &SysHeaderTok)
HandlePragmaSystemHeader - Implement #pragma GCC system_header.
void DiscardUntilEndOfDirective()
Read and discard all tokens remaining on the current line until the tok::eod token is found...
unsigned getLength() const
bool isPreprocessedOutput() const
Returns true if the preprocessor is responsible for generating output, false if it is producing token...
void DumpLocation(SourceLocation Loc) const
void setCounterValue(unsigned V)
ArrayRef< PPConditionalInfo > getPreambleConditionalStack() const
bool hadMacroDefinition() const
Returns true if this identifier was #defined to some value at any moment.
void setSkipMainFilePreamble(unsigned Bytes, bool StartOfLine)
Instruct the preprocessor to skip part of the main source file.
A trivial tuple used to represent a source range.
bool GetSuppressIncludeNotFoundError()
Directive - Abstract class representing a parsed verify directive.
MacroInfo * getMacroInfo(const IdentifierInfo *II)
void EnterAnnotationToken(SourceRange Range, tok::TokenKind Kind, void *AnnotationVal)
Enter an annotation token into the token stream.
Callback handler that receives notifications when performing code completion within the preprocessor...
PTHManager * getPTHManager()
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
SourceLocation getPragmaARCCFCodeAuditedLoc() const
The location of the currently-active #pragma clang arc_cf_code_audited begin.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
void setRecordedPreambleConditionalStack(ArrayRef< PPConditionalInfo > s)
This class handles loading and caching of source files into memory.
Defines enum values for all the target-independent builtin functions.
bool SawDateOrTime() const
Returns true if the preprocessor has seen a use of DATE or TIME in the file so far.
void AddPragmaHandler(PragmaHandler *Handler)
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
const DefMacroDirective * getDirective() const
bool isBacktrackEnabled() const
True if EnableBacktrackAtThisPos() was called and caching of tokens is on.
void setDiagnostics(DiagnosticsEngine &D)
bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
IdentifierInfo * getIdentifierInfo() const