22 #include "llvm/ADT/StringSwitch.h"
23 #include "llvm/Support/FileSystem.h"
24 #include "llvm/Support/MemoryBuffer.h"
25 #include "llvm/Support/Path.h"
26 using namespace clang;
38 return IncludeMacroStack.empty();
41 assert(IsFileLexer(IncludeMacroStack[0]) &&
42 "Top level include stack isn't our primary lexer?");
43 return std::none_of(IncludeMacroStack.begin() + 1, IncludeMacroStack.end(),
44 [
this](
const IncludeStackInfo &ISI) ->
bool {
45 return IsFileLexer(ISI);
57 for (
const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
59 return ISI.ThePPLexer;
73 assert(!CurTokenLexer &&
"Cannot #include a file inside a macro!");
74 ++NumEnteredSourceFiles;
76 if (MaxIncludeStackDepth < IncludeMacroStack.size())
77 MaxIncludeStackDepth = IncludeMacroStack.size();
80 if (
PTHLexer *PL = PTH->CreateLexer(FID)) {
81 EnterSourceFileWithPTH(PL, CurDir);
88 const llvm::MemoryBuffer *InputFile =
92 Diag(Loc, diag::err_pp_error_opening_file)
104 EnterSourceFileWithLexer(
new Lexer(FID, InputFile, *
this), CurDir);
110 void Preprocessor::EnterSourceFileWithLexer(
Lexer *TheLexer,
114 if (CurPPLexer || CurTokenLexer)
115 PushIncludeMacroStack();
117 CurLexer.reset(TheLexer);
118 CurPPLexer = TheLexer;
119 CurDirLookup = CurDir;
120 CurLexerSubmodule =
nullptr;
121 if (CurLexerKind != CLK_LexAfterModuleImport)
122 CurLexerKind = CLK_Lexer;
125 if (Callbacks && !CurLexer->Is_PragmaLexer) {
129 Callbacks->FileChanged(CurLexer->getFileLoc(),
136 void Preprocessor::EnterSourceFileWithPTH(
PTHLexer *PL,
139 if (CurPPLexer || CurTokenLexer)
140 PushIncludeMacroStack();
142 CurDirLookup = CurDir;
143 CurPTHLexer.reset(PL);
144 CurPPLexer = CurPTHLexer.get();
145 CurLexerSubmodule =
nullptr;
146 if (CurLexerKind != CLK_LexAfterModuleImport)
147 CurLexerKind = CLK_PTHLexer;
163 std::unique_ptr<TokenLexer> TokLexer;
164 if (NumCachedTokenLexers == 0) {
165 TokLexer = llvm::make_unique<TokenLexer>(Tok, ILEnd, Macro, Args, *
this);
167 TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
168 TokLexer->Init(Tok, ILEnd, Macro, Args);
171 PushIncludeMacroStack();
172 CurDirLookup =
nullptr;
173 CurTokenLexer = std::move(TokLexer);
174 if (CurLexerKind != CLK_LexAfterModuleImport)
175 CurLexerKind = CLK_TokenLexer;
190 void Preprocessor::EnterTokenStream(
const Token *Toks,
unsigned NumToks,
191 bool DisableMacroExpansion,
193 if (CurLexerKind == CLK_CachingLexer) {
198 Toks, Toks + NumToks);
206 ExitCachingLexMode();
207 EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
208 EnterCachingLexMode();
213 std::unique_ptr<TokenLexer> TokLexer;
214 if (NumCachedTokenLexers == 0) {
215 TokLexer = llvm::make_unique<TokenLexer>(
216 Toks, NumToks, DisableMacroExpansion, OwnsTokens, *
this);
218 TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
219 TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
223 PushIncludeMacroStack();
224 CurDirLookup =
nullptr;
225 CurTokenLexer = std::move(TokLexer);
226 if (CurLexerKind != CLK_LexAfterModuleImport)
227 CurLexerKind = CLK_TokenLexer;
239 while (!Path.empty()) {
242 Result = FilePath.substr(Path.size());
243 llvm::sys::path::append(Result,
244 llvm::sys::path::filename(File->
getName()));
249 Path = llvm::sys::path::parent_path(Path);
255 void Preprocessor::PropagateLineStartLeadingSpaceInfo(
Token &Result) {
257 CurTokenLexer->PropagateLineStartLeadingSpaceInfo(Result);
261 CurLexer->PropagateLineStartLeadingSpaceInfo(Result);
274 const char *Preprocessor::getCurLexerEndPos() {
275 const char *EndPos = CurLexer->BufferEnd;
276 if (EndPos != CurLexer->BufferStart &&
277 (EndPos[-1] ==
'\n' || EndPos[-1] ==
'\r')) {
281 if (EndPos != CurLexer->BufferStart &&
282 (EndPos[-1] ==
'\n' || EndPos[-1] ==
'\r') &&
283 EndPos[-1] != EndPos[0])
293 SubMods.push_back(&Mod);
298 void Preprocessor::diagnoseMissingHeaderInUmbrellaDir(
const Module &Mod) {
310 Entry !=
End && !EC; Entry.increment(EC)) {
311 using llvm::StringSwitch;
315 if (!StringSwitch<bool>(llvm::sys::path::extension(Entry->getName()))
316 .Cases(
".h",
".H",
".hh",
".hpp",
true)
326 Diag(StartLoc, diag::warn_uncovered_module_header)
337 assert(!CurTokenLexer &&
338 "Ending a file when currently in a macro!");
343 const bool LeavingSubmodule = CurLexer && CurLexerSubmodule;
344 if ((LeavingSubmodule || IncludeMacroStack.empty()) &&
345 !BuildingSubmoduleStack.empty() &&
346 BuildingSubmoduleStack.back().IsPragma) {
347 Diag(BuildingSubmoduleStack.back().ImportLoc,
348 diag::err_pp_module_begin_without_module_end);
352 const char *EndPos = getCurLexerEndPos();
353 CurLexer->BufferPtr = EndPos;
354 CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
368 getMacroInfo(const_cast<IdentifierInfo*>(ControllingMacro)))
369 MI->setUsedForHeaderGuard(
true);
373 DefinedMacro != ControllingMacro &&
382 const StringRef ControllingMacroName = ControllingMacro->getName();
383 const StringRef DefinedMacroName = DefinedMacro->getName();
384 const size_t MaxHalfLength =
std::max(ControllingMacroName.size(),
385 DefinedMacroName.size()) / 2;
386 const unsigned ED = ControllingMacroName.edit_distance(
387 DefinedMacroName,
true, MaxHalfLength);
388 if (ED <= MaxHalfLength) {
391 diag::warn_header_guard)
394 diag::note_header_guard)
399 ControllingMacro->getName());
410 if (PragmaARCCFCodeAuditedLoc.
isValid() &&
411 !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
412 Diag(PragmaARCCFCodeAuditedLoc, diag::err_pp_eof_in_arc_cf_code_audited);
421 if (PragmaAssumeNonNullLoc.
isValid() &&
422 !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
423 Diag(PragmaAssumeNonNullLoc, diag::err_pp_eof_in_assume_nonnull);
431 if (!IncludeMacroStack.empty()) {
436 CodeCompletionFileLoc) {
439 CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd,
tok::eof);
442 assert(CurPTHLexer &&
"Got EOF but no current lexer set!");
443 CurPTHLexer->getEOF(Result);
447 CurPPLexer =
nullptr;
451 if (!isEndOfMacro && CurPPLexer &&
461 bool ExitedFromPredefinesFile =
false;
463 if (!isEndOfMacro && CurPPLexer) {
466 assert(PredefinesFileID.
isValid() &&
467 "HandleEndOfFile is called before PredefinesFileId is set");
468 ExitedFromPredefinesFile = (PredefinesFileID == ExitedFID);
471 if (LeavingSubmodule) {
476 const char *EndPos = getCurLexerEndPos();
478 CurLexer->BufferPtr = EndPos;
479 CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
488 PropagateLineStartLeadingSpaceInfo(Result);
491 if (Callbacks && !isEndOfMacro && CurPPLexer) {
500 if (ExitedFromPredefinesFile)
501 replayPreambleConditionalStack();
504 return LeavingSubmodule;
509 const char *EndPos = getCurLexerEndPos();
511 CurLexer->BufferPtr = EndPos;
512 CurLexer->FormTokenWithChars(Result, EndPos,
tok::eof);
521 if (CurLexer->getFileLoc() == CodeCompletionFileLoc)
529 assert(CurPTHLexer &&
"Got EOF but no current lexer set!");
530 CurPTHLexer->getEOF(Result);
535 CurPPLexer =
nullptr;
541 for (WarnUnusedMacroLocsTy::iterator
542 I=WarnUnusedMacroLocs.begin(),
E=WarnUnusedMacroLocs.end();
544 Diag(*
I, diag::pp_macro_not_used);
554 for (
auto *M : AllMods)
555 diagnoseMissingHeaderInUmbrellaDir(*M);
564 assert(CurTokenLexer && !CurPPLexer &&
565 "Ending a macro when currently in a #include file!");
567 if (!MacroExpandingLexersStack.empty() &&
568 MacroExpandingLexersStack.back().first == CurTokenLexer.get())
569 removeCachedMacroExpandedTokensOfLastLexer();
572 if (NumCachedTokenLexers == TokenLexerCacheSize)
573 CurTokenLexer.reset();
575 TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
585 assert(!IncludeMacroStack.empty() &&
"Ran out of stack entries to load");
589 if (NumCachedTokenLexers == TokenLexerCacheSize)
590 CurTokenLexer.reset();
592 TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
595 PopIncludeMacroStack();
602 assert(CurTokenLexer && !CurPPLexer &&
603 "Pasted comment can only be formed from macro");
608 bool LexerWasInPPMode =
false;
609 for (
const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
610 if (ISI.ThePPLexer ==
nullptr)
continue;
618 FoundLexer = ISI.ThePPLexer;
640 if (Tok.
is(tok::eod)) {
641 assert(FoundLexer &&
"Can't get end of line without an active lexer");
647 if (LexerWasInPPMode)
return;
658 assert(!FoundLexer &&
"Lexer should return EOD before EOF in PP mode");
665 BuildingSubmoduleStack.push_back(
666 BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
667 PendingModuleMacroNames.size()));
681 auto R = Submodules.insert(std::make_pair(M, SubmoduleState()));
682 auto &
State = R.first->second;
683 bool FirstTime = R.second;
691 auto &StartingMacros = NullSubmoduleState.Macros;
695 for (
auto &Macro : StartingMacros) {
697 if (!Macro.second.getLatest() &&
698 Macro.second.getOverriddenMacros().empty())
701 MacroState MS(Macro.second.getLatest());
702 MS.setOverriddenMacros(*
this, Macro.second.getOverriddenMacros());
703 State.Macros.insert(std::make_pair(Macro.first, std::move(MS)));
708 BuildingSubmoduleStack.push_back(
709 BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
710 PendingModuleMacroNames.size()));
713 CurSubmoduleState = &
State;
720 bool Preprocessor::needModuleMacros()
const {
722 if (BuildingSubmoduleStack.empty())
734 if (BuildingSubmoduleStack.empty() ||
735 BuildingSubmoduleStack.back().IsPragma != ForPragma) {
736 assert(ForPragma &&
"non-pragma module enter/leave mismatch");
740 auto &Info = BuildingSubmoduleStack.back();
742 Module *LeavingMod = Info.M;
745 if (!needModuleMacros() ||
751 BuildingSubmoduleStack.pop_back();
757 llvm::SmallPtrSet<const IdentifierInfo*, 8> VisitedMacros;
758 for (
unsigned I = Info.OuterPendingModuleMacroNames;
759 I != PendingModuleMacroNames.size(); ++
I) {
761 if (!VisitedMacros.insert(II).second)
764 auto MacroIt = CurSubmoduleState->Macros.find(II);
765 if (MacroIt == CurSubmoduleState->Macros.end())
767 auto &Macro = MacroIt->second;
771 auto *OldState = Info.OuterSubmoduleState;
773 OldState = &NullSubmoduleState;
774 if (OldState && OldState != CurSubmoduleState) {
777 auto &OldMacros = OldState->Macros;
778 auto OldMacroIt = OldMacros.find(II);
779 if (OldMacroIt == OldMacros.end())
782 OldMD = OldMacroIt->second.getLatest();
787 bool ExplicitlyPublic =
false;
788 for (
auto *MD = Macro.getLatest(); MD != OldMD; MD = MD->
getPrevious()) {
789 assert(MD &&
"broken macro directive chain");
791 if (
auto *VisMD = dyn_cast<VisibilityMacroDirective>(MD)) {
794 if (VisMD->isPublic())
795 ExplicitlyPublic =
true;
796 else if (!ExplicitlyPublic)
802 Def = DefMD->getInfo();
809 if (Def || !Macro.getOverriddenMacros().empty())
811 Macro.getOverriddenMacros(), IsNew);
816 Macro.setLatest(
nullptr);
817 Macro.setOverriddenMacros(*
this, {});
823 PendingModuleMacroNames.resize(Info.OuterPendingModuleMacroNames);
832 CurSubmoduleState = Info.OuterSubmoduleState;
834 BuildingSubmoduleStack.pop_back();
SourceManager & getSourceManager() const
Header getUmbrellaHeader() const
Retrieve the header that serves as the umbrella header for this module.
unsigned getInitialNumSLocEntries() const
Number of SLocEntries before lexing the file.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Implements support for file system lookup, file system caching, and directory search management...
unsigned local_sloc_entry_size() const
Get the number of local SLocEntries we have.
Defines the clang::FileManager interface and associated types.
bool isHeaderInUnavailableModule(const FileEntry *Header) const
Determine whether the given header is part of a module marked 'unavailable'.
Defines the SourceManager interface.
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
llvm::MemoryBuffer * getBuffer(FileID FID, SourceLocation Loc, bool *Invalid=nullptr) const
Return the buffer for the specified FileID.
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool resolveUses(Module *Mod, bool Complain)
Resolve all of the unresolved uses in the given module.
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
SourceLocation GetDefinedLocation() const
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
bool hasFileInfo(const FileEntry *File) const
The virtual file system interface.
One of these records is kept for each identifier that is lexed.
A directive for a defined macro or a macro imported from a module.
bool ParsingPreprocessorDirective
True when parsing #XXX; turns '\n' into a tok::eod token.
bool resolveConflicts(Module *Mod, bool Complain)
Resolve all of the unresolved conflicts in the given module.
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
Module * LeaveSubmodule(bool ForPragma)
StringRef getBufferName(SourceLocation Loc, bool *Invalid=nullptr) const
Return the filename or buffer identifier of the buffer the location is in.
const MacroInfo * getMacroInfo(const IdentifierInfo *II) const
const LangOptions & getLangOpts() const
std::string getFullModuleName(bool AllowStringLiterals=false) const
Retrieve the full name of this module, including the path from its top-level module.
Token - This structure provides full information about a lexed token.
An input iterator over the recursive contents of a virtual path, similar to llvm::sys::fs::recursive_...
Describes a module or submodule.
FileManager & getFileManager() const
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
IntrusiveRefCntPtr< vfs::FileSystem > getVirtualFileSystem() const
HeaderSearch & getHeaderSearchInfo() const
static void collectAllSubModulesWithUmbrellaHeader(const Module &Mod, SmallVectorImpl< const Module * > &SubMods)
StringRef getName() const
const FileEntry * getFileEntry() const
getFileEntry - Return the FileEntry corresponding to this FileID.
detail::InMemoryDirectory::const_iterator I
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
const FileEntry * getFile(StringRef Filename, bool OpenFile=false, bool CacheFailure=true)
Lookup, cache, and verify the specified file (real or virtual).
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
std::string CurrentModule
The name of the current module, of which the main source file is a part.
const DirectoryEntry * getDirectory(StringRef DirName, bool CacheFailure=true)
Lookup, cache, and verify the specified directory (real or virtual).
const DirectoryEntry * Entry
ModuleMacro * addModuleMacro(Module *Mod, IdentifierInfo *II, MacroInfo *Macro, ArrayRef< ModuleMacro * > Overrides, bool &IsNew)
Register an exported macro for a module and identifier.
void setAnnotationValue(void *val)
bool LexingRawMode
True if in raw mode.
void makeModuleVisible(Module *M, SourceLocation Loc)
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
Defines the clang::Preprocessor interface.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
bool resolveExports(Module *Mod, bool Complain)
Resolve all of the unresolved exports in the given module.
MultipleIncludeOpt MIOpt
A state machine that detects the #ifndef-wrapping a file idiom for the multiple-include optimization...
static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir, const FileEntry *File, SmallString< 128 > &Result)
Compute the relative path that names the given file relative to the given directory.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
bool isNot(tok::TokenKind K) const
SourceLocation getIncludeLoc(FileID FID) const
Returns the include location if FID is a #include'd file otherwise it returns an invalid location...
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
DirectoryName getUmbrellaDir() const
Retrieve the directory for which this module serves as the umbrella.
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
StringRef getTopLevelModuleName() const
Retrieve the name of the top-level module.
virtual SourceLocation getSourceLocation()=0
Return the source location for the next observable location.
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Encodes a location in the source.
bool isIncrementalProcessingEnabled() const
Returns true if incremental processing is enabled.
bool isValid() const
Return true if this is a valid SourceLocation object.
void setAnnotationEndLoc(SourceLocation L)
Cached information about one file (either on disk or in the virtual file system). ...
void Lex(Token &Result)
Lex the next token for this preprocessor.
void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
FileID getMainFileID() const
Returns the FileID of the main source file.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
bool isIgnored(unsigned DiagID, SourceLocation Loc) const
Determine whether the diagnostic is known to be ignored.
DiagnosticsEngine & getDiagnostics() const
bool isMacroDefined(StringRef Id)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
llvm::iterator_range< submodule_iterator > submodules()
SrcMgr::CharacteristicKind getFileCharacteristic(SourceLocation Loc) const
Return the file characteristic of the specified source location, indicating whether this is a normal ...
const IdentifierInfo * GetDefinedMacro() const
If the ControllingMacro is followed by a macro definition, return the macro that was defined...
StringRef getName() const
detail::InMemoryDirectory::const_iterator E
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
Encapsulates the data about a macro definition (e.g.
const IdentifierInfo * GetControllingMacroAtEndOfFile() const
Once the entire file has been lexed, if there is a controlling macro, return it.
bool isCodeCompletionEnabled() const
Determine if we are performing code completion.
Cached information about one directory (either on disk or in the virtual file system).
char __ovld __cnfn max(char x, char y)
Returns y if x < y, otherwise it returns x.
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
static FixItHint CreateReplacement(CharSourceRange RemoveRange, StringRef Code)
Create a code modification hint that replaces the given source range with the given code string...
void setNumCreatedFIDsForFileID(FileID FID, unsigned NumFIDs) const
Set the number of FileIDs (files and macros) that were created during preprocessing of FID...
The translation unit is a complete translation unit.
SourceLocation GetMacroLocation() const
bool isCompilingModule() const
Are we compiling a module interface (.cppm or module map)?
SourceLocation getLocForStartOfFile(FileID FID) const
Return the source location corresponding to the first byte of the specified file. ...
void setLocation(SourceLocation L)
const DirectoryEntry * getDir() const
Return the directory the file lives in.
void startToken()
Reset all flags to cleared.
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...