23 #include "llvm/ADT/StringSwitch.h" 24 #include "llvm/Support/FileSystem.h" 25 #include "llvm/Support/MemoryBuffer.h" 26 #include "llvm/Support/Path.h" 27 using namespace clang;
39 return IncludeMacroStack.empty();
42 assert(IsFileLexer(IncludeMacroStack[0]) &&
43 "Top level include stack isn't our primary lexer?");
45 IncludeMacroStack.begin() + 1, IncludeMacroStack.end(),
46 [&](
const IncludeStackInfo &ISI) ->
bool {
return IsFileLexer(ISI); });
57 for (
const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
59 return ISI.ThePPLexer;
73 assert(!CurTokenLexer &&
"Cannot #include a file inside a macro!");
74 ++NumEnteredSourceFiles;
76 if (MaxIncludeStackDepth < IncludeMacroStack.size())
77 MaxIncludeStackDepth = IncludeMacroStack.size();
80 if (
PTHLexer *PL = PTH->CreateLexer(FID)) {
81 EnterSourceFileWithPTH(PL, CurDir);
88 const llvm::MemoryBuffer *InputFile =
89 getSourceManager().getBuffer(FID, Loc, &Invalid);
92 Diag(Loc, diag::err_pp_error_opening_file)
93 << std::string(SourceMgr.getBufferName(FileStart)) <<
"";
97 if (isCodeCompletionEnabled() &&
98 SourceMgr.getFileEntryForID(FID) == CodeCompletionFile) {
99 CodeCompletionFileLoc = SourceMgr.getLocForStartOfFile(FID);
101 CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset);
104 EnterSourceFileWithLexer(
new Lexer(FID, InputFile, *
this), CurDir);
110 void Preprocessor::EnterSourceFileWithLexer(
Lexer *TheLexer,
114 if (CurPPLexer || CurTokenLexer)
115 PushIncludeMacroStack();
117 CurLexer.reset(TheLexer);
118 CurPPLexer = TheLexer;
119 CurDirLookup = CurDir;
120 CurLexerSubmodule =
nullptr;
121 if (CurLexerKind != CLK_LexAfterModuleImport)
122 CurLexerKind = CLK_Lexer;
125 if (Callbacks && !CurLexer->Is_PragmaLexer) {
127 SourceMgr.getFileCharacteristic(CurLexer->getFileLoc());
129 Callbacks->FileChanged(CurLexer->getFileLoc(),
136 void Preprocessor::EnterSourceFileWithPTH(
PTHLexer *PL,
139 if (CurPPLexer || CurTokenLexer)
140 PushIncludeMacroStack();
142 CurDirLookup = CurDir;
143 CurPTHLexer.reset(PL);
144 CurPPLexer = CurPTHLexer.get();
145 CurLexerSubmodule =
nullptr;
146 if (CurLexerKind != CLK_LexAfterModuleImport)
147 CurLexerKind = CLK_PTHLexer;
151 FileID FID = CurPPLexer->getFileID();
154 SourceMgr.getFileCharacteristic(EnterLoc);
163 std::unique_ptr<TokenLexer> TokLexer;
164 if (NumCachedTokenLexers == 0) {
165 TokLexer = llvm::make_unique<TokenLexer>(
Tok, ILEnd, Macro, Args, *
this);
167 TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
168 TokLexer->Init(Tok, ILEnd, Macro, Args);
171 PushIncludeMacroStack();
172 CurDirLookup =
nullptr;
173 CurTokenLexer = std::move(TokLexer);
174 if (CurLexerKind != CLK_LexAfterModuleImport)
175 CurLexerKind = CLK_TokenLexer;
190 void Preprocessor::EnterTokenStream(
const Token *Toks,
unsigned NumToks,
191 bool DisableMacroExpansion,
193 if (CurLexerKind == CLK_CachingLexer) {
198 Toks, Toks + NumToks);
206 ExitCachingLexMode();
207 EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
208 EnterCachingLexMode();
213 std::unique_ptr<TokenLexer> TokLexer;
214 if (NumCachedTokenLexers == 0) {
215 TokLexer = llvm::make_unique<TokenLexer>(
216 Toks, NumToks, DisableMacroExpansion, OwnsTokens, *
this);
218 TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
219 TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
223 PushIncludeMacroStack();
224 CurDirLookup =
nullptr;
225 CurTokenLexer = std::move(TokLexer);
226 if (CurLexerKind != CLK_LexAfterModuleImport)
227 CurLexerKind = CLK_TokenLexer;
238 StringRef Path = FilePath;
239 while (!Path.empty()) {
242 Result = FilePath.substr(Path.size());
243 llvm::sys::path::append(Result,
244 llvm::sys::path::filename(File->
getName()));
249 Path = llvm::sys::path::parent_path(Path);
255 void Preprocessor::PropagateLineStartLeadingSpaceInfo(
Token &
Result) {
257 CurTokenLexer->PropagateLineStartLeadingSpaceInfo(Result);
261 CurLexer->PropagateLineStartLeadingSpaceInfo(Result);
274 const char *Preprocessor::getCurLexerEndPos() {
275 const char *EndPos = CurLexer->BufferEnd;
276 if (EndPos != CurLexer->BufferStart &&
277 (EndPos[-1] ==
'\n' || EndPos[-1] ==
'\r')) {
281 if (EndPos != CurLexer->BufferStart &&
282 (EndPos[-1] ==
'\n' || EndPos[-1] ==
'\r') &&
283 EndPos[-1] != EndPos[0])
293 SubMods.push_back(&Mod);
298 void Preprocessor::diagnoseMissingHeaderInUmbrellaDir(
const Module &Mod) {
301 SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID());
302 if (getDiagnostics().isIgnored(diag::warn_uncovered_module_header, StartLoc))
305 ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap();
310 Entry !=
End && !EC; Entry.increment(EC)) {
311 using llvm::StringSwitch;
315 if (!StringSwitch<bool>(llvm::sys::path::extension(Entry->getName()))
316 .Cases(
".h",
".H",
".hh",
".hpp",
true)
320 if (
const FileEntry *Header = getFileManager().getFile(Entry->getName()))
321 if (!getSourceManager().hasFileInfo(Header)) {
326 Diag(StartLoc, diag::warn_uncovered_module_header)
337 assert(!CurTokenLexer &&
338 "Ending a file when currently in a macro!");
343 const bool LeavingSubmodule = CurLexer && CurLexerSubmodule;
344 if ((LeavingSubmodule || IncludeMacroStack.empty()) &&
345 !BuildingSubmoduleStack.empty() &&
346 BuildingSubmoduleStack.back().IsPragma) {
347 Diag(BuildingSubmoduleStack.back().ImportLoc,
348 diag::err_pp_module_begin_without_module_end);
349 Module *M = LeaveSubmodule(
true);
352 const char *EndPos = getCurLexerEndPos();
353 CurLexer->BufferPtr = EndPos;
354 CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
363 CurPPLexer->MIOpt.GetControllingMacroAtEndOfFile()) {
365 if (
const FileEntry *FE = CurPPLexer->getFileEntry()) {
366 HeaderInfo.SetFileControllingMacro(FE, ControllingMacro);
368 getMacroInfo(const_cast<IdentifierInfo*>(ControllingMacro)))
369 MI->setUsedForHeaderGuard(
true);
371 CurPPLexer->MIOpt.GetDefinedMacro()) {
373 DefinedMacro != ControllingMacro &&
374 HeaderInfo.FirstTimeLexingFile(FE)) {
382 const StringRef ControllingMacroName = ControllingMacro->getName();
383 const StringRef DefinedMacroName = DefinedMacro->getName();
384 const size_t MaxHalfLength =
std::max(ControllingMacroName.size(),
385 DefinedMacroName.size()) / 2;
386 const unsigned ED = ControllingMacroName.edit_distance(
387 DefinedMacroName,
true, MaxHalfLength);
388 if (ED <= MaxHalfLength) {
390 Diag(CurPPLexer->MIOpt.GetMacroLocation(),
391 diag::warn_header_guard)
392 << CurPPLexer->MIOpt.GetMacroLocation() << ControllingMacro;
393 Diag(CurPPLexer->MIOpt.GetDefinedLocation(),
394 diag::note_header_guard)
395 << CurPPLexer->MIOpt.GetDefinedLocation() << DefinedMacro
398 CurPPLexer->MIOpt.GetDefinedLocation(),
399 ControllingMacro->getName());
410 if (PragmaARCCFCodeAuditedLoc.isValid() &&
411 !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
412 Diag(PragmaARCCFCodeAuditedLoc, diag::err_pp_eof_in_arc_cf_code_audited);
421 if (PragmaAssumeNonNullLoc.isValid() &&
422 !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
423 Diag(PragmaAssumeNonNullLoc, diag::err_pp_eof_in_assume_nonnull);
429 bool LeavingPCHThroughHeader =
false;
433 if (!IncludeMacroStack.empty()) {
436 if (isCodeCompletionEnabled() && CurPPLexer &&
437 SourceMgr.getLocForStartOfFile(CurPPLexer->getFileID()) ==
438 CodeCompletionFileLoc) {
441 CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd,
tok::eof);
444 assert(CurPTHLexer &&
"Got EOF but no current lexer set!");
445 CurPTHLexer->getEOF(Result);
449 CurPPLexer =
nullptr;
450 recomputeCurLexerKind();
454 if (!isEndOfMacro && CurPPLexer &&
455 SourceMgr.getIncludeLoc(CurPPLexer->getFileID()).isValid()) {
459 SourceMgr.local_sloc_entry_size() -
460 CurPPLexer->getInitialNumSLocEntries() + 1;
461 SourceMgr.setNumCreatedFIDsForFileID(CurPPLexer->getFileID(), NumFIDs);
464 bool ExitedFromPredefinesFile =
false;
466 if (!isEndOfMacro && CurPPLexer) {
467 ExitedFID = CurPPLexer->getFileID();
469 assert(PredefinesFileID.isValid() &&
470 "HandleEndOfFile is called before PredefinesFileId is set");
471 ExitedFromPredefinesFile = (PredefinesFileID == ExitedFID);
474 if (LeavingSubmodule) {
476 Module *M = LeaveSubmodule(
false);
479 const char *EndPos = getCurLexerEndPos();
481 CurLexer->BufferPtr = EndPos;
482 CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
487 bool FoundPCHThroughHeader =
false;
488 if (CurPPLexer && creatingPCHWithThroughHeader() &&
490 SourceMgr.getFileEntryForID(CurPPLexer->getFileID())))
491 FoundPCHThroughHeader =
true;
494 RemoveTopOfLexerStack();
497 PropagateLineStartLeadingSpaceInfo(Result);
500 if (Callbacks && !isEndOfMacro && CurPPLexer) {
502 SourceMgr.getFileCharacteristic(CurPPLexer->getSourceLocation());
503 Callbacks->FileChanged(CurPPLexer->getSourceLocation(),
509 if (ExitedFromPredefinesFile)
510 replayPreambleConditionalStack();
512 if (!isEndOfMacro && CurPPLexer && FoundPCHThroughHeader &&
513 (isInPrimaryFile() ||
514 CurPPLexer->getFileID() == getPredefinesFileID())) {
517 LeavingPCHThroughHeader =
true;
520 return LeavingSubmodule;
526 const char *EndPos = getCurLexerEndPos();
528 CurLexer->BufferPtr = EndPos;
529 CurLexer->FormTokenWithChars(Result, EndPos,
tok::eof);
531 if (isCodeCompletionEnabled()) {
538 if (CurLexer->getFileLoc() == CodeCompletionFileLoc)
542 if (creatingPCHWithThroughHeader() && !LeavingPCHThroughHeader) {
544 Diag(CurLexer->getFileLoc(), diag::err_pp_through_header_not_seen)
545 << PPOpts->PCHThroughHeader << 0;
548 if (!isIncrementalProcessingEnabled())
552 assert(CurPTHLexer &&
"Got EOF but no current lexer set!");
553 CurPTHLexer->getEOF(Result);
557 if (!isIncrementalProcessingEnabled())
558 CurPPLexer =
nullptr;
564 for (WarnUnusedMacroLocsTy::iterator
565 I=WarnUnusedMacroLocs.begin(), E=WarnUnusedMacroLocs.end();
567 Diag(*I, diag::pp_macro_not_used);
574 if (
Module *Mod = getCurrentModule()) {
577 for (
auto *M : AllMods)
578 diagnoseMissingHeaderInUmbrellaDir(*M);
587 assert(CurTokenLexer && !CurPPLexer &&
588 "Ending a macro when currently in a #include file!");
590 if (!MacroExpandingLexersStack.empty() &&
591 MacroExpandingLexersStack.back().first == CurTokenLexer.get())
592 removeCachedMacroExpandedTokensOfLastLexer();
595 if (NumCachedTokenLexers == TokenLexerCacheSize)
596 CurTokenLexer.reset();
598 TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
601 return HandleEndOfFile(Result,
true);
608 assert(!IncludeMacroStack.empty() &&
"Ran out of stack entries to load");
612 if (NumCachedTokenLexers == TokenLexerCacheSize)
613 CurTokenLexer.reset();
615 TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
618 PopIncludeMacroStack();
625 assert(CurTokenLexer && !CurPPLexer &&
626 "Pasted comment can only be formed from macro");
631 bool LexerWasInPPMode =
false;
632 for (
const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
633 if (ISI.ThePPLexer ==
nullptr)
continue;
641 FoundLexer = ISI.ThePPLexer;
651 if (!HandleEndOfTokenLexer(Tok)) Lex(Tok);
663 if (Tok.
is(tok::eod)) {
664 assert(FoundLexer &&
"Can't get end of line without an active lexer");
670 if (LexerWasInPPMode)
return;
681 assert(!FoundLexer &&
"Lexer should return EOD before EOF in PP mode");
686 if (!getLangOpts().ModulesLocalVisibility) {
688 BuildingSubmoduleStack.push_back(
689 BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
690 PendingModuleMacroNames.size()));
698 ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap();
704 auto R = Submodules.insert(std::make_pair(M, SubmoduleState()));
705 auto &
State = R.first->second;
706 bool FirstTime = R.second;
714 auto &StartingMacros = NullSubmoduleState.Macros;
718 for (
auto &Macro : StartingMacros) {
720 if (!Macro.second.getLatest() &&
721 Macro.second.getOverriddenMacros().empty())
724 MacroState MS(Macro.second.getLatest());
725 MS.setOverriddenMacros(*
this, Macro.second.getOverriddenMacros());
726 State.Macros.insert(std::make_pair(Macro.first, std::move(MS)));
731 BuildingSubmoduleStack.push_back(
732 BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
733 PendingModuleMacroNames.size()));
736 CurSubmoduleState = &
State;
740 makeModuleVisible(M, ImportLoc);
743 bool Preprocessor::needModuleMacros()
const {
745 if (BuildingSubmoduleStack.empty())
749 if (getLangOpts().ModulesLocalVisibility)
753 return getLangOpts().isCompilingModule();
757 if (BuildingSubmoduleStack.empty() ||
758 BuildingSubmoduleStack.back().IsPragma != ForPragma) {
759 assert(ForPragma &&
"non-pragma module enter/leave mismatch");
763 auto &Info = BuildingSubmoduleStack.back();
765 Module *LeavingMod = Info.M;
768 if (!needModuleMacros() ||
769 (!getLangOpts().ModulesLocalVisibility &&
774 BuildingSubmoduleStack.pop_back();
775 makeModuleVisible(LeavingMod, ImportLoc);
780 llvm::SmallPtrSet<const IdentifierInfo*, 8> VisitedMacros;
781 for (
unsigned I = Info.OuterPendingModuleMacroNames;
782 I != PendingModuleMacroNames.size(); ++I) {
783 auto *II =
const_cast<IdentifierInfo*
>(PendingModuleMacroNames[I]);
784 if (!VisitedMacros.insert(II).second)
787 auto MacroIt = CurSubmoduleState->Macros.find(II);
788 if (MacroIt == CurSubmoduleState->Macros.end())
790 auto &Macro = MacroIt->second;
794 auto *OldState = Info.OuterSubmoduleState;
795 if (getLangOpts().ModulesLocalVisibility)
796 OldState = &NullSubmoduleState;
797 if (OldState && OldState != CurSubmoduleState) {
800 auto &OldMacros = OldState->Macros;
801 auto OldMacroIt = OldMacros.find(II);
802 if (OldMacroIt == OldMacros.end())
805 OldMD = OldMacroIt->second.getLatest();
810 bool ExplicitlyPublic =
false;
811 for (
auto *MD = Macro.getLatest(); MD != OldMD; MD = MD->
getPrevious()) {
812 assert(MD &&
"broken macro directive chain");
814 if (
auto *VisMD = dyn_cast<VisibilityMacroDirective>(MD)) {
817 if (VisMD->isPublic())
818 ExplicitlyPublic =
true;
819 else if (!ExplicitlyPublic)
825 Def = DefMD->getInfo();
832 if (Def || !Macro.getOverriddenMacros().empty())
833 addModuleMacro(LeavingMod, II, Def,
834 Macro.getOverriddenMacros(), IsNew);
836 if (!getLangOpts().ModulesLocalVisibility) {
839 Macro.setLatest(
nullptr);
840 Macro.setOverriddenMacros(*
this, {});
846 PendingModuleMacroNames.resize(Info.OuterPendingModuleMacroNames);
854 if (getLangOpts().ModulesLocalVisibility)
855 CurSubmoduleState = Info.OuterSubmoduleState;
857 BuildingSubmoduleStack.pop_back();
860 makeModuleVisible(LeavingMod, ImportLoc);
static DiagnosticBuilder Diag(DiagnosticsEngine *Diags, const LangOptions &Features, FullSourceLoc TokLoc, const char *TokBegin, const char *TokRangeBegin, const char *TokRangeEnd, unsigned DiagID)
Produce a diagnostic highlighting some portion of a literal.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
Implements support for file system lookup, file system caching, and directory search management...
Defines the clang::FileManager interface and associated types.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Defines the SourceManager interface.
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool resolveUses(Module *Mod, bool Complain)
Resolve all of the unresolved uses in the given module.
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
The virtual file system interface.
One of these records is kept for each identifier that is lexed.
A directive for a defined macro or a macro imported from a module.
bool ParsingPreprocessorDirective
True when parsing #XXX; turns '\n' into a tok::eod token.
bool resolveConflicts(Module *Mod, bool Complain)
Resolve all of the unresolved conflicts in the given module.
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
Module * LeaveSubmodule(bool ForPragma)
Header getUmbrellaHeader() const
Retrieve the header that serves as the umbrella header for this module.
Token - This structure provides full information about a lexed token.
An input iterator over the recursive contents of a virtual path, similar to llvm::sys::fs::recursive_...
Describes a module or submodule.
StringRef getTopLevelModuleName() const
Retrieve the name of the top-level module.
std::string getFullModuleName(bool AllowStringLiterals=false) const
Retrieve the full name of this module, including the path from its top-level module.
const DirectoryEntry * getDirectory(StringRef DirName, bool CacheFailure=true)
Lookup, cache, and verify the specified directory (real or virtual).
const DirectoryEntry * Entry
void setAnnotationValue(void *val)
bool LexingRawMode
True if in raw mode.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Defines the clang::Preprocessor interface.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
bool resolveExports(Module *Mod, bool Complain)
Resolve all of the unresolved exports in the given module.
static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir, const FileEntry *File, SmallString< 128 > &Result)
Compute the relative path that names the given file relative to the given directory.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
The result type of a method or function.
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
const DirectoryEntry * getDir() const
Return the directory the file lives in.
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Encodes a location in the source.
StringRef getName() const
void setAnnotationEndLoc(SourceLocation L)
Cached information about one file (either on disk or in the virtual file system). ...
void EnterMacro(Token &Identifier, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Dataflow Directional Tag Classes.
llvm::iterator_range< submodule_iterator > submodules()
Encapsulates the data about a macro definition (e.g.
DirectoryName getUmbrellaDir() const
Retrieve the directory for which this module serves as the umbrella.
Cached information about one directory (either on disk or in the virtual file system).
static void collectAllSubModulesWithUmbrellaHeader(const Module &Mod, SmallVectorImpl< const Module *> &SubMods)
static bool isMacroDefined(const Sema &S, SourceLocation Loc, StringRef Name)
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
static FixItHint CreateReplacement(CharSourceRange RemoveRange, StringRef Code)
Create a code modification hint that replaces the given source range with the given code string...
The translation unit is a complete translation unit.
bool isHeaderInUnavailableModule(const FileEntry *Header) const
Determine whether the given header is part of a module marked 'unavailable'.
__DEVICE__ int max(int __a, int __b)
void setLocation(SourceLocation L)
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
void startToken()
Reset all flags to cleared.
StringRef getName() const
bool EnterSourceFile(FileID CurFileID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...