21 #include "llvm/ADT/StringSwitch.h" 22 #include "llvm/Support/FileSystem.h" 23 #include "llvm/Support/MemoryBuffer.h" 24 #include "llvm/Support/Path.h" 25 using namespace clang;
37 return IncludeMacroStack.empty();
40 assert(IsFileLexer(IncludeMacroStack[0]) &&
41 "Top level include stack isn't our primary lexer?");
43 IncludeMacroStack.begin() + 1, IncludeMacroStack.end(),
44 [&](
const IncludeStackInfo &ISI) ->
bool {
return IsFileLexer(ISI); });
55 for (
const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
57 return ISI.ThePPLexer;
71 assert(!CurTokenLexer &&
"Cannot #include a file inside a macro!");
72 ++NumEnteredSourceFiles;
74 if (MaxIncludeStackDepth < IncludeMacroStack.size())
75 MaxIncludeStackDepth = IncludeMacroStack.size();
79 const llvm::MemoryBuffer *InputFile =
80 getSourceManager().getBuffer(FID, Loc, &Invalid);
83 Diag(Loc, diag::err_pp_error_opening_file)
84 << std::string(SourceMgr.getBufferName(FileStart)) <<
"";
88 if (isCodeCompletionEnabled() &&
89 SourceMgr.getFileEntryForID(FID) == CodeCompletionFile) {
90 CodeCompletionFileLoc = SourceMgr.getLocForStartOfFile(FID);
92 CodeCompletionFileLoc.getLocWithOffset(CodeCompletionOffset);
95 EnterSourceFileWithLexer(
new Lexer(FID, InputFile, *
this), CurDir);
101 void Preprocessor::EnterSourceFileWithLexer(
Lexer *TheLexer,
105 if (CurPPLexer || CurTokenLexer)
106 PushIncludeMacroStack();
108 CurLexer.reset(TheLexer);
109 CurPPLexer = TheLexer;
110 CurDirLookup = CurDir;
111 CurLexerSubmodule =
nullptr;
112 if (CurLexerKind != CLK_LexAfterModuleImport)
113 CurLexerKind = CLK_Lexer;
116 if (Callbacks && !CurLexer->Is_PragmaLexer) {
118 SourceMgr.getFileCharacteristic(CurLexer->getFileLoc());
120 Callbacks->FileChanged(CurLexer->getFileLoc(),
129 std::unique_ptr<TokenLexer> TokLexer;
130 if (NumCachedTokenLexers == 0) {
131 TokLexer = llvm::make_unique<TokenLexer>(
Tok, ILEnd, Macro, Args, *
this);
133 TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
134 TokLexer->Init(Tok, ILEnd, Macro, Args);
137 PushIncludeMacroStack();
138 CurDirLookup =
nullptr;
139 CurTokenLexer = std::move(TokLexer);
140 if (CurLexerKind != CLK_LexAfterModuleImport)
141 CurLexerKind = CLK_TokenLexer;
156 void Preprocessor::EnterTokenStream(
const Token *Toks,
unsigned NumToks,
157 bool DisableMacroExpansion,
bool OwnsTokens,
159 if (CurLexerKind == CLK_CachingLexer) {
161 assert(IsReinject &&
"new tokens in the middle of cached stream");
165 Toks, Toks + NumToks);
173 ExitCachingLexMode();
174 EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
176 EnterCachingLexMode();
181 std::unique_ptr<TokenLexer> TokLexer;
182 if (NumCachedTokenLexers == 0) {
183 TokLexer = llvm::make_unique<TokenLexer>(
184 Toks, NumToks, DisableMacroExpansion, OwnsTokens, IsReinject, *
this);
186 TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
187 TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
192 PushIncludeMacroStack();
193 CurDirLookup =
nullptr;
194 CurTokenLexer = std::move(TokLexer);
195 if (CurLexerKind != CLK_LexAfterModuleImport)
196 CurLexerKind = CLK_TokenLexer;
207 StringRef Path = FilePath;
208 while (!Path.empty()) {
211 Result = FilePath.substr(Path.size());
212 llvm::sys::path::append(Result,
213 llvm::sys::path::filename(File->
getName()));
218 Path = llvm::sys::path::parent_path(Path);
224 void Preprocessor::PropagateLineStartLeadingSpaceInfo(
Token &
Result) {
226 CurTokenLexer->PropagateLineStartLeadingSpaceInfo(Result);
230 CurLexer->PropagateLineStartLeadingSpaceInfo(Result);
243 const char *Preprocessor::getCurLexerEndPos() {
244 const char *EndPos = CurLexer->BufferEnd;
245 if (EndPos != CurLexer->BufferStart &&
246 (EndPos[-1] ==
'\n' || EndPos[-1] ==
'\r')) {
250 if (EndPos != CurLexer->BufferStart &&
251 (EndPos[-1] ==
'\n' || EndPos[-1] ==
'\r') &&
252 EndPos[-1] != EndPos[0])
262 SubMods.push_back(&Mod);
267 void Preprocessor::diagnoseMissingHeaderInUmbrellaDir(
const Module &Mod) {
270 SourceMgr.getLocForStartOfFile(SourceMgr.getMainFileID());
271 if (getDiagnostics().isIgnored(diag::warn_uncovered_module_header, StartLoc))
274 ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap();
276 llvm::vfs::FileSystem &FS = FileMgr.getVirtualFileSystem();
278 for (llvm::vfs::recursive_directory_iterator Entry(FS, Dir->
getName(), EC),
280 Entry !=
End && !EC; Entry.increment(EC)) {
281 using llvm::StringSwitch;
285 if (!StringSwitch<bool>(llvm::sys::path::extension(Entry->path()))
286 .Cases(
".h",
".H",
".hh",
".hpp",
true)
290 if (
const FileEntry *Header = getFileManager().getFile(Entry->path()))
291 if (!getSourceManager().hasFileInfo(Header)) {
296 Diag(StartLoc, diag::warn_uncovered_module_header)
307 assert(!CurTokenLexer &&
308 "Ending a file when currently in a macro!");
312 const bool LeavingSubmodule = CurLexer && CurLexerSubmodule;
313 if ((LeavingSubmodule || IncludeMacroStack.empty()) &&
314 !BuildingSubmoduleStack.empty() &&
315 BuildingSubmoduleStack.back().IsPragma) {
316 Diag(BuildingSubmoduleStack.back().ImportLoc,
317 diag::err_pp_module_begin_without_module_end);
318 Module *M = LeaveSubmodule(
true);
321 const char *EndPos = getCurLexerEndPos();
322 CurLexer->BufferPtr = EndPos;
323 CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
332 CurPPLexer->MIOpt.GetControllingMacroAtEndOfFile()) {
334 if (
const FileEntry *FE = CurPPLexer->getFileEntry()) {
335 HeaderInfo.SetFileControllingMacro(FE, ControllingMacro);
337 getMacroInfo(const_cast<IdentifierInfo*>(ControllingMacro)))
338 MI->setUsedForHeaderGuard(
true);
340 CurPPLexer->MIOpt.GetDefinedMacro()) {
342 DefinedMacro != ControllingMacro &&
343 HeaderInfo.FirstTimeLexingFile(FE)) {
351 const StringRef ControllingMacroName = ControllingMacro->getName();
352 const StringRef DefinedMacroName = DefinedMacro->getName();
353 const size_t MaxHalfLength =
std::max(ControllingMacroName.size(),
354 DefinedMacroName.size()) / 2;
355 const unsigned ED = ControllingMacroName.edit_distance(
356 DefinedMacroName,
true, MaxHalfLength);
357 if (ED <= MaxHalfLength) {
359 Diag(CurPPLexer->MIOpt.GetMacroLocation(),
360 diag::warn_header_guard)
361 << CurPPLexer->MIOpt.GetMacroLocation() << ControllingMacro;
362 Diag(CurPPLexer->MIOpt.GetDefinedLocation(),
363 diag::note_header_guard)
364 << CurPPLexer->MIOpt.GetDefinedLocation() << DefinedMacro
367 CurPPLexer->MIOpt.GetDefinedLocation(),
368 ControllingMacro->getName());
379 if (PragmaARCCFCodeAuditedLoc.isValid() &&
380 !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
381 Diag(PragmaARCCFCodeAuditedLoc, diag::err_pp_eof_in_arc_cf_code_audited);
390 if (PragmaAssumeNonNullLoc.isValid() &&
391 !isEndOfMacro && !(CurLexer && CurLexer->Is_PragmaLexer)) {
392 Diag(PragmaAssumeNonNullLoc, diag::err_pp_eof_in_assume_nonnull);
398 bool LeavingPCHThroughHeader =
false;
402 if (!IncludeMacroStack.empty()) {
405 if (isCodeCompletionEnabled() && CurPPLexer &&
406 SourceMgr.getLocForStartOfFile(CurPPLexer->getFileID()) ==
407 CodeCompletionFileLoc) {
408 assert(CurLexer &&
"Got EOF but no current lexer set!");
410 CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd,
tok::eof);
413 CurPPLexer =
nullptr;
414 recomputeCurLexerKind();
418 if (!isEndOfMacro && CurPPLexer &&
419 SourceMgr.getIncludeLoc(CurPPLexer->getFileID()).isValid()) {
423 SourceMgr.local_sloc_entry_size() -
424 CurPPLexer->getInitialNumSLocEntries() + 1;
425 SourceMgr.setNumCreatedFIDsForFileID(CurPPLexer->getFileID(), NumFIDs);
428 bool ExitedFromPredefinesFile =
false;
430 if (!isEndOfMacro && CurPPLexer) {
431 ExitedFID = CurPPLexer->getFileID();
433 assert(PredefinesFileID.isValid() &&
434 "HandleEndOfFile is called before PredefinesFileId is set");
435 ExitedFromPredefinesFile = (PredefinesFileID == ExitedFID);
438 if (LeavingSubmodule) {
440 Module *M = LeaveSubmodule(
false);
443 const char *EndPos = getCurLexerEndPos();
445 CurLexer->BufferPtr = EndPos;
446 CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
451 bool FoundPCHThroughHeader =
false;
452 if (CurPPLexer && creatingPCHWithThroughHeader() &&
454 SourceMgr.getFileEntryForID(CurPPLexer->getFileID())))
455 FoundPCHThroughHeader =
true;
458 RemoveTopOfLexerStack();
461 PropagateLineStartLeadingSpaceInfo(Result);
464 if (Callbacks && !isEndOfMacro && CurPPLexer) {
466 SourceMgr.getFileCharacteristic(CurPPLexer->getSourceLocation());
467 Callbacks->FileChanged(CurPPLexer->getSourceLocation(),
473 if (ExitedFromPredefinesFile)
474 replayPreambleConditionalStack();
476 if (!isEndOfMacro && CurPPLexer && FoundPCHThroughHeader &&
477 (isInPrimaryFile() ||
478 CurPPLexer->getFileID() == getPredefinesFileID())) {
481 LeavingPCHThroughHeader =
true;
484 return LeavingSubmodule;
489 assert(CurLexer &&
"Got EOF but no current lexer set!");
490 const char *EndPos = getCurLexerEndPos();
492 CurLexer->BufferPtr = EndPos;
493 CurLexer->FormTokenWithChars(Result, EndPos,
tok::eof);
495 if (isCodeCompletionEnabled()) {
502 if (CurLexer->getFileLoc() == CodeCompletionFileLoc)
506 if (creatingPCHWithThroughHeader() && !LeavingPCHThroughHeader) {
508 Diag(CurLexer->getFileLoc(), diag::err_pp_through_header_not_seen)
509 << PPOpts->PCHThroughHeader << 0;
512 if (!isIncrementalProcessingEnabled())
516 if (!isIncrementalProcessingEnabled())
517 CurPPLexer =
nullptr;
523 for (WarnUnusedMacroLocsTy::iterator
524 I=WarnUnusedMacroLocs.begin(), E=WarnUnusedMacroLocs.end();
526 Diag(*I, diag::pp_macro_not_used);
533 if (
Module *Mod = getCurrentModule()) {
536 for (
auto *M : AllMods)
537 diagnoseMissingHeaderInUmbrellaDir(*M);
546 assert(CurTokenLexer && !CurPPLexer &&
547 "Ending a macro when currently in a #include file!");
549 if (!MacroExpandingLexersStack.empty() &&
550 MacroExpandingLexersStack.back().first == CurTokenLexer.get())
551 removeCachedMacroExpandedTokensOfLastLexer();
554 if (NumCachedTokenLexers == TokenLexerCacheSize)
555 CurTokenLexer.reset();
557 TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
560 return HandleEndOfFile(Result,
true);
567 assert(!IncludeMacroStack.empty() &&
"Ran out of stack entries to load");
571 if (NumCachedTokenLexers == TokenLexerCacheSize)
572 CurTokenLexer.reset();
574 TokenLexerCache[NumCachedTokenLexers++] = std::move(CurTokenLexer);
577 PopIncludeMacroStack();
584 assert(CurTokenLexer && !CurPPLexer &&
585 "Pasted comment can only be formed from macro");
590 bool LexerWasInPPMode =
false;
591 for (
const IncludeStackInfo &ISI : llvm::reverse(IncludeMacroStack)) {
592 if (ISI.ThePPLexer ==
nullptr)
continue;
600 FoundLexer = ISI.ThePPLexer;
610 if (!HandleEndOfTokenLexer(Tok)) Lex(Tok);
622 if (Tok.
is(tok::eod)) {
623 assert(FoundLexer &&
"Can't get end of line without an active lexer");
629 if (LexerWasInPPMode)
return;
640 assert(!FoundLexer &&
"Lexer should return EOD before EOF in PP mode");
645 if (!getLangOpts().ModulesLocalVisibility) {
647 BuildingSubmoduleStack.push_back(
648 BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
649 PendingModuleMacroNames.size()));
651 Callbacks->EnteredSubmodule(M, ImportLoc, ForPragma);
659 ModuleMap &ModMap = getHeaderSearchInfo().getModuleMap();
665 auto R = Submodules.insert(std::make_pair(M, SubmoduleState()));
666 auto &
State = R.first->second;
667 bool FirstTime = R.second;
675 auto &StartingMacros = NullSubmoduleState.Macros;
679 for (
auto &Macro : StartingMacros) {
681 if (!Macro.second.getLatest() &&
682 Macro.second.getOverriddenMacros().empty())
685 MacroState MS(Macro.second.getLatest());
686 MS.setOverriddenMacros(*
this, Macro.second.getOverriddenMacros());
687 State.Macros.insert(std::make_pair(Macro.first, std::move(MS)));
692 BuildingSubmoduleStack.push_back(
693 BuildingSubmoduleInfo(M, ImportLoc, ForPragma, CurSubmoduleState,
694 PendingModuleMacroNames.size()));
697 Callbacks->EnteredSubmodule(M, ImportLoc, ForPragma);
700 CurSubmoduleState = &
State;
704 makeModuleVisible(M, ImportLoc);
707 bool Preprocessor::needModuleMacros()
const {
709 if (BuildingSubmoduleStack.empty())
713 if (getLangOpts().ModulesLocalVisibility)
717 return getLangOpts().isCompilingModule();
721 if (BuildingSubmoduleStack.empty() ||
722 BuildingSubmoduleStack.back().IsPragma != ForPragma) {
723 assert(ForPragma &&
"non-pragma module enter/leave mismatch");
727 auto &Info = BuildingSubmoduleStack.back();
729 Module *LeavingMod = Info.M;
732 if (!needModuleMacros() ||
733 (!getLangOpts().ModulesLocalVisibility &&
738 BuildingSubmoduleStack.pop_back();
741 Callbacks->LeftSubmodule(LeavingMod, ImportLoc, ForPragma);
743 makeModuleVisible(LeavingMod, ImportLoc);
748 llvm::SmallPtrSet<const IdentifierInfo*, 8> VisitedMacros;
749 for (
unsigned I = Info.OuterPendingModuleMacroNames;
750 I != PendingModuleMacroNames.size(); ++I) {
751 auto *II =
const_cast<IdentifierInfo*
>(PendingModuleMacroNames[I]);
752 if (!VisitedMacros.insert(II).second)
755 auto MacroIt = CurSubmoduleState->Macros.find(II);
756 if (MacroIt == CurSubmoduleState->Macros.end())
758 auto &Macro = MacroIt->second;
762 auto *OldState = Info.OuterSubmoduleState;
763 if (getLangOpts().ModulesLocalVisibility)
764 OldState = &NullSubmoduleState;
765 if (OldState && OldState != CurSubmoduleState) {
768 auto &OldMacros = OldState->Macros;
769 auto OldMacroIt = OldMacros.find(II);
770 if (OldMacroIt == OldMacros.end())
773 OldMD = OldMacroIt->second.getLatest();
778 bool ExplicitlyPublic =
false;
779 for (
auto *MD = Macro.getLatest(); MD != OldMD; MD = MD->
getPrevious()) {
780 assert(MD &&
"broken macro directive chain");
782 if (
auto *VisMD = dyn_cast<VisibilityMacroDirective>(MD)) {
785 if (VisMD->isPublic())
786 ExplicitlyPublic =
true;
787 else if (!ExplicitlyPublic)
793 Def = DefMD->getInfo();
800 if (Def || !Macro.getOverriddenMacros().empty())
801 addModuleMacro(LeavingMod, II, Def,
802 Macro.getOverriddenMacros(), IsNew);
804 if (!getLangOpts().ModulesLocalVisibility) {
807 Macro.setLatest(
nullptr);
808 Macro.setOverriddenMacros(*
this, {});
814 PendingModuleMacroNames.resize(Info.OuterPendingModuleMacroNames);
822 if (getLangOpts().ModulesLocalVisibility)
823 CurSubmoduleState = Info.OuterSubmoduleState;
825 BuildingSubmoduleStack.pop_back();
828 Callbacks->LeftSubmodule(LeavingMod, ImportLoc, ForPragma);
831 makeModuleVisible(LeavingMod, ImportLoc);
static DiagnosticBuilder Diag(DiagnosticsEngine *Diags, const LangOptions &Features, FullSourceLoc TokLoc, const char *TokBegin, const char *TokRangeBegin, const char *TokRangeEnd, unsigned DiagID)
Produce a diagnostic highlighting some portion of a literal.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
Implements support for file system lookup, file system caching, and directory search management...
Defines the clang::FileManager interface and associated types.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Defines the SourceManager interface.
bool isInPrimaryFile() const
Return true if we're in the top-level file, not in a #include.
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool resolveUses(Module *Mod, bool Complain)
Resolve all of the unresolved uses in the given module.
CharacteristicKind
Indicates whether a file or directory holds normal user code, system code, or system code which is im...
const MacroDirective * getPrevious() const
Get previous definition of the macro with the same name.
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
One of these records is kept for each identifier that is lexed.
A directive for a defined macro or a macro imported from a module.
bool ParsingPreprocessorDirective
True when parsing #XXX; turns '\n' into a tok::eod token.
bool resolveConflicts(Module *Mod, bool Complain)
Resolve all of the unresolved conflicts in the given module.
void EnterSubmodule(Module *M, SourceLocation ImportLoc, bool ForPragma)
Module * LeaveSubmodule(bool ForPragma)
Header getUmbrellaHeader() const
Retrieve the header that serves as the umbrella header for this module.
Token - This structure provides full information about a lexed token.
__DEVICE__ int max(int __a, int __b)
Describes a module or submodule.
StringRef getTopLevelModuleName() const
Retrieve the name of the top-level module.
std::string getFullModuleName(bool AllowStringLiterals=false) const
Retrieve the full name of this module, including the path from its top-level module.
const DirectoryEntry * getDirectory(StringRef DirName, bool CacheFailure=true)
Lookup, cache, and verify the specified directory (real or virtual).
const DirectoryEntry * Entry
void setAnnotationValue(void *val)
bool LexingRawMode
True if in raw mode.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Defines the clang::Preprocessor interface.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
bool resolveExports(Module *Mod, bool Complain)
Resolve all of the unresolved exports in the given module.
static void computeRelativePath(FileManager &FM, const DirectoryEntry *Dir, const FileEntry *File, SmallString< 128 > &Result)
Compute the relative path that names the given file relative to the given directory.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
The result type of a method or function.
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
const DirectoryEntry * getDir() const
Return the directory the file lives in.
Encapsulates changes to the "macros namespace" (the location where the macro name became active...
Encodes a location in the source.
StringRef getName() const
void setAnnotationEndLoc(SourceLocation L)
Cached information about one file (either on disk or in the virtual file system). ...
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
Dataflow Directional Tag Classes.
llvm::iterator_range< submodule_iterator > submodules()
Encapsulates the data about a macro definition (e.g.
DirectoryName getUmbrellaDir() const
Retrieve the directory for which this module serves as the umbrella.
Cached information about one directory (either on disk or in the virtual file system).
static void collectAllSubModulesWithUmbrellaHeader(const Module &Mod, SmallVectorImpl< const Module *> &SubMods)
static bool isMacroDefined(const Sema &S, SourceLocation Loc, StringRef Name)
void RemoveTopOfLexerStack()
Pop the current lexer/macro exp off the top of the lexer stack.
static FixItHint CreateReplacement(CharSourceRange RemoveRange, StringRef Code)
Create a code modification hint that replaces the given source range with the given code string...
The translation unit is a complete translation unit.
bool isHeaderInUnavailableModule(const FileEntry *Header) const
Determine whether the given header is part of a module marked 'unavailable'.
void EnterMacro(Token &Tok, SourceLocation ILEnd, MacroInfo *Macro, MacroArgs *Args)
Add a Macro to the top of the include stack and start lexing tokens from it instead of the current bu...
void setLocation(SourceLocation L)
PreprocessorLexer * getCurrentFileLexer() const
Return the current file lexer being lexed from.
void startToken()
Reset all flags to cleared.
StringRef getName() const