53 #include "llvm/ADT/APInt.h" 54 #include "llvm/ADT/ArrayRef.h" 55 #include "llvm/ADT/DenseMap.h" 56 #include "llvm/ADT/SmallString.h" 57 #include "llvm/ADT/SmallVector.h" 58 #include "llvm/ADT/STLExtras.h" 59 #include "llvm/ADT/StringRef.h" 60 #include "llvm/ADT/StringSwitch.h" 61 #include "llvm/Support/Capacity.h" 62 #include "llvm/Support/ErrorHandling.h" 63 #include "llvm/Support/MemoryBuffer.h" 64 #include "llvm/Support/raw_ostream.h" 72 using namespace clang;
84 : PPOpts(
std::move(PPOpts)), Diags(&diags), LangOpts(opts),
85 FileMgr(Headers.getFileMgr()), SourceMgr(SM),
86 ScratchBuf(new
ScratchBuffer(SourceMgr)), HeaderInfo(Headers),
87 TheModuleLoader(TheModuleLoader), ExternalSource(nullptr),
91 Identifiers(IILookup), PragmaHandlers(new
PragmaNamespace(StringRef())),
92 TUKind(TUKind), SkipMainFilePreamble(0,
true),
93 CurSubmoduleState(&NullSubmoduleState) {
94 OwnsHeaderSearch = OwnsHeaders;
98 KeepMacroComments =
false;
99 SuppressIncludeNotFoundError =
false;
102 DisableMacroExpansion =
false;
103 MacroExpansionInDirectivesOverride =
false;
106 InMacroArgPreExpansion =
false;
107 NumCachedTokenLexers = 0;
108 PragmasEnabled =
true;
109 ParsingIfOrElifDirective =
false;
110 PreprocessedOutput =
false;
113 ReadMacrosFromExternalSource =
false;
123 Ident__VA_OPT__ =
nullptr;
127 RegisterBuiltinPragmas();
130 RegisterBuiltinMacros();
132 if(LangOpts.Borland) {
143 Ident__exception_info = Ident__exception_code =
nullptr;
144 Ident__abnormal_termination = Ident___exception_info =
nullptr;
145 Ident___exception_code = Ident___abnormal_termination =
nullptr;
146 Ident_GetExceptionInfo = Ident_GetExceptionCode =
nullptr;
147 Ident_AbnormalTermination =
nullptr;
152 SkippingUntilPragmaHdrStop =
true;
155 if (!this->PPOpts->PCHThroughHeader.empty() &&
156 !this->PPOpts->ImplicitPCHInclude.empty())
157 SkippingUntilPCHThroughHeader =
true;
159 if (this->PPOpts->GeneratePreamble)
160 PreambleConditionalStack.startRecording();
164 assert(BacktrackPositions.empty() &&
"EnableBacktrack/Backtrack imbalance!");
166 IncludeMacroStack.clear();
169 while (MacroInfoChain *I = MIChainHead) {
170 MIChainHead = I->Next;
171 I->~MacroInfoChain();
177 std::fill(TokenLexerCache, TokenLexerCache + NumCachedTokenLexers,
nullptr);
178 CurTokenLexer.reset();
181 for (
MacroArgs *ArgList = MacroArgCache; ArgList;)
182 ArgList = ArgList->deallocate();
185 if (OwnsHeaderSearch)
191 assert((!this->Target || this->Target == &Target) &&
192 "Invalid override of target information");
193 this->Target = &Target;
195 assert((!this->AuxTarget || this->AuxTarget == AuxTarget) &&
196 "Invalid override of aux target information.");
197 this->AuxTarget = AuxTarget;
208 NumEnteredSourceFiles = 0;
211 PragmaHandlersBackup = std::move(PragmaHandlers);
212 PragmaHandlers = llvm::make_unique<PragmaNamespace>(StringRef());
213 RegisterBuiltinPragmas();
216 PredefinesFileID =
FileID();
220 NumEnteredSourceFiles = 1;
222 PragmaHandlers = std::move(PragmaHandlersBackup);
229 if (!DumpFlags)
return;
231 llvm::errs() <<
"\t";
233 llvm::errs() <<
" [StartOfLine]";
235 llvm::errs() <<
" [LeadingSpace]";
237 llvm::errs() <<
" [ExpandDisabled]";
240 llvm::errs() <<
" [UnClean='" << StringRef(Start, Tok.
getLength())
244 llvm::errs() <<
"\tLoc=<";
250 Loc.
print(llvm::errs(), SourceMgr);
254 llvm::errs() <<
"MACRO: ";
259 llvm::errs() <<
"\n";
263 llvm::errs() <<
"\n*** Preprocessor Stats:\n";
264 llvm::errs() << NumDirectives <<
" directives found:\n";
265 llvm::errs() <<
" " << NumDefined <<
" #define.\n";
266 llvm::errs() <<
" " << NumUndefined <<
" #undef.\n";
267 llvm::errs() <<
" #include/#include_next/#import:\n";
268 llvm::errs() <<
" " << NumEnteredSourceFiles <<
" source files entered.\n";
269 llvm::errs() <<
" " << MaxIncludeStackDepth <<
" max include stack depth\n";
270 llvm::errs() <<
" " << NumIf <<
" #if/#ifndef/#ifdef.\n";
271 llvm::errs() <<
" " << NumElse <<
" #else/#elif.\n";
272 llvm::errs() <<
" " << NumEndif <<
" #endif.\n";
273 llvm::errs() <<
" " << NumPragma <<
" #pragma.\n";
274 llvm::errs() << NumSkipped <<
" #if/#ifndef#ifdef regions skipped\n";
276 llvm::errs() << NumMacroExpanded <<
"/" << NumFnMacroExpanded <<
"/" 277 << NumBuiltinMacroExpanded <<
" obj/fn/builtin macros expanded, " 278 << NumFastMacroExpanded <<
" on the fast path.\n";
279 llvm::errs() << (NumFastTokenPaste+NumTokenPaste)
280 <<
" token paste (##) operations performed, " 281 << NumFastTokenPaste <<
" on the fast path.\n";
283 llvm::errs() <<
"\nPreprocessor Memory: " <<
getTotalMemory() <<
"B total";
285 llvm::errs() <<
"\n BumpPtr: " << BP.getTotalMemory();
286 llvm::errs() <<
"\n Macro Expanded Tokens: " 287 << llvm::capacity_in_bytes(MacroExpandedTokens);
288 llvm::errs() <<
"\n Predefines Buffer: " << Predefines.capacity();
290 llvm::errs() <<
"\n Macros: " 291 << llvm::capacity_in_bytes(CurSubmoduleState->Macros);
292 llvm::errs() <<
"\n #pragma push_macro Info: " 293 << llvm::capacity_in_bytes(PragmaPushMacroInfo);
294 llvm::errs() <<
"\n Poison Reasons: " 295 << llvm::capacity_in_bytes(PoisonReasons);
296 llvm::errs() <<
"\n Comment Handlers: " 297 << llvm::capacity_in_bytes(CommentHandlers) <<
"\n";
302 if (IncludeExternalMacros && ExternalSource &&
303 !ReadMacrosFromExternalSource) {
304 ReadMacrosFromExternalSource =
true;
310 CurSubmoduleState->Macros.insert(std::make_pair(Macro.II, MacroState()));
312 return CurSubmoduleState->Macros.begin();
316 return BP.getTotalMemory()
317 + llvm::capacity_in_bytes(MacroExpandedTokens)
318 + Predefines.capacity()
321 + llvm::capacity_in_bytes(CurSubmoduleState->Macros)
322 + llvm::capacity_in_bytes(PragmaPushMacroInfo)
323 + llvm::capacity_in_bytes(PoisonReasons)
324 + llvm::capacity_in_bytes(CommentHandlers);
329 if (IncludeExternalMacros && ExternalSource &&
330 !ReadMacrosFromExternalSource) {
331 ReadMacrosFromExternalSource =
true;
335 return CurSubmoduleState->Macros.end();
342 std::equal(Tokens.begin(), Tokens.end(), MI->
tokens_begin());
349 StringRef BestSpelling;
353 Def = I->second.findDirectiveAtLoc(Loc, SourceMgr);
365 BestLocation = Location;
366 BestSpelling = I->first->getName();
374 CurLexerKind = CLK_Lexer;
375 else if (CurTokenLexer)
376 CurLexerKind = CLK_TokenLexer;
378 CurLexerKind = CLK_CachingLexer;
382 unsigned CompleteLine,
383 unsigned CompleteColumn) {
385 assert(CompleteLine && CompleteColumn &&
"Starts from 1:1");
386 assert(!CodeCompletionFile &&
"Already set");
388 using llvm::MemoryBuffer;
391 bool Invalid =
false;
397 const char *Position = Buffer->getBufferStart();
399 for (; *Position; ++Position) {
400 if (*Position !=
'\r' && *Position !=
'\n')
404 if ((Position[1] ==
'\r' || Position[1] ==
'\n') &&
405 Position[0] != Position[1])
412 Position += CompleteColumn - 1;
416 if (SkipMainFilePreamble.first &&
418 if (Position - Buffer->getBufferStart() < SkipMainFilePreamble.first)
419 Position = Buffer->getBufferStart() + SkipMainFilePreamble.first;
422 if (Position > Buffer->getBufferEnd())
423 Position = Buffer->getBufferEnd();
425 CodeCompletionFile = File;
426 CodeCompletionOffset = Position - Buffer->getBufferStart();
428 auto NewBuffer = llvm::WritableMemoryBuffer::getNewUninitMemBuffer(
429 Buffer->getBufferSize() + 1, Buffer->getBufferIdentifier());
430 char *NewBuf = NewBuffer->getBufferStart();
431 char *NewPos = std::copy(Buffer->getBufferStart(), Position, NewBuf);
433 std::copy(Position, Buffer->getBufferEnd(), NewPos+1);
457 bool *Invalid)
const {
459 if (Tok.
isNot(tok::raw_identifier) && !Tok.
hasUCN()) {
462 return II->getName();
469 const char *Ptr = Buffer.data();
471 return StringRef(Ptr, Len);
483 SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
485 if (ExpansionLocStart.
isValid())
487 ExpansionLocEnd, Str.size());
491 if (Tok.
is(tok::raw_identifier))
501 bool Invalid =
false;
509 ScratchBuf->getToken(Buffer.data() + LocInfo.second, Length, DestPtr);
530 assert(NumEnteredSourceFiles == 0 &&
"Cannot reenter the main file!");
541 if (SkipMainFilePreamble.first > 0)
542 CurLexer->SetByteOffset(SkipMainFilePreamble.first,
543 SkipMainFilePreamble.second);
552 std::unique_ptr<llvm::MemoryBuffer> SB =
553 llvm::MemoryBuffer::getMemBufferCopy(Predefines,
"<built-in>");
554 assert(SB &&
"Cannot create predefined source buffer");
556 assert(FID.
isValid() &&
"Could not create FileID for predefines?");
557 setPredefinesFileID(FID);
562 if (!PPOpts->PCHThroughHeader.empty()) {
568 false,
nullptr,
nullptr, CurDir,
574 << PPOpts->PCHThroughHeader;
577 setPCHThroughHeaderFileID(
587 void Preprocessor::setPCHThroughHeaderFileID(
FileID FID) {
588 assert(PCHThroughHeaderFileID.
isInvalid() &&
589 "PCHThroughHeaderFileID already set!");
590 PCHThroughHeaderFileID = FID;
594 assert(PCHThroughHeaderFileID.
isValid() &&
595 "Invalid PCH through header FileID");
600 return TUKind ==
TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
601 PCHThroughHeaderFileID.
isValid();
605 return TUKind !=
TU_Prefix && !PPOpts->PCHThroughHeader.empty() &&
606 PCHThroughHeaderFileID.
isValid();
610 return TUKind ==
TU_Prefix && PPOpts->PCHWithHdrStop;
614 return TUKind !=
TU_Prefix && PPOpts->PCHWithHdrStop;
623 bool ReachedMainFileEOF =
false;
624 bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
625 bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
630 switch (CurLexerKind) {
635 CurTokenLexer->Lex(Tok);
637 case CLK_CachingLexer:
640 case CLK_LexAfterModuleImport:
645 ReachedMainFileEOF =
true;
648 if (UsingPCHThroughHeader && !SkippingUntilPCHThroughHeader)
650 if (UsingPragmaHdrStop && !SkippingUntilPragmaHdrStop)
653 if (ReachedMainFileEOF) {
654 if (UsingPCHThroughHeader)
656 << PPOpts->PCHThroughHeader << 1;
657 else if (!PPOpts->PCHWithHdrStopCreate)
662 void Preprocessor::replayPreambleConditionalStack() {
664 if (PreambleConditionalStack.isReplaying()) {
666 "CurPPLexer is null when calling replayPreambleConditionalStack.");
668 PreambleConditionalStack.doneReplaying();
669 if (PreambleConditionalStack.reachedEOFWhileSkipping())
670 SkipExcludedConditionalBlock(
671 PreambleConditionalStack.SkipInfo->HashTokenLoc,
672 PreambleConditionalStack.SkipInfo->IfTokenLoc,
673 PreambleConditionalStack.SkipInfo->FoundNonSkipPortion,
674 PreambleConditionalStack.SkipInfo->FoundElse,
675 PreambleConditionalStack.SkipInfo->ElseLoc);
682 Callbacks->EndOfMainFile();
693 assert(!Identifier.
getRawIdentifier().empty() &&
"No raw identifier data!");
703 StringRef CleanedStr =
getSpelling(Identifier, IdentifierBuffer);
705 if (Identifier.
hasUCN()) {
716 if (
getLangOpts().MSVCCompat && II->isCPlusPlusOperatorKeyword() &&
718 Identifier.
setKind(tok::identifier);
720 Identifier.
setKind(II->getTokenID());
726 PoisonReasons[II] = DiagID;
730 assert(Ident__exception_code && Ident__exception_info);
731 assert(Ident___exception_code && Ident___exception_info);
745 "Can't handle identifiers without identifier info!");
746 llvm::DenseMap<IdentifierInfo*,unsigned>::const_iterator it =
748 if(it == PoisonReasons.end())
749 Diag(Identifier, diag::err_pp_used_poisoned_id);
760 if (LangOpts.CPlusPlus)
761 return llvm::StringSwitch<diag::kind>(II.
getName())
763 .Case(#NAME, diag::warn_cxx11_keyword)
764 #define CXX2A_KEYWORD(NAME, FLAGS) \ 765 .Case(#NAME, diag::warn_cxx2a_keyword) 766 #include "clang/Basic/TokenKinds.def" 770 "Keyword not known to come from a newer Standard or proposed Standard");
773 void Preprocessor::updateOutOfDateIdentifier(
IdentifierInfo &II)
const {
788 "Can't handle identifiers without identifier info!");
797 if (II.isOutOfDate()) {
798 bool CurrentIsPoisoned =
false;
799 const bool IsSpecialVariadicMacro =
800 &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
801 if (IsSpecialVariadicMacro)
804 updateOutOfDateIdentifier(II);
805 Identifier.
setKind(II.getTokenID());
807 if (IsSpecialVariadicMacro)
808 II.setIsPoisoned(CurrentIsPoisoned);
813 if (II.isPoisoned() && CurPPLexer) {
819 auto *MI = MD.getMacroInfo();
820 assert(MI &&
"macro definition with no macro info?");
821 if (!DisableMacroExpansion) {
825 if (!MI->isFunctionLike() || isNextPPTokenLParen())
826 return HandleMacroExpandedIdentifier(Identifier, MD);
832 if (MI->isObjectLike() || isNextPPTokenLParen())
833 Diag(Identifier, diag::pp_disabled_macro_expansion);
843 if (II.isFutureCompatKeyword() && !DisableMacroExpansion) {
847 II.setIsFutureCompatKeyword(
false);
854 if (II.isExtensionToken() && !DisableMacroExpansion)
855 Diag(Identifier, diag::ext_token_used);
865 if (((LastTokenWasAt && II.isModulesImport()) ||
866 Identifier.
is(tok::kw_import)) &&
867 !InMacroArgs && !DisableMacroExpansion &&
869 CurLexerKind != CLK_CachingLexer) {
871 ModuleImportPath.clear();
872 ModuleImportExpectsIdentifier =
true;
873 CurLexerKind = CLK_LexAfterModuleImport;
884 switch (CurLexerKind) {
886 ReturnedToken = CurLexer->Lex(Result);
889 ReturnedToken = CurTokenLexer->Lex(Result);
891 case CLK_CachingLexer:
893 ReturnedToken =
true;
895 case CLK_LexAfterModuleImport:
899 }
while (!ReturnedToken);
912 if (
getLangOpts().CPlusPlusModules && LexLevel == 1 &&
915 case tok::l_paren:
case tok::l_square:
case tok::l_brace:
916 ImportSeqState.handleOpenBracket();
918 case tok::r_paren:
case tok::r_square:
919 ImportSeqState.handleCloseBracket();
922 ImportSeqState.handleCloseBrace();
925 ImportSeqState.handleSemi();
927 case tok::header_name:
928 case tok::annot_header_unit:
929 ImportSeqState.handleHeaderName();
932 ImportSeqState.handleExport();
934 case tok::identifier:
936 ImportSeqState.handleImport();
937 if (ImportSeqState.afterImportSeq()) {
939 ModuleImportPath.clear();
940 ModuleImportExpectsIdentifier =
true;
941 CurLexerKind = CLK_LexAfterModuleImport;
947 ImportSeqState.handleMisc();
952 LastTokenWasAt = Result.
is(tok::at);
980 if (FilenameTok.
is(tok::less) && AllowMacroExpansion) {
987 FilenameBuffer.push_back(
'<');
993 while (FilenameTok.
isNot(tok::greater)) {
997 Diag(Start, diag::note_matching) << tok::less;
1004 if (FilenameTok.
is(tok::code_completion)) {
1013 FilenameBuffer.push_back(
' ');
1017 size_t PreAppendSize = FilenameBuffer.size();
1018 FilenameBuffer.resize(PreAppendSize + FilenameTok.
getLength());
1020 const char *BufPtr = &FilenameBuffer[PreAppendSize];
1021 unsigned ActualLen =
getSpelling(FilenameTok, BufPtr);
1024 if (BufPtr != &FilenameBuffer[PreAppendSize])
1025 memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen);
1028 if (FilenameTok.
getLength() != ActualLen)
1029 FilenameBuffer.resize(PreAppendSize + ActualLen);
1033 FilenameTok.
setKind(tok::header_name);
1038 }
else if (FilenameTok.
is(tok::string_literal) && AllowMacroExpansion) {
1049 StringRef Str =
getSpelling(FilenameTok, FilenameBuffer);
1050 if (Str.size() >= 2 && Str.front() ==
'"' && Str.back() ==
'"')
1051 FilenameTok.
setKind(tok::header_name);
1062 unsigned BracketDepth = 0;
1064 Toks.emplace_back();
1067 switch (Toks.back().getKind()) {
1068 case tok::l_paren:
case tok::l_square:
case tok::l_brace:
1072 case tok::r_paren:
case tok::r_square:
case tok::r_brace:
1073 if (BracketDepth == 0)
1079 if (BracketDepth == 0)
1122 if (ModuleImportPath.empty() &&
getLangOpts().CPlusPlusModules) {
1132 auto ToksCopy = llvm::make_unique<Token[]>(Toks.size());
1133 std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
1134 EnterTokenStream(std::move(ToksCopy), Toks.size(),
1140 if (Result.
is(tok::header_name)) {
1144 Suffix.push_back(Result);
1149 if (Suffix.back().isNot(tok::semi)) {
1151 EnterTokens(Suffix);
1160 Diag(SemiLoc, diag::err_header_import_semi_in_macro);
1165 ImportTok.
setKind(tok::kw_import);
1170 auto Action = HandleHeaderIncludeOrImport(
1172 switch (Action.Kind) {
1176 case ImportAction::ModuleBegin:
1178 Suffix.emplace_back();
1179 Suffix.back().startToken();
1180 Suffix.back().setKind(tok::annot_module_begin);
1181 Suffix.back().setLocation(SemiLoc);
1182 Suffix.back().setAnnotationEndLoc(SemiLoc);
1183 Suffix.back().setAnnotationValue(Action.ModuleForHeader);
1186 case ImportAction::ModuleImport:
1187 case ImportAction::SkippedModuleImport:
1190 Suffix[0].setKind(tok::annot_header_unit);
1191 Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
1192 Suffix[0].setAnnotationValue(Action.ModuleForHeader);
1197 EnterTokens(Suffix);
1207 if (ModuleImportExpectsIdentifier && Result.
getKind() == tok::identifier) {
1212 ModuleImportExpectsIdentifier =
false;
1213 CurLexerKind = CLK_LexAfterModuleImport;
1220 if (!ModuleImportExpectsIdentifier && Result.
getKind() == tok::period) {
1221 ModuleImportExpectsIdentifier =
true;
1222 CurLexerKind = CLK_LexAfterModuleImport;
1227 if (ModuleImportPath.empty() || Result.
is(
tok::eof))
1233 if (Result.
isNot(tok::semi)) {
1234 Suffix.push_back(Result);
1236 if (Suffix.back().isNot(tok::semi)) {
1238 EnterTokens(Suffix);
1241 SemiLoc = Suffix.back().getLocation();
1248 std::string FlatModuleName;
1250 for (
auto &Piece : ModuleImportPath) {
1251 if (!FlatModuleName.empty())
1252 FlatModuleName +=
".";
1253 FlatModuleName += Piece.first->getName();
1256 ModuleImportPath.clear();
1257 ModuleImportPath.push_back(
1261 Module *Imported =
nullptr;
1263 Imported = TheModuleLoader.
loadModule(ModuleImportLoc,
1271 Callbacks->moduleImport(ModuleImportLoc, ModuleImportPath, Imported);
1273 if (!Suffix.empty()) {
1274 EnterTokens(Suffix);
1281 CurSubmoduleState->VisibleModules.setVisible(
1286 Diag(ModuleImportLoc, diag::warn_module_conflict)
1287 << Path[0]->getFullModuleName()
1288 << Conflict->getFullModuleName()
1293 if (!BuildingSubmoduleStack.empty() && M != BuildingSubmoduleStack.back().M)
1294 BuildingSubmoduleStack.back().M->Imports.insert(M);
1298 const char *DiagnosticTag,
1299 bool AllowMacroExpansion) {
1301 if (Result.
isNot(tok::string_literal)) {
1302 Diag(Result, diag::err_expected_string_literal)
1303 << 0 << DiagnosticTag;
1310 StrToks.push_back(Result);
1313 Diag(Result, diag::err_invalid_string_udl);
1315 if (AllowMacroExpansion)
1319 }
while (Result.
is(tok::string_literal));
1323 assert(Literal.
isAscii() &&
"Didn't allow wide strings in");
1329 Diag(StrToks[0].getLocation(), diag::err_expected_string_literal)
1330 << 0 << DiagnosticTag;
1339 assert(Tok.
is(tok::numeric_constant));
1341 bool NumberInvalid =
false;
1342 StringRef Spelling =
getSpelling(Tok, IntegerBuffer, &NumberInvalid);
1346 if (Literal.hadError || !Literal.isIntegerLiteral() || Literal.hasUDSuffix())
1348 llvm::APInt APVal(64, 0);
1349 if (Literal.GetIntegerValue(APVal))
1352 Value = APVal.getLimitedValue();
1357 assert(Handler &&
"NULL comment handler");
1358 assert(llvm::find(CommentHandlers, Handler) == CommentHandlers.end() &&
1359 "Comment handler already registered");
1360 CommentHandlers.push_back(Handler);
1364 std::vector<CommentHandler *>::iterator Pos =
1365 llvm::find(CommentHandlers, Handler);
1366 assert(Pos != CommentHandlers.end() &&
"Comment handler not registered");
1367 CommentHandlers.erase(Pos);
1371 bool AnyPendingTokens =
false;
1372 for (std::vector<CommentHandler *>::iterator H = CommentHandlers.begin(),
1373 HEnd = CommentHandlers.end();
1375 if ((*H)->HandleComment(*
this, Comment))
1376 AnyPendingTokens =
true;
void setCodeCompletionTokenRange(const SourceLocation Start, const SourceLocation End)
Set the code completion token range for detecting replacement range later on.
void AddKeywords(const LangOptions &LangOpts)
Populate the identifier table with info about the language keywords for the language specified by Lan...
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
void FinalizeForModelFile()
Cleanup after model file parsing.
void SkipTokensWhileUsingPCH()
Skip tokens until after the #include of the through header or until after a #pragma hdrstop...
IdentifierInfo * getIdentifierInfo(StringRef Name) const
Return information about the specified preprocessor identifier token.
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
Defines the clang::FileManager interface and associated types.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Defines the SourceManager interface.
bool creatingPCHWithPragmaHdrStop()
True if creating a PCH with a #pragma hdrstop.
Defines the clang::Module class, which describes a module in the source code.
Module * getCurrentModule()
Retrieves the module that we're currently building, if any.
Defines the FileSystemStatCache interface.
bool isLoadedFileID(FileID FID) const
Returns true if FID came from a PCH/Module.
void EndSourceFile()
Inform the preprocessor callbacks that processing is complete.
The translation unit is a prefix to a translation unit, and is not complete.
virtual void CodeCompleteNaturalLanguage()
Callback invoked when performing code completion in a part of the file where we expect natural langua...
bool isBeforeInTranslationUnit(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the translation unit.
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
Defines the clang::MacroInfo and clang::MacroDirective classes.
A description of the current definition of a macro.
bool isOutOfDate() const
Determine whether the information for this identifier is out of date with respect to the external sou...
void setFlag(TokenFlags Flag)
Set the specified flag.
StringRef getLastMacroWithSpelling(SourceLocation Loc, ArrayRef< TokenValue > Tokens) const
Return the name of the macro defined before Loc that has spelling Tokens.
void setCodeCompletionIdentifierInfo(IdentifierInfo *Filter)
Set the code completion token for filtering purposes.
void setCodeCompletionReached()
Note that we hit the code-completion point.
Preprocessor(std::shared_ptr< PreprocessorOptions > PPOpts, DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM, HeaderSearch &Headers, ModuleLoader &TheModuleLoader, IdentifierInfoLookup *IILookup=nullptr, bool OwnsHeaderSearch=false, TranslationUnitKind TUKind=TU_Complete)
void createPreprocessingRecord()
Create a new preprocessing record, which will keep track of all macro expansions, macro definitions...
StringRef getBufferData(FileID FID, bool *Invalid=nullptr) const
Return a StringRef to the source buffer data for the specified FileID.
bool parseSimpleIntegerLiteral(Token &Tok, uint64_t &Value)
Parses a simple integer literal to get its numeric value.
tokens_iterator tokens_begin() const
tok::TokenKind getKind() const
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
void SetPoisonReason(IdentifierInfo *II, unsigned DiagID)
Specifies the reason for poisoning an identifier.
One of these records is kept for each identifier that is lexed.
Represents a macro directive exported by a module.
bool LexAfterModuleImport(Token &Result)
Lex a token following the 'import' contextual keyword.
void print(raw_ostream &OS, const SourceManager &SM) const
void setRawIdentifierData(const char *Ptr)
virtual ~ExternalPreprocessorSource()
Token - This structure provides full information about a lexed token.
void setKind(tok::TokenKind K)
virtual void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Callback invoked when performing code completion inside the filename part of an #include directive...
void removeCommentHandler(CommentHandler *Handler)
Remove the specified comment handler.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
const LangOptions & getLangOpts() const
Describes a module or submodule.
bool SetCodeCompletionPoint(const FileEntry *File, unsigned Line, unsigned Column)
Specify the point at which code-completion will be performed.
llvm::Registry< PragmaHandler > PragmaHandlerRegistry
Registry of pragma handlers added by plugins.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
A record of the steps taken while preprocessing a source file, including the various preprocessing di...
HeaderSearch & getHeaderSearchInfo() const
const Token & getReplacementToken(unsigned Tok) const
bool hasUDSuffix() const
Return true if this token is a string or character literal which has a ud-suffix. ...
macro_iterator macro_begin(bool IncludeExternalMacros=true) const
Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified...
void Initialize(const TargetInfo &Target, const TargetInfo *AuxTarget=nullptr)
Initialize the preprocessor using information about the target.
Concrete class used by the front-end to report problems and issues.
SourceLocation getSpellingLoc(SourceLocation Loc) const
Given a SourceLocation object, return the spelling location referenced by the ID. ...
macro_iterator macro_end(bool IncludeExternalMacros=true) const
bool HandleComment(Token &result, SourceRange Comment)
void LexUnexpandedToken(Token &Result)
Just like Lex, but disables macro expansion of identifier tokens.
void recomputeCurLexerKind()
Recompute the current lexer kind based on the CurLexer/ CurTokenLexer pointers.
Provides lookups to, and iteration over, IdentiferInfo objects.
Exposes information about the current target.
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
Defines the clang::LangOptions interface.
void makeModuleVisible(Module *M, SourceLocation Loc)
SourceLocation getLocation() const
bool isInSystemHeader(SourceLocation Loc) const
Returns if a SourceLocation is in a system header.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
const AnnotatedLine * Line
FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, unsigned LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
bool usingPCHWithThroughHeader()
True if using a PCH with a through header.
bool hasLeadingEmptyMacro() const
Return true if this token has an empty macro before it.
virtual void ReadDefinedMacros()=0
Read the set of macros defined by this external macro source.
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
bool isPCHThroughHeader(const FileEntry *FE)
Returns true if the FileEntry is the PCH through header.
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Defines the clang::Preprocessor interface.
void setIsPoisoned(bool Value=true)
setIsPoisoned - Mark this identifier as poisoned.
static bool MacroDefinitionEquals(const MacroInfo *MI, ArrayRef< TokenValue > Tokens)
Compares macro tokens with a specified token value sequence.
size_t getTotalMemory() const
bool getCommentRetentionState() const
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool isObjectLike() const
The result type of a method or function.
StringRef GetString() const
DirectoryLookup - This class represents one entry in the search list that specifies the search order ...
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
SourceManager & getSourceManager() const
void DumpMacro(const MacroInfo &MI) const
void overrideFileContents(const FileEntry *SourceFile, llvm::MemoryBuffer *Buffer, bool DoNotFree)
Override the contents of the given source file by providing an already-allocated buffer.
ExternalPreprocessorSource * getExternalSource() const
bool isExpandDisabled() const
Return true if this identifier token should never be expanded in the future, due to C99 6...
Encodes a location in the source.
void setLength(unsigned Len)
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, bool ExpansionIsTokenRange=true, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
MacroDefinition getMacroDefinition(const IdentifierInfo *II)
All of the names in this module are hidden.
IdentifierInfo * getIdentifierInfo() const
Cached information about one file (either on disk or in the virtual file system). ...
void setIdentifierInfo(IdentifierInfo *II)
void Lex(Token &Result)
Lex the next token for this preprocessor.
bool isPoisoned() const
Return true if this token has been poisoned.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
const FileEntry * LookupFile(SourceLocation FilenameLoc, StringRef Filename, bool isAngled, const DirectoryLookup *FromDir, const FileEntry *FromFile, const DirectoryLookup *&CurDir, SmallVectorImpl< char > *SearchPath, SmallVectorImpl< char > *RelativePath, ModuleMap::KnownHeader *SuggestedModule, bool *IsMapped, bool *IsFrameworkFound, bool SkipCache=false)
Given a "foo" or <foo> reference, look up the indicated file.
void expandUCNs(SmallVectorImpl< char > &Buf, StringRef Input)
Copy characters from Input to Buf, expanding any UCNs.
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
void DumpToken(const Token &Tok, bool DumpFlags=false) const
Print the token to stderr, used for debugging.
StringRef getName() const
Return the actual identifier string.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
bool EnterSourceFile(FileID FID, const DirectoryLookup *Dir, SourceLocation Loc)
Add a source file to the top of the include stack and start lexing tokens from it instead of the curr...
SourceLocation SplitToken(SourceLocation TokLoc, unsigned Length)
Split the first Length characters out of the token starting at TokLoc and return a location pointing ...
Dataflow Directional Tag Classes.
bool isValid() const
Return true if this is a valid SourceLocation object.
bool getFlag(TokenFlags Flag) const
Get the specified flag.
unsigned getNumTokens() const
Return the number of tokens that this macro expands to.
static diag::kind getFutureCompatDiagKind(const IdentifierInfo &II, const LangOptions &LangOpts)
Returns a diagnostic message kind for reporting a future keyword as appropriate for the identifier an...
void addCommentHandler(CommentHandler *Handler)
Add the specified comment handler to the preprocessor.
bool usingPCHWithPragmaHdrStop()
True if using a PCH with a #pragma hdrstop.
const llvm::MemoryBuffer * getMemoryBufferForFile(const FileEntry *File, bool *Invalid=nullptr)
Retrieve the memory buffer associated with the given file.
FileID getMainFileID() const
Returns the FileID of the main source file.
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
MacroInfo * getMacroInfo()
virtual ModuleLoadResult loadModule(SourceLocation ImportLoc, ModuleIdPath Path, Module::NameVisibilityKind Visibility, bool IsInclusionDirective)=0
Attempt to load the given module.
void CodeCompleteNaturalLanguage()
Hook used by the lexer to invoke the "natural language" code completion point.
Abstract interface for a module loader.
unsigned getLength() const
void PoisonSEHIdentifiers(bool Poison=true)
void setLiteralData(const char *Ptr)
Encapsulates the data about a macro definition (e.g.
void CollectPpImportSuffix(SmallVectorImpl< Token > &Toks)
Collect the tokens of a C++20 pp-import-suffix.
PragmaNamespace - This PragmaHandler subdivides the namespace of pragmas, allowing hierarchical pragm...
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
bool FinishLexStringLiteral(Token &Result, std::string &String, const char *DiagnosticTag, bool AllowMacroExpansion)
Complete the lexing of a string literal where the first token has already been lexed (see LexStringLi...
void setConditionalLevels(ArrayRef< PPConditionalInfo > CL)
virtual void updateOutOfDateIdentifier(IdentifierInfo &II)=0
Update an out-of-date identifier.
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
void InitializeForModelFile()
Initialize the preprocessor to parse a model file.
#define CXX11_KEYWORD(NAME, FLAGS)
Defines the clang::SourceLocation class and associated facilities.
FileID getPredefinesFileID() const
Returns the FileID for the preprocessor predefines.
StringRef getRawIdentifier() const
getRawIdentifier - For a raw identifier token (i.e., an identifier lexed in raw mode), returns a reference to the text substring in the buffer if known.
SourceLocation createTokenSplitLoc(SourceLocation SpellingLoc, SourceLocation TokenStart, SourceLocation TokenEnd)
Return a new SourceLocation that encodes that the token starting at TokenStart ends prematurely at To...
TranslationUnitKind
Describes the kind of translation unit being processed.
void DumpLocation(SourceLocation Loc) const
StringLiteralParser - This decodes string escape characters and performs wide string analysis and Tra...
unsigned kind
All of the diagnostics that can be emitted by the frontend.
bool creatingPCHWithThroughHeader()
True if creating a PCH with a through header.
const char * getTokenName(TokenKind Kind) LLVM_READNONE
Determines the name of a token as used within the front end.
Defines the clang::TargetInfo interface.
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
NumericLiteralParser - This performs strict semantic analysis of the content of a ppnumber...
MacroMap::const_iterator macro_iterator
bool isModulesImport() const
Determine whether this is the contextual keyword import.
void CodeCompleteIncludedFile(llvm::StringRef Dir, bool IsAngled)
Hook used by the lexer to invoke the "included file" code completion point.
void setLocation(SourceLocation L)
A trivial tuple used to represent a source range.
bool LexHeaderName(Token &Result, bool AllowMacroExpansion=true)
Lex a token, forming a header-name token if possible.
bool hasUCN() const
Returns true if this token contains a universal character name.
bool isFutureCompatKeyword() const
is/setIsFutureCompatKeyword - Initialize information about whether or not this language token is a ke...
Defines the PreprocessorLexer interface.
void LexIncludeFilename(Token &FilenameTok)
Lex a token, producing a header-name token if possible.
ScratchBuffer - This class exposes a simple interface for the dynamic construction of tokens...
void InitializeTarget(const TargetInfo &Target, const TargetInfo *AuxTarget)
Perform target-specific initialization.
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
void addPPCallbacks(std::unique_ptr< PPCallbacks > C)
This class handles loading and caching of source files into memory.
virtual ~CodeCompletionHandler()
void startToken()
Reset all flags to cleared.
std::pair< FileID, unsigned > getDecomposedLoc(SourceLocation Loc) const
Decompose the specified location into a raw FileID + Offset pair.
SourceLocation getEndLoc() const