27 #include "llvm/ADT/ArrayRef.h" 28 #include "llvm/ADT/SmallString.h" 29 #include "llvm/ADT/SmallVector.h" 30 #include "llvm/ADT/iterator_range.h" 34 using namespace clang;
52 NextTokGetsSpace =
false;
55 DisableMacroExpansion =
false;
64 assert(Tokens[0].getLocation().isValid());
65 assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
66 "Macro defined in macro?");
67 assert(ExpandLocStart.
isValid());
84 ExpandFunctionArguments();
95 bool disableMacroExpansion,
bool ownsTokens,
97 assert(!isReinject || disableMacroExpansion);
103 ActualArgs =
nullptr;
105 OwnsTokens = ownsTokens;
106 DisableMacroExpansion = disableMacroExpansion;
107 IsReinject = isReinject;
111 AtStartOfLine =
false;
112 HasLeadingSpace =
false;
113 NextTokGetsSpace =
false;
124 void TokenLexer::destroy() {
134 if (ActualArgs) ActualArgs->
destroy(PP);
137 bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
147 if (!HasPasteOperator && !PP.
getLangOpts().MSVCCompat)
160 if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
164 if (HasPasteOperator)
165 PP.
Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
168 ResultToks.pop_back();
170 if (!ResultToks.empty()) {
175 if (ResultToks.back().is(tok::hashhash))
176 ResultToks.pop_back();
183 NextTokGetsSpace =
false;
187 void TokenLexer::stringifyVAOPTContents(
191 const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt;
192 Token *
const VAOPTTokens =
193 NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] :
nullptr;
200 for (
unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens;
202 if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) {
203 assert(CurTokenIdx != 0 &&
204 "Can not have __VAOPT__ contents begin with a ##");
205 Token &LHS = VAOPTTokens[CurTokenIdx - 1];
206 pasteTokens(LHS, llvm::makeArrayRef(VAOPTTokens, NumVAOptTokens),
209 ConcatenatedVAOPTResultToks.back() = LHS;
210 if (CurTokenIdx == NumVAOptTokens)
213 ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]);
216 ConcatenatedVAOPTResultToks.push_back(VCtx.
getEOFTok());
225 getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc);
229 ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro);
236 ResultToks.resize(NumToksPriorToVAOpt + 1);
237 ResultToks.back() = StringifiedVAOPT;
242 void TokenLexer::ExpandFunctionArguments() {
248 bool MadeChange =
false;
254 for (
unsigned I = 0, E = NumTokens; I != E; ++I) {
255 const Token &CurTok = Tokens[I];
262 if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.
hasLeadingSpace())
263 NextTokGetsSpace =
true;
267 assert(Tokens[I + 1].is(tok::l_paren) &&
268 "__VA_OPT__ must be followed by '('");
290 if (Tokens[I].is(tok::l_paren))
298 if (!CalledWithVariadicArguments.hasValue()) {
299 CalledWithVariadicArguments =
302 if (!*CalledWithVariadicArguments) {
322 stringifyVAOPTContents(ResultToks, VCtx,
323 Tokens[I].getLocation());
332 if (ResultToks.size() && ResultToks.back().is(tok::hashhash)) {
333 ResultToks.pop_back();
334 }
else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) {
347 "no token paste before __VA_OPT__");
348 ResultToks.erase(ResultToks.begin() +
354 Tokens[I + 1].
is(tok::hashhash)) {
369 if (CurTok.
isOneOf(tok::hash, tok::hashat)) {
371 assert((ArgNo != -1 || VCtx.
isVAOptToken(Tokens[I + 1])) &&
372 "Token following # is not an argument or __VA_OPT__!");
377 CurTok.
is(tok::hashat));
382 getExpansionLocForMacroDefLoc(CurTok.
getLocation());
384 getExpansionLocForMacroDefLoc(Tokens[I+1].getLocation());
387 if (CurTok.
is(tok::hash))
402 if (NextTokGetsSpace)
405 ResultToks.push_back(Res);
408 NextTokGetsSpace =
false;
413 bool NonEmptyPasteBefore =
414 !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
415 bool PasteBefore = I != 0 && Tokens[I-1].
is(tok::hashhash);
416 bool PasteAfter = I+1 != E && Tokens[I+1].
is(tok::hashhash);
417 bool RParenAfter = I+1 != E && Tokens[I+1].
is(tok::r_paren);
419 assert((!NonEmptyPasteBefore || PasteBefore || VCtx.
isInVAOpt()) &&
420 "unexpected ## in ResultToks");
428 ResultToks.push_back(CurTok);
430 if (NextTokGetsSpace) {
432 NextTokGetsSpace =
false;
433 }
else if (PasteBefore && !NonEmptyPasteBefore)
448 MaybeRemoveCommaBeforeVaArgs(ResultToks,
456 if (!PasteBefore && !PasteAfter) {
457 const Token *ResultArgToks;
465 ResultArgToks = ArgTok;
469 size_t FirstResult = ResultToks.size();
471 ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
478 ResultToks.back().is(tok::comma))
483 for (
Token &
Tok : llvm::make_range(ResultToks.begin() + FirstResult,
485 if (
Tok.
is(tok::hashhash))
486 Tok.setKind(tok::unknown);
491 ResultToks.begin()+FirstResult,
501 NextTokGetsSpace =
false;
506 if (NonEmptyPasteBefore) {
509 assert(VCtx.
isInVAOpt() &&
"should only happen inside a __VA_OPT__");
523 bool VaArgsPseudoPaste =
false;
528 if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
529 ResultToks[ResultToks.size()-2].is(tok::comma) &&
532 VaArgsPseudoPaste =
true;
534 PP.
Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
537 ResultToks.append(ArgToks, ArgToks+NumToks);
541 for (
Token &
Tok : llvm::make_range(ResultToks.end() - NumToks,
543 if (
Tok.
is(tok::hashhash))
544 Tok.setKind(tok::unknown);
547 if (ExpandLocStart.
isValid()) {
549 ResultToks.end()-NumToks, ResultToks.end());
556 if (!VaArgsPseudoPaste) {
559 ResultToks[ResultToks.size() - NumToks].setFlagValue(
563 NextTokGetsSpace =
false;
585 if (NonEmptyPasteBefore) {
586 assert(ResultToks.back().is(tok::hashhash));
593 ResultToks.pop_back();
603 MaybeRemoveCommaBeforeVaArgs(ResultToks,
610 assert(!OwnsTokens &&
"This would leak if we already own the token list");
612 NumTokens = ResultToks.size();
615 Tokens = PP.cacheMacroExpandedTokens(
this, ResultToks);
624 const Token &SecondTok) {
625 return FirstTok.
is(tok::identifier) &&
641 if (CurTokenIdx == 0)
650 bool isFirstToken = CurTokenIdx == 0;
653 Tok = Tokens[CurTokenIdx++];
657 bool TokenIsFromPaste =
false;
661 if (!isAtEnd() && Macro &&
662 (Tokens[CurTokenIdx].is(tok::hashhash) ||
670 if (pasteTokens(Tok))
673 TokenIsFromPaste =
true;
681 if (ExpandLocStart.
isValid() &&
685 if (Tok.
is(tok::comment)) {
691 instLoc = getExpansionLocForMacroDefLoc(Tok.
getLocation());
708 AtStartOfLine =
false;
709 HasLeadingSpace =
false;
733 bool TokenLexer::pasteTokens(
Token &
Tok) {
734 return pasteTokens(Tok, llvm::makeArrayRef(Tokens, NumTokens), CurTokenIdx);
742 unsigned int &CurIdx) {
743 assert(CurIdx > 0 &&
"## can not be the first token within tokens");
744 assert((TokenStream[CurIdx].is(tok::hashhash) ||
747 "Token at this Index must be ## or part of the MSVC 'L " 748 "#macro-arg' pasting pair");
753 if (PP.
getLangOpts().MicrosoftExt && (CurIdx >= 2) &&
754 TokenStream[CurIdx - 2].is(tok::hashhash))
758 const char *ResultTokStrPtr =
nullptr;
762 auto IsAtEnd = [&TokenStream, &CurIdx] {
763 return TokenStream.size() == CurIdx;
768 PasteOpLoc = TokenStream[CurIdx].getLocation();
769 if (TokenStream[CurIdx].is(tok::hashhash))
771 assert(!IsAtEnd() &&
"No token on the RHS of a paste operator!");
774 const Token &RHS = TokenStream[CurIdx];
781 const char *BufPtr = &Buffer[0];
782 bool Invalid =
false;
783 unsigned LHSLen = PP.
getSpelling(LHSTok, BufPtr, &Invalid);
784 if (BufPtr != &Buffer[0])
785 memcpy(&Buffer[0], BufPtr, LHSLen);
789 BufPtr = Buffer.data() + LHSLen;
790 unsigned RHSLen = PP.
getSpelling(RHS, BufPtr, &Invalid);
793 if (RHSLen && BufPtr != &Buffer[LHSLen])
795 memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
798 Buffer.resize(LHSLen+RHSLen);
807 ResultTokTmp.
setKind(tok::string_literal);
820 Result.
setKind(tok::raw_identifier);
828 "Should be a raw location into scratch buffer");
830 FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);
832 bool Invalid =
false;
833 const char *ScratchBufStart
834 = SourceMgr.getBufferData(LocFileID, &Invalid).data();
840 Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),
842 ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
848 bool isInvalid = !TL.LexFromRawLexer(Result);
866 if (PP.
getLangOpts().MicrosoftExt && LHSTok.
is(tok::slash) &&
867 RHS.
is(tok::slash)) {
868 HandleMicrosoftCommentPaste(LHSTok, Loc);
878 : diag::err_pp_bad_paste)
888 if (Result.
is(tok::hashhash))
899 }
while (!IsAtEnd() && TokenStream[CurIdx].is(tok::hashhash));
910 StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
912 EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
914 while (SM.
getFileID(StartLoc) != MacroFID)
925 if (LHSTok.
is(tok::raw_identifier)) {
940 return Tokens[CurTokenIdx].
is(tok::l_paren);
946 return Tokens[NumTokens-1].
is(tok::eod) && !isAtEnd();
955 PP.
Diag(OpLoc, diag::ext_comment_paste_microsoft);
962 assert(Macro &&
"Token streams can't paste comments");
973 TokenLexer::getExpansionLocForMacroDefLoc(
SourceLocation loc)
const {
974 assert(ExpandLocStart.
isValid() && MacroExpansionStart.
isValid() &&
975 "Not appropriate for token streams");
979 assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
980 "Expected loc to come from the macro definition");
982 unsigned relativeOffset = 0;
998 Token *&begin_tokens,
999 Token * end_tokens) {
1000 assert(begin_tokens < end_tokens);
1015 Token *NextTok = begin_tokens + 1;
1016 for (; NextTok < end_tokens; ++NextTok) {
1026 if (RelOffs < 0 || RelOffs > 50)
1037 Token &LastConsecutiveTok = *(NextTok-1);
1038 int LastRelOffs = 0;
1041 unsigned FullLength = LastRelOffs + LastConsecutiveTok.
getLength();
1049 for (; begin_tokens < NextTok; ++begin_tokens) {
1050 Token &Tok = *begin_tokens;
1062 void TokenLexer::updateLocForMacroArgTokens(
SourceLocation ArgIdSpellLoc,
1063 Token *begin_tokens,
1064 Token *end_tokens) {
1068 getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
1070 while (begin_tokens < end_tokens) {
1072 if (end_tokens - begin_tokens == 1) {
1073 Token &Tok = *begin_tokens;
1084 void TokenLexer::PropagateLineStartLeadingSpaceInfo(
Token &
Result) {
bool isWrittenInSameFile(SourceLocation Loc1, SourceLocation Loc2) const
Returns true if the spelling locations for both SourceLocations are part of the same file buffer...
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
void sawOpeningParen(SourceLocation LParenLoc)
Call this function each time an lparen is seen.
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
bool ArgNeedsPreexpansion(const Token *ArgTok, Preprocessor &PP) const
ArgNeedsPreexpansion - If we can prove that the argument won't be affected by pre-expansion, return false.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
bool sawClosingParen()
Call this function each time an rparen is seen.
Defines the SourceManager interface.
static void updateConsecutiveMacroArgTokens(SourceManager &SM, SourceLocation InstLoc, Token *&begin_tokens, Token *end_tokens)
Finds the tokens that are consecutive (from the same FileID) creates a single SLocEntry, and assigns SourceLocations to each token that point to that SLocEntry.
A class for tracking whether we're inside a VA_OPT during a traversal of the tokens of a macro during...
bool hasCharifyBefore() const
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool isBeforeInSLocAddrSpace(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the "source location address space".
void setFlag(TokenFlags Flag)
Set the specified flag.
bool isVarargsElidedUse() const
isVarargsElidedUse - Return true if this is a C99 style varargs macro invocation and there was no arg...
bool isInSameSLocAddrSpace(SourceLocation LHS, SourceLocation RHS, int *RelativeOffset) const
Return true if both LHS and RHS are in the local source location address space or the loaded one...
bool isAnyIdentifier() const
Return true if this is a raw identifier (when lexing in raw mode) or a non-keyword identifier (when l...
unsigned getNextLocalOffset() const
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
bool getLeadingSpaceForStringifiedToken() const
tokens_iterator tokens_begin() const
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
bool hasStringifyOrCharifyBefore() const
One of these records is kept for each identifier that is lexed.
const Token & getEOFTok() const
SourceLocation getBegin() const
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
void setRawIdentifierData(const char *Ptr)
void sawVAOptFollowedByOpeningParens(const SourceLocation VAOptLoc, const unsigned int NumPriorTokens)
Token - This structure provides full information about a lexed token.
void setKind(tok::TokenKind K)
const LangOptions & getLangOpts() const
bool Lex(Token &Tok)
Lex and return a token from this macro stream.
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
SourceLocation getExpansionLoc(SourceLocation Loc) const
Given a SourceLocation object Loc, return the expansion location referenced by the ID...
Defines the Diagnostic-related interfaces.
void destroy(Preprocessor &PP)
destroy - Destroy and deallocate the memory for this object.
void Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, MacroArgs *Actuals)
Initialize this TokenLexer to expand from the specified macro with the specified argument information...
const Token & getStringifiedArgument(unsigned ArgNo, Preprocessor &PP, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
getStringifiedArgument - Compute, cache, and return the specified argument that has been 'stringified...
static bool isWideStringLiteralFromMacro(const Token &FirstTok, const Token &SecondTok)
Checks if two tokens form wide string literal.
static Token StringifyArgument(const Token *ArgToks, Preprocessor &PP, bool Charify, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of tokens into the literal string...
unsigned getNumParams() const
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
bool invokedWithVariadicArgument(const MacroInfo *const MI, Preprocessor &PP)
Returns true if the macro was defined with a variadic (ellipsis) parameter AND was invoked with at le...
Defines the clang::LangOptions interface.
bool isInSLocAddrSpace(SourceLocation Loc, SourceLocation Start, unsigned Length, unsigned *RelativeOffset=nullptr) const
Returns true if Loc is inside the [Start, +Length) chunk of the source location address space...
SourceLocation getVAOptLoc() const
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
SourceLocation createMacroArgExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLoc, unsigned TokLength)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Defines the clang::Preprocessor interface.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
CharSourceRange getImmediateExpansionRange(SourceLocation Loc) const
Return the start/end of the expansion information for an expansion location.
The result type of a method or function.
void HandlePoisonedIdentifier(Token &Identifier)
Display reason for poisoned identifier.
SourceManager & getSourceManager() const
bool isVAOptToken(const Token &T) const
Encodes a location in the source.
void setLength(unsigned Len)
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, bool ExpansionIsTokenRange=true, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
bool beginsWithPlaceholder() const
IdentifierInfo * getIdentifierInfo() const
bool isInVAOpt() const
Returns true if we have seen the VA_OPT and '(' but before having seen the matching ')'...
void hasPlaceholderBeforeRParen()
bool isPoisoned() const
Return true if this token has been poisoned.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
__DEVICE__ void * memcpy(void *__a, const void *__b, size_t __c)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
static bool isInvalid(LocType Loc, bool *Invalid)
Dataflow Directional Tag Classes.
bool isValid() const
Return true if this is a valid SourceLocation object.
bool isFunctionLike() const
const std::vector< Token > & getPreExpArgument(unsigned Arg, Preprocessor &PP)
getPreExpArgument - Return the pre-expanded form of the specified argument.
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier...
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
tokens_iterator tokens_end() const
static unsigned getArgLength(const Token *ArgPtr)
getArgLength - Given a pointer to an expanded or unexpanded argument, return the number of tokens...
void hasPlaceholderAfterHashhashAtStart()
bool endsWithPlaceholder() const
void sawHashOrHashAtBefore(const bool HasLeadingSpace, const bool IsHashAt)
unsigned getLength() const
Encapsulates the data about a macro definition (e.g.
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
FileID getFileID(SourceLocation SpellingLoc) const
Return the FileID for a SourceLocation.
int getParameterNum(const IdentifierInfo *Arg) const
Return the parameter number of the specified identifier, or -1 if the identifier is not a formal para...
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
unsigned int getNumberOfTokensPriorToVAOpt() const
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
SourceLocation getEnd() const
unsigned isNextTokenLParen() const
If the next token lexed will pop this macro off the expansion stack, return 2.
Defines the clang::TokenKind enum and support functions.
unsigned getDefinitionLength(const SourceManager &SM) const
Get length in characters of the macro definition.
Defines the clang::SourceLocation class and associated facilities.
bool stringifiedInMacro() const
Returns true if this token is formed by macro by stringizing or charizing operator.
void setLocation(SourceLocation L)
void clearFlag(TokenFlags Flag)
Unset the specified flag.
bool isParsingPreprocessorDirective() const
isParsingPreprocessorDirective - Return true if we are in the middle of a preprocessor directive...
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
This class handles loading and caching of source files into memory.
const Token * getUnexpArgument(unsigned Arg) const
getUnexpArgument - Return a pointer to the first token of the unexpanded token list for the specified...
void startToken()
Reset all flags to cleared.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.