20 #include "llvm/ADT/SmallString.h"
22 using namespace clang;
40 NextTokGetsSpace =
false;
43 DisableMacroExpansion =
false;
51 assert(Tokens[0].getLocation().isValid());
52 assert((Tokens[0].getLocation().isFileID() || Tokens[0].is(tok::comment)) &&
53 "Macro defined in macro?");
54 assert(ExpandLocStart.
isValid());
71 ExpandFunctionArguments();
82 bool disableMacroExpansion,
bool ownsTokens) {
90 OwnsTokens = ownsTokens;
91 DisableMacroExpansion = disableMacroExpansion;
95 AtStartOfLine =
false;
96 HasLeadingSpace =
false;
97 NextTokGetsSpace =
false;
108 void TokenLexer::destroy() {
118 if (ActualArgs) ActualArgs->
destroy(PP);
121 bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
131 if (!HasPasteOperator && !PP.
getLangOpts().MSVCCompat)
144 if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
148 if (HasPasteOperator)
149 PP.
Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
152 ResultToks.pop_back();
154 if (!ResultToks.empty()) {
159 if (ResultToks.back().is(tok::hashhash))
160 ResultToks.pop_back();
167 NextTokGetsSpace =
false;
173 void TokenLexer::ExpandFunctionArguments() {
179 bool MadeChange =
false;
181 for (
unsigned i = 0, e = NumTokens; i != e; ++i) {
185 const Token &CurTok = Tokens[i];
192 if (i != 0 && !Tokens[i-1].is(tok::hashhash) && CurTok.
hasLeadingSpace())
193 NextTokGetsSpace =
true;
195 if (CurTok.
isOneOf(tok::hash, tok::hashat)) {
197 assert(ArgNo != -1 &&
"Token following # is not an argument?");
200 getExpansionLocForMacroDefLoc(CurTok.
getLocation());
202 getExpansionLocForMacroDefLoc(Tokens[i+1].getLocation());
205 if (CurTok.
is(tok::hash))
220 if (NextTokGetsSpace)
223 ResultToks.push_back(Res);
226 NextTokGetsSpace =
false;
231 bool NonEmptyPasteBefore =
232 !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
233 bool PasteBefore = i != 0 && Tokens[i-1].
is(tok::hashhash);
234 bool PasteAfter = i+1 != e && Tokens[i+1].
is(tok::hashhash);
235 assert(!NonEmptyPasteBefore || PasteBefore);
243 ResultToks.push_back(CurTok);
245 if (NextTokGetsSpace) {
247 NextTokGetsSpace =
false;
248 }
else if (PasteBefore && !NonEmptyPasteBefore)
263 MaybeRemoveCommaBeforeVaArgs(ResultToks,
271 if (!PasteBefore && !PasteAfter) {
272 const Token *ResultArgToks;
280 ResultArgToks = ArgTok;
284 size_t FirstResult = ResultToks.size();
286 ResultToks.append(ResultArgToks, ResultArgToks+NumToks);
293 ResultToks.back().is(tok::comma))
298 for (
Token &Tok : llvm::make_range(ResultToks.begin() + FirstResult,
300 if (Tok.is(tok::hashhash))
301 Tok.setKind(tok::unknown);
306 ResultToks.begin()+FirstResult,
316 NextTokGetsSpace =
false;
326 bool VaArgsPseudoPaste =
false;
331 if (NonEmptyPasteBefore && ResultToks.size() >= 2 &&
332 ResultToks[ResultToks.size()-2].is(tok::comma) &&
335 VaArgsPseudoPaste =
true;
337 PP.
Diag(ResultToks.pop_back_val().getLocation(), diag::ext_paste_comma);
340 ResultToks.append(ArgToks, ArgToks+NumToks);
344 for (
Token &Tok : llvm::make_range(ResultToks.end() - NumToks,
346 if (Tok.is(tok::hashhash))
347 Tok.setKind(tok::unknown);
350 if (ExpandLocStart.
isValid()) {
352 ResultToks.end()-NumToks, ResultToks.end());
359 if (!VaArgsPseudoPaste) {
362 ResultToks[ResultToks.size() - NumToks].setFlagValue(
366 NextTokGetsSpace =
false;
385 if (NonEmptyPasteBefore) {
386 assert(ResultToks.back().is(tok::hashhash));
387 ResultToks.pop_back();
395 MaybeRemoveCommaBeforeVaArgs(ResultToks,
402 assert(!OwnsTokens &&
"This would leak if we already own the token list");
404 NumTokens = ResultToks.size();
407 Tokens = PP.cacheMacroExpandedTokens(
this, ResultToks);
416 const Token &SecondTok) {
417 return FirstTok.
is(tok::identifier) &&
443 bool isFirstToken = CurToken == 0;
446 Tok = Tokens[CurToken++];
448 bool TokenIsFromPaste =
false;
452 if (!isAtEnd() && Macro &&
453 (Tokens[CurToken].is(tok::hashhash) ||
461 if (PasteTokens(Tok))
464 TokenIsFromPaste =
true;
472 if (ExpandLocStart.
isValid() &&
476 if (Tok.
is(tok::comment)) {
482 instLoc = getExpansionLocForMacroDefLoc(Tok.
getLocation());
499 AtStartOfLine =
false;
500 HasLeadingSpace =
false;
528 bool TokenLexer::PasteTokens(
Token &Tok) {
532 if (PP.
getLangOpts().MicrosoftExt && (CurToken >= 2) &&
533 Tokens[CurToken - 2].is(tok::hashhash))
537 const char *ResultTokStrPtr =
nullptr;
543 if (Tokens[CurToken].is(tok::hashhash))
545 assert(!isAtEnd() &&
"No token on the RHS of a paste operator!");
548 const Token &RHS = Tokens[CurToken];
555 const char *BufPtr = &Buffer[0];
556 bool Invalid =
false;
557 unsigned LHSLen = PP.
getSpelling(Tok, BufPtr, &Invalid);
558 if (BufPtr != &Buffer[0])
559 memcpy(&Buffer[0], BufPtr, LHSLen);
563 BufPtr = Buffer.data() + LHSLen;
564 unsigned RHSLen = PP.
getSpelling(RHS, BufPtr, &Invalid);
567 if (RHSLen && BufPtr != &Buffer[LHSLen])
569 memcpy(&Buffer[LHSLen], BufPtr, RHSLen);
572 Buffer.resize(LHSLen+RHSLen);
581 ResultTokTmp.
setKind(tok::string_literal);
594 Result.
setKind(tok::raw_identifier);
602 "Should be a raw location into scratch buffer");
604 FileID LocFileID = SourceMgr.getFileID(ResultTokLoc);
606 bool Invalid =
false;
607 const char *ScratchBufStart
608 = SourceMgr.getBufferData(LocFileID, &Invalid).data();
614 Lexer TL(SourceMgr.getLocForStartOfFile(LocFileID),
616 ResultTokStrPtr, ResultTokStrPtr+LHSLen+RHSLen);
622 bool isInvalid = !TL.LexFromRawLexer(Result);
641 RHS.
is(tok::slash)) {
642 HandleMicrosoftCommentPaste(Tok, Loc);
652 : diag::err_pp_bad_paste)
662 if (Result.
is(tok::hashhash))
673 }
while (!isAtEnd() && Tokens[CurToken].
is(tok::hashhash));
684 StartLoc = getExpansionLocForMacroDefLoc(StartLoc);
686 EndLoc = getExpansionLocForMacroDefLoc(EndLoc);
688 while (SM.
getFileID(StartLoc) != MacroFID)
699 if (Tok.
is(tok::raw_identifier)) {
714 return Tokens[CurToken].
is(tok::l_paren);
720 return Tokens[NumTokens-1].
is(tok::eod) && !isAtEnd();
729 PP.
Diag(OpLoc, diag::ext_comment_paste_microsoft);
736 assert(Macro &&
"Token streams can't paste comments");
747 TokenLexer::getExpansionLocForMacroDefLoc(
SourceLocation loc)
const {
748 assert(ExpandLocStart.
isValid() && MacroExpansionStart.
isValid() &&
749 "Not appropriate for token streams");
753 assert(SM.isInSLocAddrSpace(loc, MacroDefStart, MacroDefLength) &&
754 "Expected loc to come from the macro definition");
756 unsigned relativeOffset = 0;
772 Token *&begin_tokens,
773 Token * end_tokens) {
774 assert(begin_tokens < end_tokens);
789 Token *NextTok = begin_tokens + 1;
790 for (; NextTok < end_tokens; ++NextTok) {
800 if (RelOffs < 0 || RelOffs > 50)
811 Token &LastConsecutiveTok = *(NextTok-1);
815 unsigned FullLength = LastRelOffs + LastConsecutiveTok.
getLength();
823 for (; begin_tokens < NextTok; ++begin_tokens) {
824 Token &Tok = *begin_tokens;
837 void TokenLexer::updateLocForMacroArgTokens(
SourceLocation ArgIdSpellLoc,
843 getExpansionLocForMacroDefLoc(ArgIdSpellLoc);
845 while (begin_tokens < end_tokens) {
847 if (end_tokens - begin_tokens == 1) {
848 Token &Tok = *begin_tokens;
859 void TokenLexer::PropagateLineStartLeadingSpaceInfo(
Token &Result) {
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
SourceManager & getSourceManager() const
bool isPoisoned() const
Return true if this token has been poisoned.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
void Init(Token &Tok, SourceLocation ILEnd, MacroInfo *MI, MacroArgs *ActualArgs)
Init - Initialize this TokenLexer to expand from the specified macro with the specified argument info...
void setFlagValue(TokenFlags Flag, bool Val)
Set a flag to either true or false.
unsigned isNextTokenLParen() const
isNextTokenLParen - If the next token lexed will pop this macro off the expansion stack...
Defines the SourceManager interface.
unsigned getNextLocalOffset() const
static void updateConsecutiveMacroArgTokens(SourceManager &SM, SourceLocation InstLoc, Token *&begin_tokens, Token *end_tokens)
Finds the tokens that are consecutive (from the same FileID) creates a single SLocEntry, and assigns SourceLocations to each token that point to that SLocEntry.
bool isParsingPreprocessorDirective() const
isParsingPreprocessorDirective - Return true if we are in the middle of a preprocessor directive...
Defines the clang::MacroInfo and clang::MacroDirective classes.
bool hasLeadingSpace() const
Return true if this token has whitespace before it.
std::unique_ptr< llvm::MemoryBuffer > Buffer
void setFlag(TokenFlags Flag)
Set the specified flag.
bool isVarargsElidedUse() const
isVarargsElidedUse - Return true if this is a C99 style varargs macro invocation and there was no arg...
bool isAnyIdentifier() const
Return true if this is a raw identifier (when lexing in raw mode) or a non-keyword identifier (when l...
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool HandleEndOfTokenLexer(Token &Result)
Callback invoked when the current TokenLexer hits the end of its token stream.
One of these records is kept for each identifier that is lexed.
bool stringifiedInMacro() const
Returns true if this token is formed by macro by stringizing or charizing operator.
void setRawIdentifierData(const char *Ptr)
const LangOptions & getLangOpts() const
Token - This structure provides full information about a lexed token.
unsigned getDefinitionLength(const SourceManager &SM) const
Get length in characters of the macro definition.
void setKind(tok::TokenKind K)
bool Lex(Token &Tok)
Lex - Lex and return a token from this macro stream.
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
const std::vector< Token > & getPreExpArgument(unsigned Arg, const MacroInfo *MI, Preprocessor &PP)
getPreExpArgument - Return the pre-expanded form of the specified argument.
void destroy(Preprocessor &PP)
destroy - Destroy and deallocate the memory for this object.
const Token * getUnexpArgument(unsigned Arg) const
getUnexpArgument - Return a pointer to the first token of the unexpanded token list for the specified...
tokens_iterator tokens_begin() const
unsigned getNumParams() const
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
const Token & getStringifiedArgument(unsigned ArgNo, Preprocessor &PP, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
getStringifiedArgument - Compute, cache, and return the specified argument that has been 'stringified...
static bool isWideStringLiteralFromMacro(const Token &FirstTok, const Token &SecondTok)
Checks if two tokens form wide string literal.
FileID getFileID(SourceLocation SpellingLoc) const
Return the FileID for a SourceLocation.
static Token StringifyArgument(const Token *ArgToks, Preprocessor &PP, bool Charify, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd)
StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of tokens into the literal string...
bool isWrittenInSameFile(SourceLocation Loc1, SourceLocation Loc2) const
Returns true if the spelling locations for both SourceLocations are part of the same file buffer...
MacroArgs - An instance of this class captures information about the formal arguments specified to a ...
SourceLocation createMacroArgExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLoc, unsigned TokLength)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
Defines the clang::Preprocessor interface.
void HandleMicrosoftCommentPaste(Token &Tok)
When the macro expander pastes together a comment (/##/) in Microsoft mode, this method handles updat...
void IncrementPasteCounter(bool isFast)
Increment the counters for the number of token paste operations performed.
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
bool isNot(tok::TokenKind K) const
SourceLocation createExpansionLoc(SourceLocation Loc, SourceLocation ExpansionLocStart, SourceLocation ExpansionLocEnd, unsigned TokLength, int LoadedID=0, unsigned LoadedOffset=0)
Return a new SourceLocation that encodes the fact that a token from SpellingLoc should actually be re...
The result type of a method or function.
bool isBeforeInSLocAddrSpace(SourceLocation LHS, SourceLocation RHS) const
Determines the order of 2 source locations in the "source location address space".
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier...
Encodes a location in the source.
void setLength(unsigned Len)
bool isValid() const
Return true if this is a valid SourceLocation object.
bool ArgNeedsPreexpansion(const Token *ArgTok, Preprocessor &PP) const
ArgNeedsPreexpansion - If we can prove that the argument won't be affected by pre-expansion, return false.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
bool isStr(const char(&Str)[StrLen]) const
Return true if this is the identifier for the specified string.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
static bool isInvalid(LocType Loc, bool *Invalid)
bool isInSLocAddrSpace(SourceLocation Loc, SourceLocation Start, unsigned Length, unsigned *RelativeOffset=nullptr) const
Returns true if Loc is inside the [Start, +Length) chunk of the source location address space...
std::pair< SourceLocation, SourceLocation > getImmediateExpansionRange(SourceLocation Loc) const
Return the start/end of the expansion information for an expansion location.
static unsigned getArgLength(const Token *ArgPtr)
getArgLength - Given a pointer to an expanded or unexpanded argument, return the number of tokens...
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
bool isFunctionLike() const
Encapsulates the data about a macro definition (e.g.
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
bool isInSameSLocAddrSpace(SourceLocation LHS, SourceLocation RHS, int *RelativeOffset) const
Return true if both LHS and RHS are in the local source location address space or the loaded one...
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
void CreateString(StringRef Str, Token &Tok, SourceLocation ExpansionLocStart=SourceLocation(), SourceLocation ExpansionLocEnd=SourceLocation())
Plop the specified string into a scratch buffer and set the specified token's location and length to ...
tokens_iterator tokens_end() const
void HandlePoisonedIdentifier(Token &Tok)
Display reason for poisoned identifier.
IdentifierInfo * LookUpIdentifierInfo(Token &Identifier) const
Given a tok::raw_identifier token, look up the identifier information for the token and install it in...
int getParameterNum(const IdentifierInfo *Arg) const
Return the parameter number of the specified identifier, or -1 if the identifier is not a formal para...
unsigned getLength() const
void setLocation(SourceLocation L)
void clearFlag(TokenFlags Flag)
Unset the specified flag.
SourceLocation getExpansionLoc(SourceLocation Loc) const
Given a SourceLocation object Loc, return the expansion location referenced by the ID...
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
This class handles loading and caching of source files into memory.
void startToken()
Reset all flags to cleared.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.
IdentifierInfo * getIdentifierInfo() const