17 #include "llvm/Support/ErrorHandling.h" 18 using namespace clang;
26 (CPlusPlus11 && (Str[0] ==
'u' || Str[0] ==
'U' || Str[0] ==
'R'))) {
33 if (Str[1] ==
'R' && Str[0] !=
'R' && Str.size() == 2 &&
CPlusPlus11)
37 if (Str[0] ==
'u' && Str[1] ==
'8') {
38 if (Str.size() == 2)
return true;
39 if (Str.size() == 3 && Str[2] ==
'R')
return true;
48 bool TokenConcatenation::IsIdentifierStringPrefix(
const Token &
Tok)
const {
57 LangOpts.CPlusPlus11);
62 const char *TokPtr = Buffer;
64 return IsStringPrefix(StringRef(TokPtr, length), LangOpts.CPlusPlus11);
71 memset(TokenInfo, 0,
sizeof(TokenInfo));
74 TokenInfo[tok::identifier ] |= aci_custom;
75 TokenInfo[tok::numeric_constant] |= aci_custom_firstchar;
76 TokenInfo[tok::period ] |= aci_custom_firstchar;
77 TokenInfo[tok::amp ] |= aci_custom_firstchar;
78 TokenInfo[tok::plus ] |= aci_custom_firstchar;
79 TokenInfo[tok::minus ] |= aci_custom_firstchar;
80 TokenInfo[tok::slash ] |= aci_custom_firstchar;
81 TokenInfo[tok::less ] |= aci_custom_firstchar;
82 TokenInfo[tok::greater ] |= aci_custom_firstchar;
83 TokenInfo[tok::pipe ] |= aci_custom_firstchar;
84 TokenInfo[tok::percent ] |= aci_custom_firstchar;
85 TokenInfo[tok::colon ] |= aci_custom_firstchar;
86 TokenInfo[tok::hash ] |= aci_custom_firstchar;
87 TokenInfo[tok::arrow ] |= aci_custom_firstchar;
91 TokenInfo[tok::string_literal ] |= aci_custom;
92 TokenInfo[tok::wide_string_literal ] |= aci_custom;
93 TokenInfo[tok::utf8_string_literal ] |= aci_custom;
94 TokenInfo[tok::utf16_string_literal] |= aci_custom;
95 TokenInfo[tok::utf32_string_literal] |= aci_custom;
96 TokenInfo[tok::char_constant ] |= aci_custom;
97 TokenInfo[tok::wide_char_constant ] |= aci_custom;
98 TokenInfo[tok::utf16_char_constant ] |= aci_custom;
99 TokenInfo[tok::utf32_char_constant ] |= aci_custom;
104 TokenInfo[tok::utf8_char_constant] |= aci_custom;
108 TokenInfo[tok::lessequal ] |= aci_custom_firstchar;
111 TokenInfo[tok::amp ] |= aci_avoid_equal;
112 TokenInfo[tok::plus ] |= aci_avoid_equal;
113 TokenInfo[tok::minus ] |= aci_avoid_equal;
114 TokenInfo[tok::slash ] |= aci_avoid_equal;
115 TokenInfo[tok::less ] |= aci_avoid_equal;
116 TokenInfo[tok::greater ] |= aci_avoid_equal;
117 TokenInfo[tok::pipe ] |= aci_avoid_equal;
118 TokenInfo[tok::percent ] |= aci_avoid_equal;
119 TokenInfo[tok::star ] |= aci_avoid_equal;
120 TokenInfo[tok::exclaim ] |= aci_avoid_equal;
121 TokenInfo[tok::lessless ] |= aci_avoid_equal;
122 TokenInfo[tok::greatergreater] |= aci_avoid_equal;
123 TokenInfo[tok::caret ] |= aci_avoid_equal;
124 TokenInfo[tok::equal ] |= aci_avoid_equal;
132 return II->getNameStart()[0];
142 const char *TokPtr = Buffer;
162 const Token &PrevTok,
163 const Token &Tok)
const {
175 PrevKind = tok::identifier;
178 unsigned ConcatInfo = TokenInfo[PrevKind];
181 if (ConcatInfo == 0)
return false;
183 if (ConcatInfo & aci_avoid_equal) {
185 if (Tok.
isOneOf(tok::equal, tok::equalequal))
187 ConcatInfo &= ~aci_avoid_equal;
191 assert(Tok.
isOneOf(tok::annot_module_include, tok::annot_module_begin,
192 tok::annot_module_end) &&
193 "unexpected annotation in AvoidConcat");
204 if (ConcatInfo & aci_custom) {
212 llvm_unreachable(
"InitAvoidConcatTokenInfo built wrong");
214 case tok::raw_identifier:
215 llvm_unreachable(
"tok::raw_identifier in non-raw lexing mode!");
217 case tok::string_literal:
218 case tok::wide_string_literal:
219 case tok::utf8_string_literal:
220 case tok::utf16_string_literal:
221 case tok::utf32_string_literal:
222 case tok::char_constant:
223 case tok::wide_char_constant:
224 case tok::utf8_char_constant:
225 case tok::utf16_char_constant:
226 case tok::utf32_char_constant:
240 case tok::identifier:
242 if (Tok.
is(tok::numeric_constant))
246 Tok.
isOneOf(tok::wide_string_literal, tok::utf8_string_literal,
247 tok::utf16_string_literal, tok::utf32_string_literal,
248 tok::wide_char_constant, tok::utf8_char_constant,
249 tok::utf16_char_constant, tok::utf32_char_constant))
253 if (Tok.
isNot(tok::char_constant) && Tok.
isNot(tok::string_literal))
258 return IsIdentifierStringPrefix(PrevTok);
260 case tok::numeric_constant:
262 FirstChar ==
'+' || FirstChar ==
'-';
264 return (FirstChar ==
'.' && PrevPrevTok.
is(tok::period)) ||
268 return FirstChar ==
'&';
270 return FirstChar ==
'+';
272 return FirstChar ==
'-' || FirstChar ==
'>';
274 return FirstChar ==
'*' || FirstChar ==
'/';
276 return FirstChar ==
'<' || FirstChar ==
':' || FirstChar ==
'%';
278 return FirstChar ==
'>';
280 return FirstChar ==
'|';
282 return FirstChar ==
'>' || FirstChar ==
':';
284 return FirstChar ==
'>' ||
287 return FirstChar ==
'#' || FirstChar ==
'@' || FirstChar ==
'%';
289 return PP.
getLangOpts().CPlusPlus && FirstChar ==
'*';
291 return PP.
getLangOpts().CPlusPlus2a && FirstChar ==
'>';
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
const char * getCharacterData(SourceLocation SL, bool *Invalid=nullptr) const
Return a pointer to the start of the specified location in the appropriate spelling MemoryBuffer...
bool isAnnotation() const
Return true if this is any of tok::annot_* kind tokens.
tok::TokenKind getKind() const
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
One of these records is kept for each identifier that is lexed.
Token - This structure provides full information about a lexed token.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
const LangOptions & getLangOpts() const
StringRef getSpelling(SourceLocation loc, SmallVectorImpl< char > &buffer, bool *invalid=nullptr) const
Return the 'spelling' of the token at the given location; does not go up to the spelling location or ...
bool hasUDSuffix() const
Return true if this token is a string or character literal which has a ud-suffix. ...
SourceLocation getSpellingLoc(SourceLocation Loc) const
Given a SourceLocation object, return the spelling location referenced by the ID. ...
__DEVICE__ void * memset(void *__a, int __b, size_t __c)
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Defines the clang::Preprocessor interface.
float __ovld __cnfn length(float p)
Return the length of vector p, i.e., sqrt(p.x2 + p.y 2 + ...)
SourceManager & getSourceManager() const
TokenConcatenation(Preprocessor &PP)
Encodes a location in the source.
IdentifierInfo * getIdentifierInfo() const
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
bool isNot(tok::TokenKind K) const
Dataflow Directional Tag Classes.
bool isOneOf(tok::TokenKind K1, tok::TokenKind K2) const
unsigned getLength() const
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
LLVM_READONLY bool isDigit(unsigned char c)
Return true if this character is an ASCII digit: [0-9].
bool AvoidConcat(const Token &PrevPrevTok, const Token &PrevTok, const Token &Tok) const
AvoidConcat - If printing PrevTok immediately followed by Tok would cause the two individual tokens t...
static bool IsStringPrefix(StringRef Str, bool CPlusPlus11)
IsStringPrefix - Return true if Str is a string prefix.
static char GetFirstChar(Preprocessor &PP, const Token &Tok)
GetFirstChar - Get the first character of the token.
bool needsCleaning() const
Return true if this token has trigraphs or escaped newlines in it.
LLVM_READONLY bool isPreprocessingNumberBody(unsigned char c)
Return true if this is the body character of a C preprocessing number, which is [a-zA-Z0-9_.
This class handles loading and caching of source files into memory.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.