27 #ifndef LLVM_CLANG_TOOLING_SYNTAX_TOKENS_H 28 #define LLVM_CLANG_TOOLING_SYNTAX_TOKENS_H 36 #include "llvm/ADT/ArrayRef.h" 37 #include "llvm/ADT/Optional.h" 38 #include "llvm/ADT/StringRef.h" 39 #include "llvm/Support/Compiler.h" 40 #include "llvm/Support/raw_ostream.h" 67 unsigned length()
const {
return End - Begin; }
71 return Begin <= Offset && Offset < End;
75 return Begin <= Offset && Offset <= End;
82 return std::tie(L.File, L.Begin, L.End) == std::tie(R.File, R.Begin, R.End);
113 unsigned length()
const {
return Length; }
135 std::string str()
const;
182 return ExpandedTokens;
241 std::string dumpForTests()
const;
255 unsigned BeginSpelled = 0;
256 unsigned EndSpelled = 0;
259 unsigned BeginExpanded = 0;
260 unsigned EndExpanded = 0;
263 std::string str()
const;
272 std::vector<syntax::Token> SpelledTokens;
274 std::vector<Mapping> Mappings;
276 unsigned BeginExpanded = 0;
277 unsigned EndExpanded = 0;
284 std::pair<const syntax::Token *, const Mapping *>
285 spelledForExpandedToken(
const syntax::Token *Expanded)
const;
290 std::vector<syntax::Token> ExpandedTokens;
291 llvm::DenseMap<FileID, MarkedFile> Files;
345 std::vector<syntax::Token> Expanded;
347 PPExpansions Expansions;
llvm::StringRef text(const SourceManager &SM) const
Gets the substring that this FileRange refers to.
const SourceManager & sourceManager() const
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
friend bool operator==(const FileRange &L, const FileRange &R)
Defines the clang::FileManager interface and associated types.
Defines the SourceManager interface.
llvm::raw_ostream & operator<<(llvm::raw_ostream &OS, NodeKind K)
For debugging purposes.
A token coming directly from a file or from a macro invocation.
FileRange(FileID File, unsigned BeginOffset, unsigned EndOffset)
EXPECTS: File.isValid() && Begin <= End.
Records information reqired to construct mappings for the token buffer that we are collecting...
llvm::ArrayRef< syntax::Token > expandedTokens() const
All tokens produced by the preprocessor after all macro replacements, directives, etc...
Token - This structure provides full information about a lexed token.
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
llvm::ArrayRef< syntax::Token > Spelled
SourceLocation location() const
Location of the first character of a token.
bool touches(unsigned Offset) const
Check Offset is inside the range or equal to its endpoint.
SourceLocation endLocation() const
Location right after the last character of a token.
TokenBuffer(const SourceManager &SourceMgr)
bool contains(unsigned Offset) const
Check if Offset is inside the range.
Defines the clang::LangOptions interface.
Encodes a location in the source.
tok::TokenKind kind() const
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
unsigned beginOffset() const
Start is a start offset (inclusive) in the corresponding file.
Dataflow Directional Tag Classes.
A half-open character range inside a particular file, the start offset is included and the end offset...
Collects tokens for the main file while running the frontend action.
std::vector< syntax::Token > tokenize(FileID FID, const SourceManager &SM, const LangOptions &LO)
Lex the text buffer, corresponding to FID, in raw mode and record the resulting spelled tokens...
Defines the clang::TokenKind enum and support functions.
Defines the clang::SourceLocation class and associated facilities.
Builds mappings and spelled tokens in the TokenBuffer based on the expanded token stream...
An expansion produced by the preprocessor, includes macro expansions and preprocessor directives...
friend bool operator!=(const FileRange &L, const FileRange &R)
A list of tokens obtained by preprocessing a text buffer and operations to map between the expanded a...
llvm::ArrayRef< syntax::Token > Expanded
unsigned endOffset() const
End offset (exclusive) in the corresponding file.
This class handles loading and caching of source files into memory.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.