23 #include "llvm/ADT/StringExtras.h"
24 #include "llvm/Support/EndianStream.h"
25 #include "llvm/Support/MemoryBuffer.h"
27 #include <system_error>
28 using namespace clang;
39 PPCond(ppcond), CurPPCondPtr(ppcond), PTHMgr(PM) {
48 using namespace llvm::support;
51 const unsigned char *CurPtrShadow = CurPtr;
54 unsigned Word0 = endian::readNext<uint32_t, little, aligned>(CurPtrShadow);
56 endian::readNext<uint32_t, little, aligned>(CurPtrShadow);
58 endian::readNext<uint32_t, little, aligned>(CurPtrShadow);
62 uint32_t Len = Word0 >> 16;
64 CurPtr = CurPtrShadow;
74 Tok.
setLocation(FileStartLoc.getLocWithOffset(FileOffset));
107 return LexEndOfFile(Tok);
118 if (TKind == tok::eod) {
143 diag::err_pp_unterminated_conditional);
160 "Must be in a preprocessing directive!");
169 const unsigned char* p = CurPtr;
188 using namespace llvm::support;
189 assert(CurPPCondPtr &&
"No cached PP conditional information.");
190 assert(LastHashTokPtr &&
"No known '#' token.");
192 const unsigned char *HashEntryI =
nullptr;
197 uint32_t
Offset = endian::readNext<uint32_t, little, aligned>(CurPPCondPtr);
200 TableIdx = endian::readNext<uint32_t, little, aligned>(CurPPCondPtr);
203 HashEntryI = TokBuf +
Offset;
209 if (HashEntryI < LastHashTokPtr && TableIdx) {
213 const unsigned char* NextPPCondPtr =
214 PPCond + TableIdx*(
sizeof(uint32_t)*2);
215 assert(NextPPCondPtr >= CurPPCondPtr);
217 const unsigned char *HashEntryJ =
218 TokBuf + endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
220 if (HashEntryJ <= LastHashTokPtr) {
222 HashEntryI = HashEntryJ;
223 TableIdx = endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
224 CurPPCondPtr = NextPPCondPtr;
228 while (HashEntryI < LastHashTokPtr);
229 assert(HashEntryI == LastHashTokPtr &&
"No PP-cond entry found for '#'");
230 assert(TableIdx &&
"No jumping from #endifs.");
233 const unsigned char* NextPPCondPtr = PPCond + TableIdx*(
sizeof(uint32_t)*2);
234 assert(NextPPCondPtr >= CurPPCondPtr);
235 CurPPCondPtr = NextPPCondPtr;
239 TokBuf + endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
240 uint32_t NextIdx = endian::readNext<uint32_t, little, aligned>(NextPPCondPtr);
244 bool isEndif = NextIdx == 0;
255 if (CurPtr > HashEntryI) {
261 LastHashTokPtr = HashEntryI;
271 LastHashTokPtr = CurPtr;
291 using namespace llvm::support;
294 uint32_t
Offset = endian::readNext<uint32_t, little, aligned>(OffsetPtr);
307 const uint32_t TokenOff;
308 const uint32_t PPCondOff;
310 PTHFileData(uint32_t tokenOff, uint32_t ppCondOff)
311 : TokenOff(tokenOff), PPCondOff(ppCondOff) {}
313 uint32_t getTokenOffset()
const {
return TokenOff; }
314 uint32_t getPPCondOffset()
const {
return PPCondOff; }
318 class PTHFileLookupCommonTrait {
320 typedef std::pair<unsigned char, StringRef> internal_key_type;
321 typedef unsigned hash_value_type;
322 typedef unsigned offset_type;
324 static hash_value_type
ComputeHash(internal_key_type x) {
325 return llvm::HashString(x.second);
328 static std::pair<unsigned, unsigned>
329 ReadKeyDataLength(
const unsigned char*& d) {
330 using namespace llvm::support;
332 (
unsigned)endian::readNext<uint16_t, little, unaligned>(d);
333 unsigned dataLen = (
unsigned) *(d++);
334 return std::make_pair(keyLen, dataLen);
337 static internal_key_type ReadKey(
const unsigned char* d,
unsigned) {
338 unsigned char k = *(d++);
339 return std::make_pair(k, (
const char*) d);
351 return std::make_pair((
unsigned char) 0x1, FE->
getName());
354 static bool EqualKey(internal_key_type a, internal_key_type b) {
355 return a.first == b.first && a.second == b.second;
358 static PTHFileData
ReadData(
const internal_key_type& k,
359 const unsigned char* d,
unsigned) {
360 assert(k.first == 0x1 &&
"Only file lookups can match!");
361 using namespace llvm::support;
362 uint32_t x = endian::readNext<uint32_t, little, unaligned>(d);
363 uint32_t y = endian::readNext<uint32_t, little, unaligned>(d);
364 return PTHFileData(x, y);
378 return (a.second == b.second) ? memcmp(a.first, b.first, a.second) == 0
383 return llvm::HashString(StringRef(a.first, a.second));
387 static const internal_key_type&
390 static std::pair<unsigned, unsigned>
392 using namespace llvm::support;
393 return std::make_pair(
394 (
unsigned)endian::readNext<uint16_t, little, unaligned>(d),
398 static std::pair<const char*, unsigned>
400 assert(n >= 2 && d[n-1] ==
'\0');
401 return std::make_pair((
const char*) d, n-1);
406 using namespace llvm::support;
407 return endian::readNext<uint32_t, little, unaligned>(d);
415 PTHManager::PTHManager(
416 std::unique_ptr<const llvm::MemoryBuffer> buf,
417 std::unique_ptr<PTHFileLookup> fileLookup,
const unsigned char *idDataTable,
418 std::unique_ptr<
IdentifierInfo *[], llvm::FreeDeleter> perIDCache,
419 std::unique_ptr<PTHStringIdLookup> stringIdLookup,
unsigned numIds,
420 const unsigned char *spellingBase,
const char *originalSourceFile)
421 : Buf(std::move(buf)), PerIDCache(std::move(perIDCache)),
422 FileLookup(std::move(fileLookup)), IdDataTable(idDataTable),
423 StringIdLookup(std::move(stringIdLookup)), NumIds(numIds),
PP(nullptr),
424 SpellingBase(spellingBase), OriginalSourceFile(originalSourceFile) {}
426 PTHManager::~PTHManager() {
435 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> FileOrErr =
436 llvm::MemoryBuffer::getFile(file);
440 Diags.
Report(diag::err_invalid_pth_file) << file;
443 std::unique_ptr<llvm::MemoryBuffer> File = std::move(FileOrErr.get());
445 using namespace llvm::support;
449 const unsigned char *BufBeg = (
const unsigned char*)File->getBufferStart();
450 const unsigned char *BufEnd = (
const unsigned char*)File->getBufferEnd();
453 if ((BufEnd - BufBeg) < (
signed)(
sizeof(
"cfe-pth") + 4 + 4) ||
454 memcmp(BufBeg,
"cfe-pth",
sizeof(
"cfe-pth")) != 0) {
455 Diags.
Report(diag::err_invalid_pth_file) << file;
460 const unsigned char *p = BufBeg + (
sizeof(
"cfe-pth"));
461 unsigned Version = endian::readNext<uint32_t, little, aligned>(p);
463 if (Version < PTHManager::Version) {
465 Version < PTHManager::Version
466 ?
"PTH file uses an older PTH format that is no longer supported"
467 :
"PTH file uses a newer PTH format that cannot be read");
472 const unsigned char *PrologueOffset = p;
474 if (PrologueOffset >= BufEnd) {
475 Diags.
Report(diag::err_invalid_pth_file) << file;
481 const unsigned char* FileTableOffset = PrologueOffset +
sizeof(uint32_t)*2;
482 const unsigned char *FileTable =
483 BufBeg + endian::readNext<uint32_t, little, aligned>(FileTableOffset);
485 if (!(FileTable > BufBeg && FileTable < BufEnd)) {
486 Diags.
Report(diag::err_invalid_pth_file) << file;
495 InvalidPTH(Diags,
"PTH file contains no cached source data");
499 const unsigned char* IDTableOffset = PrologueOffset +
sizeof(uint32_t)*0;
500 const unsigned char *IData =
501 BufBeg + endian::readNext<uint32_t, little, aligned>(IDTableOffset);
503 if (!(IData >= BufBeg && IData < BufEnd)) {
504 Diags.
Report(diag::err_invalid_pth_file) << file;
510 const unsigned char* StringIdTableOffset = PrologueOffset +
sizeof(uint32_t)*1;
511 const unsigned char *StringIdTable =
512 BufBeg + endian::readNext<uint32_t, little, aligned>(StringIdTableOffset);
513 if (!(StringIdTable >= BufBeg && StringIdTable < BufEnd)) {
514 Diags.
Report(diag::err_invalid_pth_file) << file;
518 std::unique_ptr<PTHStringIdLookup> SL(
522 const unsigned char* spellingBaseOffset = PrologueOffset +
sizeof(uint32_t)*3;
523 const unsigned char *spellingBase =
524 BufBeg + endian::readNext<uint32_t, little, aligned>(spellingBaseOffset);
525 if (!(spellingBase >= BufBeg && spellingBase < BufEnd)) {
526 Diags.
Report(diag::err_invalid_pth_file) << file;
531 uint32_t NumIds = endian::readNext<uint32_t, little, aligned>(IData);
536 std::unique_ptr<IdentifierInfo *[], llvm::FreeDeleter> PerIDCache;
539 PerIDCache.reset((
IdentifierInfo **)calloc(NumIds,
sizeof(PerIDCache[0])));
541 InvalidPTH(Diags,
"Could not allocate memory for processing PTH file");
547 const unsigned char* originalSourceBase = PrologueOffset +
sizeof(uint32_t)*4;
549 endian::readNext<uint16_t, little, unaligned>(originalSourceBase);
550 if (!len) originalSourceBase =
nullptr;
553 return new PTHManager(std::move(File), std::move(FL), IData,
554 std::move(PerIDCache), std::move(SL), NumIds,
555 spellingBase, (
const char *)originalSourceBase);
558 IdentifierInfo* PTHManager::LazilyCreateIdentifierInfo(
unsigned PersistentID) {
559 using namespace llvm::support;
561 const unsigned char* TableEntry = IdDataTable +
sizeof(uint32_t)*PersistentID;
562 const unsigned char *IDData =
563 (
const unsigned char *)Buf->getBufferStart() +
564 endian::readNext<uint32_t, little, aligned>(TableEntry);
565 assert(IDData < (
const unsigned char*)Buf->getBufferEnd());
568 std::pair<IdentifierInfo,const unsigned char*> *Mem =
569 Alloc.Allocate<std::pair<IdentifierInfo,const unsigned char*> >();
571 Mem->second = IDData;
572 assert(IDData[0] !=
'\0');
576 PerIDCache[PersistentID] = II;
577 assert(II->getNameStart() && II->getNameStart()[0] !=
'\0');
583 assert(Name.empty() || Name.back() !=
'\0');
584 PTHStringIdLookup::iterator
I =
585 StringIdLookup->find(std::make_pair(Name.data(), Name.size()));
586 if (I == StringIdLookup->end())
591 return GetIdentifierInfo(*I-1);
595 const FileEntry *FE = PP->getSourceManager().getFileEntryForID(FID);
599 using namespace llvm::support;
604 PTHFileLookup::iterator
I = FileLookup->find(FE);
606 if (
I == FileLookup->end())
611 const unsigned char *BufStart = (
const unsigned char *)Buf->getBufferStart();
613 const unsigned char* data = BufStart +
FileData.getTokenOffset();
616 const unsigned char* ppcond = BufStart +
FileData.getPPCondOffset();
617 uint32_t Len = endian::readNext<uint32_t, little, aligned>(ppcond);
618 if (Len == 0) ppcond =
nullptr;
620 assert(PP &&
"No preprocessor set yet!");
621 return new PTHLexer(*PP, FID, data, ppcond, *
this);
633 llvm::sys::fs::UniqueID UniqueID;
637 PTHStatData(uint64_t Size, time_t ModTime, llvm::sys::fs::UniqueID UniqueID,
639 : Size(Size), ModTime(ModTime), UniqueID(UniqueID), HasData(
true),
640 IsDirectory(IsDirectory) {}
642 PTHStatData() : HasData(
false) {}
645 class PTHStatLookupTrait :
public PTHFileLookupCommonTrait {
647 typedef StringRef external_key_type;
648 typedef PTHStatData data_type;
650 static internal_key_type GetInternalKey(StringRef path) {
652 return std::make_pair((
unsigned char) 0x0, path);
655 static bool EqualKey(internal_key_type a, internal_key_type b) {
658 return a.second == b.second;
661 static data_type ReadData(
const internal_key_type& k,
const unsigned char* d,
665 bool IsDirectory =
true;
666 if (k.first == 0x1 ) {
671 using namespace llvm::support;
673 uint64_t File = endian::readNext<uint64_t, little, unaligned>(d);
674 uint64_t Device = endian::readNext<uint64_t, little, unaligned>(d);
675 llvm::sys::fs::UniqueID UniqueID(Device, File);
676 time_t ModTime = endian::readNext<uint64_t, little, unaligned>(d);
677 uint64_t Size = endian::readNext<uint64_t, little, unaligned>(d);
678 return data_type(Size, ModTime, UniqueID, IsDirectory);
694 :
Cache(FL.getNumBuckets(), FL.getNumEntries(), FL.getBuckets(),
698 std::unique_ptr<vfs::File> *F,
701 CacheTy::iterator
I =
Cache.find(Path);
704 if (I ==
Cache.end())
705 return statChained(Path, Data, isFile, F, FS);
707 const PTHStatData &D = *
I;
725 std::unique_ptr<FileSystemStatCache> PTHManager::createStatCache() {
726 return llvm::make_unique<PTHStatCache>(*FileLookup);
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
SourceManager & getSourceManager() const
LookupResult getStat(StringRef Path, FileData &Data, bool isFile, std::unique_ptr< vfs::File > *F, vfs::FileSystem &FS) override
static const unsigned StoredTokenSize
const FileEntry * external_key_type
Defines the clang::FileManager interface and associated types.
bool Lex(Token &Tok)
Lex - Return the next token.
static PTHFileData ReadData(const internal_key_type &k, const unsigned char *d, unsigned)
static std::pair< unsigned, unsigned > ReadKeyDataLength(const unsigned char *&d)
Defines the FileSystemStatCache interface.
TypePropertyCache< Private > Cache
DiagnosticBuilder Report(SourceLocation Loc, unsigned DiagID)
Issue the message to the client.
void setFlag(TokenFlags Flag)
Set the specified flag.
uint32_t IdentifierID
An ID number that refers to an identifier in an AST file.
static hash_value_type ComputeHash(const internal_key_type &a)
The virtual file system interface.
One of these records is kept for each identifier that is lexed.
bool ParsingPreprocessorDirective
True when parsing #XXX; turns '\n' into a tok::eod token.
SmallVector< PPConditionalInfo, 4 > ConditionalStack
Information about the set of #if/#ifdef/#ifndef blocks we are currently in.
Token - This structure provides full information about a lexed token.
static uint32_t ReadData(const internal_key_type &k, const unsigned char *d, unsigned)
void setKind(tok::TokenKind K)
Abstract interface for introducing a FileManager cache for 'stat' system calls, which is used by prec...
SourceLocation getLocWithOffset(int Offset) const
Return a source location with the specified offset from this SourceLocation.
tok::TokenKind getTokenID() const
If this is a source-language token (e.g.
void HandleDirective(Token &Result)
Callback invoked when the lexer sees a # token at the start of a line.
Concrete class used by the front-end to report problems and issues.
StringRef getName() const
static internal_key_type GetInternalKey(const FileEntry *FE)
SourceLocation getCodeCompletionFileLoc() const
Returns the start location of the file of code-completion point.
detail::InMemoryDirectory::const_iterator I
DiagnosticBuilder Diag(SourceLocation Loc, unsigned DiagID) const
Forwarding function for diagnostics.
bool ParsingFilename
True after #include; turns <xx> into a tok::angle_string_literal token.
bool LexingRawMode
True if in raw mode.
Defines the clang::Preprocessor interface.
external_key_type internal_key_type
MultipleIncludeOpt MIOpt
A state machine that detects the #ifndef-wrapping a file idiom for the multiple-include optimization...
void DiscardToEndOfLine()
DiscardToEndOfLine - Read the rest of the current preprocessor line as an uninterpreted string...
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
bool HandleEndOfFile(Token &Result, bool isEndOfMacro=false)
Callback invoked when the lexer hits the end of the current file.
The result type of a method or function.
bool isHandleIdentifierCase() const
Return true if the Preprocessor::HandleIdentifier must be called on a token of this identifier...
SourceLocation getSourceLocation() override
getSourceLocation - Return a source location for the token in the current file.
Encodes a location in the source.
llvm::sys::fs::UniqueID UniqueID
void setLength(unsigned Len)
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr * > VL, ArrayRef< Expr * > PL, ArrayRef< Expr * > IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
Cached information about one file (either on disk or in the virtual file system). ...
void setIdentifierInfo(IdentifierInfo *II)
static bool EqualKey(const internal_key_type &a, const internal_key_type &b)
TokenKind
Provides a simple uniform namespace for tokens from all C languages.
unsigned getCustomDiagID(Level L, const char(&FormatString)[N])
Return an ID for a diagnostic with the specified format string and level.
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
unsigned ComputeHash(Selector Sel)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool SkipBlock()
SkipBlock - Used by Preprocessor to skip the current conditional block.
void setLiteralData(const char *Ptr)
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
PTHStatCache(PTHManager::PTHFileLookup &FL)
bool HandleIdentifier(Token &Identifier)
Callback invoked when the lexer reads an identifier and has filled in the tokens IdentifierInfo membe...
Defines the clang::TokenKind enum and support functions.
const std::pair< const char *, unsigned > external_key_type
SourceLocation getLocForStartOfFile(FileID FID) const
Return the source location corresponding to the first byte of the specified file. ...
void setLocation(SourceLocation L)
static void InvalidPTH(DiagnosticsEngine &Diags, const char *Msg)
static const internal_key_type & GetInternalKey(const external_key_type &x)
static std::pair< const char *, unsigned > ReadKey(const unsigned char *d, unsigned n)
static bool EqualKey(internal_key_type a, internal_key_type b)
void startToken()
Reset all flags to cleared.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.