24 #include "llvm/ADT/StringMap.h" 25 #include "llvm/Support/DJB.h" 26 #include "llvm/Support/EndianStream.h" 27 #include "llvm/Support/FileSystem.h" 28 #include "llvm/Support/MemoryBuffer.h" 29 #include "llvm/Support/OnDiskHashTable.h" 30 #include "llvm/Support/Path.h" 34 #define S_ISDIR(x) (((x)&_S_IFDIR)!=0) 37 using namespace clang;
47 Offset TokenData, PPCondData;
53 : TokenData(td), PPCondData(ppcd) {}
55 Offset getTokenOffset()
const {
return TokenData; }
56 Offset getPPCondTableOffset()
const {
return PPCondData; }
60 class PTHEntryKeyVariant {
67 enum { IsFE = 0x1, IsDE = 0x2, IsNoExist = 0x0 }
Kind;
71 PTHEntryKeyVariant(
const FileEntry *fe) : FE(fe),
Kind(IsFE), Data(
nullptr) {}
73 PTHEntryKeyVariant(
FileData *Data, StringRef Path)
74 : PathPtr(Path.data()), PathSize(Path.size()),
Kind(IsDE),
77 explicit PTHEntryKeyVariant(StringRef Path)
78 : PathPtr(Path.data()), PathSize(Path.size()),
Kind(IsNoExist),
81 bool isFile()
const {
return Kind == IsFE; }
83 StringRef getString()
const {
84 return Kind == IsFE ? FE->getName() : StringRef(PathPtr, PathSize);
87 unsigned getKind()
const {
return (
unsigned)
Kind; }
89 void EmitData(raw_ostream& Out) {
90 using namespace llvm::support;
91 endian::Writer LE(Out, little);
95 llvm::sys::fs::UniqueID UID = FE->getUniqueID();
96 LE.write<uint64_t>(UID.getFile());
97 LE.write<uint64_t>(UID.getDevice());
98 LE.write<uint64_t>(FE->getModificationTime());
99 LE.write<uint64_t>(FE->getSize());
103 LE.write<uint64_t>(Data->
UniqueID.getFile());
104 LE.write<uint64_t>(Data->
UniqueID.getDevice());
105 LE.write<uint64_t>(Data->
ModTime);
106 LE.write<uint64_t>(Data->
Size);
114 unsigned getRepresentationLength()
const {
115 return Kind == IsNoExist ? 0 : 4 * 8;
119 class FileEntryPTHEntryInfo {
121 typedef PTHEntryKeyVariant key_type;
122 typedef key_type key_type_ref;
124 typedef PTHEntry data_type;
125 typedef const PTHEntry& data_type_ref;
127 typedef unsigned hash_value_type;
128 typedef unsigned offset_type;
130 static hash_value_type
ComputeHash(PTHEntryKeyVariant V) {
131 return llvm::djbHash(V.getString());
134 static std::pair<unsigned,unsigned>
135 EmitKeyDataLength(raw_ostream& Out, PTHEntryKeyVariant V,
137 using namespace llvm::support;
138 endian::Writer LE(Out, little);
140 unsigned n = V.getString().size() + 1 + 1;
141 LE.write<uint16_t>(n);
143 unsigned m = V.getRepresentationLength() + (V.isFile() ? 4 + 4 : 0);
144 LE.write<uint8_t>(m);
146 return std::make_pair(n, m);
149 static void EmitKey(raw_ostream& Out, PTHEntryKeyVariant V,
unsigned n){
150 using namespace llvm::support;
152 Out << char(V.getKind());
154 Out.write(V.getString().data(), n - 1);
157 static void EmitData(raw_ostream& Out, PTHEntryKeyVariant V,
158 const PTHEntry& E,
unsigned) {
159 using namespace llvm::support;
160 endian::Writer LE(Out, little);
165 LE.write<uint32_t>(E.getTokenOffset());
166 LE.write<uint32_t>(E.getPPCondTableOffset());
178 OffsetOpt() : valid(
false) {}
179 bool hasOffset()
const {
return valid; }
180 Offset getOffset()
const { assert(valid);
return off; }
181 void setOffset(
Offset o) { off = o; valid =
true; }
185 typedef llvm::OnDiskChainedHashTableGenerator<FileEntryPTHEntryInfo>
PTHMap;
189 typedef llvm::DenseMap<const IdentifierInfo*,uint32_t> IDMap;
190 typedef llvm::StringMap<OffsetOpt, llvm::BumpPtrAllocator> CachedStrsTy;
192 raw_pwrite_stream &Out;
195 std::vector<llvm::StringMapEntry<OffsetOpt>*> StrEntries;
197 CachedStrsTy CachedStrs;
205 void EmitToken(
const Token& T);
207 void Emit8(uint32_t V) {
211 void Emit16(uint32_t V) {
212 using namespace llvm::support;
213 endian::write<uint16_t>(Out, V, little);
216 void Emit32(uint32_t V) {
217 using namespace llvm::support;
218 endian::write<uint32_t>(Out, V, little);
221 void EmitBuf(
const char *Ptr,
unsigned NumBytes) {
222 Out.write(Ptr, NumBytes);
226 using namespace llvm::support;
227 endian::write<uint16_t>(Out, V.size(), little);
228 EmitBuf(V.data(), V.size());
235 std::pair<Offset, Offset> EmitIdentifierTable();
239 Offset EmitFileTable() {
return PM.Emit(Out); }
241 PTHEntry LexTokens(
Lexer& L);
242 Offset EmitCachedSpellings();
246 : Out(out), PP(pp), idcount(0), CurStrOffset(0) {}
248 PTHMap &getPM() {
return PM; }
258 IDMap::iterator I = IM.find(II);
266 void PTHWriter::EmitToken(
const Token& T) {
279 auto &E = *CachedStrs.insert(std::make_pair(s, OffsetOpt())).first;
282 if (!E.second.hasOffset()) {
283 E.second.setOffset(CurStrOffset);
284 StrEntries.push_back(&E);
285 CurStrOffset += s.size() + 1;
289 Emit32(E.second.getOffset());
294 Emit32(PP.getSourceManager().getFileOffset(T.
getLocation()));
297 PTHEntry PTHWriter::LexTokens(
Lexer& L) {
300 using namespace llvm::support;
301 endian::Writer LE(Out, little);
302 uint32_t TokenOff = Out.tell();
303 for (uint64_t N = llvm::OffsetToAlignment(TokenOff, 4); N; --N, ++TokenOff)
304 LE.write<uint8_t>(0);
307 typedef std::vector<std::pair<Offset, unsigned> > PPCondTable;
309 std::vector<unsigned> PPStartCond;
310 bool ParsingPreprocessorDirective =
false;
318 ParsingPreprocessorDirective) {
328 ParsingPreprocessorDirective =
false;
331 if (Tok.
is(tok::raw_identifier)) {
332 PP.LookUpIdentifierInfo(Tok);
340 assert(!ParsingPreprocessorDirective);
357 if (Tok.
isNot(tok::raw_identifier)) {
365 ParsingPreprocessorDirective =
true;
368 case tok::pp_not_keyword:
374 case tok::pp_include:
376 case tok::pp_include_next: {
384 if (Tok.
is(tok::raw_identifier))
385 PP.LookUpIdentifierInfo(Tok);
391 case tok::pp_ifndef: {
394 PPStartCond.push_back(PPCond.size());
395 PPCond.push_back(std::make_pair(HashOff, 0U));
398 case tok::pp_endif: {
402 unsigned index = PPCond.size();
404 assert(!PPStartCond.empty());
405 assert(PPCond.size() > PPStartCond.back());
406 assert(PPCond[PPStartCond.back()].second == 0);
407 PPCond[PPStartCond.back()].second = index;
408 PPStartCond.pop_back();
410 PPCond.push_back(std::make_pair(HashOff, index));
427 unsigned index = PPCond.size();
429 assert(!PPStartCond.empty());
430 assert(PPCond.size() > PPStartCond.back());
431 assert(PPCond[PPStartCond.back()].second == 0);
432 PPCond[PPStartCond.back()].second = index;
433 PPStartCond.pop_back();
435 PPCond.push_back(std::make_pair(HashOff, 0U));
436 PPStartCond.push_back(index);
446 assert(PPStartCond.empty() &&
"Error: imblanced preprocessor conditionals.");
452 Emit32(PPCond.size());
454 for (
unsigned i = 0, e = PPCond.size(); i!=e; ++i) {
455 Emit32(PPCond[i].first - TokenOff);
456 uint32_t x = PPCond[i].second;
457 assert(x != 0 &&
"PPCond entry not backpatched.");
460 Emit32(x == i ? 0 : x);
463 return PTHEntry(TokenOff, PPCondOff);
466 Offset PTHWriter::EmitCachedSpellings() {
468 Offset SpellingsOff = Out.tell();
470 for (std::vector<llvm::StringMapEntry<OffsetOpt>*>::iterator
471 I = StrEntries.begin(), E = StrEntries.end(); I!=E; ++I)
472 EmitBuf((*I)->getKeyData(), (*I)->getKeyLength()+1 );
478 return llvm::support::endian::byte_swap<uint32_t, llvm::support::little>(
X);
481 static void pwrite32le(raw_pwrite_stream &OS, uint32_t Val, uint64_t &Off) {
483 OS.pwrite(reinterpret_cast<const char *>(&LEVal), 4, Off);
489 Out <<
"cfe-pth" <<
'\0';
493 Offset PrologueOffset = Out.tell();
494 for (
unsigned i = 0; i < 4; ++i)
498 if (!MainFile.empty()) {
517 if (llvm::sys::path::is_relative(FE->
getName()))
520 const llvm::MemoryBuffer *B = C.
getBuffer(PP.getDiagnostics(),
SM);
524 const llvm::MemoryBuffer *FromFile = SM.
getBuffer(FID);
525 Lexer L(FID, FromFile, SM, LOpts);
526 PM.insert(FE, LexTokens(L));
530 const std::pair<Offset,Offset> &IdTableOff = EmitIdentifierTable();
533 Offset SpellingOff = EmitCachedSpellings();
536 Offset FileTableOff = EmitFileTable();
539 uint64_t Off = PrologueOffset;
555 StatListener(
PTHMap &pm) : PM(pm) {}
556 ~StatListener()
override {}
559 std::unique_ptr<vfs::File> *F,
563 if (Result == CacheMissing)
564 PM.insert(PTHEntryKeyVariant(Path), PTHEntry());
567 if (llvm::sys::path::is_relative(Path))
570 PM.insert(PTHEntryKeyVariant(&Data, Path), PTHEntry());
584 llvm::sys::fs::make_absolute(MainFilePath);
587 PTHWriter PW(*OS, PP);
590 auto StatCacheOwner = llvm::make_unique<StatListener>(PW.getPM());
591 StatListener *StatCache = StatCacheOwner.get();
603 PW.GeneratePTH(MainFilePath.str());
615 class PTHIdentifierTableTrait {
617 typedef PTHIdKey* key_type;
618 typedef key_type key_type_ref;
620 typedef uint32_t data_type;
621 typedef data_type data_type_ref;
623 typedef unsigned hash_value_type;
624 typedef unsigned offset_type;
626 static hash_value_type
ComputeHash(PTHIdKey* key) {
627 return llvm::djbHash(key->II->getName());
630 static std::pair<unsigned,unsigned>
631 EmitKeyDataLength(raw_ostream& Out,
const PTHIdKey* key, uint32_t) {
632 using namespace llvm::support;
633 unsigned n = key->II->getLength() + 1;
634 endian::write<uint16_t>(Out, n, little);
635 return std::make_pair(n,
sizeof(uint32_t));
638 static void EmitKey(raw_ostream& Out, PTHIdKey* key,
unsigned n) {
641 key->FileOffset = Out.tell();
642 Out.write(key->II->getNameStart(), n);
645 static void EmitData(raw_ostream& Out, PTHIdKey*, uint32_t pID,
647 using namespace llvm::support;
648 endian::write<uint32_t>(Out, pID, little);
658 std::pair<Offset,Offset> PTHWriter::EmitIdentifierTable() {
664 PTHIdKey *IIDMap =
static_cast<PTHIdKey*
>(
665 llvm::safe_calloc(idcount,
sizeof(PTHIdKey)));
668 llvm::OnDiskChainedHashTableGenerator<PTHIdentifierTableTrait> IIOffMap;
671 for (IDMap::iterator I = IM.begin(), E = IM.end(); I != E; ++I) {
674 assert(I->second > 0);
675 assert(I->second-1 < idcount);
676 unsigned idx = I->second-1;
679 IIDMap[idx].II = I->first;
682 IIOffMap.insert(&IIDMap[idx], I->second);
688 Offset StringTableOffset = IIOffMap.Emit(Out);
691 Offset IDOff = Out.tell();
693 for (
unsigned i = 0 ; i < idcount; ++i)
694 Emit32(IIDMap[i].FileOffset);
699 return std::make_pair(IDOff, StringTableOffset);
const FileEntry * OrigEntry
Reference to the file entry representing this ContentCache.
Lexer - This provides a simple interface that turns a text buffer into a stream of tokens...
Defines the clang::FileManager interface and associated types.
bool LexFromRawLexer(Token &Result)
LexFromRawLexer - Lex a token from a designated raw lexer (one with no associated preprocessor object...
bool is(tok::TokenKind K) const
is/isNot - Predicates to check if this token is a specific kind, as in "if (Tok.is(tok::l_brace)) {...
Defines the SourceManager interface.
FileManager & getFileManager() const
fileinfo_iterator fileinfo_end() const
Defines the FileSystemStatCache interface.
tok::TokenKind getKind() const
bool isLiteral() const
Return true if this is a "literal", like a numeric constant, string, etc.
The virtual file system interface.
One of these records is kept for each identifier that is lexed.
One instance of this struct is kept for every file loaded or used.
Token - This structure provides full information about a lexed token.
static void pwrite32le(raw_pwrite_stream &OS, uint32_t Val, uint64_t &Off)
llvm::OnDiskChainedHashTableGenerator< FileEntryPTHEntryInfo > PTHMap
void setKind(tok::TokenKind K)
Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...
Abstract interface for introducing a FileManager cache for 'stat' system calls, which is used by prec...
Represents the results of name lookup.
Defines the Diagnostic-related interfaces.
void setParsingPreprocessorDirective(bool f)
Inform the lexer whether or not we are currently lexing a preprocessor directive. ...
void CacheTokens(Preprocessor &PP, raw_pwrite_stream *OS)
Cache tokens for use with PCH. Note that this requires a seekable stream.
llvm::MemoryBuffer * getBuffer(DiagnosticsEngine &Diag, const SourceManager &SM, SourceLocation Loc=SourceLocation(), bool *Invalid=nullptr) const
Returns the memory buffer for the associated content.
const FileEntry * getFileEntryForID(FileID FID) const
Returns the FileEntry record for the provided FileID.
FileID createFileID(const FileEntry *SourceFile, SourceLocation IncludePos, SrcMgr::CharacteristicKind FileCharacter, int LoadedID=0, unsigned LoadedOffset=0)
Create a new FileID that represents the specified file being #included from the specified IncludePosi...
void EnterMainSourceFile()
Enter the specified FileID as the main source file, which implicitly adds the builtin defines etc...
SourceLocation getLocation() const
Return a source location identifier for the specified offset in the current file. ...
Defines the clang::Preprocessor interface.
PPKeywordKind
Provides a namespace for preprocessor keywords which start with a '#' at the beginning of the line...
Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.
fileinfo_iterator fileinfo_begin() const
The result type of a method or function.
SourceManager & getSourceManager() const
static uint32_t swap32le(uint32_t X)
llvm::MemoryBuffer * getBuffer(FileID FID, SourceLocation Loc, bool *Invalid=nullptr) const
Return the buffer for the specified FileID.
void addStatCache(std::unique_ptr< FileSystemStatCache > statCache, bool AtBeginning=false)
Installs the provided FileSystemStatCache object within the FileManager.
Encodes a location in the source.
llvm::sys::fs::UniqueID UniqueID
StringRef getName() const
IdentifierInfo * getIdentifierInfo() const
Cached information about one file (either on disk or in the virtual file system). ...
void setIdentifierInfo(IdentifierInfo *II)
void Lex(Token &Result)
Lex the next token for this preprocessor.
bool isAtStartOfLine() const
isAtStartOfLine - Return true if this token is at the start of a line.
llvm::DenseMap< const FileEntry *, SrcMgr::ContentCache * >::const_iterator fileinfo_iterator
unsigned ComputeHash(Selector Sel)
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
bool isNot(tok::TokenKind K) const
Dataflow Directional Tag Classes.
void removeStatCache(FileSystemStatCache *statCache)
Removes the specified FileSystemStatCache object from the manager.
FileID getMainFileID() const
Returns the FileID of the main source file.
tok::PPKeywordKind getPPKeywordID() const
Return the preprocessor keyword ID for this identifier.
unsigned getLength() const
const char * getLiteralData() const
getLiteralData - For a literal token (numeric constant, string, etc), this returns a pointer to the s...
void LexIncludeFilename(Token &Result)
After the preprocessor has parsed a #include, lex and (potentially) macro expand the filename...
raw_ostream & EmitString(raw_ostream &o, StringRef s)
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
static Decl::Kind getKind(const Decl *D)
Generate pre-tokenized header.
unsigned getFlags() const
Return the internal represtation of the flags.
void clearFlag(TokenFlags Flag)
Unset the specified flag.
This class handles loading and caching of source files into memory.
Engages in a tight little dance with the lexer to efficiently preprocess tokens.