18 #ifndef LLVM_CLANG_LIB_SERIALIZATION_MULTIONDISKHASHTABLE_H
19 #define LLVM_CLANG_LIB_SERIALIZATION_MULTIONDISKHASHTABLE_H
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/ADT/DenseSet.h"
23 #include "llvm/ADT/PointerUnion.h"
24 #include "llvm/ADT/STLExtras.h"
25 #include "llvm/ADT/TinyPtrVector.h"
26 #include "llvm/Support/EndianStream.h"
27 #include "llvm/Support/OnDiskHashTable.h"
30 namespace serialization {
56 OnDiskTable(
file_type File,
unsigned NumBuckets,
unsigned NumEntries,
60 Table(NumBuckets, NumEntries, Buckets, Payload, Base, InfoObj) {}
64 std::vector<file_type> Files;
65 llvm::DenseMap<internal_key_type, data_type> Data;
68 typedef llvm::PointerUnion<OnDiskTable*, MergedTable*> Table;
69 typedef llvm::TinyPtrVector<void*> TableVector;
80 llvm::TinyPtrVector<file_type> PendingOverrides;
82 struct AsOnDiskTable {
83 typedef OnDiskTable *result_type;
84 result_type operator()(
void *
P)
const {
85 return Table::getFromOpaqueValue(P).template get<OnDiskTable *>();
88 typedef llvm::mapped_iterator<TableVector::iterator, AsOnDiskTable>
90 typedef llvm::iterator_range<table_iterator> table_range;
93 table_range tables() {
94 auto Begin = Tables.begin(),
End = Tables.end();
97 return llvm::make_range(llvm::map_iterator(
Begin, AsOnDiskTable()),
98 llvm::map_iterator(
End, AsOnDiskTable()));
101 MergedTable *getMergedTable()
const {
103 return Tables.empty() ?
nullptr : Table::getFromOpaqueValue(*Tables.begin())
104 .
template dyn_cast<MergedTable*>();
109 for (
auto *T : tables())
111 if (
auto *M = getMergedTable())
116 void removeOverriddenTables() {
118 Files.insert(PendingOverrides.begin(), PendingOverrides.end());
120 auto ShouldRemove = [&Files](
void *T) ->
bool {
121 auto *ODT = Table::getFromOpaqueValue(T).template get<OnDiskTable *>();
122 bool Remove = Files.count(ODT->File);
127 Tables.erase(std::remove_if(tables().begin().getCurrent(), Tables.end(),
130 PendingOverrides.clear();
134 MergedTable *Merged = getMergedTable();
136 Merged =
new MergedTable;
140 for (
auto *ODT : tables()) {
141 auto &HT = ODT->Table;
142 Info &InfoObj = HT.getInfoObj();
144 for (
auto I = HT.data_begin(),
E = HT.data_end();
I !=
E; ++
I) {
145 auto *LocalPtr =
I.getItem();
148 auto L = InfoObj.ReadKeyDataLength(LocalPtr);
151 InfoObj.ReadDataInto(Key, LocalPtr + L.first, L.second,
155 Merged->Files.push_back(ODT->File);
160 Tables.push_back(Table(Merged).getOpaqueValue());
164 template<
typename ReaderInfo,
typename WriterInfo>
170 : Tables(std::move(O.Tables)),
171 PendingOverrides(std::move(O.PendingOverrides)) {
178 Tables = std::move(O.Tables);
180 PendingOverrides = std::move(O.PendingOverrides);
187 using namespace llvm::support;
190 uint32_t BucketOffset = endian::readNext<uint32_t, little, unaligned>(Ptr);
193 uint32_t NumFiles = endian::readNext<uint32_t, little, unaligned>(Ptr);
197 OverriddenFiles.reserve(NumFiles);
198 for (; NumFiles != 0; --NumFiles)
199 OverriddenFiles.push_back(InfoObj.ReadFileRef(Ptr));
200 PendingOverrides.insert(PendingOverrides.end(), OverriddenFiles.begin(),
201 OverriddenFiles.end());
205 auto NumBucketsAndEntries =
206 OnDiskTable::HashTable::readNumBucketsAndEntries(Buckets);
209 Table NewTable =
new OnDiskTable(File, NumBucketsAndEntries.first,
210 NumBucketsAndEntries.second,
211 Buckets, Ptr, Data, std::move(InfoObj));
212 Tables.push_back(NewTable.getOpaqueValue());
219 if (!PendingOverrides.empty())
220 removeOverriddenTables();
222 if (Tables.size() >
static_cast<unsigned>(Info::MaxTables))
228 if (MergedTable *M = getMergedTable()) {
229 auto It = M->Data.find(Key);
230 if (It != M->Data.end())
236 for (
auto *ODT : tables()) {
237 auto &HT = ODT->Table;
238 auto It = HT.find_hashed(Key, KeyHash);
240 HT.getInfoObj().ReadDataInto(Key, It.getDataPtr(), It.getDataLen(),
253 if (!PendingOverrides.empty())
254 removeOverriddenTables();
256 if (MergedTable *M = getMergedTable()) {
257 for (
auto &KV : M->Data)
258 Info::MergeDataInto(KV.second, ResultBuilder);
261 for (
auto *ODT : tables()) {
262 auto &HT = ODT->Table;
263 Info &InfoObj = HT.getInfoObj();
264 for (
auto I = HT.data_begin(),
E = HT.data_end();
I !=
E; ++
I) {
265 auto *LocalPtr =
I.getItem();
268 auto L = InfoObj.ReadKeyDataLength(LocalPtr);
270 InfoObj.ReadDataInto(Key, LocalPtr + L.first, L.second, ResultBuilder);
279 template<
typename ReaderInfo,
typename WriterInfo>
282 typedef llvm::OnDiskChainedHashTableGenerator<WriterInfo> Generator;
289 void insert(
typename WriterInfo::key_type_ref Key,
290 typename WriterInfo::data_type_ref Data, WriterInfo &Info) {
291 Gen.insert(Key, Data, Info);
296 using namespace llvm::support;
297 llvm::raw_svector_ostream OutStream(Out);
301 endian::Writer<little> Writer(OutStream);
304 Writer.write<uint32_t>(0);
306 if (
auto *Merged = Base ? Base->getMergedTable() :
nullptr) {
308 Writer.write<uint32_t>(Merged->Files.size());
309 for (
const auto &F : Merged->Files)
310 Info.EmitFileRef(OutStream, F);
313 for (
auto &KV : Merged->Data) {
314 if (!Gen.contains(KV.first, Info))
315 Gen.insert(KV.first, Info.ImportData(KV.second), Info);
318 Writer.write<uint32_t>(0);
323 uint32_t BucketOffset = Gen.Emit(OutStream, Info);
326 endian::write32le(Out.data(), BucketOffset);
Info::data_type_builder data_type_builder
Writer for the on-disk hash table.
void insert(typename WriterInfo::key_type_ref Key, typename WriterInfo::data_type_ref Data, WriterInfo &Info)
data_type find(const external_key_type &EKey)
Find and read the lookup results for EKey.
A collection of on-disk hash tables, merged when relevant for performance.
detail::InMemoryDirectory::const_iterator I
Info::internal_key_type internal_key_type
Information about a module that has been loaded by the ASTReader.
const unsigned char * storage_type
A pointer to an on-disk representation of the hash table.
unsigned ComputeHash(Selector Sel)
Info::external_key_type external_key_type
MultiOnDiskHashTable(MultiOnDiskHashTable &&O)
Info::data_type data_type
void emit(llvm::SmallVectorImpl< char > &Out, WriterInfo &Info, const BaseTable *Base)
MultiOnDiskHashTable & operator=(MultiOnDiskHashTable &&O)
DeclarationName - The name of a declaration.
detail::InMemoryDirectory::const_iterator E
A key used when looking up entities by DeclarationName.
data_type findAll()
Read all the lookup results into a single value.
void add(file_type File, storage_type Data, Info InfoObj=Info())
Add the table Data loaded from file File.
Info::file_type file_type
A handle to a file, used when overriding tables.
MultiOnDiskHashTableGenerator()