LLVM  4.0.0
MappedBlockStream.cpp
Go to the documentation of this file.
1 //===- MappedBlockStream.cpp - Reads stream data from an MSF file ---------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 
11 
16 
17 using namespace llvm;
18 using namespace llvm::msf;
19 
20 namespace {
21 template <typename Base> class MappedBlockStreamImpl : public Base {
22 public:
23  template <typename... Args>
24  MappedBlockStreamImpl(Args &&... Params)
25  : Base(std::forward<Args>(Params)...) {}
26 };
27 }
28 
29 static void initializeFpmStreamLayout(const MSFLayout &Layout,
30  MSFStreamLayout &FpmLayout) {
31  uint32_t NumFpmIntervals = msf::getNumFpmIntervals(Layout);
32  support::ulittle32_t FpmBlock = Layout.SB->FreeBlockMapBlock;
33  assert(FpmBlock == 1 || FpmBlock == 2);
34  while (NumFpmIntervals > 0) {
35  FpmLayout.Blocks.push_back(FpmBlock);
36  FpmBlock += msf::getFpmIntervalLength(Layout);
37  --NumFpmIntervals;
38  }
39  FpmLayout.Length = msf::getFullFpmByteSize(Layout);
40 }
41 
42 typedef std::pair<uint32_t, uint32_t> Interval;
43 static Interval intersect(const Interval &I1, const Interval &I2) {
44  return std::make_pair(std::max(I1.first, I2.first),
45  std::min(I1.second, I2.second));
46 }
47 
49  const MSFStreamLayout &Layout,
50  const ReadableStream &MsfData)
51  : BlockSize(BlockSize), NumBlocks(NumBlocks), StreamLayout(Layout),
52  MsfData(MsfData) {}
53 
54 std::unique_ptr<MappedBlockStream>
56  const MSFStreamLayout &Layout,
57  const ReadableStream &MsfData) {
58  return llvm::make_unique<MappedBlockStreamImpl<MappedBlockStream>>(
59  BlockSize, NumBlocks, Layout, MsfData);
60 }
61 
62 std::unique_ptr<MappedBlockStream>
64  const ReadableStream &MsfData,
65  uint32_t StreamIndex) {
66  assert(StreamIndex < Layout.StreamMap.size() && "Invalid stream index");
67  MSFStreamLayout SL;
68  SL.Blocks = Layout.StreamMap[StreamIndex];
69  SL.Length = Layout.StreamSizes[StreamIndex];
70  return llvm::make_unique<MappedBlockStreamImpl<MappedBlockStream>>(
71  Layout.SB->BlockSize, Layout.SB->NumBlocks, SL, MsfData);
72 }
73 
74 std::unique_ptr<MappedBlockStream>
76  const ReadableStream &MsfData) {
77  MSFStreamLayout SL;
78  SL.Blocks = Layout.DirectoryBlocks;
79  SL.Length = Layout.SB->NumDirectoryBytes;
80  return createStream(Layout.SB->BlockSize, Layout.SB->NumBlocks, SL, MsfData);
81 }
82 
83 std::unique_ptr<MappedBlockStream>
85  const ReadableStream &MsfData) {
86  MSFStreamLayout SL;
87  initializeFpmStreamLayout(Layout, SL);
88  return createStream(Layout.SB->BlockSize, Layout.SB->NumBlocks, SL, MsfData);
89 }
90 
92  ArrayRef<uint8_t> &Buffer) const {
93  // Make sure we aren't trying to read beyond the end of the stream.
94  if (Size > StreamLayout.Length)
95  return make_error<MSFError>(msf_error_code::insufficient_buffer);
96  if (Offset > StreamLayout.Length - Size)
97  return make_error<MSFError>(msf_error_code::insufficient_buffer);
98 
99  if (tryReadContiguously(Offset, Size, Buffer))
100  return Error::success();
101 
102  auto CacheIter = CacheMap.find(Offset);
103  if (CacheIter != CacheMap.end()) {
104  // Try to find an alloc that was large enough for this request.
105  for (auto &Entry : CacheIter->second) {
106  if (Entry.size() >= Size) {
107  Buffer = Entry.slice(0, Size);
108  return Error::success();
109  }
110  }
111  }
112 
113  // We couldn't find a buffer that started at the correct offset (the most
114  // common scenario). Try to see if there is a buffer that starts at some
115  // other offset but overlaps the desired range.
116  for (auto &CacheItem : CacheMap) {
117  Interval RequestExtent = std::make_pair(Offset, Offset + Size);
118 
119  // We already checked this one on the fast path above.
120  if (CacheItem.first == Offset)
121  continue;
122  // If the initial extent of the cached item is beyond the ending extent
123  // of the request, there is no overlap.
124  if (CacheItem.first >= Offset + Size)
125  continue;
126 
127  // We really only have to check the last item in the list, since we append
128  // in order of increasing length.
129  if (CacheItem.second.empty())
130  continue;
131 
132  auto CachedAlloc = CacheItem.second.back();
133  // If the initial extent of the request is beyond the ending extent of
134  // the cached item, there is no overlap.
135  Interval CachedExtent =
136  std::make_pair(CacheItem.first, CacheItem.first + CachedAlloc.size());
137  if (RequestExtent.first >= CachedExtent.first + CachedExtent.second)
138  continue;
139 
140  Interval Intersection = intersect(CachedExtent, RequestExtent);
141  // Only use this if the entire request extent is contained in the cached
142  // extent.
143  if (Intersection != RequestExtent)
144  continue;
145 
146  uint32_t CacheRangeOffset =
147  AbsoluteDifference(CachedExtent.first, Intersection.first);
148  Buffer = CachedAlloc.slice(CacheRangeOffset, Size);
149  return Error::success();
150  }
151 
152  // Otherwise allocate a large enough buffer in the pool, memcpy the data
153  // into it, and return an ArrayRef to that. Do not touch existing pool
154  // allocations, as existing clients may be holding a pointer which must
155  // not be invalidated.
156  uint8_t *WriteBuffer = static_cast<uint8_t *>(Pool.Allocate(Size, 8));
157  if (auto EC = readBytes(Offset, MutableArrayRef<uint8_t>(WriteBuffer, Size)))
158  return EC;
159 
160  if (CacheIter != CacheMap.end()) {
161  CacheIter->second.emplace_back(WriteBuffer, Size);
162  } else {
163  std::vector<CacheEntry> List;
164  List.emplace_back(WriteBuffer, Size);
165  CacheMap.insert(std::make_pair(Offset, List));
166  }
167  Buffer = ArrayRef<uint8_t>(WriteBuffer, Size);
168  return Error::success();
169 }
170 
172  uint32_t Offset, ArrayRef<uint8_t> &Buffer) const {
173  // Make sure we aren't trying to read beyond the end of the stream.
174  if (Offset >= StreamLayout.Length)
175  return make_error<MSFError>(msf_error_code::insufficient_buffer);
176  uint32_t First = Offset / BlockSize;
177  uint32_t Last = First;
178 
179  while (Last < NumBlocks - 1) {
180  if (StreamLayout.Blocks[Last] != StreamLayout.Blocks[Last + 1] - 1)
181  break;
182  ++Last;
183  }
184 
185  uint32_t OffsetInFirstBlock = Offset % BlockSize;
186  uint32_t BytesFromFirstBlock = BlockSize - OffsetInFirstBlock;
187  uint32_t BlockSpan = Last - First + 1;
188  uint32_t ByteSpan = BytesFromFirstBlock + (BlockSpan - 1) * BlockSize;
189 
190  ArrayRef<uint8_t> BlockData;
191  uint32_t MsfOffset = blockToOffset(StreamLayout.Blocks[First], BlockSize);
192  if (auto EC = MsfData.readBytes(MsfOffset, BlockSize, BlockData))
193  return EC;
194 
195  BlockData = BlockData.drop_front(OffsetInFirstBlock);
196  Buffer = ArrayRef<uint8_t>(BlockData.data(), ByteSpan);
197  return Error::success();
198 }
199 
200 uint32_t MappedBlockStream::getLength() const { return StreamLayout.Length; }
201 
202 bool MappedBlockStream::tryReadContiguously(uint32_t Offset, uint32_t Size,
203  ArrayRef<uint8_t> &Buffer) const {
204  if (Size == 0) {
205  Buffer = ArrayRef<uint8_t>();
206  return true;
207  }
208  // Attempt to fulfill the request with a reference directly into the stream.
209  // This can work even if the request crosses a block boundary, provided that
210  // all subsequent blocks are contiguous. For example, a 10k read with a 4k
211  // block size can be filled with a reference if, from the starting offset,
212  // 3 blocks in a row are contiguous.
213  uint32_t BlockNum = Offset / BlockSize;
214  uint32_t OffsetInBlock = Offset % BlockSize;
215  uint32_t BytesFromFirstBlock = std::min(Size, BlockSize - OffsetInBlock);
216  uint32_t NumAdditionalBlocks =
217  llvm::alignTo(Size - BytesFromFirstBlock, BlockSize) / BlockSize;
218 
219  uint32_t RequiredContiguousBlocks = NumAdditionalBlocks + 1;
220  uint32_t E = StreamLayout.Blocks[BlockNum];
221  for (uint32_t I = 0; I < RequiredContiguousBlocks; ++I, ++E) {
222  if (StreamLayout.Blocks[I + BlockNum] != E)
223  return false;
224  }
225 
226  // Read out the entire block where the requested offset starts. Then drop
227  // bytes from the beginning so that the actual starting byte lines up with
228  // the requested starting byte. Then, since we know this is a contiguous
229  // cross-block span, explicitly resize the ArrayRef to cover the entire
230  // request length.
231  ArrayRef<uint8_t> BlockData;
232  uint32_t FirstBlockAddr = StreamLayout.Blocks[BlockNum];
233  uint32_t MsfOffset = blockToOffset(FirstBlockAddr, BlockSize);
234  if (auto EC = MsfData.readBytes(MsfOffset, BlockSize, BlockData)) {
235  consumeError(std::move(EC));
236  return false;
237  }
238  BlockData = BlockData.drop_front(OffsetInBlock);
239  Buffer = ArrayRef<uint8_t>(BlockData.data(), Size);
240  return true;
241 }
242 
244  MutableArrayRef<uint8_t> Buffer) const {
245  uint32_t BlockNum = Offset / BlockSize;
246  uint32_t OffsetInBlock = Offset % BlockSize;
247 
248  // Make sure we aren't trying to read beyond the end of the stream.
249  if (Buffer.size() > StreamLayout.Length)
250  return make_error<MSFError>(msf_error_code::insufficient_buffer);
251  if (Offset > StreamLayout.Length - Buffer.size())
252  return make_error<MSFError>(msf_error_code::insufficient_buffer);
253 
254  uint32_t BytesLeft = Buffer.size();
255  uint32_t BytesWritten = 0;
256  uint8_t *WriteBuffer = Buffer.data();
257  while (BytesLeft > 0) {
258  uint32_t StreamBlockAddr = StreamLayout.Blocks[BlockNum];
259 
260  ArrayRef<uint8_t> BlockData;
261  uint32_t Offset = blockToOffset(StreamBlockAddr, BlockSize);
262  if (auto EC = MsfData.readBytes(Offset, BlockSize, BlockData))
263  return EC;
264 
265  const uint8_t *ChunkStart = BlockData.data() + OffsetInBlock;
266  uint32_t BytesInChunk = std::min(BytesLeft, BlockSize - OffsetInBlock);
267  ::memcpy(WriteBuffer + BytesWritten, ChunkStart, BytesInChunk);
268 
269  BytesWritten += BytesInChunk;
270  BytesLeft -= BytesInChunk;
271  ++BlockNum;
272  OffsetInBlock = 0;
273  }
274 
275  return Error::success();
276 }
277 
279  return static_cast<uint32_t>(Pool.getBytesAllocated());
280 }
281 
282 void MappedBlockStream::invalidateCache() { CacheMap.shrink_and_clear(); }
283 
284 void MappedBlockStream::fixCacheAfterWrite(uint32_t Offset,
285  ArrayRef<uint8_t> Data) const {
286  // If this write overlapped a read which previously came from the pool,
287  // someone may still be holding a pointer to that alloc which is now invalid.
288  // Compute the overlapping range and update the cache entry, so any
289  // outstanding buffers are automatically updated.
290  for (const auto &MapEntry : CacheMap) {
291  // If the end of the written extent precedes the beginning of the cached
292  // extent, ignore this map entry.
293  if (Offset + Data.size() < MapEntry.first)
294  continue;
295  for (const auto &Alloc : MapEntry.second) {
296  // If the end of the cached extent precedes the beginning of the written
297  // extent, ignore this alloc.
298  if (MapEntry.first + Alloc.size() < Offset)
299  continue;
300 
301  // If we get here, they are guaranteed to overlap.
302  Interval WriteInterval = std::make_pair(Offset, Offset + Data.size());
303  Interval CachedInterval =
304  std::make_pair(MapEntry.first, MapEntry.first + Alloc.size());
305  // If they overlap, we need to write the new data into the overlapping
306  // range.
307  auto Intersection = intersect(WriteInterval, CachedInterval);
308  assert(Intersection.first <= Intersection.second);
309 
310  uint32_t Length = Intersection.second - Intersection.first;
311  uint32_t SrcOffset =
312  AbsoluteDifference(WriteInterval.first, Intersection.first);
313  uint32_t DestOffset =
314  AbsoluteDifference(CachedInterval.first, Intersection.first);
315  ::memcpy(Alloc.data() + DestOffset, Data.data() + SrcOffset, Length);
316  }
317  }
318 }
319 
321  uint32_t BlockSize, uint32_t NumBlocks, const MSFStreamLayout &Layout,
322  const WritableStream &MsfData)
323  : ReadInterface(BlockSize, NumBlocks, Layout, MsfData),
324  WriteInterface(MsfData) {}
325 
326 std::unique_ptr<WritableMappedBlockStream>
328  const MSFStreamLayout &Layout,
329  const WritableStream &MsfData) {
330  return llvm::make_unique<MappedBlockStreamImpl<WritableMappedBlockStream>>(
331  BlockSize, NumBlocks, Layout, MsfData);
332 }
333 
334 std::unique_ptr<WritableMappedBlockStream>
336  const WritableStream &MsfData,
337  uint32_t StreamIndex) {
338  assert(StreamIndex < Layout.StreamMap.size() && "Invalid stream index");
339  MSFStreamLayout SL;
340  SL.Blocks = Layout.StreamMap[StreamIndex];
341  SL.Length = Layout.StreamSizes[StreamIndex];
342  return createStream(Layout.SB->BlockSize, Layout.SB->NumBlocks, SL, MsfData);
343 }
344 
345 std::unique_ptr<WritableMappedBlockStream>
347  const MSFLayout &Layout, const WritableStream &MsfData) {
348  MSFStreamLayout SL;
349  SL.Blocks = Layout.DirectoryBlocks;
350  SL.Length = Layout.SB->NumDirectoryBytes;
351  return createStream(Layout.SB->BlockSize, Layout.SB->NumBlocks, SL, MsfData);
352 }
353 
354 std::unique_ptr<WritableMappedBlockStream>
356  const WritableStream &MsfData) {
357  MSFStreamLayout SL;
358  initializeFpmStreamLayout(Layout, SL);
359  return createStream(Layout.SB->BlockSize, Layout.SB->NumBlocks, SL, MsfData);
360 }
361 
363  ArrayRef<uint8_t> &Buffer) const {
364  return ReadInterface.readBytes(Offset, Size, Buffer);
365 }
366 
368  uint32_t Offset, ArrayRef<uint8_t> &Buffer) const {
369  return ReadInterface.readLongestContiguousChunk(Offset, Buffer);
370 }
371 
373  return ReadInterface.getLength();
374 }
375 
377  ArrayRef<uint8_t> Buffer) const {
378  // Make sure we aren't trying to write beyond the end of the stream.
379  if (Buffer.size() > getStreamLength())
380  return make_error<MSFError>(msf_error_code::insufficient_buffer);
381 
382  if (Offset > getStreamLayout().Length - Buffer.size())
383  return make_error<MSFError>(msf_error_code::insufficient_buffer);
384 
385  uint32_t BlockNum = Offset / getBlockSize();
386  uint32_t OffsetInBlock = Offset % getBlockSize();
387 
388  uint32_t BytesLeft = Buffer.size();
389  uint32_t BytesWritten = 0;
390  while (BytesLeft > 0) {
391  uint32_t StreamBlockAddr = getStreamLayout().Blocks[BlockNum];
392  uint32_t BytesToWriteInChunk =
393  std::min(BytesLeft, getBlockSize() - OffsetInBlock);
394 
395  const uint8_t *Chunk = Buffer.data() + BytesWritten;
396  ArrayRef<uint8_t> ChunkData(Chunk, BytesToWriteInChunk);
397  uint32_t MsfOffset = blockToOffset(StreamBlockAddr, getBlockSize());
398  MsfOffset += OffsetInBlock;
399  if (auto EC = WriteInterface.writeBytes(MsfOffset, ChunkData))
400  return EC;
401 
402  BytesLeft -= BytesToWriteInChunk;
403  BytesWritten += BytesToWriteInChunk;
404  ++BlockNum;
405  OffsetInBlock = 0;
406  }
407 
408  ReadInterface.fixCacheAfterWrite(Offset, Buffer);
409 
410  return Error::success();
411 }
412 
414  return WriteInterface.commit();
415 }
static std::unique_ptr< MappedBlockStream > createStream(uint32_t BlockSize, uint32_t NumBlocks, const MSFStreamLayout &Layout, const ReadableStream &MsfData)
Error writeBytes(uint32_t Offset, ArrayRef< uint8_t > Buffer) const override
static void initializeFpmStreamLayout(const MSFLayout &Layout, MSFStreamLayout &FpmLayout)
size_t getBytesAllocated() const
Definition: Allocator.h:284
Interval Class - An Interval is a set of nodes defined such that every node in the interval has all o...
Definition: Interval.h:37
uint64_t blockToOffset(uint64_t BlockNumber, uint64_t BlockSize)
Definition: MSFCommon.h:83
T * data() const
Definition: ArrayRef.h:322
static std::unique_ptr< WritableMappedBlockStream > createDirectoryStream(const MSFLayout &Layout, const WritableStream &MsfData)
static Interval intersect(const Interval &I1, const Interval &I2)
uint64_t alignTo(uint64_t Value, uint64_t Align, uint64_t Skew=0)
Returns the next integer (mod 2**64) that is greater than or equal to Value and is a multiple of Alig...
Definition: MathExtras.h:664
uint32_t getLength() const override
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array...
Definition: ArrayRef.h:171
Describes the layout of a stream in an MSF layout.
Error readLongestContiguousChunk(uint32_t Offset, ArrayRef< uint8_t > &Buffer) const override
Error readBytes(uint32_t Offset, uint32_t Size, ArrayRef< uint8_t > &Buffer) const override
uint32_t getFpmIntervalLength(const MSFLayout &L)
Definition: MSFCommon.h:87
static std::unique_ptr< WritableMappedBlockStream > createFpmStream(const MSFLayout &Layout, const WritableStream &MsfData)
static std::unique_ptr< MappedBlockStream > createDirectoryStream(const MSFLayout &Layout, const ReadableStream &MsfData)
static std::unique_ptr< WritableMappedBlockStream > createStream(uint32_t BlockSize, uint32_t NumBlocks, const MSFStreamLayout &Layout, const WritableStream &MsfData)
std::vector< support::ulittle32_t > Blocks
static std::unique_ptr< MappedBlockStream > createFpmStream(const MSFLayout &Layout, const ReadableStream &MsfData)
support::ulittle32_t BlockSize
Definition: MSFCommon.h:37
Error readBytes(uint32_t Offset, uint32_t Size, ArrayRef< uint8_t > &Buffer) const override
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:141
Maximum length of the test input libFuzzer tries to guess a good value based on the corpus and reports it always prefer smaller inputs during the corpus shuffle When libFuzzer itself reports a bug this exit code will be used If indicates the maximal total time in seconds to run the fuzzer minimizes the provided crash input Use with etc Experimental Use value profile to guide fuzzing Number of simultaneous worker processes to run the jobs If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
MappedBlockStream(uint32_t BlockSize, uint32_t NumBlocks, const MSFStreamLayout &StreamLayout, const ReadableStream &MsfData)
LLVM_ATTRIBUTE_RETURNS_NONNULL LLVM_ATTRIBUTE_RETURNS_NOALIAS void * Allocate(size_t Size, size_t Alignment)
Allocate space at the specified alignment.
Definition: Allocator.h:212
uint32_t Offset
virtual Error commit() const =0
void consumeError(Error Err)
Consume a Error without doing anything.
uint32_t getNumFpmIntervals(const MSFLayout &L)
Definition: MSFCommon.h:91
static const int BlockSize
Definition: TarWriter.cpp:34
std::pair< uint32_t, uint32_t > Interval
static ErrorSuccess success()
Create a success value.
WritableMappedBlockStream(uint32_t BlockSize, uint32_t NumBlocks, const MSFStreamLayout &StreamLayout, const WritableStream &MsfData)
ArrayRef< support::ulittle32_t > DirectoryBlocks
Definition: MSFCommon.h:56
static std::unique_ptr< MappedBlockStream > createIndexedStream(const MSFLayout &Layout, const ReadableStream &MsfData, uint32_t StreamIndex)
Error readLongestContiguousChunk(uint32_t Offset, ArrayRef< uint8_t > &Buffer) const override
virtual Error readBytes(uint32_t Offset, uint32_t Size, ArrayRef< uint8_t > &Buffer) const =0
const NodeList & List
Definition: RDFGraph.cpp:205
#define I(x, y, z)
Definition: MD5.cpp:54
std::enable_if< std::is_unsigned< T >::value, T >::type AbsoluteDifference(T X, T Y)
Subtract two unsigned integers, X and Y, of type T and return the absolute value of the result...
Definition: MathExtras.h:741
std::vector< ArrayRef< support::ulittle32_t > > StreamMap
Definition: MSFCommon.h:58
static std::unique_ptr< WritableMappedBlockStream > createIndexedStream(const MSFLayout &Layout, const WritableStream &MsfData, uint32_t StreamIndex)
uint32_t getFullFpmByteSize(const MSFLayout &L)
Definition: MSFCommon.h:96
ArrayRef< support::ulittle32_t > StreamSizes
Definition: MSFCommon.h:57
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
Definition: ArrayRef.h:180
support::ulittle32_t FreeBlockMapBlock
Definition: MSFCommon.h:39
Lightweight error class with error context and mandatory checking.
const SuperBlock * SB
Definition: MSFCommon.h:54
support::ulittle32_t NumDirectoryBytes
Definition: MSFCommon.h:45
const T * data() const
Definition: ArrayRef.h:138
support::ulittle32_t NumBlocks
Definition: MSFCommon.h:43
const MSFStreamLayout & getStreamLayout() const
virtual Error writeBytes(uint32_t Offset, ArrayRef< uint8_t > Data) const =0