LLVM  3.7.0
MemoryBuffer.cpp
Go to the documentation of this file.
1 //===--- MemoryBuffer.cpp - Memory Buffer implementation ------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the MemoryBuffer interface.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "llvm/ADT/SmallString.h"
16 #include "llvm/Config/config.h"
17 #include "llvm/Support/Errc.h"
18 #include "llvm/Support/Errno.h"
21 #include "llvm/Support/Path.h"
22 #include "llvm/Support/Process.h"
23 #include "llvm/Support/Program.h"
24 #include <cassert>
25 #include <cerrno>
26 #include <cstring>
27 #include <new>
28 #include <sys/types.h>
29 #include <system_error>
30 #if !defined(_MSC_VER) && !defined(__MINGW32__)
31 #include <unistd.h>
32 #else
33 #include <io.h>
34 #endif
35 using namespace llvm;
36 
37 //===----------------------------------------------------------------------===//
38 // MemoryBuffer implementation itself.
39 //===----------------------------------------------------------------------===//
40 
42 
43 /// init - Initialize this MemoryBuffer as a reference to externally allocated
44 /// memory, memory that we know is already null terminated.
45 void MemoryBuffer::init(const char *BufStart, const char *BufEnd,
46  bool RequiresNullTerminator) {
47  assert((!RequiresNullTerminator || BufEnd[0] == 0) &&
48  "Buffer is not null terminated!");
49  BufferStart = BufStart;
50  BufferEnd = BufEnd;
51 }
52 
53 //===----------------------------------------------------------------------===//
54 // MemoryBufferMem implementation.
55 //===----------------------------------------------------------------------===//
56 
57 /// CopyStringRef - Copies contents of a StringRef into a block of memory and
58 /// null-terminates it.
59 static void CopyStringRef(char *Memory, StringRef Data) {
60  if (!Data.empty())
61  memcpy(Memory, Data.data(), Data.size());
62  Memory[Data.size()] = 0; // Null terminate string.
63 }
64 
65 namespace {
66 struct NamedBufferAlloc {
67  const Twine &Name;
68  NamedBufferAlloc(const Twine &Name) : Name(Name) {}
69 };
70 }
71 
72 void *operator new(size_t N, const NamedBufferAlloc &Alloc) {
73  SmallString<256> NameBuf;
74  StringRef NameRef = Alloc.Name.toStringRef(NameBuf);
75 
76  char *Mem = static_cast<char *>(operator new(N + NameRef.size() + 1));
77  CopyStringRef(Mem + N, NameRef);
78  return Mem;
79 }
80 
81 namespace {
82 /// MemoryBufferMem - Named MemoryBuffer pointing to a block of memory.
83 class MemoryBufferMem : public MemoryBuffer {
84 public:
85  MemoryBufferMem(StringRef InputData, bool RequiresNullTerminator) {
86  init(InputData.begin(), InputData.end(), RequiresNullTerminator);
87  }
88 
89  const char *getBufferIdentifier() const override {
90  // The name is stored after the class itself.
91  return reinterpret_cast<const char*>(this + 1);
92  }
93 
94  BufferKind getBufferKind() const override {
95  return MemoryBuffer_Malloc;
96  }
97 };
98 }
99 
101 getFileAux(const Twine &Filename, int64_t FileSize, uint64_t MapSize,
102  uint64_t Offset, bool RequiresNullTerminator, bool IsVolatileSize);
103 
104 std::unique_ptr<MemoryBuffer>
106  bool RequiresNullTerminator) {
107  auto *Ret = new (NamedBufferAlloc(BufferName))
108  MemoryBufferMem(InputData, RequiresNullTerminator);
109  return std::unique_ptr<MemoryBuffer>(Ret);
110 }
111 
112 std::unique_ptr<MemoryBuffer>
113 MemoryBuffer::getMemBuffer(MemoryBufferRef Ref, bool RequiresNullTerminator) {
114  return std::unique_ptr<MemoryBuffer>(getMemBuffer(
115  Ref.getBuffer(), Ref.getBufferIdentifier(), RequiresNullTerminator));
116 }
117 
118 std::unique_ptr<MemoryBuffer>
119 MemoryBuffer::getMemBufferCopy(StringRef InputData, const Twine &BufferName) {
120  std::unique_ptr<MemoryBuffer> Buf =
121  getNewUninitMemBuffer(InputData.size(), BufferName);
122  if (!Buf)
123  return nullptr;
124  memcpy(const_cast<char*>(Buf->getBufferStart()), InputData.data(),
125  InputData.size());
126  return Buf;
127 }
128 
129 std::unique_ptr<MemoryBuffer>
130 MemoryBuffer::getNewUninitMemBuffer(size_t Size, const Twine &BufferName) {
131  // Allocate space for the MemoryBuffer, the data and the name. It is important
132  // that MemoryBuffer and data are aligned so PointerIntPair works with them.
133  // TODO: Is 16-byte alignment enough? We copy small object files with large
134  // alignment expectations into this buffer.
135  SmallString<256> NameBuf;
136  StringRef NameRef = BufferName.toStringRef(NameBuf);
137  size_t AlignedStringLen =
138  RoundUpToAlignment(sizeof(MemoryBufferMem) + NameRef.size() + 1, 16);
139  size_t RealLen = AlignedStringLen + Size + 1;
140  char *Mem = static_cast<char*>(operator new(RealLen, std::nothrow));
141  if (!Mem)
142  return nullptr;
143 
144  // The name is stored after the class itself.
145  CopyStringRef(Mem + sizeof(MemoryBufferMem), NameRef);
146 
147  // The buffer begins after the name and must be aligned.
148  char *Buf = Mem + AlignedStringLen;
149  Buf[Size] = 0; // Null terminate buffer.
150 
151  auto *Ret = new (Mem) MemoryBufferMem(StringRef(Buf, Size), true);
152  return std::unique_ptr<MemoryBuffer>(Ret);
153 }
154 
155 std::unique_ptr<MemoryBuffer>
156 MemoryBuffer::getNewMemBuffer(size_t Size, StringRef BufferName) {
157  std::unique_ptr<MemoryBuffer> SB = getNewUninitMemBuffer(Size, BufferName);
158  if (!SB)
159  return nullptr;
160  memset(const_cast<char*>(SB->getBufferStart()), 0, Size);
161  return SB;
162 }
163 
165 MemoryBuffer::getFileOrSTDIN(const Twine &Filename, int64_t FileSize) {
166  SmallString<256> NameBuf;
167  StringRef NameRef = Filename.toStringRef(NameBuf);
168 
169  if (NameRef == "-")
170  return getSTDIN();
171  return getFile(Filename, FileSize);
172 }
173 
175 MemoryBuffer::getFileSlice(const Twine &FilePath, uint64_t MapSize,
176  uint64_t Offset) {
177  return getFileAux(FilePath, -1, MapSize, Offset, false, false);
178 }
179 
180 
181 //===----------------------------------------------------------------------===//
182 // MemoryBuffer::getFile implementation.
183 //===----------------------------------------------------------------------===//
184 
185 namespace {
186 /// \brief Memory maps a file descriptor using sys::fs::mapped_file_region.
187 ///
188 /// This handles converting the offset into a legal offset on the platform.
189 class MemoryBufferMMapFile : public MemoryBuffer {
191 
192  static uint64_t getLegalMapOffset(uint64_t Offset) {
193  return Offset & ~(sys::fs::mapped_file_region::alignment() - 1);
194  }
195 
196  static uint64_t getLegalMapSize(uint64_t Len, uint64_t Offset) {
197  return Len + (Offset - getLegalMapOffset(Offset));
198  }
199 
200  const char *getStart(uint64_t Len, uint64_t Offset) {
201  return MFR.const_data() + (Offset - getLegalMapOffset(Offset));
202  }
203 
204 public:
205  MemoryBufferMMapFile(bool RequiresNullTerminator, int FD, uint64_t Len,
206  uint64_t Offset, std::error_code &EC)
207  : MFR(FD, sys::fs::mapped_file_region::readonly,
208  getLegalMapSize(Len, Offset), getLegalMapOffset(Offset), EC) {
209  if (!EC) {
210  const char *Start = getStart(Len, Offset);
211  init(Start, Start + Len, RequiresNullTerminator);
212  }
213  }
214 
215  const char *getBufferIdentifier() const override {
216  // The name is stored after the class itself.
217  return reinterpret_cast<const char *>(this + 1);
218  }
219 
220  BufferKind getBufferKind() const override {
221  return MemoryBuffer_MMap;
222  }
223 };
224 }
225 
227 getMemoryBufferForStream(int FD, const Twine &BufferName) {
228  const ssize_t ChunkSize = 4096*4;
229  SmallString<ChunkSize> Buffer;
230  ssize_t ReadBytes;
231  // Read into Buffer until we hit EOF.
232  do {
233  Buffer.reserve(Buffer.size() + ChunkSize);
234  ReadBytes = read(FD, Buffer.end(), ChunkSize);
235  if (ReadBytes == -1) {
236  if (errno == EINTR) continue;
237  return std::error_code(errno, std::generic_category());
238  }
239  Buffer.set_size(Buffer.size() + ReadBytes);
240  } while (ReadBytes != 0);
241 
242  return MemoryBuffer::getMemBufferCopy(Buffer, BufferName);
243 }
244 
245 
247 MemoryBuffer::getFile(const Twine &Filename, int64_t FileSize,
248  bool RequiresNullTerminator, bool IsVolatileSize) {
249  return getFileAux(Filename, FileSize, FileSize, 0,
250  RequiresNullTerminator, IsVolatileSize);
251 }
252 
254 getOpenFileImpl(int FD, const Twine &Filename, uint64_t FileSize,
255  uint64_t MapSize, int64_t Offset, bool RequiresNullTerminator,
256  bool IsVolatileSize);
257 
259 getFileAux(const Twine &Filename, int64_t FileSize, uint64_t MapSize,
260  uint64_t Offset, bool RequiresNullTerminator, bool IsVolatileSize) {
261  int FD;
262  std::error_code EC = sys::fs::openFileForRead(Filename, FD);
263  if (EC)
264  return EC;
265 
267  getOpenFileImpl(FD, Filename, FileSize, MapSize, Offset,
268  RequiresNullTerminator, IsVolatileSize);
269  close(FD);
270  return Ret;
271 }
272 
273 static bool shouldUseMmap(int FD,
274  size_t FileSize,
275  size_t MapSize,
276  off_t Offset,
277  bool RequiresNullTerminator,
278  int PageSize,
279  bool IsVolatileSize) {
280  // mmap may leave the buffer without null terminator if the file size changed
281  // by the time the last page is mapped in, so avoid it if the file size is
282  // likely to change.
283  if (IsVolatileSize)
284  return false;
285 
286  // We don't use mmap for small files because this can severely fragment our
287  // address space.
288  if (MapSize < 4 * 4096 || MapSize < (unsigned)PageSize)
289  return false;
290 
291  if (!RequiresNullTerminator)
292  return true;
293 
294 
295  // If we don't know the file size, use fstat to find out. fstat on an open
296  // file descriptor is cheaper than stat on a random path.
297  // FIXME: this chunk of code is duplicated, but it avoids a fstat when
298  // RequiresNullTerminator = false and MapSize != -1.
299  if (FileSize == size_t(-1)) {
301  if (sys::fs::status(FD, Status))
302  return false;
303  FileSize = Status.getSize();
304  }
305 
306  // If we need a null terminator and the end of the map is inside the file,
307  // we cannot use mmap.
308  size_t End = Offset + MapSize;
309  assert(End <= FileSize);
310  if (End != FileSize)
311  return false;
312 
313  // Don't try to map files that are exactly a multiple of the system page size
314  // if we need a null terminator.
315  if ((FileSize & (PageSize -1)) == 0)
316  return false;
317 
318 #if defined(__CYGWIN__)
319  // Don't try to map files that are exactly a multiple of the physical page size
320  // if we need a null terminator.
321  // FIXME: We should reorganize again getPageSize() on Win32.
322  if ((FileSize & (4096 - 1)) == 0)
323  return false;
324 #endif
325 
326  return true;
327 }
328 
330 getOpenFileImpl(int FD, const Twine &Filename, uint64_t FileSize,
331  uint64_t MapSize, int64_t Offset, bool RequiresNullTerminator,
332  bool IsVolatileSize) {
333  static int PageSize = sys::Process::getPageSize();
334 
335  // Default is to map the full file.
336  if (MapSize == uint64_t(-1)) {
337  // If we don't know the file size, use fstat to find out. fstat on an open
338  // file descriptor is cheaper than stat on a random path.
339  if (FileSize == uint64_t(-1)) {
341  std::error_code EC = sys::fs::status(FD, Status);
342  if (EC)
343  return EC;
344 
345  // If this not a file or a block device (e.g. it's a named pipe
346  // or character device), we can't trust the size. Create the memory
347  // buffer by copying off the stream.
348  sys::fs::file_type Type = Status.type();
349  if (Type != sys::fs::file_type::regular_file &&
351  return getMemoryBufferForStream(FD, Filename);
352 
353  FileSize = Status.getSize();
354  }
355  MapSize = FileSize;
356  }
357 
358  if (shouldUseMmap(FD, FileSize, MapSize, Offset, RequiresNullTerminator,
359  PageSize, IsVolatileSize)) {
360  std::error_code EC;
361  std::unique_ptr<MemoryBuffer> Result(
362  new (NamedBufferAlloc(Filename))
363  MemoryBufferMMapFile(RequiresNullTerminator, FD, MapSize, Offset, EC));
364  if (!EC)
365  return std::move(Result);
366  }
367 
368  std::unique_ptr<MemoryBuffer> Buf =
369  MemoryBuffer::getNewUninitMemBuffer(MapSize, Filename);
370  if (!Buf) {
371  // Failed to create a buffer. The only way it can fail is if
372  // new(std::nothrow) returns 0.
374  }
375 
376  char *BufPtr = const_cast<char *>(Buf->getBufferStart());
377 
378  size_t BytesLeft = MapSize;
379 #ifndef HAVE_PREAD
380  if (lseek(FD, Offset, SEEK_SET) == -1)
381  return std::error_code(errno, std::generic_category());
382 #endif
383 
384  while (BytesLeft) {
385 #ifdef HAVE_PREAD
386  ssize_t NumRead = ::pread(FD, BufPtr, BytesLeft, MapSize-BytesLeft+Offset);
387 #else
388  ssize_t NumRead = ::read(FD, BufPtr, BytesLeft);
389 #endif
390  if (NumRead == -1) {
391  if (errno == EINTR)
392  continue;
393  // Error while reading.
394  return std::error_code(errno, std::generic_category());
395  }
396  if (NumRead == 0) {
397  memset(BufPtr, 0, BytesLeft); // zero-initialize rest of the buffer.
398  break;
399  }
400  BytesLeft -= NumRead;
401  BufPtr += NumRead;
402  }
403 
404  return std::move(Buf);
405 }
406 
408 MemoryBuffer::getOpenFile(int FD, const Twine &Filename, uint64_t FileSize,
409  bool RequiresNullTerminator, bool IsVolatileSize) {
410  return getOpenFileImpl(FD, Filename, FileSize, FileSize, 0,
411  RequiresNullTerminator, IsVolatileSize);
412 }
413 
415 MemoryBuffer::getOpenFileSlice(int FD, const Twine &Filename, uint64_t MapSize,
416  int64_t Offset) {
417  assert(MapSize != uint64_t(-1));
418  return getOpenFileImpl(FD, Filename, -1, MapSize, Offset, false,
419  /*IsVolatileSize*/ false);
420 }
421 
423  // Read in all of the data from stdin, we cannot mmap stdin.
424  //
425  // FIXME: That isn't necessarily true, we should try to mmap stdin and
426  // fallback if it fails.
428 
429  return getMemoryBufferForStream(0, "<stdin>");
430 }
431 
434  StringRef Identifier = getBufferIdentifier();
435  return MemoryBufferRef(Data, Identifier);
436 }
MemoryBufferRef getMemBufferRef() const
void init(const char *BufStart, const char *BufEnd, bool RequiresNullTerminator)
init - Initialize this MemoryBuffer as a reference to externally allocated memory, memory that we know is already null terminated.
Represents either an error or a value T.
Definition: ErrorOr.h:82
size_t size() const
size - Get the string size.
Definition: StringRef.h:113
value_type read(const void *memory)
Read a value of a particular endianness from memory.
Definition: Endian.h:49
This class provides various memory handling functions that manipulate MemoryBlock instances...
Definition: Memory.h:45
static std::unique_ptr< MemoryBuffer > getMemBuffer(StringRef InputData, StringRef BufferName="", bool RequiresNullTerminator=true)
Open the specified memory range as a MemoryBuffer.
static ErrorOr< std::unique_ptr< MemoryBuffer > > getMemoryBufferForStream(int FD, const Twine &BufferName)
This class represents a memory mapped file.
Definition: FileSystem.h:627
StringRef getBuffer() const
Definition: MemoryBuffer.h:54
static std::unique_ptr< MemoryBuffer > getNewUninitMemBuffer(size_t Size, const Twine &BufferName="")
Allocate a new MemoryBuffer of the specified size that is not initialized.
static ErrorOr< std::unique_ptr< MemoryBuffer > > getOpenFile(int FD, const Twine &Filename, uint64_t FileSize, bool RequiresNullTerminator=true, bool IsVolatileSize=false)
Given an already-open file descriptor, read the file and return a MemoryBuffer.
void reserve(size_type N)
Definition: SmallVector.h:401
static std::unique_ptr< MemoryBuffer > getNewMemBuffer(size_t Size, StringRef BufferName="")
Allocate a new zero-initialized MemoryBuffer of the specified size.
const char * const_data() const
Get a const view of the data.
static bool shouldUseMmap(int FD, size_t FileSize, size_t MapSize, off_t Offset, bool RequiresNullTerminator, int PageSize, bool IsVolatileSize)
file_status - Represents the result of a call to stat and friends.
Definition: FileSystem.h:138
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFileAux(const Twine &Filename, int64_t FileSize, uint64_t MapSize, uint64_t Offset, bool RequiresNullTerminator, bool IsVolatileSize)
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFileSlice(const Twine &Filename, uint64_t MapSize, uint64_t Offset)
Map a subrange of the specified file as a MemoryBuffer.
virtual const char * getBufferIdentifier() const
Return an identifier for this buffer, typically the filename it was read from.
Definition: MemoryBuffer.h:60
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:79
std::error_code make_error_code(BitcodeError E)
Definition: ReaderWriter.h:150
const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:107
static void CopyStringRef(char *Memory, StringRef Data)
CopyStringRef - Copies contents of a StringRef into a block of memory and null-terminates it...
iterator begin() const
Definition: StringRef.h:90
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:325
static ErrorOr< std::unique_ptr< MemoryBuffer > > getSTDIN()
Read all of stdin into a file buffer, and return it.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
static unsigned getPageSize()
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFileOrSTDIN(const Twine &Filename, int64_t FileSize=-1)
Open the specified file as a MemoryBuffer, or open stdin if the Filename is "-".
StringRef getBuffer() const
Definition: MemoryBuffer.h:157
StringRef toStringRef(SmallVectorImpl< char > &Out) const
This returns the twine as a single StringRef if it can be represented as such.
Definition: Twine.h:454
This interface provides simple read-only access to a block of memory, and provides simple methods for...
Definition: MemoryBuffer.h:37
static std::unique_ptr< MemoryBuffer > getMemBufferCopy(StringRef InputData, const Twine &BufferName="")
Open the specified memory range as a MemoryBuffer, copying the contents and taking ownership of it...
uint64_t RoundUpToAlignment(uint64_t Value, uint64_t Align)
Returns the next integer (mod 2**64) that is greater than or equal to Value and is a multiple of Alig...
Definition: MathExtras.h:609
std::error_code ChangeStdinToBinary()
static ErrorOr< std::unique_ptr< MemoryBuffer > > getOpenFileImpl(int FD, const Twine &Filename, uint64_t FileSize, uint64_t MapSize, int64_t Offset, bool RequiresNullTerminator, bool IsVolatileSize)
void set_size(size_type N)
Set the array size to N, which the current array must have enough capacity for.
Definition: SmallVector.h:685
Provides a library for accessing information about this process and other processes on the operating ...
static cl::opt< unsigned > PageSize("imp-null-check-page-size", cl::desc("The page size of the target in ""bytes"), cl::init(4096))
file_type
An enumeration for the file system's view of the type.
Definition: FileSystem.h:53
virtual ~MemoryBuffer()
std::error_code openFileForRead(const Twine &Name, int &ResultFD)
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFile(const Twine &Filename, int64_t FileSize=-1, bool RequiresNullTerminator=true, bool IsVolatileSize=false)
Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful, otherwise returning null.
#define N
StringRef getBufferIdentifier() const
Definition: MemoryBuffer.h:159
iterator end() const
Definition: StringRef.h:92
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:40
std::error_code status(const Twine &path, file_status &result)
Get file status as if by POSIX stat().
file_type type() const
Definition: FileSystem.h:196
static ErrorOr< std::unique_ptr< MemoryBuffer > > getOpenFileSlice(int FD, const Twine &Filename, uint64_t MapSize, int64_t Offset)
Given an already-open file descriptor, map some slice of it into a MemoryBuffer.
bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:110