LLVM  9.0.0svn
Caching.cpp
Go to the documentation of this file.
1 //===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the Caching for ThinLTO.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/LTO/Caching.h"
14 #include "llvm/ADT/StringExtras.h"
15 #include "llvm/Support/Errc.h"
17 #include "llvm/Support/Path.h"
18 #include "llvm/Support/Process.h"
20 
21 #if !defined(_MSC_VER) && !defined(__MINGW32__)
22 #include <unistd.h>
23 #else
24 #include <io.h>
25 #endif
26 
27 using namespace llvm;
28 using namespace llvm::lto;
29 
31  AddBufferFn AddBuffer) {
32  if (std::error_code EC = sys::fs::create_directories(CacheDirectoryPath))
33  return errorCodeToError(EC);
34 
35  return [=](unsigned Task, StringRef Key) -> AddStreamFn {
36  // This choice of file name allows the cache to be pruned (see pruneCache()
37  // in include/llvm/Support/CachePruning.h).
38  SmallString<64> EntryPath;
39  sys::path::append(EntryPath, CacheDirectoryPath, "llvmcache-" + Key);
40  // First, see if we have a cache hit.
41  int FD;
42  SmallString<64> ResultPath;
43  std::error_code EC = sys::fs::openFileForRead(
44  Twine(EntryPath), FD, sys::fs::OF_UpdateAtime, &ResultPath);
45  if (!EC) {
47  MemoryBuffer::getOpenFile(FD, EntryPath,
48  /*FileSize*/ -1,
49  /*RequiresNullTerminator*/ false);
50  close(FD);
51  if (MBOrErr) {
52  AddBuffer(Task, std::move(*MBOrErr));
53  return AddStreamFn();
54  }
55  EC = MBOrErr.getError();
56  }
57 
58  // On Windows we can fail to open a cache file with a permission denied
59  // error. This generally means that another process has requested to delete
60  // the file while it is still open, but it could also mean that another
61  // process has opened the file without the sharing permissions we need.
62  // Since the file is probably being deleted we handle it in the same way as
63  // if the file did not exist at all.
65  report_fatal_error(Twine("Failed to open cache file ") + EntryPath +
66  ": " + EC.message() + "\n");
67 
68  // This native object stream is responsible for commiting the resulting
69  // file to the cache and calling AddBuffer to add it to the link.
70  struct CacheStream : NativeObjectStream {
71  AddBufferFn AddBuffer;
72  sys::fs::TempFile TempFile;
73  std::string EntryPath;
74  unsigned Task;
75 
76  CacheStream(std::unique_ptr<raw_pwrite_stream> OS, AddBufferFn AddBuffer,
77  sys::fs::TempFile TempFile, std::string EntryPath,
78  unsigned Task)
79  : NativeObjectStream(std::move(OS)), AddBuffer(std::move(AddBuffer)),
80  TempFile(std::move(TempFile)), EntryPath(std::move(EntryPath)),
81  Task(Task) {}
82 
83  ~CacheStream() {
84  // Make sure the stream is closed before committing it.
85  OS.reset();
86 
87  // Open the file first to avoid racing with a cache pruner.
89  MemoryBuffer::getOpenFile(TempFile.FD, TempFile.TmpName,
90  /*FileSize*/ -1,
91  /*RequiresNullTerminator*/ false);
92  if (!MBOrErr)
93  report_fatal_error(Twine("Failed to open new cache file ") +
94  TempFile.TmpName + ": " +
95  MBOrErr.getError().message() + "\n");
96 
97  // On POSIX systems, this will atomically replace the destination if
98  // it already exists. We try to emulate this on Windows, but this may
99  // fail with a permission denied error (for example, if the destination
100  // is currently opened by another process that does not give us the
101  // sharing permissions we need). Since the existing file should be
102  // semantically equivalent to the one we are trying to write, we give
103  // AddBuffer a copy of the bytes we wrote in that case. We do this
104  // instead of just using the existing file, because the pruner might
105  // delete the file before we get a chance to use it.
106  Error E = TempFile.keep(EntryPath);
107  E = handleErrors(std::move(E), [&](const ECError &E) -> Error {
108  std::error_code EC = E.convertToErrorCode();
109  if (EC != errc::permission_denied)
110  return errorCodeToError(EC);
111 
112  auto MBCopy = MemoryBuffer::getMemBufferCopy((*MBOrErr)->getBuffer(),
113  EntryPath);
114  MBOrErr = std::move(MBCopy);
115 
116  // FIXME: should we consume the discard error?
117  consumeError(TempFile.discard());
118 
119  return Error::success();
120  });
121 
122  if (E)
123  report_fatal_error(Twine("Failed to rename temporary file ") +
124  TempFile.TmpName + " to " + EntryPath + ": " +
125  toString(std::move(E)) + "\n");
126 
127  AddBuffer(Task, std::move(*MBOrErr));
128  }
129  };
130 
131  return [=](size_t Task) -> std::unique_ptr<NativeObjectStream> {
132  // Write to a temporary to avoid race condition
133  SmallString<64> TempFilenameModel;
134  sys::path::append(TempFilenameModel, CacheDirectoryPath, "Thin-%%%%%%.tmp.o");
136  TempFilenameModel, sys::fs::owner_read | sys::fs::owner_write);
137  if (!Temp) {
138  errs() << "Error: " << toString(Temp.takeError()) << "\n";
139  report_fatal_error("ThinLTO: Can't get a temporary file");
140  }
141 
142  // This CacheStream will move the temporary file into the cache when done.
143  return llvm::make_unique<CacheStream>(
144  llvm::make_unique<raw_fd_ostream>(Temp->FD, /* ShouldClose */ false),
145  AddBuffer, std::move(*Temp), EntryPath.str(), Task);
146  };
147  };
148 }
std::error_code create_directories(const Twine &path, bool IgnoreExisting=true, perms Perms=owner_all|group_all)
Create all the non-existent directories in path.
Definition: Path.cpp:915
Represents either an error or a value T.
Definition: ErrorOr.h:56
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:23
std::error_code openFileForRead(const Twine &Name, int &ResultFD, OpenFlags Flags=OF_None, SmallVectorImpl< char > *RealPath=nullptr)
Opens the file with the given name in a read-only mode, returning its open file descriptor.
Error takeError()
Take ownership of the stored error.
Definition: Error.h:552
static Expected< TempFile > create(const Twine &Model, unsigned Mode=all_read|all_write)
This creates a temporary file with createUniqueFile and schedules it for deletion with sys::RemoveFil...
Definition: Path.cpp:1227
Error keep(const Twine &Name)
Definition: Path.cpp:1159
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Error.h:1072
void append(SmallVectorImpl< char > &path, const Twine &a, const Twine &b="", const Twine &c="", const Twine &d="")
Append to path.
Definition: Path.cpp:455
std::string toString(Error E)
Write all error messages (if any) in E to a string.
Definition: Error.h:966
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
Represents a temporary file.
Definition: FileSystem.h:826
Tagged union holding either a T or a Error.
Definition: CachePruning.h:22
StringRef str() const
Explicit conversion to StringRef.
Definition: SmallString.h:266
Key
PAL metadata keys.
std::function< void(unsigned Task, std::unique_ptr< MemoryBuffer > MB)> AddBufferFn
This type defines the callback to add a pre-existing native object file (e.g.
Definition: Caching.h:28
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Definition: Error.cpp:87
This class wraps an output stream for a native object.
Definition: LTO.h:189
Expected< NativeObjectCache > localCache(StringRef CacheDirectoryPath, AddBufferFn AddBuffer)
Create a local file system cache which uses the given cache directory and file callback.
Definition: Caching.cpp:30
std::error_code getError() const
Definition: ErrorOr.h:159
void consumeError(Error Err)
Consume a Error without doing anything.
Definition: Error.h:981
Force files Atime to be updated on access. Only makes a difference on windows.
Definition: FileSystem.h:764
std::function< std::unique_ptr< NativeObjectStream >(unsigned Task)> AddStreamFn
This type defines the callback to add a native object that is generated on the fly.
Definition: LTO.h:201
static ErrorSuccess success()
Create a success value.
Definition: Error.h:326
static std::unique_ptr< MemoryBuffer > getMemBufferCopy(StringRef InputData, const Twine &BufferName="")
Open the specified memory range as a MemoryBuffer, copying the contents and taking ownership of it...
Provides a library for accessing information about this process and other processes on the operating ...
Lightweight error class with error context and mandatory checking.
Definition: Error.h:157
Error handleErrors(Error E, HandlerTs &&... Hs)
Pass the ErrorInfo(s) contained in E to their respective handlers.
Definition: Error.h:881
This class wraps a std::error_code in a Error.
Definition: Error.h:1065
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
static ErrorOr< std::unique_ptr< MemoryBuffer > > getOpenFile(int FD, const Twine &Filename, uint64_t FileSize, bool RequiresNullTerminator=true, bool IsVolatile=false)
Given an already-open file descriptor, read the file and return a MemoryBuffer.