LLVM  10.0.0svn
Caching.cpp
Go to the documentation of this file.
1 //===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the Caching for ThinLTO.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/LTO/Caching.h"
14 #include "llvm/ADT/StringExtras.h"
15 #include "llvm/Support/Errc.h"
17 #include "llvm/Support/Path.h"
18 #include "llvm/Support/Process.h"
20 
21 #if !defined(_MSC_VER) && !defined(__MINGW32__)
22 #include <unistd.h>
23 #else
24 #include <io.h>
25 #endif
26 
27 using namespace llvm;
28 using namespace llvm::lto;
29 
31  AddBufferFn AddBuffer) {
32  if (std::error_code EC = sys::fs::create_directories(CacheDirectoryPath))
33  return errorCodeToError(EC);
34 
35  return [=](unsigned Task, StringRef Key) -> AddStreamFn {
36  // This choice of file name allows the cache to be pruned (see pruneCache()
37  // in include/llvm/Support/CachePruning.h).
38  SmallString<64> EntryPath;
39  sys::path::append(EntryPath, CacheDirectoryPath, "llvmcache-" + Key);
40  // First, see if we have a cache hit.
41  SmallString<64> ResultPath;
43  Twine(EntryPath), sys::fs::OF_UpdateAtime, &ResultPath);
44  std::error_code EC;
45  if (FDOrErr) {
47  MemoryBuffer::getOpenFile(*FDOrErr, EntryPath,
48  /*FileSize=*/-1,
49  /*RequiresNullTerminator=*/false);
50  sys::fs::closeFile(*FDOrErr);
51  if (MBOrErr) {
52  AddBuffer(Task, std::move(*MBOrErr));
53  return AddStreamFn();
54  }
55  EC = MBOrErr.getError();
56  } else {
57  EC = errorToErrorCode(FDOrErr.takeError());
58  }
59 
60  // On Windows we can fail to open a cache file with a permission denied
61  // error. This generally means that another process has requested to delete
62  // the file while it is still open, but it could also mean that another
63  // process has opened the file without the sharing permissions we need.
64  // Since the file is probably being deleted we handle it in the same way as
65  // if the file did not exist at all.
67  report_fatal_error(Twine("Failed to open cache file ") + EntryPath +
68  ": " + EC.message() + "\n");
69 
70  // This native object stream is responsible for commiting the resulting
71  // file to the cache and calling AddBuffer to add it to the link.
72  struct CacheStream : NativeObjectStream {
73  AddBufferFn AddBuffer;
74  sys::fs::TempFile TempFile;
75  std::string EntryPath;
76  unsigned Task;
77 
78  CacheStream(std::unique_ptr<raw_pwrite_stream> OS, AddBufferFn AddBuffer,
79  sys::fs::TempFile TempFile, std::string EntryPath,
80  unsigned Task)
81  : NativeObjectStream(std::move(OS)), AddBuffer(std::move(AddBuffer)),
82  TempFile(std::move(TempFile)), EntryPath(std::move(EntryPath)),
83  Task(Task) {}
84 
85  ~CacheStream() {
86  // Make sure the stream is closed before committing it.
87  OS.reset();
88 
89  // Open the file first to avoid racing with a cache pruner.
92  sys::fs::convertFDToNativeFile(TempFile.FD), TempFile.TmpName,
93  /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
94  if (!MBOrErr)
95  report_fatal_error(Twine("Failed to open new cache file ") +
96  TempFile.TmpName + ": " +
97  MBOrErr.getError().message() + "\n");
98 
99  // On POSIX systems, this will atomically replace the destination if
100  // it already exists. We try to emulate this on Windows, but this may
101  // fail with a permission denied error (for example, if the destination
102  // is currently opened by another process that does not give us the
103  // sharing permissions we need). Since the existing file should be
104  // semantically equivalent to the one we are trying to write, we give
105  // AddBuffer a copy of the bytes we wrote in that case. We do this
106  // instead of just using the existing file, because the pruner might
107  // delete the file before we get a chance to use it.
108  Error E = TempFile.keep(EntryPath);
109  E = handleErrors(std::move(E), [&](const ECError &E) -> Error {
110  std::error_code EC = E.convertToErrorCode();
111  if (EC != errc::permission_denied)
112  return errorCodeToError(EC);
113 
114  auto MBCopy = MemoryBuffer::getMemBufferCopy((*MBOrErr)->getBuffer(),
115  EntryPath);
116  MBOrErr = std::move(MBCopy);
117 
118  // FIXME: should we consume the discard error?
119  consumeError(TempFile.discard());
120 
121  return Error::success();
122  });
123 
124  if (E)
125  report_fatal_error(Twine("Failed to rename temporary file ") +
126  TempFile.TmpName + " to " + EntryPath + ": " +
127  toString(std::move(E)) + "\n");
128 
129  AddBuffer(Task, std::move(*MBOrErr));
130  }
131  };
132 
133  return [=](size_t Task) -> std::unique_ptr<NativeObjectStream> {
134  // Write to a temporary to avoid race condition
135  SmallString<64> TempFilenameModel;
136  sys::path::append(TempFilenameModel, CacheDirectoryPath, "Thin-%%%%%%.tmp.o");
138  TempFilenameModel, sys::fs::owner_read | sys::fs::owner_write);
139  if (!Temp) {
140  errs() << "Error: " << toString(Temp.takeError()) << "\n";
141  report_fatal_error("ThinLTO: Can't get a temporary file");
142  }
143 
144  // This CacheStream will move the temporary file into the cache when done.
145  return std::make_unique<CacheStream>(
146  std::make_unique<raw_fd_ostream>(Temp->FD, /* ShouldClose */ false),
147  AddBuffer, std::move(*Temp), EntryPath.str(), Task);
148  };
149  };
150 }
std::error_code create_directories(const Twine &path, bool IgnoreExisting=true, perms Perms=owner_all|group_all)
Create all the non-existent directories in path.
Definition: Path.cpp:915
Represents either an error or a value T.
Definition: ErrorOr.h:56
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:23
Error takeError()
Take ownership of the stored error.
Definition: Error.h:552
static Expected< TempFile > create(const Twine &Model, unsigned Mode=all_read|all_write)
This creates a temporary file with createUniqueFile and schedules it for deletion with sys::RemoveFil...
Definition: Path.cpp:1227
Error keep(const Twine &Name)
Definition: Path.cpp:1159
std::error_code convertToErrorCode() const override
Convert this error to a std::error_code.
Definition: Error.h:1106
file_t convertFDToNativeFile(int FD)
Converts from a Posix file descriptor number to a native file handle.
Definition: FileSystem.h:979
void append(SmallVectorImpl< char > &path, const Twine &a, const Twine &b="", const Twine &c="", const Twine &d="")
Append to path.
Definition: Path.cpp:455
std::string toString(Error E)
Write all error messages (if any) in E to a string.
Definition: Error.h:986
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
Represents a temporary file.
Definition: FileSystem.h:844
static ErrorOr< std::unique_ptr< MemoryBuffer > > getOpenFile(sys::fs::file_t FD, const Twine &Filename, uint64_t FileSize, bool RequiresNullTerminator=true, bool IsVolatile=false)
Given an already-open file descriptor, read the file and return a MemoryBuffer.
Tagged union holding either a T or a Error.
Definition: yaml2obj.h:21
StringRef str() const
Explicit conversion to StringRef.
Definition: SmallString.h:266
Key
PAL metadata keys.
std::function< void(unsigned Task, std::unique_ptr< MemoryBuffer > MB)> AddBufferFn
This type defines the callback to add a pre-existing native object file (e.g.
Definition: Caching.h:28
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Definition: Error.cpp:87
This class wraps an output stream for a native object.
Definition: LTO.h:192
Expected< NativeObjectCache > localCache(StringRef CacheDirectoryPath, AddBufferFn AddBuffer)
Create a local file system cache which uses the given cache directory and file callback.
Definition: Caching.cpp:30
std::error_code getError() const
Definition: ErrorOr.h:159
void consumeError(Error Err)
Consume a Error without doing anything.
Definition: Error.h:1001
Force files Atime to be updated on access. Only makes a difference on windows.
Definition: FileSystem.h:782
std::function< std::unique_ptr< NativeObjectStream >(unsigned Task)> AddStreamFn
This type defines the callback to add a native object that is generated on the fly.
Definition: LTO.h:204
static ErrorSuccess success()
Create a success value.
Definition: Error.h:326
Expected< file_t > openNativeFileForRead(const Twine &Name, OpenFlags Flags=OF_None, SmallVectorImpl< char > *RealPath=nullptr)
Opens the file with the given name in a read-only mode, returning its open file descriptor.
std::error_code closeFile(file_t &F)
Close the file object.
static std::unique_ptr< MemoryBuffer > getMemBufferCopy(StringRef InputData, const Twine &BufferName="")
Open the specified memory range as a MemoryBuffer, copying the contents and taking ownership of it...
Provides a library for accessing information about this process and other processes on the operating ...
Lightweight error class with error context and mandatory checking.
Definition: Error.h:157
Error handleErrors(Error E, HandlerTs &&... Hs)
Pass the ErrorInfo(s) contained in E to their respective handlers.
Definition: Error.h:901
This class wraps a std::error_code in a Error.
Definition: Error.h:1099
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
std::error_code errorToErrorCode(Error Err)
Helper for converting an ECError to a std::error_code.
Definition: Error.cpp:93