LLVM  9.0.0svn
CachePruning.cpp
Go to the documentation of this file.
1 //===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the pruning of a directory based on least recently used.
10 //
11 //===----------------------------------------------------------------------===//
12 
14 
15 #include "llvm/Support/Debug.h"
16 #include "llvm/Support/Errc.h"
17 #include "llvm/Support/Error.h"
19 #include "llvm/Support/Path.h"
21 
22 #define DEBUG_TYPE "cache-pruning"
23 
24 #include <set>
25 #include <system_error>
26 
27 using namespace llvm;
28 
29 namespace {
30 struct FileInfo {
31  sys::TimePoint<> Time;
32  uint64_t Size;
33  std::string Path;
34 
35  /// Used to determine which files to prune first. Also used to determine
36  /// set membership, so must take into account all fields.
37  bool operator<(const FileInfo &Other) const {
38  if (Time < Other.Time)
39  return true;
40  else if (Other.Time < Time)
41  return false;
42  if (Other.Size < Size)
43  return true;
44  else if (Size < Other.Size)
45  return false;
46  return Path < Other.Path;
47  }
48 };
49 } // anonymous namespace
50 
51 /// Write a new timestamp file with the given path. This is used for the pruning
52 /// interval option.
53 static void writeTimestampFile(StringRef TimestampFile) {
54  std::error_code EC;
55  raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None);
56 }
57 
59  if (Duration.empty())
60  return make_error<StringError>("Duration must not be empty",
62 
63  StringRef NumStr = Duration.slice(0, Duration.size()-1);
64  uint64_t Num;
65  if (NumStr.getAsInteger(0, Num))
66  return make_error<StringError>("'" + NumStr + "' not an integer",
68 
69  switch (Duration.back()) {
70  case 's':
71  return std::chrono::seconds(Num);
72  case 'm':
73  return std::chrono::minutes(Num);
74  case 'h':
75  return std::chrono::hours(Num);
76  default:
77  return make_error<StringError>("'" + Duration +
78  "' must end with one of 's', 'm' or 'h'",
80  }
81 }
82 
85  CachePruningPolicy Policy;
86  std::pair<StringRef, StringRef> P = {"", PolicyStr};
87  while (!P.second.empty()) {
88  P = P.second.split(':');
89 
91  std::tie(Key, Value) = P.first.split('=');
92  if (Key == "prune_interval") {
93  auto DurationOrErr = parseDuration(Value);
94  if (!DurationOrErr)
95  return DurationOrErr.takeError();
96  Policy.Interval = *DurationOrErr;
97  } else if (Key == "prune_after") {
98  auto DurationOrErr = parseDuration(Value);
99  if (!DurationOrErr)
100  return DurationOrErr.takeError();
101  Policy.Expiration = *DurationOrErr;
102  } else if (Key == "cache_size") {
103  if (Value.back() != '%')
104  return make_error<StringError>("'" + Value + "' must be a percentage",
106  StringRef SizeStr = Value.drop_back();
107  uint64_t Size;
108  if (SizeStr.getAsInteger(0, Size))
109  return make_error<StringError>("'" + SizeStr + "' not an integer",
111  if (Size > 100)
112  return make_error<StringError>("'" + SizeStr +
113  "' must be between 0 and 100",
116  } else if (Key == "cache_size_bytes") {
117  uint64_t Mult = 1;
118  switch (tolower(Value.back())) {
119  case 'k':
120  Mult = 1024;
121  Value = Value.drop_back();
122  break;
123  case 'm':
124  Mult = 1024 * 1024;
125  Value = Value.drop_back();
126  break;
127  case 'g':
128  Mult = 1024 * 1024 * 1024;
129  Value = Value.drop_back();
130  break;
131  }
132  uint64_t Size;
133  if (Value.getAsInteger(0, Size))
134  return make_error<StringError>("'" + Value + "' not an integer",
136  Policy.MaxSizeBytes = Size * Mult;
137  } else if (Key == "cache_size_files") {
138  if (Value.getAsInteger(0, Policy.MaxSizeFiles))
139  return make_error<StringError>("'" + Value + "' not an integer",
141  } else {
142  return make_error<StringError>("Unknown key: '" + Key + "'",
144  }
145  }
146 
147  return Policy;
148 }
149 
150 /// Prune the cache of files that haven't been accessed in a long time.
152  using namespace std::chrono;
153 
154  if (Path.empty())
155  return false;
156 
157  bool isPathDir;
158  if (sys::fs::is_directory(Path, isPathDir))
159  return false;
160 
161  if (!isPathDir)
162  return false;
163 
165  std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u);
166 
167  if (Policy.Expiration == seconds(0) &&
168  Policy.MaxSizePercentageOfAvailableSpace == 0 &&
169  Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) {
170  LLVM_DEBUG(dbgs() << "No pruning settings set, exit early\n");
171  // Nothing will be pruned, early exit
172  return false;
173  }
174 
175  // Try to stat() the timestamp file.
176  SmallString<128> TimestampFile(Path);
177  sys::path::append(TimestampFile, "llvmcache.timestamp");
178  sys::fs::file_status FileStatus;
179  const auto CurrentTime = system_clock::now();
180  if (auto EC = sys::fs::status(TimestampFile, FileStatus)) {
182  // If the timestamp file wasn't there, create one now.
183  writeTimestampFile(TimestampFile);
184  } else {
185  // Unknown error?
186  return false;
187  }
188  } else {
189  if (!Policy.Interval)
190  return false;
191  if (Policy.Interval != seconds(0)) {
192  // Check whether the time stamp is older than our pruning interval.
193  // If not, do nothing.
194  const auto TimeStampModTime = FileStatus.getLastModificationTime();
195  auto TimeStampAge = CurrentTime - TimeStampModTime;
196  if (TimeStampAge <= *Policy.Interval) {
197  LLVM_DEBUG(dbgs() << "Timestamp file too recent ("
198  << duration_cast<seconds>(TimeStampAge).count()
199  << "s old), do not prune.\n");
200  return false;
201  }
202  }
203  // Write a new timestamp file so that nobody else attempts to prune.
204  // There is a benign race condition here, if two processes happen to
205  // notice at the same time that the timestamp is out-of-date.
206  writeTimestampFile(TimestampFile);
207  }
208 
209  // Keep track of files to delete to get below the size limit.
210  // Order by time of last use so that recently used files are preserved.
211  std::set<FileInfo> FileInfos;
212  uint64_t TotalSize = 0;
213 
214  // Walk the entire directory cache, looking for unused files.
215  std::error_code EC;
216  SmallString<128> CachePathNative;
217  sys::path::native(Path, CachePathNative);
218  // Walk all of the files within this directory.
219  for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
220  File != FileEnd && !EC; File.increment(EC)) {
221  // Ignore any files not beginning with the string "llvmcache-". This
222  // includes the timestamp file as well as any files created by the user.
223  // This acts as a safeguard against data loss if the user specifies the
224  // wrong directory as their cache directory.
225  if (!sys::path::filename(File->path()).startswith("llvmcache-"))
226  continue;
227 
228  // Look at this file. If we can't stat it, there's nothing interesting
229  // there.
230  ErrorOr<sys::fs::basic_file_status> StatusOrErr = File->status();
231  if (!StatusOrErr) {
232  LLVM_DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n");
233  continue;
234  }
235 
236  // If the file hasn't been used recently enough, delete it
237  const auto FileAccessTime = StatusOrErr->getLastAccessedTime();
238  auto FileAge = CurrentTime - FileAccessTime;
239  if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) {
240  LLVM_DEBUG(dbgs() << "Remove " << File->path() << " ("
241  << duration_cast<seconds>(FileAge).count()
242  << "s old)\n");
243  sys::fs::remove(File->path());
244  continue;
245  }
246 
247  // Leave it here for now, but add it to the list of size-based pruning.
248  TotalSize += StatusOrErr->getSize();
249  FileInfos.insert({FileAccessTime, StatusOrErr->getSize(), File->path()});
250  }
251 
252  auto FileInfo = FileInfos.begin();
253  size_t NumFiles = FileInfos.size();
254 
255  auto RemoveCacheFile = [&]() {
256  // Remove the file.
257  sys::fs::remove(FileInfo->Path);
258  // Update size
259  TotalSize -= FileInfo->Size;
260  NumFiles--;
261  LLVM_DEBUG(dbgs() << " - Remove " << FileInfo->Path << " (size "
262  << FileInfo->Size << "), new occupancy is " << TotalSize
263  << "%\n");
264  ++FileInfo;
265  };
266 
267  // Prune for number of files.
268  if (Policy.MaxSizeFiles)
269  while (NumFiles > Policy.MaxSizeFiles)
270  RemoveCacheFile();
271 
272  // Prune for size now if needed
273  if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) {
274  auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
275  if (!ErrOrSpaceInfo) {
276  report_fatal_error("Can't get available size");
277  }
278  sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
279  auto AvailableSpace = TotalSize + SpaceInfo.free;
280 
281  if (Policy.MaxSizePercentageOfAvailableSpace == 0)
283  if (Policy.MaxSizeBytes == 0)
284  Policy.MaxSizeBytes = AvailableSpace;
285  auto TotalSizeTarget = std::min<uint64_t>(
286  AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,
287  Policy.MaxSizeBytes);
288 
289  LLVM_DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
290  << "% target is: "
291  << Policy.MaxSizePercentageOfAvailableSpace << "%, "
292  << Policy.MaxSizeBytes << " bytes\n");
293 
294  // Remove the oldest accessed files first, till we get below the threshold.
295  while (TotalSize > TotalSizeTarget && FileInfo != FileInfos.end())
296  RemoveCacheFile();
297  }
298  return true;
299 }
uint64_t MaxSizeBytes
The maximum size for the cache directory in bytes.
Definition: CachePruning.h:48
Represents either an error or a value T.
Definition: ErrorOr.h:56
void native(const Twine &path, SmallVectorImpl< char > &result, Style style=Style::native)
Convert path to the native form.
Definition: Path.cpp:546
space_info - Self explanatory.
Definition: FileSystem.h:80
LLVM_NODISCARD std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:218
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:23
std::error_code remove(const Twine &path, bool IgnoreNonExisting=true)
Remove path.
std::chrono::seconds Expiration
The expiration for a file.
Definition: CachePruning.h:36
bool is_directory(const basic_file_status &status)
Does status represent a directory?
Definition: Path.cpp:1036
Represents the result of a call to sys::fs::status().
Definition: FileSystem.h:246
uint64_t MaxSizeFiles
The maximum number of files in the cache directory.
Definition: CachePruning.h:59
void append(SmallVectorImpl< char > &path, const Twine &a, const Twine &b="", const Twine &c="", const Twine &d="")
Append to path.
Definition: Path.cpp:479
LLVM_NODISCARD StringRef slice(size_t Start, size_t End) const
Return a reference to the substring from [Start, End).
Definition: StringRef.h:679
Tagged union holding either a T or a Error.
Definition: CachePruning.h:22
LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:126
ELFYAML::ELF_STO Other
Definition: ELFYAML.cpp:810
static bool startswith(StringRef Magic, const char(&S)[N])
Definition: Magic.cpp:29
Key
PAL metadata keys.
unsigned MaxSizePercentageOfAvailableSpace
The maximum size for the cache directory, in terms of percentage of the available space on the disk...
Definition: CachePruning.h:43
LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:130
Expected< CachePruningPolicy > parseCachePruningPolicy(StringRef PolicyStr)
Parse the given string as a cache pruning policy.
auto count(R &&Range, const E &Element) -> typename std::iterator_traits< decltype(adl_begin(Range))>::difference_type
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:1251
#define P(N)
static void writeTimestampFile(StringRef TimestampFile)
Write a new timestamp file with the given path.
std::error_code status(const Twine &path, file_status &result, bool follow=true)
Get file status as if by POSIX stat().
TimePoint getLastModificationTime() const
The file modification time as reported from the underlying file system.
static sys::TimePoint< std::chrono::seconds > now(bool Deterministic)
bool pruneCache(StringRef Path, CachePruningPolicy Policy)
Peform pruning using the supplied policy, returns true if pruning occurred, i.e.
LLVM_NODISCARD char back() const
back - Get the last character in the string.
Definition: StringRef.h:141
static Expected< std::chrono::seconds > parseDuration(StringRef Duration)
std::enable_if< std::numeric_limits< T >::is_signed, bool >::type getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:478
llvm::Optional< std::chrono::seconds > Interval
The pruning interval.
Definition: CachePruning.h:31
LLVM_NODISCARD std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:696
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
Policy for the pruneCache() function.
Definition: CachePruning.h:26
FileInfo(const GCOV::Options &Options)
Definition: GCOV.h:427
ErrorOr< space_info > disk_space(const Twine &Path)
Get disk space usage information.
directory_iterator - Iterates through the entries in path.
Definition: FileSystem.h:1196
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:365
StringRef filename(StringRef path, Style style=Style::native)
Get filename.
Definition: Path.cpp:589
uint32_t Size
Definition: Profile.cpp:46
bool operator<(int64_t V1, const APSInt &V2)
Definition: APSInt.h:325
LLVM Value Representation.
Definition: Value.h:72
LLVM_NODISCARD StringRef drop_back(size_t N=1) const
Return a StringRef equal to &#39;this&#39; but with the last N elements dropped.
Definition: StringRef.h:628
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
#define LLVM_DEBUG(X)
Definition: Debug.h:122
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:77
std::chrono::time_point< std::chrono::system_clock, D > TimePoint
A time point on the system clock.
Definition: Chrono.h:33