LLVM  10.0.0svn
CachePruning.cpp
Go to the documentation of this file.
1 //===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the pruning of a directory based on least recently used.
10 //
11 //===----------------------------------------------------------------------===//
12 
14 
15 #include "llvm/Support/Debug.h"
16 #include "llvm/Support/Errc.h"
17 #include "llvm/Support/Error.h"
19 #include "llvm/Support/Path.h"
21 
22 #define DEBUG_TYPE "cache-pruning"
23 
24 #include <set>
25 #include <system_error>
26 
27 using namespace llvm;
28 
29 namespace {
30 struct FileInfo {
31  sys::TimePoint<> Time;
32  uint64_t Size;
33  std::string Path;
34 
35  /// Used to determine which files to prune first. Also used to determine
36  /// set membership, so must take into account all fields.
37  bool operator<(const FileInfo &Other) const {
38  return std::tie(Time, Other.Size, Path) <
39  std::tie(Other.Time, Size, Other.Path);
40  }
41 };
42 } // anonymous namespace
43 
44 /// Write a new timestamp file with the given path. This is used for the pruning
45 /// interval option.
46 static void writeTimestampFile(StringRef TimestampFile) {
47  std::error_code EC;
48  raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::OF_None);
49 }
50 
52  if (Duration.empty())
53  return make_error<StringError>("Duration must not be empty",
55 
56  StringRef NumStr = Duration.slice(0, Duration.size()-1);
57  uint64_t Num;
58  if (NumStr.getAsInteger(0, Num))
59  return make_error<StringError>("'" + NumStr + "' not an integer",
61 
62  switch (Duration.back()) {
63  case 's':
64  return std::chrono::seconds(Num);
65  case 'm':
66  return std::chrono::minutes(Num);
67  case 'h':
68  return std::chrono::hours(Num);
69  default:
70  return make_error<StringError>("'" + Duration +
71  "' must end with one of 's', 'm' or 'h'",
73  }
74 }
75 
78  CachePruningPolicy Policy;
79  std::pair<StringRef, StringRef> P = {"", PolicyStr};
80  while (!P.second.empty()) {
81  P = P.second.split(':');
82 
84  std::tie(Key, Value) = P.first.split('=');
85  if (Key == "prune_interval") {
86  auto DurationOrErr = parseDuration(Value);
87  if (!DurationOrErr)
88  return DurationOrErr.takeError();
89  Policy.Interval = *DurationOrErr;
90  } else if (Key == "prune_after") {
91  auto DurationOrErr = parseDuration(Value);
92  if (!DurationOrErr)
93  return DurationOrErr.takeError();
94  Policy.Expiration = *DurationOrErr;
95  } else if (Key == "cache_size") {
96  if (Value.back() != '%')
97  return make_error<StringError>("'" + Value + "' must be a percentage",
99  StringRef SizeStr = Value.drop_back();
100  uint64_t Size;
101  if (SizeStr.getAsInteger(0, Size))
102  return make_error<StringError>("'" + SizeStr + "' not an integer",
104  if (Size > 100)
105  return make_error<StringError>("'" + SizeStr +
106  "' must be between 0 and 100",
109  } else if (Key == "cache_size_bytes") {
110  uint64_t Mult = 1;
111  switch (tolower(Value.back())) {
112  case 'k':
113  Mult = 1024;
114  Value = Value.drop_back();
115  break;
116  case 'm':
117  Mult = 1024 * 1024;
118  Value = Value.drop_back();
119  break;
120  case 'g':
121  Mult = 1024 * 1024 * 1024;
122  Value = Value.drop_back();
123  break;
124  }
125  uint64_t Size;
126  if (Value.getAsInteger(0, Size))
127  return make_error<StringError>("'" + Value + "' not an integer",
129  Policy.MaxSizeBytes = Size * Mult;
130  } else if (Key == "cache_size_files") {
131  if (Value.getAsInteger(0, Policy.MaxSizeFiles))
132  return make_error<StringError>("'" + Value + "' not an integer",
134  } else {
135  return make_error<StringError>("Unknown key: '" + Key + "'",
137  }
138  }
139 
140  return Policy;
141 }
142 
143 /// Prune the cache of files that haven't been accessed in a long time.
145  using namespace std::chrono;
146 
147  if (Path.empty())
148  return false;
149 
150  bool isPathDir;
151  if (sys::fs::is_directory(Path, isPathDir))
152  return false;
153 
154  if (!isPathDir)
155  return false;
156 
158  std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u);
159 
160  if (Policy.Expiration == seconds(0) &&
161  Policy.MaxSizePercentageOfAvailableSpace == 0 &&
162  Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) {
163  LLVM_DEBUG(dbgs() << "No pruning settings set, exit early\n");
164  // Nothing will be pruned, early exit
165  return false;
166  }
167 
168  // Try to stat() the timestamp file.
169  SmallString<128> TimestampFile(Path);
170  sys::path::append(TimestampFile, "llvmcache.timestamp");
171  sys::fs::file_status FileStatus;
172  const auto CurrentTime = system_clock::now();
173  if (auto EC = sys::fs::status(TimestampFile, FileStatus)) {
175  // If the timestamp file wasn't there, create one now.
176  writeTimestampFile(TimestampFile);
177  } else {
178  // Unknown error?
179  return false;
180  }
181  } else {
182  if (!Policy.Interval)
183  return false;
184  if (Policy.Interval != seconds(0)) {
185  // Check whether the time stamp is older than our pruning interval.
186  // If not, do nothing.
187  const auto TimeStampModTime = FileStatus.getLastModificationTime();
188  auto TimeStampAge = CurrentTime - TimeStampModTime;
189  if (TimeStampAge <= *Policy.Interval) {
190  LLVM_DEBUG(dbgs() << "Timestamp file too recent ("
191  << duration_cast<seconds>(TimeStampAge).count()
192  << "s old), do not prune.\n");
193  return false;
194  }
195  }
196  // Write a new timestamp file so that nobody else attempts to prune.
197  // There is a benign race condition here, if two processes happen to
198  // notice at the same time that the timestamp is out-of-date.
199  writeTimestampFile(TimestampFile);
200  }
201 
202  // Keep track of files to delete to get below the size limit.
203  // Order by time of last use so that recently used files are preserved.
204  std::set<FileInfo> FileInfos;
205  uint64_t TotalSize = 0;
206 
207  // Walk the entire directory cache, looking for unused files.
208  std::error_code EC;
209  SmallString<128> CachePathNative;
210  sys::path::native(Path, CachePathNative);
211  // Walk all of the files within this directory.
212  for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
213  File != FileEnd && !EC; File.increment(EC)) {
214  // Ignore any files not beginning with the string "llvmcache-". This
215  // includes the timestamp file as well as any files created by the user.
216  // This acts as a safeguard against data loss if the user specifies the
217  // wrong directory as their cache directory.
218  if (!sys::path::filename(File->path()).startswith("llvmcache-"))
219  continue;
220 
221  // Look at this file. If we can't stat it, there's nothing interesting
222  // there.
223  ErrorOr<sys::fs::basic_file_status> StatusOrErr = File->status();
224  if (!StatusOrErr) {
225  LLVM_DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n");
226  continue;
227  }
228 
229  // If the file hasn't been used recently enough, delete it
230  const auto FileAccessTime = StatusOrErr->getLastAccessedTime();
231  auto FileAge = CurrentTime - FileAccessTime;
232  if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) {
233  LLVM_DEBUG(dbgs() << "Remove " << File->path() << " ("
234  << duration_cast<seconds>(FileAge).count()
235  << "s old)\n");
236  sys::fs::remove(File->path());
237  continue;
238  }
239 
240  // Leave it here for now, but add it to the list of size-based pruning.
241  TotalSize += StatusOrErr->getSize();
242  FileInfos.insert({FileAccessTime, StatusOrErr->getSize(), File->path()});
243  }
244 
245  auto FileInfo = FileInfos.begin();
246  size_t NumFiles = FileInfos.size();
247 
248  auto RemoveCacheFile = [&]() {
249  // Remove the file.
250  sys::fs::remove(FileInfo->Path);
251  // Update size
252  TotalSize -= FileInfo->Size;
253  NumFiles--;
254  LLVM_DEBUG(dbgs() << " - Remove " << FileInfo->Path << " (size "
255  << FileInfo->Size << "), new occupancy is " << TotalSize
256  << "%\n");
257  ++FileInfo;
258  };
259 
260  // Prune for number of files.
261  if (Policy.MaxSizeFiles)
262  while (NumFiles > Policy.MaxSizeFiles)
263  RemoveCacheFile();
264 
265  // Prune for size now if needed
266  if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) {
267  auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
268  if (!ErrOrSpaceInfo) {
269  report_fatal_error("Can't get available size");
270  }
271  sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
272  auto AvailableSpace = TotalSize + SpaceInfo.free;
273 
274  if (Policy.MaxSizePercentageOfAvailableSpace == 0)
276  if (Policy.MaxSizeBytes == 0)
277  Policy.MaxSizeBytes = AvailableSpace;
278  auto TotalSizeTarget = std::min<uint64_t>(
279  AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,
280  Policy.MaxSizeBytes);
281 
282  LLVM_DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
283  << "% target is: "
284  << Policy.MaxSizePercentageOfAvailableSpace << "%, "
285  << Policy.MaxSizeBytes << " bytes\n");
286 
287  // Remove the oldest accessed files first, till we get below the threshold.
288  while (TotalSize > TotalSizeTarget && FileInfo != FileInfos.end())
289  RemoveCacheFile();
290  }
291  return true;
292 }
uint64_t MaxSizeBytes
The maximum size for the cache directory in bytes.
Definition: CachePruning.h:48
Represents either an error or a value T.
Definition: ErrorOr.h:56
void native(const Twine &path, SmallVectorImpl< char > &result, Style style=Style::native)
Convert path to the native form.
Definition: Path.cpp:522
space_info - Self explanatory.
Definition: FileSystem.h:80
LLVM_NODISCARD std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:218
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:23
std::error_code remove(const Twine &path, bool IgnoreNonExisting=true)
Remove path.
std::chrono::seconds Expiration
The expiration for a file.
Definition: CachePruning.h:36
bool is_directory(const basic_file_status &status)
Does status represent a directory?
Definition: Path.cpp:1040
Represents the result of a call to sys::fs::status().
Definition: FileSystem.h:246
uint64_t MaxSizeFiles
The maximum number of files in the cache directory.
Definition: CachePruning.h:59
void append(SmallVectorImpl< char > &path, const Twine &a, const Twine &b="", const Twine &c="", const Twine &d="")
Append to path.
Definition: Path.cpp:455
LLVM_NODISCARD StringRef slice(size_t Start, size_t End) const
Return a reference to the substring from [Start, End).
Definition: StringRef.h:679
Tagged union holding either a T or a Error.
Definition: yaml2obj.h:21
LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:126
ELFYAML::ELF_STO Other
Definition: ELFYAML.cpp:877
static bool startswith(StringRef Magic, const char(&S)[N])
Definition: Magic.cpp:29
Key
PAL metadata keys.
unsigned MaxSizePercentageOfAvailableSpace
The maximum size for the cache directory, in terms of percentage of the available space on the disk...
Definition: CachePruning.h:43
LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:130
Expected< CachePruningPolicy > parseCachePruningPolicy(StringRef PolicyStr)
Parse the given string as a cache pruning policy.
auto count(R &&Range, const E &Element) -> typename std::iterator_traits< decltype(adl_begin(Range))>::difference_type
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:1243
#define P(N)
static void writeTimestampFile(StringRef TimestampFile)
Write a new timestamp file with the given path.
std::error_code status(const Twine &path, file_status &result, bool follow=true)
Get file status as if by POSIX stat().
Instrumentation for Order File
TimePoint getLastModificationTime() const
The file modification time as reported from the underlying file system.
static sys::TimePoint< std::chrono::seconds > now(bool Deterministic)
bool pruneCache(StringRef Path, CachePruningPolicy Policy)
Peform pruning using the supplied policy, returns true if pruning occurred, i.e.
LLVM_NODISCARD char back() const
back - Get the last character in the string.
Definition: StringRef.h:141
static Expected< std::chrono::seconds > parseDuration(StringRef Duration)
std::enable_if< std::numeric_limits< T >::is_signed, bool >::type getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:478
llvm::Optional< std::chrono::seconds > Interval
The pruning interval.
Definition: CachePruning.h:31
LLVM_NODISCARD std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:696
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
Policy for the pruneCache() function.
Definition: CachePruning.h:26
FileInfo(const GCOV::Options &Options)
Definition: GCOV.h:423
ErrorOr< space_info > disk_space(const Twine &Path)
Get disk space usage information.
directory_iterator - Iterates through the entries in path.
Definition: FileSystem.h:1282
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:384
StringRef filename(StringRef path, Style style=Style::native)
Get filename.
Definition: Path.cpp:565
uint32_t Size
Definition: Profile.cpp:46
bool operator<(int64_t V1, const APSInt &V2)
Definition: APSInt.h:343
LLVM_NODISCARD StringRef drop_back(size_t N=1) const
Return a StringRef equal to &#39;this&#39; but with the last N elements dropped.
Definition: StringRef.h:628
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
#define LLVM_DEBUG(X)
Definition: Debug.h:122
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:77
std::chrono::time_point< std::chrono::system_clock, D > TimePoint
A time point on the system clock.
Definition: Chrono.h:33