LLVM  7.0.0svn
CachePruning.cpp
Go to the documentation of this file.
1 //===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the pruning of a directory based on least recently used.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 
16 #include "llvm/Support/Debug.h"
17 #include "llvm/Support/Errc.h"
18 #include "llvm/Support/Error.h"
20 #include "llvm/Support/Path.h"
22 
23 #define DEBUG_TYPE "cache-pruning"
24 
25 #include <set>
26 #include <system_error>
27 
28 using namespace llvm;
29 
30 /// Write a new timestamp file with the given path. This is used for the pruning
31 /// interval option.
32 static void writeTimestampFile(StringRef TimestampFile) {
33  std::error_code EC;
34  raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None);
35 }
36 
38  if (Duration.empty())
39  return make_error<StringError>("Duration must not be empty",
41 
42  StringRef NumStr = Duration.slice(0, Duration.size()-1);
43  uint64_t Num;
44  if (NumStr.getAsInteger(0, Num))
45  return make_error<StringError>("'" + NumStr + "' not an integer",
47 
48  switch (Duration.back()) {
49  case 's':
50  return std::chrono::seconds(Num);
51  case 'm':
52  return std::chrono::minutes(Num);
53  case 'h':
54  return std::chrono::hours(Num);
55  default:
56  return make_error<StringError>("'" + Duration +
57  "' must end with one of 's', 'm' or 'h'",
59  }
60 }
61 
64  CachePruningPolicy Policy;
65  std::pair<StringRef, StringRef> P = {"", PolicyStr};
66  while (!P.second.empty()) {
67  P = P.second.split(':');
68 
70  std::tie(Key, Value) = P.first.split('=');
71  if (Key == "prune_interval") {
72  auto DurationOrErr = parseDuration(Value);
73  if (!DurationOrErr)
74  return DurationOrErr.takeError();
75  Policy.Interval = *DurationOrErr;
76  } else if (Key == "prune_after") {
77  auto DurationOrErr = parseDuration(Value);
78  if (!DurationOrErr)
79  return DurationOrErr.takeError();
80  Policy.Expiration = *DurationOrErr;
81  } else if (Key == "cache_size") {
82  if (Value.back() != '%')
83  return make_error<StringError>("'" + Value + "' must be a percentage",
85  StringRef SizeStr = Value.drop_back();
86  uint64_t Size;
87  if (SizeStr.getAsInteger(0, Size))
88  return make_error<StringError>("'" + SizeStr + "' not an integer",
90  if (Size > 100)
91  return make_error<StringError>("'" + SizeStr +
92  "' must be between 0 and 100",
95  } else if (Key == "cache_size_bytes") {
96  uint64_t Mult = 1;
97  switch (tolower(Value.back())) {
98  case 'k':
99  Mult = 1024;
100  Value = Value.drop_back();
101  break;
102  case 'm':
103  Mult = 1024 * 1024;
104  Value = Value.drop_back();
105  break;
106  case 'g':
107  Mult = 1024 * 1024 * 1024;
108  Value = Value.drop_back();
109  break;
110  }
111  uint64_t Size;
112  if (Value.getAsInteger(0, Size))
113  return make_error<StringError>("'" + Value + "' not an integer",
115  Policy.MaxSizeBytes = Size * Mult;
116  } else if (Key == "cache_size_files") {
117  if (Value.getAsInteger(0, Policy.MaxSizeFiles))
118  return make_error<StringError>("'" + Value + "' not an integer",
120  } else {
121  return make_error<StringError>("Unknown key: '" + Key + "'",
123  }
124  }
125 
126  return Policy;
127 }
128 
129 /// Prune the cache of files that haven't been accessed in a long time.
131  using namespace std::chrono;
132 
133  if (Path.empty())
134  return false;
135 
136  bool isPathDir;
137  if (sys::fs::is_directory(Path, isPathDir))
138  return false;
139 
140  if (!isPathDir)
141  return false;
142 
144  std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u);
145 
146  if (Policy.Expiration == seconds(0) &&
147  Policy.MaxSizePercentageOfAvailableSpace == 0 &&
148  Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) {
149  DEBUG(dbgs() << "No pruning settings set, exit early\n");
150  // Nothing will be pruned, early exit
151  return false;
152  }
153 
154  // Try to stat() the timestamp file.
155  SmallString<128> TimestampFile(Path);
156  sys::path::append(TimestampFile, "llvmcache.timestamp");
157  sys::fs::file_status FileStatus;
158  const auto CurrentTime = system_clock::now();
159  if (auto EC = sys::fs::status(TimestampFile, FileStatus)) {
161  // If the timestamp file wasn't there, create one now.
162  writeTimestampFile(TimestampFile);
163  } else {
164  // Unknown error?
165  return false;
166  }
167  } else {
168  if (!Policy.Interval)
169  return false;
170  if (Policy.Interval != seconds(0)) {
171  // Check whether the time stamp is older than our pruning interval.
172  // If not, do nothing.
173  const auto TimeStampModTime = FileStatus.getLastModificationTime();
174  auto TimeStampAge = CurrentTime - TimeStampModTime;
175  if (TimeStampAge <= *Policy.Interval) {
176  DEBUG(dbgs() << "Timestamp file too recent ("
177  << duration_cast<seconds>(TimeStampAge).count()
178  << "s old), do not prune.\n");
179  return false;
180  }
181  }
182  // Write a new timestamp file so that nobody else attempts to prune.
183  // There is a benign race condition here, if two processes happen to
184  // notice at the same time that the timestamp is out-of-date.
185  writeTimestampFile(TimestampFile);
186  }
187 
188  // Keep track of space. Needs to be kept ordered by size for determinism.
189  std::set<std::pair<uint64_t, std::string>> FileSizes;
190  uint64_t TotalSize = 0;
191 
192  // Walk the entire directory cache, looking for unused files.
193  std::error_code EC;
194  SmallString<128> CachePathNative;
195  sys::path::native(Path, CachePathNative);
196  // Walk all of the files within this directory.
197  for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
198  File != FileEnd && !EC; File.increment(EC)) {
199  // Ignore any files not beginning with the string "llvmcache-". This
200  // includes the timestamp file as well as any files created by the user.
201  // This acts as a safeguard against data loss if the user specifies the
202  // wrong directory as their cache directory.
203  if (!sys::path::filename(File->path()).startswith("llvmcache-"))
204  continue;
205 
206  // Look at this file. If we can't stat it, there's nothing interesting
207  // there.
208  ErrorOr<sys::fs::basic_file_status> StatusOrErr = File->status();
209  if (!StatusOrErr) {
210  DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n");
211  continue;
212  }
213 
214  // If the file hasn't been used recently enough, delete it
215  const auto FileAccessTime = StatusOrErr->getLastAccessedTime();
216  auto FileAge = CurrentTime - FileAccessTime;
217  if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) {
218  DEBUG(dbgs() << "Remove " << File->path() << " ("
219  << duration_cast<seconds>(FileAge).count() << "s old)\n");
220  sys::fs::remove(File->path());
221  continue;
222  }
223 
224  // Leave it here for now, but add it to the list of size-based pruning.
225  TotalSize += StatusOrErr->getSize();
226  FileSizes.insert({StatusOrErr->getSize(), std::string(File->path())});
227  }
228 
229  auto FileAndSize = FileSizes.rbegin();
230  size_t NumFiles = FileSizes.size();
231 
232  auto RemoveCacheFile = [&]() {
233  // Remove the file.
234  sys::fs::remove(FileAndSize->second);
235  // Update size
236  TotalSize -= FileAndSize->first;
237  NumFiles--;
238  DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size "
239  << FileAndSize->first << "), new occupancy is " << TotalSize
240  << "%\n");
241  ++FileAndSize;
242  };
243 
244  // Prune for number of files.
245  if (Policy.MaxSizeFiles)
246  while (NumFiles > Policy.MaxSizeFiles)
247  RemoveCacheFile();
248 
249  // Prune for size now if needed
250  if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) {
251  auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
252  if (!ErrOrSpaceInfo) {
253  report_fatal_error("Can't get available size");
254  }
255  sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
256  auto AvailableSpace = TotalSize + SpaceInfo.free;
257 
258  if (Policy.MaxSizePercentageOfAvailableSpace == 0)
260  if (Policy.MaxSizeBytes == 0)
261  Policy.MaxSizeBytes = AvailableSpace;
262  auto TotalSizeTarget = std::min<uint64_t>(
263  AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,
264  Policy.MaxSizeBytes);
265 
266  DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
267  << "% target is: " << Policy.MaxSizePercentageOfAvailableSpace
268  << "%, " << Policy.MaxSizeBytes << " bytes\n");
269 
270  // Remove the oldest accessed files first, till we get below the threshold.
271  while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend())
272  RemoveCacheFile();
273  }
274  return true;
275 }
uint64_t MaxSizeBytes
The maximum size for the cache directory in bytes.
Definition: CachePruning.h:49
Represents either an error or a value T.
Definition: ErrorOr.h:69
void native(const Twine &path, SmallVectorImpl< char > &result, Style style=Style::native)
Convert path to the native form.
Definition: Path.cpp:533
space_info - Self explanatory.
Definition: FileSystem.h:71
LLVM_NODISCARD std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:228
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:115
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
std::error_code remove(const Twine &path, bool IgnoreNonExisting=true)
Remove path.
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE size_t size() const
size - Get the string size.
Definition: StringRef.h:138
std::chrono::seconds Expiration
The expiration for a file.
Definition: CachePruning.h:37
bool is_directory(const basic_file_status &status)
Does status represent a directory?
Definition: Path.cpp:977
Represents the result of a call to sys::fs::status().
Definition: FileSystem.h:219
uint64_t MaxSizeFiles
The maximum number of files in the cache directory.
Definition: CachePruning.h:58
void append(SmallVectorImpl< char > &path, const Twine &a, const Twine &b="", const Twine &c="", const Twine &d="")
Append to path.
Definition: Path.cpp:466
Tagged union holding either a T or a Error.
Definition: CachePruning.h:23
static bool startswith(StringRef Magic, const char(&S)[N])
Definition: Magic.cpp:29
Key
PAL metadata keys.
unsigned MaxSizePercentageOfAvailableSpace
The maximum size for the cache directory, in terms of percentage of the available space on the the di...
Definition: CachePruning.h:44
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:133
Expected< CachePruningPolicy > parseCachePruningPolicy(StringRef PolicyStr)
Parse the given string as a cache pruning policy.
auto count(R &&Range, const E &Element) -> typename std::iterator_traits< decltype(adl_begin(Range))>::difference_type
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:880
#define P(N)
static void writeTimestampFile(StringRef TimestampFile)
Write a new timestamp file with the given path.
std::error_code status(const Twine &path, file_status &result, bool follow=true)
Get file status as if by POSIX stat().
TimePoint getLastModificationTime() const
static sys::TimePoint< std::chrono::seconds > now(bool Deterministic)
bool pruneCache(StringRef Path, CachePruningPolicy Policy)
Peform pruning using the supplied policy, returns true if pruning occured, i.e.
LLVM_NODISCARD char back() const
back - Get the last character in the string.
Definition: StringRef.h:149
static Expected< std::chrono::seconds > parseDuration(StringRef Duration)
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef drop_back(size_t N=1) const
Return a StringRef equal to &#39;this&#39; but with the last N elements dropped.
Definition: StringRef.h:654
std::enable_if< std::numeric_limits< T >::is_signed, bool >::type getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:497
llvm::Optional< std::chrono::seconds > Interval
The pruning interval.
Definition: CachePruning.h:32
LLVM_NODISCARD std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:727
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef slice(size_t Start, size_t End) const
Return a reference to the substring from [Start, End).
Definition: StringRef.h:710
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
Policy for the pruneCache() function.
Definition: CachePruning.h:27
ErrorOr< space_info > disk_space(const Twine &Path)
Get disk space usage information.
directory_iterator - Iterates through the entries in path.
Definition: FileSystem.h:915
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:361
StringRef filename(StringRef path, Style style=Style::native)
Get filename.
Definition: Path.cpp:576
LLVM Value Representation.
Definition: Value.h:73
constexpr char Size[]
Key for Kernel::Arg::Metadata::mSize.
#define DEBUG(X)
Definition: Debug.h:118
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:73