LLVM  6.0.0svn
CachePruning.cpp
Go to the documentation of this file.
1 //===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the pruning of a directory based on least recently used.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 
16 #include "llvm/Support/Debug.h"
17 #include "llvm/Support/Errc.h"
18 #include "llvm/Support/Error.h"
20 #include "llvm/Support/Path.h"
22 
23 #define DEBUG_TYPE "cache-pruning"
24 
25 #include <set>
26 #include <system_error>
27 
28 using namespace llvm;
29 
30 /// Write a new timestamp file with the given path. This is used for the pruning
31 /// interval option.
32 static void writeTimestampFile(StringRef TimestampFile) {
33  std::error_code EC;
34  raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None);
35 }
36 
38  if (Duration.empty())
39  return make_error<StringError>("Duration must not be empty",
41 
42  StringRef NumStr = Duration.slice(0, Duration.size()-1);
43  uint64_t Num;
44  if (NumStr.getAsInteger(0, Num))
45  return make_error<StringError>("'" + NumStr + "' not an integer",
47 
48  switch (Duration.back()) {
49  case 's':
50  return std::chrono::seconds(Num);
51  case 'm':
52  return std::chrono::minutes(Num);
53  case 'h':
54  return std::chrono::hours(Num);
55  default:
56  return make_error<StringError>("'" + Duration +
57  "' must end with one of 's', 'm' or 'h'",
59  }
60 }
61 
64  CachePruningPolicy Policy;
65  std::pair<StringRef, StringRef> P = {"", PolicyStr};
66  while (!P.second.empty()) {
67  P = P.second.split(':');
68 
70  std::tie(Key, Value) = P.first.split('=');
71  if (Key == "prune_interval") {
72  auto DurationOrErr = parseDuration(Value);
73  if (!DurationOrErr)
74  return DurationOrErr.takeError();
75  Policy.Interval = *DurationOrErr;
76  } else if (Key == "prune_after") {
77  auto DurationOrErr = parseDuration(Value);
78  if (!DurationOrErr)
79  return DurationOrErr.takeError();
80  Policy.Expiration = *DurationOrErr;
81  } else if (Key == "cache_size") {
82  if (Value.back() != '%')
83  return make_error<StringError>("'" + Value + "' must be a percentage",
85  StringRef SizeStr = Value.drop_back();
86  uint64_t Size;
87  if (SizeStr.getAsInteger(0, Size))
88  return make_error<StringError>("'" + SizeStr + "' not an integer",
90  if (Size > 100)
91  return make_error<StringError>("'" + SizeStr +
92  "' must be between 0 and 100",
95  } else if (Key == "cache_size_bytes") {
96  uint64_t Mult = 1;
97  switch (tolower(Value.back())) {
98  case 'k':
99  Mult = 1024;
100  Value = Value.drop_back();
101  break;
102  case 'm':
103  Mult = 1024 * 1024;
104  Value = Value.drop_back();
105  break;
106  case 'g':
107  Mult = 1024 * 1024 * 1024;
108  Value = Value.drop_back();
109  break;
110  }
111  uint64_t Size;
112  if (Value.getAsInteger(0, Size))
113  return make_error<StringError>("'" + Value + "' not an integer",
115  Policy.MaxSizeBytes = Size * Mult;
116  } else {
117  return make_error<StringError>("Unknown key: '" + Key + "'",
119  }
120  }
121 
122  return Policy;
123 }
124 
125 /// Prune the cache of files that haven't been accessed in a long time.
127  using namespace std::chrono;
128 
129  if (Path.empty())
130  return false;
131 
132  bool isPathDir;
133  if (sys::fs::is_directory(Path, isPathDir))
134  return false;
135 
136  if (!isPathDir)
137  return false;
138 
140  std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u);
141 
142  if (Policy.Expiration == seconds(0) &&
143  Policy.MaxSizePercentageOfAvailableSpace == 0 &&
144  Policy.MaxSizeBytes == 0) {
145  DEBUG(dbgs() << "No pruning settings set, exit early\n");
146  // Nothing will be pruned, early exit
147  return false;
148  }
149 
150  // Try to stat() the timestamp file.
151  SmallString<128> TimestampFile(Path);
152  sys::path::append(TimestampFile, "llvmcache.timestamp");
153  sys::fs::file_status FileStatus;
154  const auto CurrentTime = system_clock::now();
155  if (auto EC = sys::fs::status(TimestampFile, FileStatus)) {
157  // If the timestamp file wasn't there, create one now.
158  writeTimestampFile(TimestampFile);
159  } else {
160  // Unknown error?
161  return false;
162  }
163  } else {
164  if (Policy.Interval != seconds(0)) {
165  // Check whether the time stamp is older than our pruning interval.
166  // If not, do nothing.
167  const auto TimeStampModTime = FileStatus.getLastModificationTime();
168  auto TimeStampAge = CurrentTime - TimeStampModTime;
169  if (TimeStampAge <= Policy.Interval) {
170  DEBUG(dbgs() << "Timestamp file too recent ("
171  << duration_cast<seconds>(TimeStampAge).count()
172  << "s old), do not prune.\n");
173  return false;
174  }
175  }
176  // Write a new timestamp file so that nobody else attempts to prune.
177  // There is a benign race condition here, if two processes happen to
178  // notice at the same time that the timestamp is out-of-date.
179  writeTimestampFile(TimestampFile);
180  }
181 
182  bool ShouldComputeSize =
183  (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0);
184 
185  // Keep track of space. Needs to be kept ordered by size for determinism.
186  std::set<std::pair<uint64_t, std::string>> FileSizes;
187  uint64_t TotalSize = 0;
188 
189  // Walk the entire directory cache, looking for unused files.
190  std::error_code EC;
191  SmallString<128> CachePathNative;
192  sys::path::native(Path, CachePathNative);
193  // Walk all of the files within this directory.
194  for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
195  File != FileEnd && !EC; File.increment(EC)) {
196  // Ignore any files not beginning with the string "llvmcache-". This
197  // includes the timestamp file as well as any files created by the user.
198  // This acts as a safeguard against data loss if the user specifies the
199  // wrong directory as their cache directory.
200  if (!sys::path::filename(File->path()).startswith("llvmcache-"))
201  continue;
202 
203  // Look at this file. If we can't stat it, there's nothing interesting
204  // there.
205  ErrorOr<sys::fs::basic_file_status> StatusOrErr = File->status();
206  if (!StatusOrErr) {
207  DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n");
208  continue;
209  }
210 
211  // If the file hasn't been used recently enough, delete it
212  const auto FileAccessTime = StatusOrErr->getLastAccessedTime();
213  auto FileAge = CurrentTime - FileAccessTime;
214  if (FileAge > Policy.Expiration) {
215  DEBUG(dbgs() << "Remove " << File->path() << " ("
216  << duration_cast<seconds>(FileAge).count() << "s old)\n");
217  sys::fs::remove(File->path());
218  continue;
219  }
220 
221  // Leave it here for now, but add it to the list of size-based pruning.
222  if (!ShouldComputeSize)
223  continue;
224  TotalSize += StatusOrErr->getSize();
225  FileSizes.insert({StatusOrErr->getSize(), std::string(File->path())});
226  }
227 
228  // Prune for size now if needed
229  if (ShouldComputeSize) {
230  auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
231  if (!ErrOrSpaceInfo) {
232  report_fatal_error("Can't get available size");
233  }
234  sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
235  auto AvailableSpace = TotalSize + SpaceInfo.free;
236 
237  if (Policy.MaxSizePercentageOfAvailableSpace == 0)
239  if (Policy.MaxSizeBytes == 0)
240  Policy.MaxSizeBytes = AvailableSpace;
241  auto TotalSizeTarget = std::min<uint64_t>(
242  AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,
243  Policy.MaxSizeBytes);
244 
245  DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
246  << "% target is: " << Policy.MaxSizePercentageOfAvailableSpace
247  << "%, " << Policy.MaxSizeBytes << " bytes\n");
248 
249  auto FileAndSize = FileSizes.rbegin();
250  // Remove the oldest accessed files first, till we get below the threshold
251  while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend()) {
252  // Remove the file.
253  sys::fs::remove(FileAndSize->second);
254  // Update size
255  TotalSize -= FileAndSize->first;
256  DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size "
257  << FileAndSize->first << "), new occupancy is " << TotalSize
258  << "%\n");
259  ++FileAndSize;
260  }
261  }
262  return true;
263 }
uint64_t MaxSizeBytes
The maximum size for the cache directory in bytes.
Definition: CachePruning.h:48
Represents either an error or a value T.
Definition: ErrorOr.h:69
void native(const Twine &path, SmallVectorImpl< char > &result, Style style=Style::native)
Convert path to the native form.
Definition: Path.cpp:532
space_info - Self explanatory.
Definition: FileSystem.h:71
LLVM_NODISCARD std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:228
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:115
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
std::error_code remove(const Twine &path, bool IgnoreNonExisting=true)
Remove path.
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE size_t size() const
size - Get the string size.
Definition: StringRef.h:138
std::chrono::seconds Expiration
The expiration for a file.
Definition: CachePruning.h:36
std::chrono::seconds Interval
The pruning interval.
Definition: CachePruning.h:31
bool is_directory(const basic_file_status &status)
Does status represent a directory?
Definition: Path.cpp:1048
Represents the result of a call to sys::fs::status().
Definition: FileSystem.h:219
void append(SmallVectorImpl< char > &path, const Twine &a, const Twine &b="", const Twine &c="", const Twine &d="")
Append to path.
Definition: Path.cpp:465
Tagged union holding either a T or a Error.
Definition: CachePruning.h:23
static bool startswith(StringRef Magic, const char(&S)[N])
Definition: Magic.cpp:29
Key
PAL metadata keys.
unsigned MaxSizePercentageOfAvailableSpace
The maximum size for the cache directory, in terms of percentage of the available space on the the di...
Definition: CachePruning.h:43
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:133
Expected< CachePruningPolicy > parseCachePruningPolicy(StringRef PolicyStr)
Parse the given string as a cache pruning policy.
auto count(R &&Range, const E &Element) -> typename std::iterator_traits< decltype(std::begin(Range))>::difference_type
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:829
#define P(N)
static void writeTimestampFile(StringRef TimestampFile)
Write a new timestamp file with the given path.
std::error_code status(const Twine &path, file_status &result, bool follow=true)
Get file status as if by POSIX stat().
TimePoint getLastModificationTime() const
static sys::TimePoint< std::chrono::seconds > now(bool Deterministic)
bool pruneCache(StringRef Path, CachePruningPolicy Policy)
Peform pruning using the supplied policy, returns true if pruning occured, i.e.
LLVM_NODISCARD char back() const
back - Get the last character in the string.
Definition: StringRef.h:149
static Expected< std::chrono::seconds > parseDuration(StringRef Duration)
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef drop_back(size_t N=1) const
Return a StringRef equal to &#39;this&#39; but with the last N elements dropped.
Definition: StringRef.h:654
std::enable_if< std::numeric_limits< T >::is_signed, bool >::type getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:497
LLVM_NODISCARD std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:727
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef slice(size_t Start, size_t End) const
Return a reference to the substring from [Start, End).
Definition: StringRef.h:710
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
Policy for the pruneCache() function.
Definition: CachePruning.h:27
ErrorOr< space_info > disk_space(const Twine &Path)
Get disk space usage information.
directory_iterator - Iterates through the entries in path.
Definition: FileSystem.h:910
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:361
StringRef filename(StringRef path, Style style=Style::native)
Get filename.
Definition: Path.cpp:575
LLVM Value Representation.
Definition: Value.h:73
constexpr char Size[]
Key for Kernel::Arg::Metadata::mSize.
#define DEBUG(X)
Definition: Debug.h:118
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:73