LLVM  8.0.0svn
CachePruning.cpp
Go to the documentation of this file.
1 //===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements the pruning of a directory based on least recently used.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 
16 #include "llvm/Support/Debug.h"
17 #include "llvm/Support/Errc.h"
18 #include "llvm/Support/Error.h"
20 #include "llvm/Support/Path.h"
22 
23 #define DEBUG_TYPE "cache-pruning"
24 
25 #include <set>
26 #include <system_error>
27 
28 using namespace llvm;
29 
30 namespace {
31 struct FileInfo {
32  sys::TimePoint<> Time;
33  uint64_t Size;
34  std::string Path;
35 
36  /// Used to determine which files to prune first. Also used to determine
37  /// set membership, so must take into account all fields.
38  bool operator<(const FileInfo &Other) const {
39  if (Time < Other.Time)
40  return true;
41  else if (Other.Time < Time)
42  return false;
43  if (Other.Size < Size)
44  return true;
45  else if (Size < Other.Size)
46  return false;
47  return Path < Other.Path;
48  }
49 };
50 } // anonymous namespace
51 
52 /// Write a new timestamp file with the given path. This is used for the pruning
53 /// interval option.
54 static void writeTimestampFile(StringRef TimestampFile) {
55  std::error_code EC;
56  raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None);
57 }
58 
60  if (Duration.empty())
61  return make_error<StringError>("Duration must not be empty",
63 
64  StringRef NumStr = Duration.slice(0, Duration.size()-1);
65  uint64_t Num;
66  if (NumStr.getAsInteger(0, Num))
67  return make_error<StringError>("'" + NumStr + "' not an integer",
69 
70  switch (Duration.back()) {
71  case 's':
72  return std::chrono::seconds(Num);
73  case 'm':
74  return std::chrono::minutes(Num);
75  case 'h':
76  return std::chrono::hours(Num);
77  default:
78  return make_error<StringError>("'" + Duration +
79  "' must end with one of 's', 'm' or 'h'",
81  }
82 }
83 
86  CachePruningPolicy Policy;
87  std::pair<StringRef, StringRef> P = {"", PolicyStr};
88  while (!P.second.empty()) {
89  P = P.second.split(':');
90 
92  std::tie(Key, Value) = P.first.split('=');
93  if (Key == "prune_interval") {
94  auto DurationOrErr = parseDuration(Value);
95  if (!DurationOrErr)
96  return DurationOrErr.takeError();
97  Policy.Interval = *DurationOrErr;
98  } else if (Key == "prune_after") {
99  auto DurationOrErr = parseDuration(Value);
100  if (!DurationOrErr)
101  return DurationOrErr.takeError();
102  Policy.Expiration = *DurationOrErr;
103  } else if (Key == "cache_size") {
104  if (Value.back() != '%')
105  return make_error<StringError>("'" + Value + "' must be a percentage",
107  StringRef SizeStr = Value.drop_back();
108  uint64_t Size;
109  if (SizeStr.getAsInteger(0, Size))
110  return make_error<StringError>("'" + SizeStr + "' not an integer",
112  if (Size > 100)
113  return make_error<StringError>("'" + SizeStr +
114  "' must be between 0 and 100",
117  } else if (Key == "cache_size_bytes") {
118  uint64_t Mult = 1;
119  switch (tolower(Value.back())) {
120  case 'k':
121  Mult = 1024;
122  Value = Value.drop_back();
123  break;
124  case 'm':
125  Mult = 1024 * 1024;
126  Value = Value.drop_back();
127  break;
128  case 'g':
129  Mult = 1024 * 1024 * 1024;
130  Value = Value.drop_back();
131  break;
132  }
133  uint64_t Size;
134  if (Value.getAsInteger(0, Size))
135  return make_error<StringError>("'" + Value + "' not an integer",
137  Policy.MaxSizeBytes = Size * Mult;
138  } else if (Key == "cache_size_files") {
139  if (Value.getAsInteger(0, Policy.MaxSizeFiles))
140  return make_error<StringError>("'" + Value + "' not an integer",
142  } else {
143  return make_error<StringError>("Unknown key: '" + Key + "'",
145  }
146  }
147 
148  return Policy;
149 }
150 
151 /// Prune the cache of files that haven't been accessed in a long time.
153  using namespace std::chrono;
154 
155  if (Path.empty())
156  return false;
157 
158  bool isPathDir;
159  if (sys::fs::is_directory(Path, isPathDir))
160  return false;
161 
162  if (!isPathDir)
163  return false;
164 
166  std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u);
167 
168  if (Policy.Expiration == seconds(0) &&
169  Policy.MaxSizePercentageOfAvailableSpace == 0 &&
170  Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) {
171  LLVM_DEBUG(dbgs() << "No pruning settings set, exit early\n");
172  // Nothing will be pruned, early exit
173  return false;
174  }
175 
176  // Try to stat() the timestamp file.
177  SmallString<128> TimestampFile(Path);
178  sys::path::append(TimestampFile, "llvmcache.timestamp");
179  sys::fs::file_status FileStatus;
180  const auto CurrentTime = system_clock::now();
181  if (auto EC = sys::fs::status(TimestampFile, FileStatus)) {
183  // If the timestamp file wasn't there, create one now.
184  writeTimestampFile(TimestampFile);
185  } else {
186  // Unknown error?
187  return false;
188  }
189  } else {
190  if (!Policy.Interval)
191  return false;
192  if (Policy.Interval != seconds(0)) {
193  // Check whether the time stamp is older than our pruning interval.
194  // If not, do nothing.
195  const auto TimeStampModTime = FileStatus.getLastModificationTime();
196  auto TimeStampAge = CurrentTime - TimeStampModTime;
197  if (TimeStampAge <= *Policy.Interval) {
198  LLVM_DEBUG(dbgs() << "Timestamp file too recent ("
199  << duration_cast<seconds>(TimeStampAge).count()
200  << "s old), do not prune.\n");
201  return false;
202  }
203  }
204  // Write a new timestamp file so that nobody else attempts to prune.
205  // There is a benign race condition here, if two processes happen to
206  // notice at the same time that the timestamp is out-of-date.
207  writeTimestampFile(TimestampFile);
208  }
209 
210  // Keep track of files to delete to get below the size limit.
211  // Order by time of last use so that recently used files are preserved.
212  std::set<FileInfo> FileInfos;
213  uint64_t TotalSize = 0;
214 
215  // Walk the entire directory cache, looking for unused files.
216  std::error_code EC;
217  SmallString<128> CachePathNative;
218  sys::path::native(Path, CachePathNative);
219  // Walk all of the files within this directory.
220  for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
221  File != FileEnd && !EC; File.increment(EC)) {
222  // Ignore any files not beginning with the string "llvmcache-". This
223  // includes the timestamp file as well as any files created by the user.
224  // This acts as a safeguard against data loss if the user specifies the
225  // wrong directory as their cache directory.
226  if (!sys::path::filename(File->path()).startswith("llvmcache-"))
227  continue;
228 
229  // Look at this file. If we can't stat it, there's nothing interesting
230  // there.
231  ErrorOr<sys::fs::basic_file_status> StatusOrErr = File->status();
232  if (!StatusOrErr) {
233  LLVM_DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n");
234  continue;
235  }
236 
237  // If the file hasn't been used recently enough, delete it
238  const auto FileAccessTime = StatusOrErr->getLastAccessedTime();
239  auto FileAge = CurrentTime - FileAccessTime;
240  if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) {
241  LLVM_DEBUG(dbgs() << "Remove " << File->path() << " ("
242  << duration_cast<seconds>(FileAge).count()
243  << "s old)\n");
244  sys::fs::remove(File->path());
245  continue;
246  }
247 
248  // Leave it here for now, but add it to the list of size-based pruning.
249  TotalSize += StatusOrErr->getSize();
250  FileInfos.insert({FileAccessTime, StatusOrErr->getSize(), File->path()});
251  }
252 
253  auto FileInfo = FileInfos.begin();
254  size_t NumFiles = FileInfos.size();
255 
256  auto RemoveCacheFile = [&]() {
257  // Remove the file.
258  sys::fs::remove(FileInfo->Path);
259  // Update size
260  TotalSize -= FileInfo->Size;
261  NumFiles--;
262  LLVM_DEBUG(dbgs() << " - Remove " << FileInfo->Path << " (size "
263  << FileInfo->Size << "), new occupancy is " << TotalSize
264  << "%\n");
265  ++FileInfo;
266  };
267 
268  // Prune for number of files.
269  if (Policy.MaxSizeFiles)
270  while (NumFiles > Policy.MaxSizeFiles)
271  RemoveCacheFile();
272 
273  // Prune for size now if needed
274  if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) {
275  auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
276  if (!ErrOrSpaceInfo) {
277  report_fatal_error("Can't get available size");
278  }
279  sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
280  auto AvailableSpace = TotalSize + SpaceInfo.free;
281 
282  if (Policy.MaxSizePercentageOfAvailableSpace == 0)
284  if (Policy.MaxSizeBytes == 0)
285  Policy.MaxSizeBytes = AvailableSpace;
286  auto TotalSizeTarget = std::min<uint64_t>(
287  AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,
288  Policy.MaxSizeBytes);
289 
290  LLVM_DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
291  << "% target is: "
292  << Policy.MaxSizePercentageOfAvailableSpace << "%, "
293  << Policy.MaxSizeBytes << " bytes\n");
294 
295  // Remove the oldest accessed files first, till we get below the threshold.
296  while (TotalSize > TotalSizeTarget && FileInfo != FileInfos.end())
297  RemoveCacheFile();
298  }
299  return true;
300 }
uint64_t MaxSizeBytes
The maximum size for the cache directory in bytes.
Definition: CachePruning.h:49
Represents either an error or a value T.
Definition: ErrorOr.h:57
void native(const Twine &path, SmallVectorImpl< char > &result, Style style=Style::native)
Convert path to the native form.
Definition: Path.cpp:547
space_info - Self explanatory.
Definition: FileSystem.h:81
LLVM_NODISCARD std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:228
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:24
std::error_code remove(const Twine &path, bool IgnoreNonExisting=true)
Remove path.
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE size_t size() const
size - Get the string size.
Definition: StringRef.h:138
std::chrono::seconds Expiration
The expiration for a file.
Definition: CachePruning.h:37
bool is_directory(const basic_file_status &status)
Does status represent a directory?
Definition: Path.cpp:1038
Represents the result of a call to sys::fs::status().
Definition: FileSystem.h:247
uint64_t MaxSizeFiles
The maximum number of files in the cache directory.
Definition: CachePruning.h:60
void append(SmallVectorImpl< char > &path, const Twine &a, const Twine &b="", const Twine &c="", const Twine &d="")
Append to path.
Definition: Path.cpp:480
Tagged union holding either a T or a Error.
Definition: CachePruning.h:23
ELFYAML::ELF_STO Other
Definition: ELFYAML.cpp:783
static bool startswith(StringRef Magic, const char(&S)[N])
Definition: Magic.cpp:30
Key
PAL metadata keys.
unsigned MaxSizePercentageOfAvailableSpace
The maximum size for the cache directory, in terms of percentage of the available space on the disk...
Definition: CachePruning.h:44
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:133
Expected< CachePruningPolicy > parseCachePruningPolicy(StringRef PolicyStr)
Parse the given string as a cache pruning policy.
auto count(R &&Range, const E &Element) -> typename std::iterator_traits< decltype(adl_begin(Range))>::difference_type
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:1246
#define P(N)
static void writeTimestampFile(StringRef TimestampFile)
Write a new timestamp file with the given path.
std::error_code status(const Twine &path, file_status &result, bool follow=true)
Get file status as if by POSIX stat().
TimePoint getLastModificationTime() const
The file modification time as reported from the underlying file system.
static sys::TimePoint< std::chrono::seconds > now(bool Deterministic)
bool pruneCache(StringRef Path, CachePruningPolicy Policy)
Peform pruning using the supplied policy, returns true if pruning occurred, i.e.
LLVM_NODISCARD char back() const
back - Get the last character in the string.
Definition: StringRef.h:149
static Expected< std::chrono::seconds > parseDuration(StringRef Duration)
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef drop_back(size_t N=1) const
Return a StringRef equal to &#39;this&#39; but with the last N elements dropped.
Definition: StringRef.h:654
std::enable_if< std::numeric_limits< T >::is_signed, bool >::type getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:497
llvm::Optional< std::chrono::seconds > Interval
The pruning interval.
Definition: CachePruning.h:32
LLVM_NODISCARD std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:727
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef slice(size_t Start, size_t End) const
Return a reference to the substring from [Start, End).
Definition: StringRef.h:710
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:133
Policy for the pruneCache() function.
Definition: CachePruning.h:27
FileInfo(const GCOV::Options &Options)
Definition: GCOV.h:428
ErrorOr< space_info > disk_space(const Twine &Path)
Get disk space usage information.
directory_iterator - Iterates through the entries in path.
Definition: FileSystem.h:1200
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:366
StringRef filename(StringRef path, Style style=Style::native)
Get filename.
Definition: Path.cpp:590
uint32_t Size
Definition: Profile.cpp:47
bool operator<(int64_t V1, const APSInt &V2)
Definition: APSInt.h:326
LLVM Value Representation.
Definition: Value.h:73
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
#define LLVM_DEBUG(X)
Definition: Debug.h:123
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:77
std::chrono::time_point< std::chrono::system_clock, D > TimePoint
A time point on the system clock.
Definition: Chrono.h:34