Line data Source code
1 : //===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // This file implements the pruning of a directory based on least recently used.
11 : //
12 : //===----------------------------------------------------------------------===//
13 :
14 : #include "llvm/Support/CachePruning.h"
15 :
16 : #include "llvm/Support/Debug.h"
17 : #include "llvm/Support/Errc.h"
18 : #include "llvm/Support/Error.h"
19 : #include "llvm/Support/FileSystem.h"
20 : #include "llvm/Support/Path.h"
21 : #include "llvm/Support/raw_ostream.h"
22 :
23 : #define DEBUG_TYPE "cache-pruning"
24 :
25 : #include <set>
26 : #include <system_error>
27 :
28 : using namespace llvm;
29 :
30 : namespace {
31 0 : struct FileInfo {
32 : sys::TimePoint<> Time;
33 : uint64_t Size;
34 : std::string Path;
35 :
36 : /// Used to determine which files to prune first. Also used to determine
37 : /// set membership, so must take into account all fields.
38 122 : bool operator<(const FileInfo &Other) const {
39 122 : if (Time < Other.Time)
40 : return true;
41 90 : else if (Other.Time < Time)
42 : return false;
43 69 : if (Other.Size < Size)
44 : return true;
45 29 : else if (Size < Other.Size)
46 : return false;
47 18 : return Path < Other.Path;
48 : }
49 : };
50 : } // anonymous namespace
51 :
52 : /// Write a new timestamp file with the given path. This is used for the pruning
53 : /// interval option.
54 28 : static void writeTimestampFile(StringRef TimestampFile) {
55 : std::error_code EC;
56 84 : raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None);
57 28 : }
58 :
59 31 : static Expected<std::chrono::seconds> parseDuration(StringRef Duration) {
60 31 : if (Duration.empty())
61 1 : return make_error<StringError>("Duration must not be empty",
62 2 : inconvertibleErrorCode());
63 :
64 60 : StringRef NumStr = Duration.slice(0, Duration.size()-1);
65 : uint64_t Num;
66 30 : if (NumStr.getAsInteger(0, Num))
67 1 : return make_error<StringError>("'" + NumStr + "' not an integer",
68 2 : inconvertibleErrorCode());
69 :
70 58 : switch (Duration.back()) {
71 : case 's':
72 : return std::chrono::seconds(Num);
73 : case 'm':
74 : return std::chrono::minutes(Num);
75 : case 'h':
76 : return std::chrono::hours(Num);
77 1 : default:
78 1 : return make_error<StringError>("'" + Duration +
79 1 : "' must end with one of 's', 'm' or 'h'",
80 2 : inconvertibleErrorCode());
81 : }
82 : }
83 :
84 : Expected<CachePruningPolicy>
85 2726 : llvm::parseCachePruningPolicy(StringRef PolicyStr) {
86 : CachePruningPolicy Policy;
87 : std::pair<StringRef, StringRef> P = {"", PolicyStr};
88 2776 : while (!P.second.empty()) {
89 59 : P = P.second.split(':');
90 :
91 59 : StringRef Key, Value;
92 118 : std::tie(Key, Value) = P.first.split('=');
93 : if (Key == "prune_interval") {
94 20 : auto DurationOrErr = parseDuration(Value);
95 20 : if (!DurationOrErr)
96 : return DurationOrErr.takeError();
97 : Policy.Interval = *DurationOrErr;
98 : } else if (Key == "prune_after") {
99 11 : auto DurationOrErr = parseDuration(Value);
100 11 : if (!DurationOrErr)
101 : return DurationOrErr.takeError();
102 11 : Policy.Expiration = *DurationOrErr;
103 : } else if (Key == "cache_size") {
104 20 : if (Value.back() != '%')
105 1 : return make_error<StringError>("'" + Value + "' must be a percentage",
106 2 : inconvertibleErrorCode());
107 9 : StringRef SizeStr = Value.drop_back();
108 : uint64_t Size;
109 9 : if (SizeStr.getAsInteger(0, Size))
110 1 : return make_error<StringError>("'" + SizeStr + "' not an integer",
111 2 : inconvertibleErrorCode());
112 8 : if (Size > 100)
113 1 : return make_error<StringError>("'" + SizeStr +
114 1 : "' must be between 0 and 100",
115 2 : inconvertibleErrorCode());
116 7 : Policy.MaxSizePercentageOfAvailableSpace = Size;
117 : } else if (Key == "cache_size_bytes") {
118 : uint64_t Mult = 1;
119 24 : switch (tolower(Value.back())) {
120 7 : case 'k':
121 : Mult = 1024;
122 7 : Value = Value.drop_back();
123 7 : break;
124 2 : case 'm':
125 : Mult = 1024 * 1024;
126 2 : Value = Value.drop_back();
127 2 : break;
128 1 : case 'g':
129 : Mult = 1024 * 1024 * 1024;
130 1 : Value = Value.drop_back();
131 1 : break;
132 : }
133 : uint64_t Size;
134 12 : if (Value.getAsInteger(0, Size))
135 2 : return make_error<StringError>("'" + Value + "' not an integer",
136 4 : inconvertibleErrorCode());
137 10 : Policy.MaxSizeBytes = Size * Mult;
138 : } else if (Key == "cache_size_files") {
139 5 : if (Value.getAsInteger(0, Policy.MaxSizeFiles))
140 0 : return make_error<StringError>("'" + Value + "' not an integer",
141 0 : inconvertibleErrorCode());
142 : } else {
143 1 : return make_error<StringError>("Unknown key: '" + Key + "'",
144 2 : inconvertibleErrorCode());
145 : }
146 : }
147 :
148 : return Policy;
149 : }
150 :
151 : /// Prune the cache of files that haven't been accessed in a long time.
152 47 : bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
153 : using namespace std::chrono;
154 :
155 47 : if (Path.empty())
156 : return false;
157 :
158 : bool isPathDir;
159 33 : if (sys::fs::is_directory(Path, isPathDir))
160 : return false;
161 :
162 33 : if (!isPathDir)
163 : return false;
164 :
165 33 : Policy.MaxSizePercentageOfAvailableSpace =
166 33 : std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u);
167 :
168 5 : if (Policy.Expiration == seconds(0) &&
169 5 : Policy.MaxSizePercentageOfAvailableSpace == 0 &&
170 38 : Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) {
171 : LLVM_DEBUG(dbgs() << "No pruning settings set, exit early\n");
172 : // Nothing will be pruned, early exit
173 : return false;
174 : }
175 :
176 : // Try to stat() the timestamp file.
177 : SmallString<128> TimestampFile(Path);
178 31 : sys::path::append(TimestampFile, "llvmcache.timestamp");
179 31 : sys::fs::file_status FileStatus;
180 31 : const auto CurrentTime = system_clock::now();
181 31 : if (auto EC = sys::fs::status(TimestampFile, FileStatus)) {
182 : if (EC == errc::no_such_file_or_directory) {
183 : // If the timestamp file wasn't there, create one now.
184 17 : writeTimestampFile(TimestampFile);
185 : } else {
186 : // Unknown error?
187 3 : return false;
188 : }
189 : } else {
190 14 : if (!Policy.Interval)
191 : return false;
192 : if (Policy.Interval != seconds(0)) {
193 : // Check whether the time stamp is older than our pruning interval.
194 : // If not, do nothing.
195 2 : const auto TimeStampModTime = FileStatus.getLastModificationTime();
196 : auto TimeStampAge = CurrentTime - TimeStampModTime;
197 2 : if (TimeStampAge <= *Policy.Interval) {
198 : LLVM_DEBUG(dbgs() << "Timestamp file too recent ("
199 : << duration_cast<seconds>(TimeStampAge).count()
200 : << "s old), do not prune.\n");
201 1 : return false;
202 : }
203 : }
204 : // Write a new timestamp file so that nobody else attempts to prune.
205 : // There is a benign race condition here, if two processes happen to
206 : // notice at the same time that the timestamp is out-of-date.
207 11 : writeTimestampFile(TimestampFile);
208 : }
209 :
210 : // Keep track of files to delete to get below the size limit.
211 : // Order by time of last use so that recently used files are preserved.
212 : std::set<FileInfo> FileInfos;
213 28 : uint64_t TotalSize = 0;
214 :
215 : // Walk the entire directory cache, looking for unused files.
216 : std::error_code EC;
217 : SmallString<128> CachePathNative;
218 28 : sys::path::native(Path, CachePathNative);
219 : // Walk all of the files within this directory.
220 28 : for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
221 143 : File != FileEnd && !EC; File.increment(EC)) {
222 : // Ignore any files not beginning with the string "llvmcache-". This
223 : // includes the timestamp file as well as any files created by the user.
224 : // This acts as a safeguard against data loss if the user specifies the
225 : // wrong directory as their cache directory.
226 230 : if (!sys::path::filename(File->path()).startswith("llvmcache-"))
227 48 : continue;
228 :
229 : // Look at this file. If we can't stat it, there's nothing interesting
230 : // there.
231 74 : ErrorOr<sys::fs::basic_file_status> StatusOrErr = File->status();
232 74 : if (!StatusOrErr) {
233 : LLVM_DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n");
234 : continue;
235 : }
236 :
237 : // If the file hasn't been used recently enough, delete it
238 74 : const auto FileAccessTime = StatusOrErr->getLastAccessedTime();
239 : auto FileAge = CurrentTime - FileAccessTime;
240 74 : if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) {
241 : LLVM_DEBUG(dbgs() << "Remove " << File->path() << " ("
242 : << duration_cast<seconds>(FileAge).count()
243 : << "s old)\n");
244 7 : sys::fs::remove(File->path());
245 7 : continue;
246 : }
247 :
248 : // Leave it here for now, but add it to the list of size-based pruning.
249 67 : TotalSize += StatusOrErr->getSize();
250 134 : FileInfos.insert({FileAccessTime, StatusOrErr->getSize(), File->path()});
251 : }
252 :
253 28 : auto FileInfo = FileInfos.begin();
254 28 : size_t NumFiles = FileInfos.size();
255 :
256 : auto RemoveCacheFile = [&]() {
257 : // Remove the file.
258 : sys::fs::remove(FileInfo->Path);
259 : // Update size
260 : TotalSize -= FileInfo->Size;
261 : NumFiles--;
262 : LLVM_DEBUG(dbgs() << " - Remove " << FileInfo->Path << " (size "
263 : << FileInfo->Size << "), new occupancy is " << TotalSize
264 : << "%\n");
265 : ++FileInfo;
266 28 : };
267 :
268 : // Prune for number of files.
269 28 : if (Policy.MaxSizeFiles)
270 34 : while (NumFiles > Policy.MaxSizeFiles)
271 6 : RemoveCacheFile();
272 :
273 : // Prune for size now if needed
274 28 : if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) {
275 25 : auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
276 25 : if (!ErrOrSpaceInfo) {
277 0 : report_fatal_error("Can't get available size");
278 : }
279 25 : sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
280 25 : auto AvailableSpace = TotalSize + SpaceInfo.free;
281 :
282 25 : if (Policy.MaxSizePercentageOfAvailableSpace == 0)
283 0 : Policy.MaxSizePercentageOfAvailableSpace = 100;
284 25 : if (Policy.MaxSizeBytes == 0)
285 17 : Policy.MaxSizeBytes = AvailableSpace;
286 : auto TotalSizeTarget = std::min<uint64_t>(
287 50 : AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,
288 33 : Policy.MaxSizeBytes);
289 :
290 : LLVM_DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
291 : << "% target is: "
292 : << Policy.MaxSizePercentageOfAvailableSpace << "%, "
293 : << Policy.MaxSizeBytes << " bytes\n");
294 :
295 : // Remove the oldest accessed files first, till we get below the threshold.
296 31 : while (TotalSize > TotalSizeTarget && FileInfo != FileInfos.end())
297 6 : RemoveCacheFile();
298 : }
299 : return true;
300 : }
|
