LCOV - code coverage report
Current view: top level - lib/Support - CachePruning.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 127 137 92.7 %
Date: 2017-09-14 15:23:50 Functions: 5 5 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file implements the pruning of a directory based on least recently used.
      11             : //
      12             : //===----------------------------------------------------------------------===//
      13             : 
      14             : #include "llvm/Support/CachePruning.h"
      15             : 
      16             : #include "llvm/Support/Debug.h"
      17             : #include "llvm/Support/Errc.h"
      18             : #include "llvm/Support/Error.h"
      19             : #include "llvm/Support/FileSystem.h"
      20             : #include "llvm/Support/Path.h"
      21             : #include "llvm/Support/raw_ostream.h"
      22             : 
      23             : #define DEBUG_TYPE "cache-pruning"
      24             : 
      25             : #include <set>
      26             : #include <system_error>
      27             : 
      28             : using namespace llvm;
      29             : 
      30             : /// Write a new timestamp file with the given path. This is used for the pruning
      31             : /// interval option.
      32          10 : static void writeTimestampFile(StringRef TimestampFile) {
      33          10 :   std::error_code EC;
      34          30 :   raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None);
      35          10 : }
      36             : 
      37          10 : static Expected<std::chrono::seconds> parseDuration(StringRef Duration) {
      38          10 :   if (Duration.empty())
      39           2 :     return make_error<StringError>("Duration must not be empty",
      40           3 :                                    inconvertibleErrorCode());
      41             : 
      42          18 :   StringRef NumStr = Duration.slice(0, Duration.size()-1);
      43             :   uint64_t Num;
      44           9 :   if (NumStr.getAsInteger(0, Num))
      45           5 :     return make_error<StringError>("'" + NumStr + "' not an integer",
      46           3 :                                    inconvertibleErrorCode());
      47             : 
      48           8 :   switch (Duration.back()) {
      49           3 :   case 's':
      50           6 :     return std::chrono::seconds(Num);
      51           1 :   case 'm':
      52           2 :     return std::chrono::minutes(Num);
      53           3 :   case 'h':
      54           6 :     return std::chrono::hours(Num);
      55           1 :   default:
      56           4 :     return make_error<StringError>("'" + Duration +
      57           2 :                                        "' must end with one of 's', 'm' or 'h'",
      58           3 :                                    inconvertibleErrorCode());
      59             :   }
      60             : }
      61             : 
      62             : Expected<CachePruningPolicy>
      63        1770 : llvm::parseCachePruningPolicy(StringRef PolicyStr) {
      64        1770 :   CachePruningPolicy Policy;
      65             :   std::pair<StringRef, StringRef> P = {"", PolicyStr};
      66        1800 :   while (!P.second.empty()) {
      67          48 :     P = P.second.split(':');
      68             : 
      69          24 :     StringRef Key, Value;
      70          72 :     std::tie(Key, Value) = P.first.split('=');
      71          30 :     if (Key == "prune_interval") {
      72           9 :       auto DurationOrErr = parseDuration(Value);
      73           6 :       if (!DurationOrErr)
      74           9 :         return DurationOrErr.takeError();
      75           3 :       Policy.Interval = *DurationOrErr;
      76          22 :     } else if (Key == "prune_after") {
      77           8 :       auto DurationOrErr = parseDuration(Value);
      78           4 :       if (!DurationOrErr)
      79           0 :         return DurationOrErr.takeError();
      80           4 :       Policy.Expiration = *DurationOrErr;
      81          19 :     } else if (Key == "cache_size") {
      82           5 :       if (Value.back() != '%')
      83           5 :         return make_error<StringError>("'" + Value + "' must be a percentage",
      84           3 :                                        inconvertibleErrorCode());
      85           4 :       StringRef SizeStr = Value.drop_back();
      86             :       uint64_t Size;
      87           4 :       if (SizeStr.getAsInteger(0, Size))
      88           5 :         return make_error<StringError>("'" + SizeStr + "' not an integer",
      89           3 :                                        inconvertibleErrorCode());
      90           3 :       if (Size > 100)
      91           4 :         return make_error<StringError>("'" + SizeStr +
      92           2 :                                            "' must be between 0 and 100",
      93           3 :                                        inconvertibleErrorCode());
      94           2 :       Policy.MaxSizePercentageOfAvailableSpace = Size;
      95          17 :     } else if (Key == "cache_size_bytes") {
      96           8 :       uint64_t Mult = 1;
      97           8 :       switch (tolower(Value.back())) {
      98           3 :       case 'k':
      99           3 :         Mult = 1024;
     100           3 :         Value = Value.drop_back();
     101           3 :         break;
     102           2 :       case 'm':
     103           2 :         Mult = 1024 * 1024;
     104           2 :         Value = Value.drop_back();
     105           2 :         break;
     106           1 :       case 'g':
     107           1 :         Mult = 1024 * 1024 * 1024;
     108           1 :         Value = Value.drop_back();
     109           1 :         break;
     110             :       }
     111             :       uint64_t Size;
     112           8 :       if (Value.getAsInteger(0, Size))
     113          10 :         return make_error<StringError>("'" + Value + "' not an integer",
     114           6 :                                        inconvertibleErrorCode());
     115           6 :       Policy.MaxSizeBytes = Size * Mult;
     116             :     } else {
     117           5 :       return make_error<StringError>("Unknown key: '" + Key + "'",
     118           3 :                                      inconvertibleErrorCode());
     119             :     }
     120             :   }
     121             : 
     122             :   return Policy;
     123             : }
     124             : 
     125             : /// Prune the cache of files that haven't been accessed in a long time.
     126          23 : bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
     127             :   using namespace std::chrono;
     128             : 
     129          23 :   if (Path.empty())
     130             :     return false;
     131             : 
     132             :   bool isPathDir;
     133          10 :   if (sys::fs::is_directory(Path, isPathDir))
     134             :     return false;
     135             : 
     136          10 :   if (!isPathDir)
     137             :     return false;
     138             : 
     139          10 :   Policy.MaxSizePercentageOfAvailableSpace =
     140          30 :       std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u);
     141             : 
     142          20 :   if (Policy.Expiration == seconds(0) &&
     143          10 :       Policy.MaxSizePercentageOfAvailableSpace == 0 &&
     144           0 :       Policy.MaxSizeBytes == 0) {
     145             :     DEBUG(dbgs() << "No pruning settings set, exit early\n");
     146             :     // Nothing will be pruned, early exit
     147             :     return false;
     148             :   }
     149             : 
     150             :   // Try to stat() the timestamp file.
     151          10 :   SmallString<128> TimestampFile(Path);
     152          40 :   sys::path::append(TimestampFile, "llvmcache.timestamp");
     153          10 :   sys::fs::file_status FileStatus;
     154          10 :   const auto CurrentTime = system_clock::now();
     155          10 :   if (auto EC = sys::fs::status(TimestampFile, FileStatus)) {
     156          16 :     if (EC == errc::no_such_file_or_directory) {
     157             :       // If the timestamp file wasn't there, create one now.
     158           8 :       writeTimestampFile(TimestampFile);
     159             :     } else {
     160             :       // Unknown error?
     161           0 :       return false;
     162             :     }
     163             :   } else {
     164           4 :     if (Policy.Interval == seconds(0)) {
     165             :       // Check whether the time stamp is older than our pruning interval.
     166             :       // If not, do nothing.
     167           0 :       const auto TimeStampModTime = FileStatus.getLastModificationTime();
     168           0 :       auto TimeStampAge = CurrentTime - TimeStampModTime;
     169           0 :       if (TimeStampAge <= Policy.Interval) {
     170             :         DEBUG(dbgs() << "Timestamp file too recent ("
     171             :                      << duration_cast<seconds>(TimeStampAge).count()
     172             :                      << "s old), do not prune.\n");
     173           0 :         return false;
     174             :       }
     175             :     }
     176             :     // Write a new timestamp file so that nobody else attempts to prune.
     177             :     // There is a benign race condition here, if two processes happen to
     178             :     // notice at the same time that the timestamp is out-of-date.
     179           2 :     writeTimestampFile(TimestampFile);
     180             :   }
     181             : 
     182          10 :   bool ShouldComputeSize =
     183          10 :       (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0);
     184             : 
     185             :   // Keep track of space
     186          10 :   std::set<std::pair<uint64_t, std::string>> FileSizes;
     187          10 :   uint64_t TotalSize = 0;
     188             :   // Helper to add a path to the set of files to consider for size-based
     189             :   // pruning, sorted by size.
     190             :   auto AddToFileListForSizePruning =
     191          18 :       [&](StringRef Path) {
     192          18 :         if (!ShouldComputeSize)
     193             :           return;
     194          54 :         TotalSize += FileStatus.getSize();
     195          36 :         FileSizes.insert(
     196         108 :             std::make_pair(FileStatus.getSize(), std::string(Path)));
     197          10 :       };
     198             : 
     199             :   // Walk the entire directory cache, looking for unused files.
     200          10 :   std::error_code EC;
     201          20 :   SmallString<128> CachePathNative;
     202          10 :   sys::path::native(Path, CachePathNative);
     203             :   // Walk all of the files within this directory.
     204          30 :   for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
     205          46 :        File != FileEnd && !EC; File.increment(EC)) {
     206             :     // Ignore any files not beginning with the string "llvmcache-". This
     207             :     // includes the timestamp file as well as any files created by the user.
     208             :     // This acts as a safeguard against data loss if the user specifies the
     209             :     // wrong directory as their cache directory.
     210         123 :     if (!sys::path::filename(File->path()).startswith("llvmcache-"))
     211          33 :       continue;
     212             : 
     213             :     // Look at this file. If we can't stat it, there's nothing interesting
     214             :     // there.
     215          63 :     if (sys::fs::status(File->path(), FileStatus)) {
     216             :       DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n");
     217           0 :       continue;
     218             :     }
     219             : 
     220             :     // If the file hasn't been used recently enough, delete it
     221          21 :     const auto FileAccessTime = FileStatus.getLastAccessedTime();
     222          21 :     auto FileAge = CurrentTime - FileAccessTime;
     223          24 :     if (FileAge > Policy.Expiration) {
     224             :       DEBUG(dbgs() << "Remove " << File->path() << " ("
     225             :                    << duration_cast<seconds>(FileAge).count() << "s old)\n");
     226           9 :       sys::fs::remove(File->path());
     227           3 :       continue;
     228             :     }
     229             : 
     230             :     // Leave it here for now, but add it to the list of size-based pruning.
     231          54 :     AddToFileListForSizePruning(File->path());
     232             :   }
     233             : 
     234             :   // Prune for size now if needed
     235          10 :   if (ShouldComputeSize) {
     236          20 :     auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
     237          10 :     if (!ErrOrSpaceInfo) {
     238           0 :       report_fatal_error("Can't get available size");
     239             :     }
     240          10 :     sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
     241          10 :     auto AvailableSpace = TotalSize + SpaceInfo.free;
     242             : 
     243          10 :     if (Policy.MaxSizePercentageOfAvailableSpace == 0)
     244           0 :       Policy.MaxSizePercentageOfAvailableSpace = 100;
     245          10 :     if (Policy.MaxSizeBytes == 0)
     246           8 :       Policy.MaxSizeBytes = AvailableSpace;
     247             :     auto TotalSizeTarget = std::min<uint64_t>(
     248          20 :         AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,
     249          10 :         Policy.MaxSizeBytes);
     250             : 
     251             :     DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
     252             :                  << "% target is: " << Policy.MaxSizePercentageOfAvailableSpace
     253             :                  << "%, " << Policy.MaxSizeBytes << " bytes\n");
     254             : 
     255             :     auto FileAndSize = FileSizes.rbegin();
     256             :     // Remove the oldest accessed files first, till we get below the threshold
     257          13 :     while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend()) {
     258             :       // Remove the file.
     259           2 :       sys::fs::remove(FileAndSize->second);
     260             :       // Update size
     261           1 :       TotalSize -= FileAndSize->first;
     262             :       DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size "
     263             :                    << FileAndSize->first << "), new occupancy is " << TotalSize
     264             :                    << "%\n");
     265             :       ++FileAndSize;
     266             :     }
     267             :   }
     268          10 :   return true;
     269             : }

Generated by: LCOV version 1.13