LCOV - code coverage report
Current view: top level - lib/Support - CachePruning.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 113 118 95.8 %
Date: 2018-09-23 13:06:45 Functions: 5 5 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file implements the pruning of a directory based on least recently used.
      11             : //
      12             : //===----------------------------------------------------------------------===//
      13             : 
      14             : #include "llvm/Support/CachePruning.h"
      15             : 
      16             : #include "llvm/Support/Debug.h"
      17             : #include "llvm/Support/Errc.h"
      18             : #include "llvm/Support/Error.h"
      19             : #include "llvm/Support/FileSystem.h"
      20             : #include "llvm/Support/Path.h"
      21             : #include "llvm/Support/raw_ostream.h"
      22             : 
      23             : #define DEBUG_TYPE "cache-pruning"
      24             : 
      25             : #include <set>
      26             : #include <system_error>
      27             : 
      28             : using namespace llvm;
      29             : 
      30             : namespace {
      31           0 : struct FileInfo {
      32             :   sys::TimePoint<> Time;
      33             :   uint64_t Size;
      34             :   std::string Path;
      35             : 
      36             :   /// Used to determine which files to prune first. Also used to determine
      37             :   /// set membership, so must take into account all fields.
      38          94 :   bool operator<(const FileInfo &Other) const {
      39          94 :     if (Time < Other.Time)
      40             :       return true;
      41          87 :     else if (Other.Time < Time)
      42             :       return false;
      43          79 :     if (Other.Size < Size)
      44             :       return true;
      45          19 :     else if (Size < Other.Size)
      46             :       return false;
      47          18 :     return Path < Other.Path;
      48             :   }
      49             : };
      50             : } // anonymous namespace
      51             : 
      52             : /// Write a new timestamp file with the given path. This is used for the pruning
      53             : /// interval option.
      54          28 : static void writeTimestampFile(StringRef TimestampFile) {
      55             :   std::error_code EC;
      56          84 :   raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None);
      57          28 : }
      58             : 
      59          31 : static Expected<std::chrono::seconds> parseDuration(StringRef Duration) {
      60          31 :   if (Duration.empty())
      61           1 :     return make_error<StringError>("Duration must not be empty",
      62           2 :                                    inconvertibleErrorCode());
      63             : 
      64          60 :   StringRef NumStr = Duration.slice(0, Duration.size()-1);
      65             :   uint64_t Num;
      66          30 :   if (NumStr.getAsInteger(0, Num))
      67           1 :     return make_error<StringError>("'" + NumStr + "' not an integer",
      68           2 :                                    inconvertibleErrorCode());
      69             : 
      70          58 :   switch (Duration.back()) {
      71             :   case 's':
      72             :     return std::chrono::seconds(Num);
      73             :   case 'm':
      74             :     return std::chrono::minutes(Num);
      75             :   case 'h':
      76             :     return std::chrono::hours(Num);
      77           1 :   default:
      78           1 :     return make_error<StringError>("'" + Duration +
      79           1 :                                        "' must end with one of 's', 'm' or 'h'",
      80           2 :                                    inconvertibleErrorCode());
      81             :   }
      82             : }
      83             : 
      84             : Expected<CachePruningPolicy>
      85        2665 : llvm::parseCachePruningPolicy(StringRef PolicyStr) {
      86             :   CachePruningPolicy Policy;
      87             :   std::pair<StringRef, StringRef> P = {"", PolicyStr};
      88        2715 :   while (!P.second.empty()) {
      89          59 :     P = P.second.split(':');
      90             : 
      91          59 :     StringRef Key, Value;
      92         118 :     std::tie(Key, Value) = P.first.split('=');
      93             :     if (Key == "prune_interval") {
      94          20 :       auto DurationOrErr = parseDuration(Value);
      95          20 :       if (!DurationOrErr)
      96             :         return DurationOrErr.takeError();
      97             :       Policy.Interval = *DurationOrErr;
      98             :     } else if (Key == "prune_after") {
      99          11 :       auto DurationOrErr = parseDuration(Value);
     100          11 :       if (!DurationOrErr)
     101             :         return DurationOrErr.takeError();
     102          11 :       Policy.Expiration = *DurationOrErr;
     103             :     } else if (Key == "cache_size") {
     104          20 :       if (Value.back() != '%')
     105           1 :         return make_error<StringError>("'" + Value + "' must be a percentage",
     106           2 :                                        inconvertibleErrorCode());
     107           9 :       StringRef SizeStr = Value.drop_back();
     108             :       uint64_t Size;
     109           9 :       if (SizeStr.getAsInteger(0, Size))
     110           1 :         return make_error<StringError>("'" + SizeStr + "' not an integer",
     111           2 :                                        inconvertibleErrorCode());
     112           8 :       if (Size > 100)
     113           1 :         return make_error<StringError>("'" + SizeStr +
     114           1 :                                            "' must be between 0 and 100",
     115           2 :                                        inconvertibleErrorCode());
     116           7 :       Policy.MaxSizePercentageOfAvailableSpace = Size;
     117             :     } else if (Key == "cache_size_bytes") {
     118             :       uint64_t Mult = 1;
     119          24 :       switch (tolower(Value.back())) {
     120           7 :       case 'k':
     121             :         Mult = 1024;
     122           7 :         Value = Value.drop_back();
     123           7 :         break;
     124           2 :       case 'm':
     125             :         Mult = 1024 * 1024;
     126           2 :         Value = Value.drop_back();
     127           2 :         break;
     128           1 :       case 'g':
     129             :         Mult = 1024 * 1024 * 1024;
     130           1 :         Value = Value.drop_back();
     131           1 :         break;
     132             :       }
     133             :       uint64_t Size;
     134          12 :       if (Value.getAsInteger(0, Size))
     135           2 :         return make_error<StringError>("'" + Value + "' not an integer",
     136           4 :                                        inconvertibleErrorCode());
     137          10 :       Policy.MaxSizeBytes = Size * Mult;
     138             :     } else if (Key == "cache_size_files") {
     139           5 :       if (Value.getAsInteger(0, Policy.MaxSizeFiles))
     140           0 :         return make_error<StringError>("'" + Value + "' not an integer",
     141           0 :                                        inconvertibleErrorCode());
     142             :     } else {
     143           1 :       return make_error<StringError>("Unknown key: '" + Key + "'",
     144           2 :                                      inconvertibleErrorCode());
     145             :     }
     146             :   }
     147             : 
     148             :   return Policy;
     149             : }
     150             : 
     151             : /// Prune the cache of files that haven't been accessed in a long time.
     152          47 : bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
     153             :   using namespace std::chrono;
     154             : 
     155          47 :   if (Path.empty())
     156             :     return false;
     157             : 
     158             :   bool isPathDir;
     159          33 :   if (sys::fs::is_directory(Path, isPathDir))
     160             :     return false;
     161             : 
     162          33 :   if (!isPathDir)
     163             :     return false;
     164             : 
     165          33 :   Policy.MaxSizePercentageOfAvailableSpace =
     166          33 :       std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u);
     167             : 
     168           5 :   if (Policy.Expiration == seconds(0) &&
     169           5 :       Policy.MaxSizePercentageOfAvailableSpace == 0 &&
     170          38 :       Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) {
     171             :     LLVM_DEBUG(dbgs() << "No pruning settings set, exit early\n");
     172             :     // Nothing will be pruned, early exit
     173             :     return false;
     174             :   }
     175             : 
     176             :   // Try to stat() the timestamp file.
     177             :   SmallString<128> TimestampFile(Path);
     178          31 :   sys::path::append(TimestampFile, "llvmcache.timestamp");
     179          31 :   sys::fs::file_status FileStatus;
     180          31 :   const auto CurrentTime = system_clock::now();
     181          31 :   if (auto EC = sys::fs::status(TimestampFile, FileStatus)) {
     182             :     if (EC == errc::no_such_file_or_directory) {
     183             :       // If the timestamp file wasn't there, create one now.
     184          17 :       writeTimestampFile(TimestampFile);
     185             :     } else {
     186             :       // Unknown error?
     187           3 :       return false;
     188             :     }
     189             :   } else {
     190          14 :     if (!Policy.Interval)
     191             :       return false;
     192             :     if (Policy.Interval != seconds(0)) {
     193             :       // Check whether the time stamp is older than our pruning interval.
     194             :       // If not, do nothing.
     195           2 :       const auto TimeStampModTime = FileStatus.getLastModificationTime();
     196             :       auto TimeStampAge = CurrentTime - TimeStampModTime;
     197           2 :       if (TimeStampAge <= *Policy.Interval) {
     198             :         LLVM_DEBUG(dbgs() << "Timestamp file too recent ("
     199             :                           << duration_cast<seconds>(TimeStampAge).count()
     200             :                           << "s old), do not prune.\n");
     201           1 :         return false;
     202             :       }
     203             :     }
     204             :     // Write a new timestamp file so that nobody else attempts to prune.
     205             :     // There is a benign race condition here, if two processes happen to
     206             :     // notice at the same time that the timestamp is out-of-date.
     207          11 :     writeTimestampFile(TimestampFile);
     208             :   }
     209             : 
     210             :   // Keep track of files to delete to get below the size limit.
     211             :   // Order by time of last use so that recently used files are preserved.
     212             :   std::set<FileInfo> FileInfos;
     213          28 :   uint64_t TotalSize = 0;
     214             : 
     215             :   // Walk the entire directory cache, looking for unused files.
     216             :   std::error_code EC;
     217             :   SmallString<128> CachePathNative;
     218          28 :   sys::path::native(Path, CachePathNative);
     219             :   // Walk all of the files within this directory.
     220          28 :   for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
     221         143 :        File != FileEnd && !EC; File.increment(EC)) {
     222             :     // Ignore any files not beginning with the string "llvmcache-". This
     223             :     // includes the timestamp file as well as any files created by the user.
     224             :     // This acts as a safeguard against data loss if the user specifies the
     225             :     // wrong directory as their cache directory.
     226         230 :     if (!sys::path::filename(File->path()).startswith("llvmcache-"))
     227          54 :       continue;
     228             : 
     229             :     // Look at this file. If we can't stat it, there's nothing interesting
     230             :     // there.
     231          74 :     ErrorOr<sys::fs::basic_file_status> StatusOrErr = File->status();
     232          74 :     if (!StatusOrErr) {
     233             :       LLVM_DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n");
     234             :       continue;
     235             :     }
     236             : 
     237             :     // If the file hasn't been used recently enough, delete it
     238          74 :     const auto FileAccessTime = StatusOrErr->getLastAccessedTime();
     239             :     auto FileAge = CurrentTime - FileAccessTime;
     240          74 :     if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) {
     241             :       LLVM_DEBUG(dbgs() << "Remove " << File->path() << " ("
     242             :                         << duration_cast<seconds>(FileAge).count()
     243             :                         << "s old)\n");
     244          13 :       sys::fs::remove(File->path());
     245          13 :       continue;
     246             :     }
     247             : 
     248             :     // Leave it here for now, but add it to the list of size-based pruning.
     249          61 :     TotalSize += StatusOrErr->getSize();
     250         122 :     FileInfos.insert({FileAccessTime, StatusOrErr->getSize(), File->path()});
     251             :   }
     252             : 
     253          28 :   auto FileInfo = FileInfos.begin();
     254          28 :   size_t NumFiles = FileInfos.size();
     255             : 
     256             :   auto RemoveCacheFile = [&]() {
     257             :     // Remove the file.
     258             :     sys::fs::remove(FileInfo->Path);
     259             :     // Update size
     260             :     TotalSize -= FileInfo->Size;
     261             :     NumFiles--;
     262             :     LLVM_DEBUG(dbgs() << " - Remove " << FileInfo->Path << " (size "
     263             :                       << FileInfo->Size << "), new occupancy is " << TotalSize
     264             :                       << "%\n");
     265             :     ++FileInfo;
     266          28 :   };
     267             : 
     268             :   // Prune for number of files.
     269          28 :   if (Policy.MaxSizeFiles)
     270          31 :     while (NumFiles > Policy.MaxSizeFiles)
     271           3 :       RemoveCacheFile();
     272             : 
     273             :   // Prune for size now if needed
     274          28 :   if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) {
     275          25 :     auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
     276          25 :     if (!ErrOrSpaceInfo) {
     277           0 :       report_fatal_error("Can't get available size");
     278             :     }
     279          25 :     sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
     280          25 :     auto AvailableSpace = TotalSize + SpaceInfo.free;
     281             : 
     282          25 :     if (Policy.MaxSizePercentageOfAvailableSpace == 0)
     283           0 :       Policy.MaxSizePercentageOfAvailableSpace = 100;
     284          25 :     if (Policy.MaxSizeBytes == 0)
     285          17 :       Policy.MaxSizeBytes = AvailableSpace;
     286             :     auto TotalSizeTarget = std::min<uint64_t>(
     287          25 :         AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,
     288          25 :         Policy.MaxSizeBytes);
     289             : 
     290             :     LLVM_DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
     291             :                       << "% target is: "
     292             :                       << Policy.MaxSizePercentageOfAvailableSpace << "%, "
     293             :                       << Policy.MaxSizeBytes << " bytes\n");
     294             : 
     295             :     // Remove the oldest accessed files first, till we get below the threshold.
     296          30 :     while (TotalSize > TotalSizeTarget && FileInfo != FileInfos.end())
     297           5 :       RemoveCacheFile();
     298             :   }
     299             :   return true;
     300             : }

Generated by: LCOV version 1.13