LCOV - code coverage report
Current view: top level - lib/Support - CachePruning.cpp (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 104 109 95.4 %
Date: 2018-02-22 04:41:24 Functions: 5 5 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : //===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
       2             : //
       3             : //                     The LLVM Compiler Infrastructure
       4             : //
       5             : // This file is distributed under the University of Illinois Open Source
       6             : // License. See LICENSE.TXT for details.
       7             : //
       8             : //===----------------------------------------------------------------------===//
       9             : //
      10             : // This file implements the pruning of a directory based on least recently used.
      11             : //
      12             : //===----------------------------------------------------------------------===//
      13             : 
      14             : #include "llvm/Support/CachePruning.h"
      15             : 
      16             : #include "llvm/Support/Debug.h"
      17             : #include "llvm/Support/Errc.h"
      18             : #include "llvm/Support/Error.h"
      19             : #include "llvm/Support/FileSystem.h"
      20             : #include "llvm/Support/Path.h"
      21             : #include "llvm/Support/raw_ostream.h"
      22             : 
      23             : #define DEBUG_TYPE "cache-pruning"
      24             : 
      25             : #include <set>
      26             : #include <system_error>
      27             : 
      28             : using namespace llvm;
      29             : 
      30             : /// Write a new timestamp file with the given path. This is used for the pruning
      31             : /// interval option.
      32          16 : static void writeTimestampFile(StringRef TimestampFile) {
      33             :   std::error_code EC;
      34          32 :   raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None);
      35          16 : }
      36             : 
      37          21 : static Expected<std::chrono::seconds> parseDuration(StringRef Duration) {
      38          21 :   if (Duration.empty())
      39             :     return make_error<StringError>("Duration must not be empty",
      40           2 :                                    inconvertibleErrorCode());
      41             : 
      42          40 :   StringRef NumStr = Duration.slice(0, Duration.size()-1);
      43             :   uint64_t Num;
      44             :   if (NumStr.getAsInteger(0, Num))
      45           1 :     return make_error<StringError>("'" + NumStr + "' not an integer",
      46           2 :                                    inconvertibleErrorCode());
      47             : 
      48          19 :   switch (Duration.back()) {
      49             :   case 's':
      50             :     return std::chrono::seconds(Num);
      51             :   case 'm':
      52             :     return std::chrono::minutes(Num);
      53             :   case 'h':
      54             :     return std::chrono::hours(Num);
      55           1 :   default:
      56           1 :     return make_error<StringError>("'" + Duration +
      57           1 :                                        "' must end with one of 's', 'm' or 'h'",
      58           2 :                                    inconvertibleErrorCode());
      59             :   }
      60             : }
      61             : 
      62             : Expected<CachePruningPolicy>
      63        2102 : llvm::parseCachePruningPolicy(StringRef PolicyStr) {
      64             :   CachePruningPolicy Policy;
      65             :   std::pair<StringRef, StringRef> P = {"", PolicyStr};
      66        2166 :   while (!P.second.empty()) {
      67          82 :     P = P.second.split(':');
      68             : 
      69          41 :     StringRef Key, Value;
      70          82 :     std::tie(Key, Value) = P.first.split('=');
      71             :     if (Key == "prune_interval") {
      72          14 :       auto DurationOrErr = parseDuration(Value);
      73          14 :       if (!DurationOrErr)
      74             :         return DurationOrErr.takeError();
      75             :       Policy.Interval = *DurationOrErr;
      76             :     } else if (Key == "prune_after") {
      77           7 :       auto DurationOrErr = parseDuration(Value);
      78           7 :       if (!DurationOrErr)
      79             :         return DurationOrErr.takeError();
      80           7 :       Policy.Expiration = *DurationOrErr;
      81             :     } else if (Key == "cache_size") {
      82           7 :       if (Value.back() != '%')
      83           1 :         return make_error<StringError>("'" + Value + "' must be a percentage",
      84           2 :                                        inconvertibleErrorCode());
      85           6 :       StringRef SizeStr = Value.drop_back();
      86             :       uint64_t Size;
      87             :       if (SizeStr.getAsInteger(0, Size))
      88           1 :         return make_error<StringError>("'" + SizeStr + "' not an integer",
      89           2 :                                        inconvertibleErrorCode());
      90           5 :       if (Size > 100)
      91           1 :         return make_error<StringError>("'" + SizeStr +
      92           1 :                                            "' must be between 0 and 100",
      93           2 :                                        inconvertibleErrorCode());
      94           4 :       Policy.MaxSizePercentageOfAvailableSpace = Size;
      95             :     } else if (Key == "cache_size_bytes") {
      96             :       uint64_t Mult = 1;
      97          10 :       switch (tolower(Value.back())) {
      98           5 :       case 'k':
      99             :         Mult = 1024;
     100           5 :         Value = Value.drop_back();
     101           5 :         break;
     102           2 :       case 'm':
     103             :         Mult = 1024 * 1024;
     104           2 :         Value = Value.drop_back();
     105           2 :         break;
     106           1 :       case 'g':
     107             :         Mult = 1024 * 1024 * 1024;
     108           1 :         Value = Value.drop_back();
     109           1 :         break;
     110             :       }
     111             :       uint64_t Size;
     112             :       if (Value.getAsInteger(0, Size))
     113           2 :         return make_error<StringError>("'" + Value + "' not an integer",
     114           4 :                                        inconvertibleErrorCode());
     115           8 :       Policy.MaxSizeBytes = Size * Mult;
     116             :     } else if (Key == "cache_size_files") {
     117             :       if (Value.getAsInteger(0, Policy.MaxSizeFiles))
     118           0 :         return make_error<StringError>("'" + Value + "' not an integer",
     119           0 :                                        inconvertibleErrorCode());
     120             :     } else {
     121           1 :       return make_error<StringError>("Unknown key: '" + Key + "'",
     122           2 :                                      inconvertibleErrorCode());
     123             :     }
     124             :   }
     125             : 
     126             :   return Policy;
     127             : }
     128             : 
     129             : /// Prune the cache of files that haven't been accessed in a long time.
     130          34 : bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) {
     131             :   using namespace std::chrono;
     132             : 
     133             :   if (Path.empty())
     134             :     return false;
     135             : 
     136             :   bool isPathDir;
     137          19 :   if (sys::fs::is_directory(Path, isPathDir))
     138             :     return false;
     139             : 
     140          19 :   if (!isPathDir)
     141             :     return false;
     142             : 
     143          19 :   Policy.MaxSizePercentageOfAvailableSpace =
     144          57 :       std::min(Policy.MaxSizePercentageOfAvailableSpace, 100u);
     145             : 
     146           2 :   if (Policy.Expiration == seconds(0) &&
     147           2 :       Policy.MaxSizePercentageOfAvailableSpace == 0 &&
     148          23 :       Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) {
     149             :     DEBUG(dbgs() << "No pruning settings set, exit early\n");
     150             :     // Nothing will be pruned, early exit
     151             :     return false;
     152             :   }
     153             : 
     154             :   // Try to stat() the timestamp file.
     155             :   SmallString<128> TimestampFile(Path);
     156          18 :   sys::path::append(TimestampFile, "llvmcache.timestamp");
     157          18 :   sys::fs::file_status FileStatus;
     158          18 :   const auto CurrentTime = system_clock::now();
     159          18 :   if (auto EC = sys::fs::status(TimestampFile, FileStatus)) {
     160             :     if (EC == errc::no_such_file_or_directory) {
     161             :       // If the timestamp file wasn't there, create one now.
     162           9 :       writeTimestampFile(TimestampFile);
     163             :     } else {
     164             :       // Unknown error?
     165           2 :       return false;
     166             :     }
     167             :   } else {
     168           9 :     if (!Policy.Interval)
     169             :       return false;
     170             :     if (Policy.Interval != seconds(0)) {
     171             :       // Check whether the time stamp is older than our pruning interval.
     172             :       // If not, do nothing.
     173           2 :       const auto TimeStampModTime = FileStatus.getLastModificationTime();
     174             :       auto TimeStampAge = CurrentTime - TimeStampModTime;
     175           2 :       if (TimeStampAge <= *Policy.Interval) {
     176             :         DEBUG(dbgs() << "Timestamp file too recent ("
     177             :                      << duration_cast<seconds>(TimeStampAge).count()
     178             :                      << "s old), do not prune.\n");
     179           1 :         return false;
     180             :       }
     181             :     }
     182             :     // Write a new timestamp file so that nobody else attempts to prune.
     183             :     // There is a benign race condition here, if two processes happen to
     184             :     // notice at the same time that the timestamp is out-of-date.
     185           7 :     writeTimestampFile(TimestampFile);
     186             :   }
     187             : 
     188             :   // Keep track of space. Needs to be kept ordered by size for determinism.
     189             :   std::set<std::pair<uint64_t, std::string>> FileSizes;
     190          16 :   uint64_t TotalSize = 0;
     191             : 
     192             :   // Walk the entire directory cache, looking for unused files.
     193             :   std::error_code EC;
     194             :   SmallString<128> CachePathNative;
     195          16 :   sys::path::native(Path, CachePathNative);
     196             :   // Walk all of the files within this directory.
     197          16 :   for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
     198          79 :        File != FileEnd && !EC; File.increment(EC)) {
     199             :     // Ignore any files not beginning with the string "llvmcache-". This
     200             :     // includes the timestamp file as well as any files created by the user.
     201             :     // This acts as a safeguard against data loss if the user specifies the
     202             :     // wrong directory as their cache directory.
     203          88 :     if (!sys::path::filename(File->path()).startswith("llvmcache-"))
     204          56 :       continue;
     205             : 
     206             :     // Look at this file. If we can't stat it, there's nothing interesting
     207             :     // there.
     208          38 :     ErrorOr<sys::fs::basic_file_status> StatusOrErr = File->status();
     209          38 :     if (!StatusOrErr) {
     210             :       DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n");
     211           0 :       continue;
     212             :     }
     213             : 
     214             :     // If the file hasn't been used recently enough, delete it
     215          38 :     const auto FileAccessTime = StatusOrErr->getLastAccessedTime();
     216             :     auto FileAge = CurrentTime - FileAccessTime;
     217          80 :     if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) {
     218             :       DEBUG(dbgs() << "Remove " << File->path() << " ("
     219             :                    << duration_cast<seconds>(FileAge).count() << "s old)\n");
     220           6 :       sys::fs::remove(File->path());
     221           6 :       continue;
     222             :     }
     223             : 
     224             :     // Leave it here for now, but add it to the list of size-based pruning.
     225          64 :     TotalSize += StatusOrErr->getSize();
     226          64 :     FileSizes.insert({StatusOrErr->getSize(), std::string(File->path())});
     227             :   }
     228             : 
     229             :   auto FileAndSize = FileSizes.rbegin();
     230          16 :   size_t NumFiles = FileSizes.size();
     231             : 
     232           3 :   auto RemoveCacheFile = [&]() {
     233             :     // Remove the file.
     234           9 :     sys::fs::remove(FileAndSize->second);
     235             :     // Update size
     236           3 :     TotalSize -= FileAndSize->first;
     237           3 :     NumFiles--;
     238             :     DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size "
     239             :                  << FileAndSize->first << "), new occupancy is " << TotalSize
     240             :                  << "%\n");
     241             :     ++FileAndSize;
     242          19 :   };
     243             : 
     244             :   // Prune for number of files.
     245          16 :   if (Policy.MaxSizeFiles)
     246          18 :     while (NumFiles > Policy.MaxSizeFiles)
     247           1 :       RemoveCacheFile();
     248             : 
     249             :   // Prune for size now if needed
     250          16 :   if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) {
     251          15 :     auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
     252          15 :     if (!ErrOrSpaceInfo) {
     253           0 :       report_fatal_error("Can't get available size");
     254             :     }
     255          15 :     sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
     256          15 :     auto AvailableSpace = TotalSize + SpaceInfo.free;
     257             : 
     258          15 :     if (Policy.MaxSizePercentageOfAvailableSpace == 0)
     259           0 :       Policy.MaxSizePercentageOfAvailableSpace = 100;
     260          15 :     if (Policy.MaxSizeBytes == 0)
     261          11 :       Policy.MaxSizeBytes = AvailableSpace;
     262             :     auto TotalSizeTarget = std::min<uint64_t>(
     263          30 :         AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,
     264          30 :         Policy.MaxSizeBytes);
     265             : 
     266             :     DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
     267             :                  << "% target is: " << Policy.MaxSizePercentageOfAvailableSpace
     268             :                  << "%, " << Policy.MaxSizeBytes << " bytes\n");
     269             : 
     270             :     // Remove the oldest accessed files first, till we get below the threshold.
     271          21 :     while (TotalSize > TotalSizeTarget && FileAndSize != FileSizes.rend())
     272           2 :       RemoveCacheFile();
     273             :   }
     274             :   return true;
     275             : }

Generated by: LCOV version 1.13