ObjectStore.h

Go to the documentation of this file.

//===- llvm/CAS/ObjectStore.h -----------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// This file contains the declaration of the ObjectStore class.
///
//===----------------------------------------------------------------------===//
 
#ifndef LLVM_CAS_OBJECTSTORE_H
#define LLVM_CAS_OBJECTSTORE_H
 
#include "llvm/ADT/StringRef.h"
#include "llvm/CAS/CASID.h"
#include "llvm/CAS/CASReference.h"
#include "llvm/CAS/ValidationResult.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
#include <cstddef>
 
namespace llvm {
 
class MemoryBuffer;
template <typename T> class unique_function;
 
namespace cas {
 
class ObjectStore;
class ObjectProxy;
class ActionCache;
 
/// Content-addressable storage for objects.
///
/// Conceptually, objects are stored in a "unique set".
///
/// - Objects are immutable ("value objects") that are defined by their
///   content. They are implicitly deduplicated by content.
/// - Each object has a unique identifier (UID) that's derived from its content,
///   called a \a CASID.
///     - This UID is a fixed-size (strong) hash of the transitive content of a
///       CAS object.
///     - It's comparable between any two CAS instances that have the same \a
///       CASIDContext::getHashSchemaIdentifier().
///     - The UID can be printed (e.g., \a CASID::toString()) and it can parsed
///       by the same or a different CAS instance with \a
///       ObjectStore::parseID().
/// - An object can be looked up by content or by UID.
///     - \a store() is "get-or-create"  methods, writing an object if it
///       doesn't exist yet, and return a ref to it in any case.
///     - \a loadObject(const CASID&) looks up an object by its UID.
/// - Objects can reference other objects, forming an arbitrary DAG.
///
/// The \a ObjectStore interface has a few ways of referencing objects:
///
/// - \a ObjectRef encapsulates a reference to something in the CAS. It is an
///   opaque type that references an object inside a specific CAS. It is
///   implementation defined if the underlying object exists or not for an
///   ObjectRef, and it can used to speed up CAS lookup as an implementation
///   detail. However, you don't know anything about the underlying objects.
///   "Loading" the object is a separate step that may not have happened
///   yet, and which can fail (e.g. due to filesystem corruption) or introduce
///   latency (if downloading from a remote store).
/// - \a ObjectHandle encapulates a *loaded* object in the CAS. You need one of
///   these to inspect the content of an object: to look at its stored
///   data and references. This is internal to CAS implementation and not
///   availble from CAS public APIs.
/// - \a CASID: the UID for an object in the CAS, obtained through \a
///   ObjectStore::getID() or \a ObjectStore::parseID(). This is a valid CAS
///   identifier, but may reference an object that is unknown to this CAS
///   instance.
/// - \a ObjectProxy pairs an ObjectHandle (subclass) with a ObjectStore, and
///   wraps access APIs to avoid having to pass extra parameters. It is the
///   object used for accessing underlying data and refs by CAS users.
///
/// Both ObjectRef and ObjectHandle are lightweight, wrapping a `uint64_t` and
/// are only valid with the associated ObjectStore instance.
///
/// There are a few options for accessing content of objects, with different
/// lifetime tradeoffs:
///
/// - \a getData() accesses data without exposing lifetime at all.
/// - \a getMemoryBuffer() returns a \a MemoryBuffer that may alias storage
///   owned by the CAS, so it must not outlive the \a ObjectStore.
/// - \a getStandaloneMemoryBuffer() returns a \a MemoryBuffer whose lifetime
///   is independent of the CAS (it can live longer).
/// - \a getDataString() return StringRef with lifetime is guaranteed to last as
///   long as \a ObjectStore.
/// - \a readRef() and \a forEachRef() iterate through the references in an
///   object. There is no lifetime assumption.
class LLVM_ABI ObjectStore {
  friend class ObjectProxy;
  void anchor();
 
public:
  /// Get a \p CASID from a \p ID, which should have been generated by \a
  /// CASID::print(). This succeeds as long as \a validateID() would pass. The
  /// object may be unknown to this CAS instance.
  ///
  /// TODO: Remove, and update callers to use \a validateID() or \a
  /// extractHashFromID().
  virtual Expected<CASID> parseID(StringRef ID) = 0;
 
  /// Store object into ObjectStore.
  virtual Expected<ObjectRef> store(ArrayRef<ObjectRef> Refs,
                                    ArrayRef<char> Data) = 0;
  /// Get an ID for \p Ref.
  virtual CASID getID(ObjectRef Ref) const = 0;
 
  /// Stores the data of a file into ObjectStore.
  ///
  /// An underlying implementation could perform optimizations that reduce I/O
  /// and disk space consumption.
  ///
  /// If there are any concurrent modifications to the file, the contents in the
  /// CAS may be corrupt.
  ///
  /// \param FilePath the path of the file data.
  virtual Expected<ObjectRef> storeFromFile(StringRef Path);
 
  /// Exports the data of an object to a file path. It does not include any
  /// references of the object.
  ///
  /// An underlying implementation could perform optimizations that reduce I/O
  /// and disk space consumption.
  ///
  /// \param Node the object to read data from.
  /// \param FilePath the path of the file data.
  virtual Error exportDataToFile(ObjectHandle Node, StringRef Path) const;
 
  /// Get an existing reference to the object called \p ID.
  ///
  /// Returns \c None if the object is not stored in this CAS.
  virtual std::optional<ObjectRef> getReference(const CASID &ID) const = 0;
 
  /// \returns true if the object is directly available from the local CAS, for
  /// implementations that have this kind of distinction.
  virtual Expected<bool> isMaterialized(ObjectRef Ref) const = 0;
 
  /// Validate the underlying object referred by CASID.
  virtual Error validateObject(const CASID &ID) = 0;
 
  /// Validate the entire ObjectStore.
  virtual Error validate(bool CheckHash) const = 0;
 
protected:
  /// Load the object referenced by \p Ref.
  ///
  /// Errors if the object cannot be loaded.
  /// \returns \c std::nullopt if the object is missing from the CAS.
  virtual Expected<std::optional<ObjectHandle>> loadIfExists(ObjectRef Ref) = 0;
 
  /// Like \c loadIfExists but returns an error if the object is missing.
  Expected<ObjectHandle> load(ObjectRef Ref);
 
  /// Get the size of some data.
  virtual uint64_t getDataSize(ObjectHandle Node) const = 0;
 
  /// Methods for handling objects. CAS implementations need to override to
  /// provide functions to access stored CAS objects and references.
  virtual Error forEachRef(ObjectHandle Node,
                           function_ref<Error(ObjectRef)> Callback) const = 0;
  virtual ObjectRef readRef(ObjectHandle Node, size_t I) const = 0;
  virtual size_t getNumRefs(ObjectHandle Node) const = 0;
  virtual ArrayRef<char> getData(ObjectHandle Node,
                                 bool RequiresNullTerminator = false) const = 0;
 
  /// Get ObjectRef from open file.
  virtual Expected<ObjectRef>
  storeFromOpenFileImpl(sys::fs::file_t FD,
                        std::optional<sys::fs::file_status> Status);
 
  /// Customization point for \a getStandaloneMemoryBuffer(). The default
  /// implementation copies the data, which always satisfies the lifetime
  /// requirement; implementations that can hand out storage outliving
  /// themselves, e.g. a mapping of a file they do not keep open, should
  /// override this to avoid the copy. Must not return \c nullptr: fall back
  /// to \c ObjectStore::getStandaloneMemoryBufferImpl() where the cheaper
  /// path does not apply.
  virtual std::unique_ptr<MemoryBuffer>
  getStandaloneMemoryBufferImpl(ObjectHandle Node, StringRef Name,
                                bool RequiresNullTerminator);
 
  /// Get a lifetime-extended StringRef pointing at \p Data.
  ///
  /// Depending on the CAS implementation, this may involve in-memory storage
  /// overhead.
  StringRef getDataString(ObjectHandle Node) {
    return toStringRef(getData(Node));
  }
 
  /// Get a MemoryBuffer pointing at \p Data.
  ///
  /// The buffer may alias storage owned by this ObjectStore, in which case it
  /// is only valid for as long as the store is.
  std::unique_ptr<MemoryBuffer>
  getMemoryBuffer(ObjectHandle Node, StringRef Name = "",
                  bool RequiresNullTerminator = true);
 
  /// Get a MemoryBuffer for \p Node that stays valid after this ObjectStore is
  /// destroyed.
  ///
  /// May be more expensive than \a getMemoryBuffer(), which is free to alias
  /// storage the store already has mapped; prefer that one whenever the buffer
  /// cannot outlive the store. Never returns \c nullptr: copying the data
  /// always satisfies the lifetime requirement.
  std::unique_ptr<MemoryBuffer>
  getStandaloneMemoryBuffer(ObjectHandle Node, StringRef Name = "",
                            bool RequiresNullTerminator = true);
 
  /// Read all the refs from object in a SmallVector.
  virtual void readRefs(ObjectHandle Node,
                        SmallVectorImpl<ObjectRef> &Refs) const;
 
  /// Allow ObjectStore implementations to create internal handles.
#define MAKE_CAS_HANDLE_CONSTRUCTOR(HandleKind)                                \
  HandleKind make##HandleKind(uint64_t InternalRef) const {                    \
    return HandleKind(*this, InternalRef);                                     \
  }
  MAKE_CAS_HANDLE_CONSTRUCTOR(ObjectHandle)
  MAKE_CAS_HANDLE_CONSTRUCTOR(ObjectRef)
#undef MAKE_CAS_HANDLE_CONSTRUCTOR
 
public:
  /// Helper functions to store object and returns a ObjectProxy.
  Expected<ObjectProxy> createProxy(ArrayRef<ObjectRef> Refs, StringRef Data);
 
  /// Store object from StringRef.
  Expected<ObjectRef> storeFromString(ArrayRef<ObjectRef> Refs,
                                      StringRef String) {
    return store(Refs, arrayRefFromStringRef<char>(String));
  }
 
  /// Default implementation reads \p FD and calls \a storeNode(). Does not
  /// take ownership of \p FD; the caller is responsible for closing it.
  ///
  /// If \p Status is sent in it is to be treated as a hint. Implementations
  /// must protect against the file size potentially growing after the status
  /// was taken (i.e., they cannot assume that an mmap will be null-terminated
  /// where \p Status implies).
  ///
  /// Returns the \a CASID and the size of the file.
  Expected<ObjectRef>
  storeFromOpenFile(sys::fs::file_t FD,
                    std::optional<sys::fs::file_status> Status = std::nullopt) {
    return storeFromOpenFileImpl(FD, Status);
  }
 
  static Error createUnknownObjectError(const CASID &ID);
 
  /// Create ObjectProxy from CASID. If the object doesn't exist, get an error.
  Expected<ObjectProxy> getProxy(const CASID &ID);
  /// Create ObjectProxy from ObjectRef. If the object can't be loaded, get an
  /// error.
  Expected<ObjectProxy> getProxy(ObjectRef Ref);
 
  /// \returns \c std::nullopt if the object is missing from the CAS.
  Expected<std::optional<ObjectProxy>> getProxyIfExists(ObjectRef Ref);
 
  /// Read the data from \p Data into \p OS.
  uint64_t readData(ObjectHandle Node, raw_ostream &OS, uint64_t Offset = 0,
                    uint64_t MaxBytes = -1ULL) const {
    ArrayRef<char> Data = getData(Node);
    assert(Offset < Data.size() && "Expected valid offset");
    Data = Data.drop_front(Offset).take_front(MaxBytes);
    OS << toStringRef(Data);
    return Data.size();
  }
 
  /// Set the size for limiting growth of on-disk storage. This has an effect
  /// for when the instance is closed.
  ///
  /// Implementations may leave this unimplemented.
  virtual Error setSizeLimit(std::optional<uint64_t> SizeLimit) {
    return Error::success();
  }
 
  /// \returns the storage size of the on-disk CAS data.
  ///
  /// Implementations that don't have an implementation for this should return
  /// \p std::nullopt.
  virtual Expected<std::optional<uint64_t>> getStorageSize() const {
    return std::nullopt;
  }
 
  /// Prune local storage to reduce its size according to the desired size
  /// limit. Pruning can happen concurrently with other operations.
  ///
  /// Implementations may leave this unimplemented.
  virtual Error pruneStorageData() { return Error::success(); }
 
  /// Validate the whole node tree.
  Error validateTree(ObjectRef Ref);
 
  /// Import object from another CAS. This will import the full tree from the
  /// other CAS.
  Expected<ObjectRef> importObject(ObjectStore &Upstream, ObjectRef Other);
 
  /// Print the ObjectStore internals for debugging purpose.
  virtual void print(raw_ostream &) const {}
  void dump() const;
 
  /// Get CASContext
  const CASContext &getContext() const { return Context; }
 
  virtual ~ObjectStore() = default;
 
protected:
  ObjectStore(const CASContext &Context) : Context(Context) {}
 
private:
  const CASContext &Context;
};
 
/// Reference to an abstract hierarchical node, with data and references.
/// Reference is passed by value and is expected to be valid as long as the \a
/// ObjectStore is.
class ObjectProxy {
public:
  ObjectStore &getCAS() const { return *CAS; }
  CASID getID() const { return CAS->getID(Ref); }
  ObjectRef getRef() const { return Ref; }
  size_t getNumReferences() const { return CAS->getNumRefs(H); }
  ObjectRef getReference(size_t I) const { return CAS->readRef(H, I); }
 
  operator CASID() const { return getID(); }
  CASID getReferenceID(size_t I) const {
    std::optional<CASID> ID = getCAS().getID(getReference(I));
    assert(ID && "Expected reference to be first-class object");
    return *ID;
  }
 
  /// Visit each reference in order, returning an error from \p Callback to
  /// stop early.
  Error forEachReference(function_ref<Error(ObjectRef)> Callback) const {
    return CAS->forEachRef(H, Callback);
  }
 
  LLVM_ABI std::unique_ptr<MemoryBuffer>
  getMemoryBuffer(StringRef Name = "",
                  bool RequiresNullTerminator = true) const;
 
  /// Get a MemoryBuffer that stays valid after the CAS is destroyed.
  LLVM_ABI std::unique_ptr<MemoryBuffer>
  getStandaloneMemoryBuffer(StringRef Name = "",
                            bool RequiresNullTerminator = true) const;
 
  /// Get the content of the node. Valid as long as the CAS is valid.
  StringRef getData() const { return CAS->getDataString(H); }
 
  /// Exports the data of an object to a file path.
  Error exportDataToFile(StringRef Path) const {
    return CAS->exportDataToFile(H, Path);
  }
 
  friend bool operator==(const ObjectProxy &Proxy, ObjectRef Ref) {
    return Proxy.getRef() == Ref;
  }
  friend bool operator==(ObjectRef Ref, const ObjectProxy &Proxy) {
    return Proxy.getRef() == Ref;
  }
  friend bool operator!=(const ObjectProxy &Proxy, ObjectRef Ref) {
    return !(Proxy.getRef() == Ref);
  }
  friend bool operator!=(ObjectRef Ref, const ObjectProxy &Proxy) {
    return !(Proxy.getRef() == Ref);
  }
 
public:
  ObjectProxy() = delete;
 
  static ObjectProxy load(ObjectStore &CAS, ObjectRef Ref, ObjectHandle Node) {
    return ObjectProxy(CAS, Ref, Node);
  }
 
private:
  ObjectProxy(ObjectStore &CAS, ObjectRef Ref, ObjectHandle H)
      : CAS(&CAS), Ref(Ref), H(H) {}
 
  ObjectStore *CAS;
  ObjectRef Ref;
  ObjectHandle H;
};
 
/// Create an in memory CAS.
LLVM_ABI std::unique_ptr<ObjectStore> createInMemoryCAS();
 
/// \returns true if \c LLVM_ENABLE_ONDISK_CAS configuration was enabled.
LLVM_ABI bool isOnDiskCASEnabled();
 
/// Create a persistent on-disk path at \p Path.
LLVM_ABI Expected<std::unique_ptr<ObjectStore>>
createOnDiskCAS(const Twine &Path);
 
/// Create \c ObjectStore and \c ActionCache instances backed by a plugin that
/// implements the C API in \c "llvm-c/CAS/PluginAPI_functions.h".
///
/// \param PluginPath path of the dynamic library to load.
/// \param OnDiskPath local path that the plugin should use for any on-disk
/// resources/caches.
/// \param PluginArgs name/value pairs passed to the plugin as custom options;
/// they are opaque to the client.
LLVM_ABI Expected<
    std::pair<std::shared_ptr<ObjectStore>, std::shared_ptr<ActionCache>>>
createPluginCASDatabases(
    StringRef PluginPath, StringRef OnDiskPath,
    ArrayRef<std::pair<std::string, std::string>> PluginArgs);
 
/// Validate the on-disk data of a plugin-backed CAS in-process if needed, by
/// calling the plugin's \c llcas_cas_validate_if_needed. The plugin decides
/// whether validation is needed.
///
/// Clients that want to be resilient to unexpected crashes during validation
/// may call this from a separate process (e.g. via
/// \c llvm-cas -validate-if-needed) and call \c recoverPluginCASDatabases if
/// it fails.
///
/// \param PluginPath path of the dynamic library to load.
/// \param OnDiskPath local path that the plugin uses for any on-disk
/// resources/caches.
/// \param PluginArgs name/value pairs passed to the plugin as custom options;
/// they are opaque to the client.
/// \param CheckHash Whether to validate hashes match the data.
/// \param ForceValidation Whether to force validation to occur even if it
/// should not be necessary.
///
/// \returns \c Valid if the data is valid, \c Skipped if validation is not
/// needed, or an \c Error if validation cannot be performed (including if the
/// plugin does not support it) or the data is invalid.
LLVM_ABI Expected<ValidationResult> validatePluginCASDatabasesIfNeeded(
    StringRef PluginPath, StringRef OnDiskPath,
    ArrayRef<std::pair<std::string, std::string>> PluginArgs, bool CheckHash,
    bool ForceValidation);
 
/// Recover the on-disk data of a plugin-backed CAS after a failed
/// \c validatePluginCASDatabasesIfNeeded, by calling the plugin's
/// \c llcas_cas_recover_ondisk_data.
///
/// \returns \c Recovered if the data has been recovered, \c Skipped if
/// recovery is not needed (e.g. a concurrent process already recovered), or an
/// \c Error if recovery cannot be performed (including if the plugin does not
/// support it).
LLVM_ABI Expected<ValidationResult> recoverPluginCASDatabases(
    StringRef PluginPath, StringRef OnDiskPath,
    ArrayRef<std::pair<std::string, std::string>> PluginArgs);
 
} // namespace cas
} // namespace llvm
 
#endif // LLVM_CAS_OBJECTSTORE_H