Introduce libgit2

flake.nix

@@ -193,6 +193,7 @@
             bzip2 xz brotli editline
             openssl sqlite
             libarchive
+            libgit2
             boost
             lowdown-nix
             libsodium

src/libfetchers/cache.cc

@@ -19,6 +19,9 @@ create table if not exists Cache (
 );
 )sql";
 
+// FIXME: we should periodically purge/nuke this cache to prevent it
+// from growing too big.
+
 struct CacheImpl : Cache
 {
     struct State
@@ -47,6 +50,60 @@ struct CacheImpl : Cache
             "select info, path, immutable, timestamp from Cache where input = ?");
     }
 
+    void upsert(
+        const Attrs & inAttrs,
+        const Attrs & infoAttrs) override
+    {
+        _state.lock()->add.use()
+            (attrsToJSON(inAttrs).dump())
+            (attrsToJSON(infoAttrs).dump())
+            ("") // no path
+            (false)
+            (time(0)).exec();
+    }
+
+    std::optional<Attrs> lookup(const Attrs & inAttrs) override
+    {
+        if (auto res = lookupExpired(inAttrs))
+            return std::move(res->infoAttrs);
+        return {};
+    }
+
+    std::optional<Attrs> lookupWithTTL(const Attrs & inAttrs) override
+    {
+        if (auto res = lookupExpired(inAttrs)) {
+            if (!res->expired)
+                return std::move(res->infoAttrs);
+            debug("ignoring expired cache entry '%s'",
+                attrsToJSON(inAttrs).dump());
+        }
+        return {};
+    }
+
+    std::optional<Result2> lookupExpired(const Attrs & inAttrs) override
+    {
+        auto state(_state.lock());
+
+        auto inAttrsJSON = attrsToJSON(inAttrs).dump();
+
+        auto stmt(state->lookup.use()(inAttrsJSON));
+        if (!stmt.next()) {
+            debug("did not find cache entry for '%s'", inAttrsJSON);
+            return {};
+        }
+
+        auto infoJSON = stmt.getStr(0);
+        auto locked = stmt.getInt(2) != 0;
+        auto timestamp = stmt.getInt(3);
+
+        debug("using cache entry '%s' -> '%s'", inAttrsJSON, infoJSON);
+
+        return Result2 {
+            .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
+            .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)),
+        };
+    }
+
     void add(
         ref<Store> store,
         const Attrs & inAttrs,

src/libfetchers/cache.hh

@@ -10,6 +10,44 @@ struct Cache
 {
     virtual ~Cache() { }
 
+    /* A cache for arbitrary Attrs -> Attrs mappings with a timestamp
+       for expiration. */
+
+    /*
+     * Add a value to the cache. The cache is an arbitrary mapping of
+     * Attrs to Attrs.
+     */
+    virtual void upsert(
+        const Attrs & inAttrs,
+        const Attrs & infoAttrs) = 0;
+
+    /*
+     * Look up a key with infinite TTL.
+     */
+    virtual std::optional<Attrs> lookup(
+        const Attrs & inAttrs) = 0;
+
+    /*
+     * Look up a key. Return nothing if its TTL has exceeded
+     * `settings.tarballTTL`.
+     */
+    virtual std::optional<Attrs> lookupWithTTL(
+        const Attrs & inAttrs) = 0;
+
+    struct Result2
+    {
+        bool expired = false;
+        Attrs infoAttrs;
+    };
+
+    /*
+     * Look up a key. Return a bool denoting whether its TTL has
+     * exceeded `settings.tarballTTL`.
+     */
+    virtual std::optional<Result2> lookupExpired(
+        const Attrs & inAttrs) = 0;
+
+    /* Old cache for things that have a store path. */
     virtual void add(
         ref<Store> store,
         const Attrs & inAttrs,

src/libfetchers/fetchers.cc

@@ -1,5 +1,6 @@
 #include "fetchers.hh"
 #include "store-api.hh"
+#include "input-accessor.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -355,6 +356,18 @@ void InputScheme::clone(const Input & input, const Path & destDir) const
     throw Error("do not know how to clone input '%s'", input.to_string());
 }
 
+std::pair<StorePath, Input> InputScheme::fetch(ref<Store> store, const Input & input)
+{
+    auto [accessor, input2] = getAccessor(store, input);
+    auto storePath = accessor->root().fetchToStore(store, input2.getName());
+    return {storePath, input2};
+}
+
+std::pair<ref<InputAccessor>, Input> InputScheme::getAccessor(ref<Store> store, const Input & input) const
+{
+    throw UnimplementedError("InputScheme must implement fetch() or getAccessor()");
+}
+
 std::optional<ExperimentalFeature> InputScheme::experimentalFeature() const
 {
     return {};

src/libfetchers/fetchers.hh

@@ -10,7 +10,7 @@
 #include <memory>
 #include <nlohmann/json_fwd.hpp>
 
-namespace nix { class Store; class StorePath; }
+namespace nix { class Store; class StorePath; struct InputAccessor; }
 
 namespace nix::fetchers {
 
@@ -167,7 +167,9 @@ struct InputScheme
         std::string_view contents,
         std::optional<std::string> commitMsg) const;
 
-    virtual std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) = 0;
+    virtual std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input);
+
+    virtual std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & input) const;
 
     /**
      * Is this `InputScheme` part of an experimental feature?