From e1b8442fa1dbd2e69598dbeb701da4df8e6d2c38 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Oct 2023 08:20:31 +0200
Subject: [PATCH 01/20] Fetcher cache: Add support for caching facts not
 related to store paths

---
 src/libfetchers/cache.cc | 57 ++++++++++++++++++++++++++++++++++++++++
 src/libfetchers/cache.hh | 38 +++++++++++++++++++++++++++
 2 files changed, 95 insertions(+)

diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc
index 0c8ecac9d48..8a3e462d313 100644
--- a/src/libfetchers/cache.cc
+++ b/src/libfetchers/cache.cc
@@ -19,6 +19,9 @@ create table if not exists Cache (
 );
 )sql";
 
+// FIXME: we should periodically purge/nuke this cache to prevent it
+// from growing too big.
+
 struct CacheImpl : Cache
 {
     struct State
@@ -47,6 +50,60 @@ struct CacheImpl : Cache
             "select info, path, immutable, timestamp from Cache where input = ?");
     }
 
+    void upsert(
+        const Attrs & inAttrs,
+        const Attrs & infoAttrs) override
+    {
+        _state.lock()->add.use()
+            (attrsToJSON(inAttrs).dump())
+            (attrsToJSON(infoAttrs).dump())
+            ("") // no path
+            (false)
+            (time(0)).exec();
+    }
+
+    std::optional<Attrs> lookup(const Attrs & inAttrs) override
+    {
+        if (auto res = lookupExpired(inAttrs))
+            return std::move(res->infoAttrs);
+        return {};
+    }
+
+    std::optional<Attrs> lookupWithTTL(const Attrs & inAttrs) override
+    {
+        if (auto res = lookupExpired(inAttrs)) {
+            if (!res->expired)
+                return std::move(res->infoAttrs);
+            debug("ignoring expired cache entry '%s'",
+                attrsToJSON(inAttrs).dump());
+        }
+        return {};
+    }
+
+    std::optional<Result2> lookupExpired(const Attrs & inAttrs) override
+    {
+        auto state(_state.lock());
+
+        auto inAttrsJSON = attrsToJSON(inAttrs).dump();
+
+        auto stmt(state->lookup.use()(inAttrsJSON));
+        if (!stmt.next()) {
+            debug("did not find cache entry for '%s'", inAttrsJSON);
+            return {};
+        }
+
+        auto infoJSON = stmt.getStr(0);
+        auto locked = stmt.getInt(2) != 0;
+        auto timestamp = stmt.getInt(3);
+
+        debug("using cache entry '%s' -> '%s'", inAttrsJSON, infoJSON);
+
+        return Result2 {
+            .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
+            .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)),
+        };
+    }
+
     void add(
         ref<Store> store,
         const Attrs & inAttrs,
diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh
index af34e66ce13..b517d496ed3 100644
--- a/src/libfetchers/cache.hh
+++ b/src/libfetchers/cache.hh
@@ -10,6 +10,44 @@ struct Cache
 {
     virtual ~Cache() { }
 
+    /* A cache for arbitrary Attrs -> Attrs mappings with a timestamp
+       for expiration. */
+
+    /*
+     * Add a value to the cache. The cache is an arbitrary mapping of
+     * Attrs to Attrs.
+     */
+    virtual void upsert(
+        const Attrs & inAttrs,
+        const Attrs & infoAttrs) = 0;
+
+    /*
+     * Look up a key with infinite TTL.
+     */
+    virtual std::optional<Attrs> lookup(
+        const Attrs & inAttrs) = 0;
+
+    /*
+     * Look up a key. Return nothing if its TTL has exceeded
+     * `settings.tarballTTL`.
+     */
+    virtual std::optional<Attrs> lookupWithTTL(
+        const Attrs & inAttrs) = 0;
+
+    struct Result2
+    {
+        bool expired = false;
+        Attrs infoAttrs;
+    };
+
+    /*
+     * Look up a key. Return a bool denoting whether its TTL has
+     * exceeded `settings.tarballTTL`.
+     */
+    virtual std::optional<Result2> lookupExpired(
+        const Attrs & inAttrs) = 0;
+
+    /* Old cache for things that have a store path. */
     virtual void add(
         ref<Store> store,
         const Attrs & inAttrs,

From 1d0e3d84b6ed693c140c3b7fd6a72ef8a8a26ec3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 24 Oct 2023 10:23:46 +0200
Subject: [PATCH 02/20] Provide a InputScheme::fetch() built on top of
 InputScheme::getAccessor()

This is for graceful migration to lazy-trees fetchers (which are all
accessor-based). Eventually fetch() will be removed.
---
 src/libfetchers/fetchers.cc | 13 +++++++++++++
 src/libfetchers/fetchers.hh |  6 ++++--
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index c339c441ba6..3e654dd5321 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -1,5 +1,6 @@
 #include "fetchers.hh"
 #include "store-api.hh"
+#include "input-accessor.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -312,6 +313,18 @@ void InputScheme::clone(const Input & input, const Path & destDir) const
     throw Error("do not know how to clone input '%s'", input.to_string());
 }
 
+std::pair<StorePath, Input> InputScheme::fetch(ref<Store> store, const Input & input)
+{
+    auto [accessor, input2] = getAccessor(store, input);
+    auto storePath = accessor->root().fetchToStore(store, input2.getName());
+    return {storePath, input2};
+}
+
+std::pair<ref<InputAccessor>, Input> InputScheme::getAccessor(ref<Store> store, const Input & input) const
+{
+    throw UnimplementedError("InputScheme must implement fetch() or getAccessor()");
+}
+
 std::optional<ExperimentalFeature> InputScheme::experimentalFeature()
 {
     return {};
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 4212a3e1fce..7b70ab6e223 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -9,7 +9,7 @@
 
 #include <memory>
 
-namespace nix { class Store; class StorePath; }
+namespace nix { class Store; class StorePath; struct InputAccessor; }
 
 namespace nix::fetchers {
 
@@ -148,7 +148,9 @@ struct InputScheme
         std::string_view contents,
         std::optional<std::string> commitMsg) const;
 
-    virtual std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) = 0;
+    virtual std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input);
+
+    virtual std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & input) const;
 
     /**
      * Is this `InputScheme` part of an experimental feature?

From ee36a44bf272c8cca62a2ce96a017a8150c4d35b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 25 Oct 2023 18:55:08 +0200
Subject: [PATCH 03/20] GitInputScheme: Use libgit2

This replaces most calls to the "git" binary with libgit2.
---
 flake.nix                                     |   1 +
 src/libfetchers/git-utils.cc                  | 498 +++++++++++++++
 src/libfetchers/git-utils.hh                  |  56 ++
 src/libfetchers/git.cc                        | 599 ++++++++----------
 src/libfetchers/local.mk                      |   2 +-
 tests/functional/fetchGit.sh                  |   7 +-
 tests/functional/flakes/flake-in-submodule.sh |   5 +-
 7 files changed, 834 insertions(+), 334 deletions(-)
 create mode 100644 src/libfetchers/git-utils.cc
 create mode 100644 src/libfetchers/git-utils.hh

diff --git a/flake.nix b/flake.nix
index 398ba10a078..3472bf7a894 100644
--- a/flake.nix
+++ b/flake.nix
@@ -191,6 +191,7 @@
             bzip2 xz brotli editline
             openssl sqlite
             libarchive
+            libgit2
             boost
             lowdown-nix
             libsodium
diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
new file mode 100644
index 00000000000..68e39580f6e
--- /dev/null
+++ b/src/libfetchers/git-utils.cc
@@ -0,0 +1,498 @@
+#include "git-utils.hh"
+#include "input-accessor.hh"
+#include "cache.hh"
+
+#include <boost/core/span.hpp>
+
+#include <git2/blob.h>
+#include <git2/commit.h>
+#include <git2/describe.h>
+#include <git2/errors.h>
+#include <git2/global.h>
+#include <git2/object.h>
+#include <git2/refs.h>
+#include <git2/remote.h>
+#include <git2/repository.h>
+#include <git2/status.h>
+#include <git2/tree.h>
+
+#include <unordered_set>
+#include <queue>
+
+namespace std {
+
+template<> struct hash<git_oid>
+{
+    size_t operator()(const git_oid & oid) const
+    {
+        return * (size_t *) oid.id;
+    }
+};
+
+}
+
+std::ostream & operator << (std::ostream & str, const git_oid & oid)
+{
+    str << git_oid_tostr_s(&oid);
+    return str;
+}
+
+bool operator == (const git_oid & oid1, const git_oid & oid2)
+{
+    return git_oid_equal(&oid1, &oid2);
+}
+
+namespace nix {
+
+// Some wrapper types that ensure that the git_*_free functions get called.
+template<auto del>
+struct Deleter
+{
+    template <typename T>
+    void operator()(T * p) const { del(p); };
+};
+
+typedef std::unique_ptr<git_repository, Deleter<git_repository_free>> Repository;
+typedef std::unique_ptr<git_tree_entry, Deleter<git_tree_entry_free>> TreeEntry;
+typedef std::unique_ptr<git_tree, Deleter<git_tree_free>> Tree;
+typedef std::unique_ptr<git_treebuilder, Deleter<git_treebuilder_free>> TreeBuilder;
+typedef std::unique_ptr<git_blob, Deleter<git_blob_free>> Blob;
+typedef std::unique_ptr<git_object, Deleter<git_object_free>> Object;
+typedef std::unique_ptr<git_commit, Deleter<git_commit_free>> Commit;
+typedef std::unique_ptr<git_reference, Deleter<git_reference_free>> Reference;
+typedef std::unique_ptr<git_describe_result, Deleter<git_describe_result_free>> DescribeResult;
+typedef std::unique_ptr<git_status_list, Deleter<git_status_list_free>> StatusList;
+typedef std::unique_ptr<git_remote, Deleter<git_remote_free>> Remote;
+
+// A helper to ensure that we don't leak objects returned by libgit2.
+template<typename T>
+struct Setter
+{
+    T & t;
+    typename T::pointer p = nullptr;
+
+    Setter(T & t) : t(t) { }
+
+    ~Setter() { if (p) t = T(p); }
+
+    operator typename T::pointer * () { return &p; }
+};
+
+Hash toHash(const git_oid & oid)
+{
+    #ifdef GIT_EXPERIMENTAL_SHA256
+    assert(oid.type == GIT_OID_SHA1);
+    #endif
+    Hash hash(htSHA1);
+    memcpy(hash.hash, oid.id, hash.hashSize);
+    return hash;
+}
+
+static void initLibGit2()
+{
+    if (git_libgit2_init() < 0)
+        throw Error("initialising libgit2: %s", git_error_last()->message);
+}
+
+git_oid hashToOID(const Hash & hash)
+{
+    git_oid oid;
+    if (git_oid_fromstr(&oid, hash.gitRev().c_str()))
+        throw Error("cannot convert '%s' to a Git OID", hash.gitRev());
+    return oid;
+}
+
+Object lookupObject(git_repository * repo, const git_oid & oid)
+{
+    Object obj;
+    if (git_object_lookup(Setter(obj), repo, &oid, GIT_OBJECT_ANY)) {
+        auto err = git_error_last();
+        throw Error("getting Git object '%s': %s", oid, err->message);
+    }
+    return obj;
+}
+
+template<typename T>
+T peelObject(git_repository * repo, git_object * obj, git_object_t type)
+{
+    T obj2;
+    if (git_object_peel((git_object * *) (typename T::pointer *) Setter(obj2), obj, type)) {
+        auto err = git_error_last();
+        throw Error("peeling Git object '%s': %s", git_object_id(obj), err->message);
+    }
+    return obj2;
+}
+
+int statusCallbackTrampoline(const char * path, unsigned int statusFlags, void * payload)
+{
+    return (*((std::function<int(const char * path, unsigned int statusFlags)> *) payload))(path, statusFlags);
+}
+
+struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
+{
+    CanonPath path;
+    Repository repo;
+
+    GitRepoImpl(CanonPath _path, bool create, bool bare)
+        : path(std::move(_path))
+    {
+        initLibGit2();
+
+        if (pathExists(path.abs())) {
+            if (git_repository_open(Setter(repo), path.c_str()))
+                throw Error("opening Git repository '%s': %s", path, git_error_last()->message);
+        } else {
+            if (git_repository_init(Setter(repo), path.c_str(), bare))
+                throw Error("creating Git repository '%s': %s", path, git_error_last()->message);
+        }
+    }
+
+    operator git_repository * ()
+    {
+        return repo.get();
+    }
+
+    uint64_t getRevCount(const Hash & rev) override
+    {
+        std::unordered_set<git_oid> done;
+        std::queue<Commit> todo;
+
+        todo.push(peelObject<Commit>(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT));
+
+        while (auto commit = pop(todo)) {
+            if (!done.insert(*git_commit_id(commit->get())).second) continue;
+
+            for (size_t n = 0; n < git_commit_parentcount(commit->get()); ++n) {
+                git_commit * parent;
+                if (git_commit_parent(&parent, commit->get(), n))
+                    throw Error("getting parent of Git commit '%s': %s", *git_commit_id(commit->get()), git_error_last()->message);
+                todo.push(Commit(parent));
+            }
+        }
+
+        return done.size();
+    }
+
+    uint64_t getLastModified(const Hash & rev) override
+    {
+        auto commit = peelObject<Commit>(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT);
+
+        return git_commit_time(commit.get());
+    }
+
+    bool isShallow() override
+    {
+        return git_repository_is_shallow(*this);
+    }
+
+    Hash resolveRef(std::string ref) override
+    {
+        // Handle revisions used as refs.
+        {
+            git_oid oid;
+            if (git_oid_fromstr(&oid, ref.c_str()) == 0)
+                return toHash(oid);
+        }
+
+        // Resolve short names like 'master'.
+        Reference ref2;
+        if (!git_reference_dwim(Setter(ref2), *this, ref.c_str()))
+            ref = git_reference_name(ref2.get());
+
+        // Resolve full references like 'refs/heads/master'.
+        Reference ref3;
+        if (git_reference_lookup(Setter(ref3), *this, ref.c_str()))
+            throw Error("resolving Git reference '%s': %s", ref, git_error_last()->message);
+
+        auto oid = git_reference_target(ref3.get());
+        if (!oid)
+            throw Error("cannot get OID for Git reference '%s'", git_reference_name(ref3.get()));
+
+        return toHash(*oid);
+    }
+
+    WorkdirInfo getWorkdirInfo() override
+    {
+        WorkdirInfo info;
+
+        /* Get the head revision, if any. */
+        git_oid headRev;
+        if (auto err = git_reference_name_to_id(&headRev, *this, "HEAD")) {
+            if (err != GIT_ENOTFOUND)
+                throw Error("resolving HEAD: %s", git_error_last()->message);
+        } else
+            info.headRev = toHash(headRev);
+
+        /* Get all tracked files and determine whether the working
+           directory is dirty. */
+        std::function<int(const char * path, unsigned int statusFlags)> statusCallback = [&](const char * path, unsigned int statusFlags)
+        {
+            if (!(statusFlags & GIT_STATUS_INDEX_DELETED) &&
+                !(statusFlags & GIT_STATUS_WT_DELETED))
+                info.files.insert(CanonPath(path));
+            if (statusFlags != GIT_STATUS_CURRENT)
+                info.isDirty = true;
+            return 0;
+        };
+
+        git_status_options options = GIT_STATUS_OPTIONS_INIT;
+        options.flags |= GIT_STATUS_OPT_INCLUDE_UNMODIFIED;
+        options.flags |= GIT_STATUS_OPT_EXCLUDE_SUBMODULES;
+        if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback))
+            throw Error("getting working directory status: %s", git_error_last()->message);
+
+        return info;
+    }
+
+    std::optional<std::string> getWorkdirRef() override
+    {
+        Reference ref;
+        if (git_reference_lookup(Setter(ref), *this, "HEAD"))
+            throw Error("looking up HEAD: %s", git_error_last()->message);
+
+        if (auto target = git_reference_symbolic_target(ref.get()))
+            return target;
+
+        return std::nullopt;
+    }
+
+    bool hasObject(const Hash & oid_) override
+    {
+        auto oid = hashToOID(oid_);
+
+        Object obj;
+        if (auto errCode = git_object_lookup(Setter(obj), *this, &oid, GIT_OBJECT_ANY)) {
+            if (errCode == GIT_ENOTFOUND) return false;
+            auto err = git_error_last();
+            throw Error("getting Git object '%s': %s", oid, err->message);
+        }
+
+        return true;
+    }
+
+    ref<InputAccessor> getAccessor(const Hash & rev) override;
+
+    void fetch(
+        const std::string & url,
+        const std::string & refspec) override
+    {
+        /* FIXME: use libgit2. Unfortunately, it doesn't support
+           ssh_config at the moment. */
+        #if 0
+        Remote remote;
+
+        if (git_remote_create_anonymous(Setter(remote), *this, url.c_str()))
+            throw Error("cannot create Git remote '%s': %s", url, git_error_last()->message);
+
+        char * refspecs[] = {(char *) refspec.c_str()};
+        git_strarray refspecs2 {
+            .strings = refspecs,
+            .count = 1
+        };
+
+        if (git_remote_fetch(remote.get(), &refspecs2, nullptr, nullptr))
+            throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message);
+        #endif
+
+        // FIXME: git stderr messes up our progress indicator, so
+        // we're using --quiet for now. Should process its stderr.
+        runProgram("git", true,
+            { "-C", path.abs(),
+              "--bare",
+              "fetch",
+              "--quiet",
+              "--force",
+              "--",
+              url,
+              refspec
+            }, {}, true);
+    }
+};
+
+ref<GitRepo> GitRepo::openRepo(const CanonPath & path, bool create, bool bare)
+{
+    return make_ref<GitRepoImpl>(path, create, bare);
+}
+
+struct GitInputAccessor : InputAccessor
+{
+    ref<GitRepoImpl> repo;
+    Tree root;
+
+    GitInputAccessor(ref<GitRepoImpl> repo_, const Hash & rev)
+        : repo(repo_)
+        , root(peelObject<Tree>(*repo, lookupObject(*repo, hashToOID(rev)).get(), GIT_OBJECT_TREE))
+    {
+    }
+
+    std::string readBlob(const CanonPath & path, bool symlink)
+    {
+        auto blob = getBlob(path, symlink);
+
+        auto data = std::string_view((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get()));
+
+        return std::string(data);
+    }
+
+    std::string readFile(const CanonPath & path) override
+    {
+        return readBlob(path, false);
+    }
+
+    bool pathExists(const CanonPath & path) override
+    {
+        return path.isRoot() ? true : (bool) lookup(path);
+    }
+
+    Stat lstat(const CanonPath & path) override
+    {
+        if (path.isRoot())
+            return Stat { .type = tDirectory };
+
+        auto entry = need(path);
+
+        auto mode = git_tree_entry_filemode(entry);
+
+        if (mode == GIT_FILEMODE_TREE)
+            return Stat { .type = tDirectory };
+
+        else if (mode == GIT_FILEMODE_BLOB)
+            return Stat { .type = tRegular };
+
+        else if (mode == GIT_FILEMODE_BLOB_EXECUTABLE)
+            return Stat { .type = tRegular, .isExecutable = true };
+
+        else if (mode == GIT_FILEMODE_LINK)
+            return Stat { .type = tSymlink };
+
+        else if (mode == GIT_FILEMODE_COMMIT)
+            // Treat submodules as an empty directory.
+            return Stat { .type = tDirectory };
+
+        else
+            throw Error("file '%s' has an unsupported Git file type");
+    }
+
+    DirEntries readDirectory(const CanonPath & path) override
+    {
+        return std::visit(overloaded {
+            [&](Tree tree) {
+                DirEntries res;
+
+                auto count = git_tree_entrycount(tree.get());
+
+                for (size_t n = 0; n < count; ++n) {
+                    auto entry = git_tree_entry_byindex(tree.get(), n);
+                    // FIXME: add to cache
+                    res.emplace(std::string(git_tree_entry_name(entry)), DirEntry{});
+                }
+
+                return res;
+            },
+            [&](Submodule) {
+                return DirEntries();
+            }
+        }, getTree(path));
+    }
+
+    std::string readLink(const CanonPath & path) override
+    {
+        return readBlob(path, true);
+    }
+
+    std::map<CanonPath, TreeEntry> lookupCache;
+
+    /* Recursively look up 'path' relative to the root. */
+    git_tree_entry * lookup(const CanonPath & path)
+    {
+        if (path.isRoot()) return nullptr;
+
+        auto i = lookupCache.find(path);
+        if (i == lookupCache.end()) {
+            TreeEntry entry;
+            if (auto err = git_tree_entry_bypath(Setter(entry), root.get(), std::string(path.rel()).c_str())) {
+                if (err != GIT_ENOTFOUND)
+                    throw Error("looking up '%s': %s", showPath(path), git_error_last()->message);
+            }
+
+            i = lookupCache.emplace(path, std::move(entry)).first;
+        }
+
+        return &*i->second;
+    }
+
+    git_tree_entry * need(const CanonPath & path)
+    {
+        auto entry = lookup(path);
+        if (!entry)
+            throw Error("'%s' does not exist", showPath(path));
+        return entry;
+    }
+
+    struct Submodule { };
+
+    std::variant<Tree, Submodule> getTree(const CanonPath & path)
+    {
+        if (path.isRoot()) {
+            Tree tree;
+            if (git_tree_dup(Setter(tree), root.get()))
+                throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message);
+            return tree;
+        }
+
+        auto entry = need(path);
+
+        if (git_tree_entry_type(entry) == GIT_OBJECT_COMMIT)
+            return Submodule();
+
+        if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
+            throw Error("'%s' is not a directory", showPath(path));
+
+        Tree tree;
+        if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(tree), *repo, entry))
+            throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message);
+
+        return tree;
+    }
+
+    Blob getBlob(const CanonPath & path, bool expectSymlink)
+    {
+        auto notExpected = [&]()
+        {
+            throw Error(
+                expectSymlink
+                ? "'%s' is not a symlink"
+                : "'%s' is not a regular file",
+                showPath(path));
+        };
+
+        if (path.isRoot()) notExpected();
+
+        auto entry = need(path);
+
+        if (git_tree_entry_type(entry) != GIT_OBJECT_BLOB)
+            notExpected();
+
+        auto mode = git_tree_entry_filemode(entry);
+        if (expectSymlink) {
+            if (mode != GIT_FILEMODE_LINK)
+                notExpected();
+        } else {
+            if (mode != GIT_FILEMODE_BLOB && mode != GIT_FILEMODE_BLOB_EXECUTABLE)
+                notExpected();
+        }
+
+        Blob blob;
+        if (git_tree_entry_to_object((git_object * *) (git_blob * *) Setter(blob), *repo, entry))
+            throw Error("looking up file '%s': %s", showPath(path), git_error_last()->message);
+
+        return blob;
+    }
+};
+
+ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev)
+{
+    return make_ref<GitInputAccessor>(ref<GitRepoImpl>(shared_from_this()), rev);
+}
+
+}
diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
new file mode 100644
index 00000000000..dd2c066725b
--- /dev/null
+++ b/src/libfetchers/git-utils.hh
@@ -0,0 +1,56 @@
+#pragma once
+
+#include "input-accessor.hh"
+
+namespace nix {
+
+struct GitRepo
+{
+    virtual ~GitRepo()
+    { }
+
+    static ref<GitRepo> openRepo(const CanonPath & path, bool create = false, bool bare = false);
+
+    virtual uint64_t getRevCount(const Hash & rev) = 0;
+
+    virtual uint64_t getLastModified(const Hash & rev) = 0;
+
+    virtual bool isShallow() = 0;
+
+    /* Return the commit hash to which a ref points. */
+    virtual Hash resolveRef(std::string ref) = 0;
+
+    struct WorkdirInfo
+    {
+        bool isDirty = false;
+
+        /* The checked out commit, or nullopt if there are no commits
+           in the repo yet. */
+        std::optional<Hash> headRev;
+
+        /* All files in the working directory that are unchanged,
+           modified or added, but excluding deleted files. */
+        std::set<CanonPath> files;
+    };
+
+    virtual WorkdirInfo getWorkdirInfo() = 0;
+
+    /* Get the ref that HEAD points to. */
+    virtual std::optional<std::string> getWorkdirRef() = 0;
+
+    struct TarballInfo
+    {
+        Hash treeHash;
+        time_t lastModified;
+    };
+
+    virtual bool hasObject(const Hash & oid) = 0;
+
+    virtual ref<InputAccessor> getAccessor(const Hash & rev) = 0;
+
+    virtual void fetch(
+        const std::string & url,
+        const std::string & refspec) = 0;
+};
+
+}
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 4bfd53b0e9b..55d3a8ebed6 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -7,6 +7,8 @@
 #include "pathlocks.hh"
 #include "util.hh"
 #include "git.hh"
+#include "fs-input-accessor.hh"
+#include "git-utils.hh"
 
 #include "fetch-settings.hh"
 
@@ -137,121 +139,6 @@ bool isNotDotGitDirectory(const Path & path)
     return baseNameOf(path) != ".git";
 }
 
-struct WorkdirInfo
-{
-    bool clean = false;
-    bool hasHead = false;
-};
-
-// Returns whether a git workdir is clean and has commits.
-WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir)
-{
-    const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
-    std::string gitDir(".git");
-
-    auto env = getEnv();
-    // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong
-    // that way unknown errors can lead to a failure instead of continuing through the wrong code path
-    env["LC_ALL"] = "C";
-
-    /* Check whether HEAD points to something that looks like a commit,
-       since that is the refrence we want to use later on. */
-    auto result = runProgram(RunOptions {
-        .program = "git",
-        .args = { "-C", workdir, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" },
-        .environment = env,
-        .mergeStderrToStdout = true
-    });
-    auto exitCode = WEXITSTATUS(result.first);
-    auto errorMessage = result.second;
-
-    if (errorMessage.find("fatal: not a git repository") != std::string::npos) {
-        throw Error("'%s' is not a Git repository", workdir);
-    } else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) {
-        // indicates that the repo does not have any commits
-        // we want to proceed and will consider it dirty later
-    } else if (exitCode != 0) {
-        // any other errors should lead to a failure
-        throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", workdir, exitCode, errorMessage);
-    }
-
-    bool clean = false;
-    bool hasHead = exitCode == 0;
-
-    try {
-        if (hasHead) {
-            // Using git diff is preferrable over lower-level operations here,
-            // because its conceptually simpler and we only need the exit code anyways.
-            auto gitDiffOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "diff", "HEAD", "--quiet"});
-            if (!submodules) {
-                // Changes in submodules should only make the tree dirty
-                // when those submodules will be copied as well.
-                gitDiffOpts.emplace_back("--ignore-submodules");
-            }
-            gitDiffOpts.emplace_back("--");
-            runProgram("git", true, gitDiffOpts);
-
-            clean = true;
-        }
-    } catch (ExecError & e) {
-        if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw;
-    }
-
-    return WorkdirInfo { .clean = clean, .hasHead = hasHead };
-}
-
-std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo)
-{
-    const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
-    auto gitDir = ".git";
-
-    if (!fetchSettings.allowDirty)
-        throw Error("Git tree '%s' is dirty", workdir);
-
-    if (fetchSettings.warnDirty)
-        warn("Git tree '%s' is dirty", workdir);
-
-    auto gitOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "ls-files", "-z" });
-    if (submodules)
-        gitOpts.emplace_back("--recurse-submodules");
-
-    auto files = tokenizeString<std::set<std::string>>(
-        runProgram("git", true, gitOpts), "\0"s);
-
-    Path actualPath(absPath(workdir));
-
-    PathFilter filter = [&](const Path & p) -> bool {
-        assert(hasPrefix(p, actualPath));
-        std::string file(p, actualPath.size() + 1);
-
-        auto st = lstat(p);
-
-        if (S_ISDIR(st.st_mode)) {
-            auto prefix = file + "/";
-            auto i = files.lower_bound(prefix);
-            return i != files.end() && hasPrefix(*i, prefix);
-        }
-
-        return files.count(file);
-    };
-
-    auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter);
-
-    // FIXME: maybe we should use the timestamp of the last
-    // modified dirty file?
-    input.attrs.insert_or_assign(
-        "lastModified",
-        workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
-
-    if (workdirInfo.hasHead) {
-        input.attrs.insert_or_assign("dirtyRev", chomp(
-            runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "HEAD" })) + "-dirty");
-        input.attrs.insert_or_assign("dirtyShortRev", chomp(
-            runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "--short", "HEAD" })) + "-dirty");
-    }
-
-    return {std::move(storePath), input};
-}
 }  // end namespace
 
 struct GitInputScheme : InputScheme
@@ -336,11 +223,11 @@ struct GitInputScheme : InputScheme
 
     void clone(const Input & input, const Path & destDir) const override
     {
-        auto [isLocal, actualUrl] = getActualUrl(input);
+        auto repoInfo = getRepoInfo(input);
 
         Strings args = {"clone"};
 
-        args.push_back(actualUrl);
+        args.push_back(repoInfo.url);
 
         if (auto ref = input.getRef()) {
             args.push_back("--branch");
@@ -356,10 +243,9 @@ struct GitInputScheme : InputScheme
 
     std::optional<Path> getSourcePath(const Input & input) const override
     {
-        auto url = parseURL(getStrAttr(input.attrs, "url"));
-        if (url.scheme == "file" && !input.getRef() && !input.getRev())
-            return url.path;
-        return {};
+        auto repoInfo = getRepoInfo(input);
+        if (repoInfo.isLocal) return repoInfo.url;
+        return std::nullopt;
     }
 
     void putFile(
@@ -368,24 +254,79 @@ struct GitInputScheme : InputScheme
         std::string_view contents,
         std::optional<std::string> commitMsg) const override
     {
-        auto root = getSourcePath(input);
-        if (!root)
+        auto repoInfo = getRepoInfo(input);
+        if (!repoInfo.isLocal)
             throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string());
 
-        writeFile((CanonPath(*root) + path).abs(), contents);
-
-        auto gitDir = ".git";
+        writeFile((CanonPath(repoInfo.url) + path).abs(), contents);
 
         runProgram("git", true,
-            { "-C", *root, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
+            { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) });
 
         if (commitMsg)
             runProgram("git", true,
-                { "-C", *root, "--git-dir", gitDir, "commit", std::string(path.rel()), "-m", *commitMsg });
+                { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-m", *commitMsg });
     }
 
-    std::pair<bool, std::string> getActualUrl(const Input & input) const
+    struct RepoInfo
     {
+        bool shallow = false;
+        bool submodules = false;
+        bool allRefs = false;
+
+        std::string cacheType;
+
+        /* Whether this is a local, non-bare repository. */
+        bool isLocal = false;
+
+        /* Working directory info: the complete list of files, and
+           whether the working directory is dirty compared to HEAD. */
+        GitRepo::WorkdirInfo workdirInfo;
+
+        /* URL of the repo, or its path if isLocal. */
+        std::string url;
+
+        void warnDirty() const
+        {
+            if (workdirInfo.isDirty) {
+                if (!fetchSettings.allowDirty)
+                    throw Error("Git tree '%s' is dirty", url);
+
+                if (fetchSettings.warnDirty)
+                    warn("Git tree '%s' is dirty", url);
+            }
+        }
+
+        std::string gitDir = ".git";
+    };
+
+    bool getSubmodulesAttr(const Input & input) const
+    {
+        return maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
+    }
+
+    RepoInfo getRepoInfo(const Input & input) const
+    {
+        auto checkHashType = [&](const std::optional<Hash> & hash)
+        {
+            if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256))
+                throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true));
+        };
+
+        if (auto rev = input.getRev())
+            checkHashType(rev);
+
+        RepoInfo repoInfo {
+            .shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false),
+            .submodules = getSubmodulesAttr(input),
+            .allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false)
+        };
+
+        repoInfo.cacheType = "git";
+        if (repoInfo.shallow) repoInfo.cacheType += "-shallow";
+        if (repoInfo.submodules) repoInfo.cacheType += "-submodules";
+        if (repoInfo.allRefs) repoInfo.cacheType += "-all-refs";
+
         // file:// URIs are normally not cloned (but otherwise treated the
         // same as remote URIs, i.e. we don't use the working tree or
         // HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git
@@ -393,153 +334,142 @@ struct GitInputScheme : InputScheme
         static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
         auto url = parseURL(getStrAttr(input.attrs, "url"));
         bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git");
-        bool isLocal = url.scheme == "file" && !forceHttp && !isBareRepository;
-        return {isLocal, isLocal ? url.path : url.base};
+        repoInfo.isLocal = url.scheme == "file" && !forceHttp && !isBareRepository;
+        repoInfo.url = repoInfo.isLocal ? url.path : url.base;
+
+        // If this is a local directory and no ref or revision is
+        // given, then allow the use of an unclean working tree.
+        if (!input.getRef() && !input.getRev() && repoInfo.isLocal)
+            repoInfo.workdirInfo = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirInfo();
+
+        return repoInfo;
     }
 
-    std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
+    uint64_t getLastModified(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const
     {
-        Input input(_input);
-        auto gitDir = ".git";
+        Attrs key{{"_what", "gitLastModified"}, {"rev", rev.gitRev()}};
 
-        std::string name = input.getName();
+        auto cache = getCache();
 
-        bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
-        bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
-        bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
+        if (auto res = cache->lookup(key))
+            return getIntAttr(*res, "lastModified");
 
-        std::string cacheType = "git";
-        if (shallow) cacheType += "-shallow";
-        if (submodules) cacheType += "-submodules";
-        if (allRefs) cacheType += "-all-refs";
+        auto lastModified = GitRepo::openRepo(CanonPath(repoDir))->getLastModified(rev);
 
-        auto checkHashType = [&](const std::optional<Hash> & hash)
+        cache->upsert(key, Attrs{{"lastModified", lastModified}});
+
+        return lastModified;
+    }
+
+    uint64_t getRevCount(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const
+    {
+        Attrs key{{"_what", "gitRevCount"}, {"rev", rev.gitRev()}};
+
+        auto cache = getCache();
+
+        if (auto revCountAttrs = cache->lookup(key))
+            return getIntAttr(*revCountAttrs, "revCount");
+
+        Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.url));
+
+        auto revCount = GitRepo::openRepo(CanonPath(repoDir))->getRevCount(rev);
+
+        cache->upsert(key, Attrs{{"revCount", revCount}});
+
+        return revCount;
+    }
+
+    std::string getDefaultRef(const RepoInfo & repoInfo) const
+    {
+        auto head = repoInfo.isLocal
+            ? GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef()
+            : readHeadCached(repoInfo.url);
+        if (!head) {
+            warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.url);
+            return "master";
+        }
+        return *head;
+    }
+
+    static MakeNotAllowedError makeNotAllowedError(std::string url)
+    {
+        return [url{std::move(url)}](const CanonPath & path) -> RestrictedPathError
         {
-            if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256))
-                throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true));
+            if (nix::pathExists(path.abs()))
+                return RestrictedPathError("access to path '%s' is forbidden because it is not under Git control; maybe you should 'git add' it to the repository '%s'?", path, url);
+            else
+                return RestrictedPathError("path '%s' does not exist in Git repository '%s'", path, url);
         };
+    }
 
-        auto getLockedAttrs = [&]()
-        {
-            checkHashType(input.getRev());
+    std::pair<ref<InputAccessor>, Input> getAccessorFromCommit(
+        ref<Store> store,
+        RepoInfo & repoInfo,
+        Input && input) const
+    {
+        assert(!repoInfo.workdirInfo.isDirty);
 
-            return Attrs({
-                {"type", cacheType},
-                {"name", name},
-                {"rev", input.getRev()->gitRev()},
-            });
-        };
+        auto origRev = input.getRev();
 
-        auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
-            -> std::pair<StorePath, Input>
+        std::string name = input.getName();
+
+        auto makeResult2 = [&](const Attrs & infoAttrs, ref<InputAccessor> accessor) -> std::pair<ref<InputAccessor>, Input>
         {
             assert(input.getRev());
-            assert(!_input.getRev() || _input.getRev() == input.getRev());
-            if (!shallow)
+            assert(!origRev || origRev == input.getRev());
+            if (!repoInfo.shallow)
                 input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
             input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
-            return {std::move(storePath), input};
+
+            return {accessor, std::move(input)};
         };
 
-        if (input.getRev()) {
-            if (auto res = getCache()->lookup(store, getLockedAttrs()))
-                return makeResult(res->first, std::move(res->second));
-        }
+        auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> std::pair<ref<InputAccessor>, Input>
+        {
+            // FIXME: remove?
+            //input.attrs.erase("narHash");
+            auto narHash = store->queryPathInfo(storePath)->narHash;
+            input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
 
-        auto [isLocal, actualUrl_] = getActualUrl(input);
-        auto actualUrl = actualUrl_; // work around clang bug
+            auto accessor = makeStorePathAccessor(store, storePath, makeNotAllowedError(repoInfo.url));
 
-        /* If this is a local directory and no ref or revision is given,
-           allow fetching directly from a dirty workdir. */
-        if (!input.getRef() && !input.getRev() && isLocal) {
-            auto workdirInfo = getWorkdirInfo(input, actualUrl);
-            if (!workdirInfo.clean) {
-                return fetchFromWorkdir(store, input, actualUrl, workdirInfo);
-            }
-        }
+            return makeResult2(infoAttrs, accessor);
+        };
 
-        Attrs unlockedAttrs({
-            {"type", cacheType},
-            {"name", name},
-            {"url", actualUrl},
-        });
+        auto originalRef = input.getRef();
+        auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo);
+        input.attrs.insert_or_assign("ref", ref);
 
         Path repoDir;
 
-        if (isLocal) {
-            if (!input.getRef()) {
-                auto head = readHead(actualUrl);
-                if (!head) {
-                    warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl);
-                    head = "master";
-                }
-                input.attrs.insert_or_assign("ref", *head);
-                unlockedAttrs.insert_or_assign("ref", *head);
-            }
-
+        if (repoInfo.isLocal) {
+            repoDir = repoInfo.url;
             if (!input.getRev())
-                input.attrs.insert_or_assign("rev",
-                    Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", *input.getRef() })), htSHA1).gitRev());
-
-            repoDir = actualUrl;
+                input.attrs.insert_or_assign("rev", GitRepo::openRepo(CanonPath(repoDir))->resolveRef(ref).gitRev());
         } else {
-            const bool useHeadRef = !input.getRef();
-            if (useHeadRef) {
-                auto head = readHeadCached(actualUrl);
-                if (!head) {
-                    warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl);
-                    head = "master";
-                }
-                input.attrs.insert_or_assign("ref", *head);
-                unlockedAttrs.insert_or_assign("ref", *head);
-            } else {
-                if (!input.getRev()) {
-                    unlockedAttrs.insert_or_assign("ref", input.getRef().value());
-                }
-            }
-
-            if (auto res = getCache()->lookup(store, unlockedAttrs)) {
-                auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1);
-                if (!input.getRev() || input.getRev() == rev2) {
-                    input.attrs.insert_or_assign("rev", rev2.gitRev());
-                    return makeResult(res->first, std::move(res->second));
-                }
-            }
-
-            Path cacheDir = getCachePath(actualUrl);
+            Path cacheDir = getCachePath(repoInfo.url);
             repoDir = cacheDir;
-            gitDir = ".";
+            repoInfo.gitDir = ".";
 
             createDirs(dirOf(cacheDir));
-            PathLocks cacheDirLock({cacheDir + ".lock"});
+            PathLocks cacheDirLock({cacheDir});
 
-            if (!pathExists(cacheDir)) {
-                runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir });
-            }
+            auto repo = GitRepo::openRepo(CanonPath(cacheDir), true, true);
 
             Path localRefFile =
-                input.getRef()->compare(0, 5, "refs/") == 0
-                ? cacheDir + "/" + *input.getRef()
-                : cacheDir + "/refs/heads/" + *input.getRef();
+                ref.compare(0, 5, "refs/") == 0
+                ? cacheDir + "/" + ref
+                : cacheDir + "/refs/heads/" + ref;
 
             bool doFetch;
             time_t now = time(0);
 
             /* If a rev was specified, we need to fetch if it's not in the
                repo. */
-            if (input.getRev()) {
-                try {
-                    runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "cat-file", "-e", input.getRev()->gitRev() });
-                    doFetch = false;
-                } catch (ExecError & e) {
-                    if (WIFEXITED(e.status)) {
-                        doFetch = true;
-                    } else {
-                        throw;
-                    }
-                }
+            if (auto rev = input.getRev()) {
+                doFetch = !repo->hasObject(*rev);
             } else {
-                if (allRefs) {
+                if (repoInfo.allRefs) {
                     doFetch = true;
                 } else {
                     /* If the local ref is older than ‘tarball-ttl’ seconds, do a
@@ -551,75 +481,80 @@ struct GitInputScheme : InputScheme
             }
 
             if (doFetch) {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl));
+                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", repoInfo.url));
 
-                // FIXME: git stderr messes up our progress indicator, so
-                // we're using --quiet for now. Should process its stderr.
                 try {
-                    auto ref = input.getRef();
-                    auto fetchRef = allRefs
+                    auto fetchRef = repoInfo.allRefs
                         ? "refs/*"
-                        : ref->compare(0, 5, "refs/") == 0
-                            ? *ref
-                            : ref == "HEAD"
-                                ? *ref
-                                : "refs/heads/" + *ref;
-                    runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, {}, true);
+                        : ref.compare(0, 5, "refs/") == 0
+                        ? ref
+                        : ref == "HEAD"
+                        ? ref
+                        : "refs/heads/" + ref;
+
+                    repo->fetch(repoInfo.url, fmt("%s:%s", fetchRef, fetchRef));
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;
-                    warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl);
+                    logError(e.info());
+                    warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.url);
                 }
 
                 if (!touchCacheFile(localRefFile, now))
                     warn("could not update mtime for file '%s': %s", localRefFile, strerror(errno));
-                if (useHeadRef && !storeCachedHead(actualUrl, *input.getRef()))
-                    warn("could not update cached head '%s' for '%s'", *input.getRef(), actualUrl);
+                if (!originalRef && !storeCachedHead(repoInfo.url, ref))
+                    warn("could not update cached head '%s' for '%s'", ref, repoInfo.url);
             }
 
-            if (!input.getRev())
+            if (auto rev = input.getRev()) {
+                if (!repo->hasObject(*rev))
+                    throw Error(
+                        "Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
+                        "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the "
+                        ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD
+                        "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".",
+                        rev->gitRev(),
+                        ref,
+                        repoInfo.url
+                        );
+            } else
                 input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev());
 
             // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder
         }
 
-        bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-parse", "--is-shallow-repository" })) == "true";
+        auto isShallow = GitRepo::openRepo(CanonPath(repoDir))->isShallow();
 
-        if (isShallow && !shallow)
-            throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified.", actualUrl);
+        if (isShallow && !repoInfo.shallow)
+            throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", repoInfo.url);
 
-        // FIXME: check whether rev is an ancestor of ref.
+        // FIXME: check whether rev is an ancestor of ref?
 
-        printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl);
+        auto rev = *input.getRev();
 
-        /* Now that we know the ref, check again whether we have it in
-           the store. */
-        if (auto res = getCache()->lookup(store, getLockedAttrs()))
-            return makeResult(res->first, std::move(res->second));
+        Attrs infoAttrs({
+            {"rev", rev.gitRev()},
+            {"lastModified", getLastModified(repoInfo, repoDir, rev)},
+        });
 
-        Path tmpDir = createTempDir();
-        AutoDelete delTmpDir(tmpDir, true);
-        PathFilter filter = defaultPathFilter;
+        if (!repoInfo.shallow)
+            infoAttrs.insert_or_assign("revCount",
+                getRevCount(repoInfo, repoDir, rev));
 
-        auto result = runProgram(RunOptions {
-            .program = "git",
-            .args = { "-C", repoDir, "--git-dir", gitDir, "cat-file", "commit", input.getRev()->gitRev() },
-            .mergeStderrToStdout = true
-        });
-        if (WEXITSTATUS(result.first) == 128
-            && result.second.find("bad file") != std::string::npos)
-        {
-            throw Error(
-                "Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
-                "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the "
-                ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD
-                "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".",
-                input.getRev()->gitRev(),
-                *input.getRef(),
-                actualUrl
-            );
+        printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.url);
+
+        if (!repoInfo.submodules) {
+            auto accessor = GitRepo::openRepo(CanonPath(repoDir))->getAccessor(rev);
+            return makeResult2(infoAttrs, accessor);
         }
 
-        if (submodules) {
+        else {
+            // FIXME: use libgit2
+            Path tmpDir = createTempDir();
+            AutoDelete delTmpDir(tmpDir, true);
+            PathFilter filter = defaultPathFilter;
+
+            Activity act(*logger, lvlChatty, actUnknown, fmt("copying Git tree '%s' to the store", input.to_string()));
+
             Path tmpGitDir = createTempDir();
             AutoDelete delTmpGitDir(tmpGitDir, true);
 
@@ -634,77 +569,89 @@ struct GitInputScheme : InputScheme
                         "--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true);
             }
 
-            runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() });
+            runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", rev.gitRev() });
 
             /* Ensure that we use the correct origin for fetching
                submodules. This matters for submodules with relative
                URLs. */
-            if (isLocal) {
-                writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + gitDir + "/config"));
+            if (repoInfo.isLocal) {
+                writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + repoInfo.gitDir + "/config"));
 
                 /* Restore the config.bare setting we may have just
                    copied erroneously from the user's repo. */
                 runProgram("git", true, { "-C", tmpDir, "config", "core.bare", "false" });
             } else
-                runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", actualUrl });
+                runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", repoInfo.url });
 
             /* As an optimisation, copy the modules directory of the
                source repo if it exists. */
-            auto modulesPath = repoDir + "/" + gitDir + "/modules";
+            auto modulesPath = repoDir + "/" + repoInfo.gitDir + "/modules";
             if (pathExists(modulesPath)) {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", actualUrl));
+                Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", repoInfo.url));
                 runProgram("cp", true, { "-R", "--", modulesPath, tmpGitDir + "/modules" });
             }
 
             {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", actualUrl));
+                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", repoInfo.url));
                 runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true);
             }
 
             filter = isNotDotGitDirectory;
+
+            auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
+
+            return makeResult(infoAttrs, std::move(storePath));
+        }
+    }
+
+    std::pair<ref<InputAccessor>, Input> getAccessorFromWorkdir(
+        RepoInfo & repoInfo,
+        Input && input) const
+    {
+        if (!repoInfo.workdirInfo.isDirty) {
+            if (auto ref = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef())
+                input.attrs.insert_or_assign("ref", *ref);
+
+            auto rev = repoInfo.workdirInfo.headRev.value();
+
+            input.attrs.insert_or_assign("rev", rev.gitRev());
+
+            input.attrs.insert_or_assign("revCount", getRevCount(repoInfo, repoInfo.url, rev));
         } else {
-            // FIXME: should pipe this, or find some better way to extract a
-            // revision.
-            auto source = sinkToSource([&](Sink & sink) {
-                runProgram2({
-                    .program = "git",
-                    .args = { "-C", repoDir, "--git-dir", gitDir, "archive", input.getRev()->gitRev() },
-                    .standardOut = &sink
-                });
-            });
-
-            unpackTarfile(*source, tmpDir);
+            repoInfo.warnDirty();
+
+            if (repoInfo.workdirInfo.headRev) {
+                input.attrs.insert_or_assign("dirtyRev",
+                    repoInfo.workdirInfo.headRev->gitRev() + "-dirty");
+                input.attrs.insert_or_assign("dirtyShortRev",
+                    repoInfo.workdirInfo.headRev->gitShortRev() + "-dirty");
+            }
         }
 
-        auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
+        input.attrs.insert_or_assign(
+            "lastModified",
+            repoInfo.workdirInfo.headRev
+            ? getLastModified(repoInfo, repoInfo.url, *repoInfo.workdirInfo.headRev)
+            : 0);
 
-        auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() }));
+        input.locked = true; // FIXME
 
-        Attrs infoAttrs({
-            {"rev", input.getRev()->gitRev()},
-            {"lastModified", lastModified},
-        });
+        return {
+            makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url)),
+            std::move(input)
+        };
+    }
 
-        if (!shallow)
-            infoAttrs.insert_or_assign("revCount",
-                std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-list", "--count", input.getRev()->gitRev() })));
-
-        if (!_input.getRev())
-            getCache()->add(
-                store,
-                unlockedAttrs,
-                infoAttrs,
-                storePath,
-                false);
-
-        getCache()->add(
-            store,
-            getLockedAttrs(),
-            infoAttrs,
-            storePath,
-            true);
-
-        return makeResult(infoAttrs, std::move(storePath));
+    std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
+    {
+        Input input(_input);
+
+        auto repoInfo = getRepoInfo(input);
+
+        if (input.getRef() || input.getRev() || !repoInfo.isLocal)
+            return getAccessorFromCommit(store, repoInfo, std::move(input));
+        else
+            return getAccessorFromWorkdir(repoInfo, std::move(input));
     }
 };
 
diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk
index 2e8869d83fa..f21651d77d4 100644
--- a/src/libfetchers/local.mk
+++ b/src/libfetchers/local.mk
@@ -8,6 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc)
 
 libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
 
-libfetchers_LDFLAGS += -pthread
+libfetchers_LDFLAGS += -pthread -lgit2 -larchive
 
 libfetchers_LIBS = libutil libstore
diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh
index fc89f2040ce..c38cd27eb81 100644
--- a/tests/functional/fetchGit.sh
+++ b/tests/functional/fetchGit.sh
@@ -185,11 +185,7 @@ path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref =
 # Nuke the cache
 rm -rf $TEST_HOME/.cache/nix
 
-# Try again, but without 'git' on PATH. This should fail.
-NIX=$(command -v nix)
-(! PATH= $NIX eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" )
-
-# Try again, with 'git' available.  This should work.
+# Try again. This should work.
 path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath")
 [[ $path3 = $path5 ]]
 
@@ -241,6 +237,7 @@ rm -rf $repo/.git
 
 # should succeed for a repo without commits
 git init $repo
+git -C $repo add hello # need to add at least one file to cause the root of the repo to be visible
 path10=$(nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath")
 
 # should succeed for a path with a space
diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh
index 21a4b52de5e..6e24a80c1bb 100644
--- a/tests/functional/flakes/flake-in-submodule.sh
+++ b/tests/functional/flakes/flake-in-submodule.sh
@@ -46,7 +46,8 @@ echo '"expression in root repo"' > $rootRepo/root.nix
 git -C $rootRepo add root.nix
 git -C $rootRepo commit -m "Add root.nix"
 
+# FIXME
 # Flake can live inside a submodule and can be accessed via ?dir=submodule
-[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#sub ) = '"expression in submodule"' ]]
+#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#sub ) = '"expression in submodule"' ]]
 # The flake can access content outside of the submodule
-[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#root ) = '"expression in root repo"' ]]
+#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#root ) = '"expression in root repo"' ]]

From d88106df24869104cc6c29c726ddfbbfda9dae10 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 27 Oct 2023 18:39:00 +0200
Subject: [PATCH 04/20] Git fetcher: Improve submodule handling

Instead of making a complete copy of the repo, fetching the
submodules, and writing the result to the store (which is all
superexpensive), we now fetch the submodules recursively using the Git
fetcher, and return a union accessor that "mounts" the accessors for
the submodules on top of the root accessor.
---
 src/libfetchers/git-utils.cc            |  78 ++++++++++++++++++
 src/libfetchers/git-utils.hh            |  12 +++
 src/libfetchers/git.cc                  | 105 +++++++-----------------
 src/libfetchers/union-input-accessor.cc |  80 ++++++++++++++++++
 src/libfetchers/union-input-accessor.hh |   9 ++
 tests/functional/fetchGitSubmodules.sh  |   8 --
 6 files changed, 210 insertions(+), 82 deletions(-)
 create mode 100644 src/libfetchers/union-input-accessor.cc
 create mode 100644 src/libfetchers/union-input-accessor.hh

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 68e39580f6e..5e3e6dae4b2 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -1,11 +1,13 @@
 #include "git-utils.hh"
 #include "input-accessor.hh"
 #include "cache.hh"
+#include "finally.hh"
 
 #include <boost/core/span.hpp>
 
 #include <git2/blob.h>
 #include <git2/commit.h>
+#include <git2/config.h>
 #include <git2/describe.h>
 #include <git2/errors.h>
 #include <git2/global.h>
@@ -14,6 +16,7 @@
 #include <git2/remote.h>
 #include <git2/repository.h>
 #include <git2/status.h>
+#include <git2/submodule.h>
 #include <git2/tree.h>
 
 #include <unordered_set>
@@ -63,6 +66,8 @@ typedef std::unique_ptr<git_reference, Deleter<git_reference_free>> Reference;
 typedef std::unique_ptr<git_describe_result, Deleter<git_describe_result_free>> DescribeResult;
 typedef std::unique_ptr<git_status_list, Deleter<git_status_list_free>> StatusList;
 typedef std::unique_ptr<git_remote, Deleter<git_remote_free>> Remote;
+typedef std::unique_ptr<git_config, Deleter<git_config_free>> GitConfig;
+typedef std::unique_ptr<git_config_iterator, Deleter<git_config_iterator_free>> ConfigIterator;
 
 // A helper to ensure that we don't leak objects returned by libgit2.
 template<typename T>
@@ -256,6 +261,17 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return std::nullopt;
     }
 
+    std::vector<Submodule> getSubmodules(const Hash & rev) override;
+
+    std::string resolveSubmoduleUrl(const std::string & url) override
+    {
+        git_buf buf = GIT_BUF_INIT;
+        if (git_submodule_resolve_url(&buf, *this, url.c_str()))
+            throw Error("resolving Git submodule URL '%s'", url);
+        Finally cleanup = [&]() { git_buf_dispose(&buf); };
+        return buf.ptr;
+    }
+
     bool hasObject(const Hash & oid_) override
     {
         auto oid = hashToOID(oid_);
@@ -400,6 +416,16 @@ struct GitInputAccessor : InputAccessor
         return readBlob(path, true);
     }
 
+    Hash getSubmoduleRev(const CanonPath & path)
+    {
+        auto entry = need(path);
+
+        if (git_tree_entry_type(entry) != GIT_OBJECT_COMMIT)
+            throw Error("'%s' is not a submodule", showPath(path));
+
+        return toHash(*git_tree_entry_id(entry));
+    }
+
     std::map<CanonPath, TreeEntry> lookupCache;
 
     /* Recursively look up 'path' relative to the root. */
@@ -495,4 +521,56 @@ ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev)
     return make_ref<GitInputAccessor>(ref<GitRepoImpl>(shared_from_this()), rev);
 }
 
+std::vector<GitRepoImpl::Submodule> GitRepoImpl::getSubmodules(const Hash & rev)
+{
+    /* Read the .gitmodules files from this revision. */
+    CanonPath modulesFile(".gitmodules");
+
+    auto accessor = getAccessor(rev);
+    if (!accessor->pathExists(modulesFile)) return {};
+
+    /* Parse it. */
+    auto configS = accessor->readFile(modulesFile);
+
+    auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules");
+    writeFull(fdTemp.get(), configS);
+
+    GitConfig config;
+    if (git_config_open_ondisk(Setter(config), pathTemp.c_str()))
+        throw Error("parsing .gitmodules file: %s", git_error_last()->message);
+
+    ConfigIterator it;
+    if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$"))
+        throw Error("iterating over .gitmodules: %s", git_error_last()->message);
+
+    std::map<std::string, std::string> entries;
+
+    while (true) {
+        git_config_entry * entry = nullptr;
+        if (auto err = git_config_next(&entry, it.get())) {
+            if (err == GIT_ITEROVER) break;
+            throw Error("iterating over .gitmodules: %s", git_error_last()->message);
+        }
+        entries.emplace(entry->name + 10, entry->value);
+    }
+
+    std::vector<Submodule> result;
+
+    for (auto & [key, value] : entries) {
+        if (!hasSuffix(key, ".path")) continue;
+        std::string key2(key, 0, key.size() - 5);
+        auto path = CanonPath(value);
+        auto rev = accessor.dynamic_pointer_cast<GitInputAccessor>()->getSubmoduleRev(path);
+        result.push_back(Submodule {
+            .path = path,
+            .url = entries[key2 + ".url"],
+            .branch = entries[key2 + ".branch"],
+            .rev = rev,
+        });
+    }
+
+    return result;
+}
+
+
 }
diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
index dd2c066725b..55e7ef969ac 100644
--- a/src/libfetchers/git-utils.hh
+++ b/src/libfetchers/git-utils.hh
@@ -38,6 +38,18 @@ struct GitRepo
     /* Get the ref that HEAD points to. */
     virtual std::optional<std::string> getWorkdirRef() = 0;
 
+    struct Submodule
+    {
+        CanonPath path;
+        std::string url;
+        std::string branch;
+        Hash rev;
+    };
+
+    virtual std::vector<Submodule> getSubmodules(const Hash & rev) = 0;
+
+    virtual std::string resolveSubmoduleUrl(const std::string & url) = 0;
+
     struct TarballInfo
     {
         Hash treeHash;
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 55d3a8ebed6..42b4aa23a6f 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -8,6 +8,7 @@
 #include "util.hh"
 #include "git.hh"
 #include "fs-input-accessor.hh"
+#include "union-input-accessor.hh"
 #include "git-utils.hh"
 
 #include "fetch-settings.hh"
@@ -134,11 +135,6 @@ std::optional<std::string> readHeadCached(const std::string & actualUrl)
     return std::nullopt;
 }
 
-bool isNotDotGitDirectory(const Path & path)
-{
-    return baseNameOf(path) != ".git";
-}
-
 }  // end namespace
 
 struct GitInputScheme : InputScheme
@@ -413,7 +409,7 @@ struct GitInputScheme : InputScheme
 
         std::string name = input.getName();
 
-        auto makeResult2 = [&](const Attrs & infoAttrs, ref<InputAccessor> accessor) -> std::pair<ref<InputAccessor>, Input>
+        auto makeResult = [&](const Attrs & infoAttrs, ref<InputAccessor> accessor) -> std::pair<ref<InputAccessor>, Input>
         {
             assert(input.getRev());
             assert(!origRev || origRev == input.getRev());
@@ -424,18 +420,6 @@ struct GitInputScheme : InputScheme
             return {accessor, std::move(input)};
         };
 
-        auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> std::pair<ref<InputAccessor>, Input>
-        {
-            // FIXME: remove?
-            //input.attrs.erase("narHash");
-            auto narHash = store->queryPathInfo(storePath)->narHash;
-            input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
-
-            auto accessor = makeStorePathAccessor(store, storePath, makeNotAllowedError(repoInfo.url));
-
-            return makeResult2(infoAttrs, accessor);
-        };
-
         auto originalRef = input.getRef();
         auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo);
         input.attrs.insert_or_assign("ref", ref);
@@ -542,66 +526,39 @@ struct GitInputScheme : InputScheme
 
         printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.url);
 
-        if (!repoInfo.submodules) {
-            auto accessor = GitRepo::openRepo(CanonPath(repoDir))->getAccessor(rev);
-            return makeResult2(infoAttrs, accessor);
-        }
-
-        else {
-            // FIXME: use libgit2
-            Path tmpDir = createTempDir();
-            AutoDelete delTmpDir(tmpDir, true);
-            PathFilter filter = defaultPathFilter;
-
-            Activity act(*logger, lvlChatty, actUnknown, fmt("copying Git tree '%s' to the store", input.to_string()));
-
-            Path tmpGitDir = createTempDir();
-            AutoDelete delTmpGitDir(tmpGitDir, true);
-
-            runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir });
-
-            {
-                // TODO: repoDir might lack the ref (it only checks if rev
-                // exists, see FIXME above) so use a big hammer and fetch
-                // everything to ensure we get the rev.
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir));
-                runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
-                        "--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true);
+        auto repo = GitRepo::openRepo(CanonPath(repoDir));
+
+        auto accessor = repo->getAccessor(rev);
+
+        /* If the repo has submodules, fetch them and return a union
+           input accessor consisting of the accessor for the top-level
+           repo and the accessors for the submodules. */
+        if (repoInfo.submodules) {
+            std::map<CanonPath, nix::ref<InputAccessor>> mounts;
+
+            for (auto & submodule : repo->getSubmodules(rev)) {
+                auto resolved = repo->resolveSubmoduleUrl(submodule.url);
+                debug("Git submodule %s: %s %s %s -> %s",
+                    submodule.path, submodule.url, submodule.branch, submodule.rev.gitRev(), resolved);
+                fetchers::Attrs attrs;
+                attrs.insert_or_assign("type", "git");
+                attrs.insert_or_assign("url", resolved);
+                if (submodule.branch != "")
+                    attrs.insert_or_assign("ref", submodule.branch);
+                attrs.insert_or_assign("rev", submodule.rev.gitRev());
+                auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
+                auto [submoduleAccessor, submoduleInput2] =
+                    submoduleInput.scheme->getAccessor(store, submoduleInput);
+                mounts.insert_or_assign(submodule.path, submoduleAccessor);
             }
 
-            runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", rev.gitRev() });
-
-            /* Ensure that we use the correct origin for fetching
-               submodules. This matters for submodules with relative
-               URLs. */
-            if (repoInfo.isLocal) {
-                writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + repoInfo.gitDir + "/config"));
-
-                /* Restore the config.bare setting we may have just
-                   copied erroneously from the user's repo. */
-                runProgram("git", true, { "-C", tmpDir, "config", "core.bare", "false" });
-            } else
-                runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", repoInfo.url });
-
-            /* As an optimisation, copy the modules directory of the
-               source repo if it exists. */
-            auto modulesPath = repoDir + "/" + repoInfo.gitDir + "/modules";
-            if (pathExists(modulesPath)) {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", repoInfo.url));
-                runProgram("cp", true, { "-R", "--", modulesPath, tmpGitDir + "/modules" });
+            if (!mounts.empty()) {
+                mounts.insert_or_assign(CanonPath::root, accessor);
+                accessor = makeUnionInputAccessor(std::move(mounts));
             }
-
-            {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", repoInfo.url));
-                runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true);
-            }
-
-            filter = isNotDotGitDirectory;
-
-            auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
-
-            return makeResult(infoAttrs, std::move(storePath));
         }
+
+        return makeResult(infoAttrs, accessor);
     }
 
     std::pair<ref<InputAccessor>, Input> getAccessorFromWorkdir(
diff --git a/src/libfetchers/union-input-accessor.cc b/src/libfetchers/union-input-accessor.cc
new file mode 100644
index 00000000000..940c0e06cc8
--- /dev/null
+++ b/src/libfetchers/union-input-accessor.cc
@@ -0,0 +1,80 @@
+#include "union-input-accessor.hh"
+
+namespace nix {
+
+struct UnionInputAccessor : InputAccessor
+{
+    std::map<CanonPath, ref<InputAccessor>> mounts;
+
+    UnionInputAccessor(std::map<CanonPath, ref<InputAccessor>> _mounts)
+        : mounts(std::move(_mounts))
+    {
+        // Currently we require a root filesystem. This could be relaxed.
+        assert(mounts.contains(CanonPath::root));
+
+        // FIXME: should check that every mount point exists. Or we
+        // could return dummy parent directories automatically.
+    }
+
+    std::string readFile(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        return accessor->readFile(subpath);
+    }
+
+    bool pathExists(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        return accessor->pathExists(subpath);
+    }
+
+    Stat lstat(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        return accessor->lstat(subpath);
+    }
+
+    DirEntries readDirectory(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        return accessor->readDirectory(subpath);
+    }
+
+    std::string readLink(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        return accessor->readLink(subpath);
+    }
+
+    std::string showPath(const CanonPath & path) override
+    {
+        auto [accessor, subpath] = resolve(path);
+        return accessor->showPath(subpath);
+    }
+
+    std::pair<ref<InputAccessor>, CanonPath> resolve(CanonPath path)
+    {
+        // Find the nearest parent of `path` that is a mount point.
+        std::vector<std::string> ss;
+        while (true) {
+            auto i = mounts.find(path);
+            if (i != mounts.end()) {
+                auto subpath = CanonPath::root;
+                for (auto j = ss.rbegin(); j != ss.rend(); ++j)
+                    subpath.push(*j);
+                return {i->second, std::move(subpath)};
+            }
+
+            assert(!path.isRoot());
+            ss.push_back(std::string(*path.baseName()));
+            path.pop();
+        }
+    }
+};
+
+ref<InputAccessor> makeUnionInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts)
+{
+    return make_ref<UnionInputAccessor>(std::move(mounts));
+}
+
+}
diff --git a/src/libfetchers/union-input-accessor.hh b/src/libfetchers/union-input-accessor.hh
new file mode 100644
index 00000000000..6a1649c1dea
--- /dev/null
+++ b/src/libfetchers/union-input-accessor.hh
@@ -0,0 +1,9 @@
+#pragma once
+
+#include "input-accessor.hh"
+
+namespace nix {
+
+ref<InputAccessor> makeUnionInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts);
+
+}
diff --git a/tests/functional/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh
index df81232e599..369cdc5db43 100644
--- a/tests/functional/fetchGitSubmodules.sh
+++ b/tests/functional/fetchGitSubmodules.sh
@@ -118,11 +118,3 @@ cloneRepo=$TEST_ROOT/a/b/gitSubmodulesClone # NB /a/b to make the relative path
 git clone $rootRepo $cloneRepo
 pathIndirect=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath")
 [[ $pathIndirect = $pathWithRelative ]]
-
-# Test that if the clone has the submodule already, we're not fetching
-# it again.
-git -C $cloneRepo submodule update --init
-rm $TEST_HOME/.cache/nix/fetcher-cache*
-rm -rf $subRepo
-pathSubmoduleGone=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath")
-[[ $pathSubmoduleGone = $pathWithRelative ]]

From 669b074f51c4fea6b362313f47eebb4a67f0e89d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Sat, 28 Oct 2023 16:16:20 +0200
Subject: [PATCH 05/20] Cleanup

---
 src/libfetchers/git.cc | 19 +++++++------------
 1 file changed, 7 insertions(+), 12 deletions(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 42b4aa23a6f..a66a51cca20 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -409,17 +409,6 @@ struct GitInputScheme : InputScheme
 
         std::string name = input.getName();
 
-        auto makeResult = [&](const Attrs & infoAttrs, ref<InputAccessor> accessor) -> std::pair<ref<InputAccessor>, Input>
-        {
-            assert(input.getRev());
-            assert(!origRev || origRev == input.getRev());
-            if (!repoInfo.shallow)
-                input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
-            input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
-
-            return {accessor, std::move(input)};
-        };
-
         auto originalRef = input.getRef();
         auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo);
         input.attrs.insert_or_assign("ref", ref);
@@ -558,7 +547,13 @@ struct GitInputScheme : InputScheme
             }
         }
 
-        return makeResult(infoAttrs, accessor);
+        assert(input.getRev());
+        assert(!origRev || origRev == input.getRev());
+        if (!repoInfo.shallow)
+            input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
+        input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
+
+        return {accessor, std::move(input)};
     }
 
     std::pair<ref<InputAccessor>, Input> getAccessorFromWorkdir(

From 0c5eac9c4550a6de2cd829d25e628f779e2a29c7 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 31 Oct 2023 15:59:25 +0100
Subject: [PATCH 06/20] Git fetcher: Handle submodules for workdirs

---
 src/libfetchers/git-utils.cc                  | 83 +++++++++++--------
 src/libfetchers/git-utils.hh                  | 27 ++++--
 src/libfetchers/git.cc                        | 49 +++++++++--
 tests/functional/flakes/flake-in-submodule.sh | 14 +++-
 4 files changed, 119 insertions(+), 54 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 5e3e6dae4b2..5b14cfdb1bd 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -216,6 +216,43 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return toHash(*oid);
     }
 
+    std::vector<Submodule> parseSubmodules(const CanonPath & configFile)
+    {
+        GitConfig config;
+        if (git_config_open_ondisk(Setter(config), configFile.abs().c_str()))
+            throw Error("parsing .gitmodules file: %s", git_error_last()->message);
+
+        ConfigIterator it;
+        if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$"))
+            throw Error("iterating over .gitmodules: %s", git_error_last()->message);
+
+        std::map<std::string, std::string> entries;
+
+        while (true) {
+            git_config_entry * entry = nullptr;
+            if (auto err = git_config_next(&entry, it.get())) {
+                if (err == GIT_ITEROVER) break;
+                throw Error("iterating over .gitmodules: %s", git_error_last()->message);
+            }
+            entries.emplace(entry->name + 10, entry->value);
+        }
+
+        std::vector<Submodule> result;
+
+        for (auto & [key, value] : entries) {
+            if (!hasSuffix(key, ".path")) continue;
+            std::string key2(key, 0, key.size() - 5);
+            auto path = CanonPath(value);
+            result.push_back(Submodule {
+                .path = path,
+                .url = entries[key2 + ".url"],
+                .branch = entries[key2 + ".branch"],
+            });
+        }
+
+        return result;
+    }
+
     WorkdirInfo getWorkdirInfo() override
     {
         WorkdirInfo info;
@@ -246,6 +283,11 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback))
             throw Error("getting working directory status: %s", git_error_last()->message);
 
+        /* Get submodule info. */
+        auto modulesFile = path + ".gitmodules";
+        if (pathExists(modulesFile.abs()))
+            info.submodules = parseSubmodules(modulesFile);
+
         return info;
     }
 
@@ -261,7 +303,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return std::nullopt;
     }
 
-    std::vector<Submodule> getSubmodules(const Hash & rev) override;
+    std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev) override;
 
     std::string resolveSubmoduleUrl(const std::string & url) override
     {
@@ -521,7 +563,7 @@ ref<InputAccessor> GitRepoImpl::getAccessor(const Hash & rev)
     return make_ref<GitInputAccessor>(ref<GitRepoImpl>(shared_from_this()), rev);
 }
 
-std::vector<GitRepoImpl::Submodule> GitRepoImpl::getSubmodules(const Hash & rev)
+std::vector<std::tuple<GitRepoImpl::Submodule, Hash>> GitRepoImpl::getSubmodules(const Hash & rev)
 {
     /* Read the .gitmodules files from this revision. */
     CanonPath modulesFile(".gitmodules");
@@ -529,44 +571,17 @@ std::vector<GitRepoImpl::Submodule> GitRepoImpl::getSubmodules(const Hash & rev)
     auto accessor = getAccessor(rev);
     if (!accessor->pathExists(modulesFile)) return {};
 
-    /* Parse it. */
+    /* Parse it and get the revision of each submodule. */
     auto configS = accessor->readFile(modulesFile);
 
     auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules");
     writeFull(fdTemp.get(), configS);
 
-    GitConfig config;
-    if (git_config_open_ondisk(Setter(config), pathTemp.c_str()))
-        throw Error("parsing .gitmodules file: %s", git_error_last()->message);
-
-    ConfigIterator it;
-    if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$"))
-        throw Error("iterating over .gitmodules: %s", git_error_last()->message);
-
-    std::map<std::string, std::string> entries;
-
-    while (true) {
-        git_config_entry * entry = nullptr;
-        if (auto err = git_config_next(&entry, it.get())) {
-            if (err == GIT_ITEROVER) break;
-            throw Error("iterating over .gitmodules: %s", git_error_last()->message);
-        }
-        entries.emplace(entry->name + 10, entry->value);
-    }
+    std::vector<std::tuple<Submodule, Hash>> result;
 
-    std::vector<Submodule> result;
-
-    for (auto & [key, value] : entries) {
-        if (!hasSuffix(key, ".path")) continue;
-        std::string key2(key, 0, key.size() - 5);
-        auto path = CanonPath(value);
-        auto rev = accessor.dynamic_pointer_cast<GitInputAccessor>()->getSubmoduleRev(path);
-        result.push_back(Submodule {
-            .path = path,
-            .url = entries[key2 + ".url"],
-            .branch = entries[key2 + ".branch"],
-            .rev = rev,
-        });
+    for (auto & submodule : parseSubmodules(CanonPath(pathTemp))) {
+        auto rev = accessor.dynamic_pointer_cast<GitInputAccessor>()->getSubmoduleRev(submodule.path);
+        result.push_back({std::move(submodule), rev});
     }
 
     return result;
diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
index 55e7ef969ac..a425e5814f8 100644
--- a/src/libfetchers/git-utils.hh
+++ b/src/libfetchers/git-utils.hh
@@ -20,6 +20,16 @@ struct GitRepo
     /* Return the commit hash to which a ref points. */
     virtual Hash resolveRef(std::string ref) = 0;
 
+    /**
+     * Info about a submodule.
+     */
+    struct Submodule
+    {
+        CanonPath path;
+        std::string url;
+        std::string branch;
+    };
+
     struct WorkdirInfo
     {
         bool isDirty = false;
@@ -31,6 +41,9 @@ struct GitRepo
         /* All files in the working directory that are unchanged,
            modified or added, but excluding deleted files. */
         std::set<CanonPath> files;
+
+        /* The submodules listed in .gitmodules of this workdir. */
+        std::vector<Submodule> submodules;
     };
 
     virtual WorkdirInfo getWorkdirInfo() = 0;
@@ -38,15 +51,11 @@ struct GitRepo
     /* Get the ref that HEAD points to. */
     virtual std::optional<std::string> getWorkdirRef() = 0;
 
-    struct Submodule
-    {
-        CanonPath path;
-        std::string url;
-        std::string branch;
-        Hash rev;
-    };
-
-    virtual std::vector<Submodule> getSubmodules(const Hash & rev) = 0;
+    /**
+     * Return the submodules of this repo at the indicated revision,
+     * along with the revision of each submodule.
+     */
+    virtual std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev) = 0;
 
     virtual std::string resolveSubmoduleUrl(const std::string & url) = 0;
 
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index a66a51cca20..5471eb260d6 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -525,16 +525,16 @@ struct GitInputScheme : InputScheme
         if (repoInfo.submodules) {
             std::map<CanonPath, nix::ref<InputAccessor>> mounts;
 
-            for (auto & submodule : repo->getSubmodules(rev)) {
+            for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev)) {
                 auto resolved = repo->resolveSubmoduleUrl(submodule.url);
                 debug("Git submodule %s: %s %s %s -> %s",
-                    submodule.path, submodule.url, submodule.branch, submodule.rev.gitRev(), resolved);
+                    submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved);
                 fetchers::Attrs attrs;
                 attrs.insert_or_assign("type", "git");
                 attrs.insert_or_assign("url", resolved);
                 if (submodule.branch != "")
                     attrs.insert_or_assign("ref", submodule.branch);
-                attrs.insert_or_assign("rev", submodule.rev.gitRev());
+                attrs.insert_or_assign("rev", submoduleRev.gitRev());
                 auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
                 auto [submoduleAccessor, submoduleInput2] =
                     submoduleInput.scheme->getAccessor(store, submoduleInput);
@@ -557,9 +557,45 @@ struct GitInputScheme : InputScheme
     }
 
     std::pair<ref<InputAccessor>, Input> getAccessorFromWorkdir(
+        ref<Store> store,
         RepoInfo & repoInfo,
         Input && input) const
     {
+        if (repoInfo.submodules)
+            /* Create mountpoints for the submodules. */
+            for (auto & submodule : repoInfo.workdirInfo.submodules)
+                repoInfo.workdirInfo.files.insert(submodule.path);
+
+        ref<InputAccessor> accessor =
+            makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url));
+
+        /* If the repo has submodules, return a union input accessor
+           consisting of the accessor for the top-level repo and the
+           accessors for the submodule workdirs. */
+        if (repoInfo.submodules && !repoInfo.workdirInfo.submodules.empty()) {
+            std::map<CanonPath, nix::ref<InputAccessor>> mounts;
+
+            for (auto & submodule : repoInfo.workdirInfo.submodules) {
+                auto submodulePath = CanonPath(repoInfo.url) + submodule.path;
+                fetchers::Attrs attrs;
+                attrs.insert_or_assign("type", "git");
+                attrs.insert_or_assign("url", submodulePath.abs());
+                auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
+                auto [submoduleAccessor, submoduleInput2] =
+                    submoduleInput.scheme->getAccessor(store, submoduleInput);
+
+                /* If the submodule is dirty, mark this repo dirty as
+                   well. */
+                if (!submoduleInput2.getRev())
+                    repoInfo.workdirInfo.isDirty = true;
+
+                mounts.insert_or_assign(submodule.path, submoduleAccessor);
+            }
+
+            mounts.insert_or_assign(CanonPath::root, accessor);
+            accessor = makeUnionInputAccessor(std::move(mounts));
+        }
+
         if (!repoInfo.workdirInfo.isDirty) {
             if (auto ref = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef())
                 input.attrs.insert_or_assign("ref", *ref);
@@ -588,10 +624,7 @@ struct GitInputScheme : InputScheme
 
         input.locked = true; // FIXME
 
-        return {
-            makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url)),
-            std::move(input)
-        };
+        return {accessor, std::move(input)};
     }
 
     std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store, const Input & _input) const override
@@ -603,7 +636,7 @@ struct GitInputScheme : InputScheme
         if (input.getRef() || input.getRev() || !repoInfo.isLocal)
             return getAccessorFromCommit(store, repoInfo, std::move(input));
         else
-            return getAccessorFromWorkdir(repoInfo, std::move(input));
+            return getAccessorFromWorkdir(store, repoInfo, std::move(input));
     }
 };
 
diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh
index 6e24a80c1bb..85a4d33898f 100644
--- a/tests/functional/flakes/flake-in-submodule.sh
+++ b/tests/functional/flakes/flake-in-submodule.sh
@@ -46,8 +46,16 @@ echo '"expression in root repo"' > $rootRepo/root.nix
 git -C $rootRepo add root.nix
 git -C $rootRepo commit -m "Add root.nix"
 
-# FIXME
+flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule
+
 # Flake can live inside a submodule and can be accessed via ?dir=submodule
-#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#sub ) = '"expression in submodule"' ]]
+[[ $(nix eval --json $flakeref#sub ) = '"expression in submodule"' ]]
+
 # The flake can access content outside of the submodule
-#[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#root ) = '"expression in root repo"' ]]
+[[ $(nix eval --json $flakeref#root ) = '"expression in root repo"' ]]
+
+# Check that dirtying a submodule makes the entire thing dirty.
+[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) != null ]]
+echo '"foo"' > $rootRepo/submodule/sub.nix
+[[ $(nix eval --json $flakeref#sub ) = '"foo"' ]]
+[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) = null ]]

From cf59ea83ec98522113bf2fd81678537a871d0339 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Fri, 10 Nov 2023 13:58:59 +0100
Subject: [PATCH 07/20] configure: Check for libgit2

---
 Makefile.config.in       | 7 ++++---
 configure.ac             | 6 ++++++
 src/libfetchers/local.mk | 2 +-
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/Makefile.config.in b/Makefile.config.in
index 19992fa2077..aadece0e114 100644
--- a/Makefile.config.in
+++ b/Makefile.config.in
@@ -8,7 +8,9 @@ CXX = @CXX@
 CXXFLAGS = @CXXFLAGS@
 CXXLTO = @CXXLTO@
 EDITLINE_LIBS = @EDITLINE_LIBS@
+ENABLE_BUILD = @ENABLE_BUILD@
 ENABLE_S3 = @ENABLE_S3@
+ENABLE_TESTS = @ENABLE_TESTS@
 GTEST_LIBS = @GTEST_LIBS@
 HAVE_LIBCPUID = @HAVE_LIBCPUID@
 HAVE_SECCOMP = @HAVE_SECCOMP@
@@ -17,6 +19,7 @@ LDFLAGS = @LDFLAGS@
 LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@
 LIBBROTLI_LIBS = @LIBBROTLI_LIBS@
 LIBCURL_LIBS = @LIBCURL_LIBS@
+LIBGIT2_LIBS = @LIBGIT2_LIBS@
 LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@
 LOWDOWN_LIBS = @LOWDOWN_LIBS@
 OPENSSL_LIBS = @OPENSSL_LIBS@
@@ -35,6 +38,7 @@ docdir = @docdir@
 embedded_sandbox_shell = @embedded_sandbox_shell@
 exec_prefix = @exec_prefix@
 includedir = @includedir@
+internal_api_docs = @internal_api_docs@
 libdir = @libdir@
 libexecdir = @libexecdir@
 localstatedir = @localstatedir@
@@ -46,6 +50,3 @@ sandbox_shell = @sandbox_shell@
 storedir = @storedir@
 sysconfdir = @sysconfdir@
 system = @system@
-ENABLE_BUILD = @ENABLE_BUILD@
-ENABLE_TESTS = @ENABLE_TESTS@
-internal_api_docs = @internal_api_docs@
diff --git a/configure.ac b/configure.ac
index 75ce7d01dc4..1cda0852a6b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -335,9 +335,15 @@ AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation
   doc_generate=$enableval, doc_generate=yes)
 AC_SUBST(doc_generate)
 
+
 # Look for lowdown library.
 PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"])
 
+
+# Look for libgit2.
+PKG_CHECK_MODULES([LIBGIT2], [libgit2])
+
+
 # Setuid installations.
 AC_CHECK_FUNCS([setresuid setreuid lchown])
 
diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk
index f21651d77d4..266e7a21149 100644
--- a/src/libfetchers/local.mk
+++ b/src/libfetchers/local.mk
@@ -8,6 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc)
 
 libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
 
-libfetchers_LDFLAGS += -pthread -lgit2 -larchive
+libfetchers_LDFLAGS += -pthread $(LIBGIT2_LIBS) -larchive
 
 libfetchers_LIBS = libutil libstore

From 21bb180547118e29a66bf091bd6b1dd911b3114d Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 13:30:51 +0100
Subject: [PATCH 08/20] Use libgit2 with ssh-exec support

See https://github.com/libgit2/libgit2/pull/6617. This ensures that we
get support for ~/.ssh/config, known_hosts etc.
---
 flake.lock                   | 17 +++++++++++++++++
 flake.nix                    |  9 +++++++--
 src/libfetchers/git-utils.cc | 17 -----------------
 3 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/flake.lock b/flake.lock
index 991cef1eef1..2b1d96e4ea6 100644
--- a/flake.lock
+++ b/flake.lock
@@ -16,6 +16,22 @@
         "type": "github"
       }
     },
+    "libgit2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1697646580,
+        "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=",
+        "owner": "libgit2",
+        "repo": "libgit2",
+        "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5",
+        "type": "github"
+      },
+      "original": {
+        "owner": "libgit2",
+        "repo": "libgit2",
+        "type": "github"
+      }
+    },
     "lowdown-src": {
       "flake": false,
       "locked": {
@@ -67,6 +83,7 @@
     "root": {
       "inputs": {
         "flake-compat": "flake-compat",
+        "libgit2": "libgit2",
         "lowdown-src": "lowdown-src",
         "nixpkgs": "nixpkgs",
         "nixpkgs-regression": "nixpkgs-regression"
diff --git a/flake.nix b/flake.nix
index e71aa53748b..d6a173081fc 100644
--- a/flake.nix
+++ b/flake.nix
@@ -7,8 +7,9 @@
   inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2";
   inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; };
   inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
+  inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; };
 
-  outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat }:
+  outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat, libgit2 }:
 
     let
       inherit (nixpkgs) lib;
@@ -194,7 +195,11 @@
             bzip2 xz brotli editline
             openssl sqlite
             libarchive
-            libgit2
+            (pkgs.libgit2.overrideAttrs (attrs: {
+              src = libgit2;
+              version = libgit2.lastModifiedDate;
+              cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"];
+            }))
             boost
             lowdown-nix
             libsodium
diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 3a0e2d02f59..1ec50099b58 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -336,9 +336,6 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         const std::string & url,
         const std::string & refspec) override
     {
-        /* FIXME: use libgit2. Unfortunately, it doesn't support
-           ssh_config at the moment. */
-        #if 0
         Remote remote;
 
         if (git_remote_create_anonymous(Setter(remote), *this, url.c_str()))
@@ -352,20 +349,6 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
         if (git_remote_fetch(remote.get(), &refspecs2, nullptr, nullptr))
             throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message);
-        #endif
-
-        // FIXME: git stderr messes up our progress indicator, so
-        // we're using --quiet for now. Should process its stderr.
-        runProgram("git", true,
-            { "-C", path.abs(),
-              "--bare",
-              "fetch",
-              "--quiet",
-              "--force",
-              "--",
-              url,
-              refspec
-            }, {}, true);
     }
 
     void verifyCommit(

From d74d2fdaa721cd7cddceca2e0b4063a1d891bb9f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 13:35:26 +0100
Subject: [PATCH 09/20] Move statusCallbackTrampoline

---
 src/libfetchers/git-utils.cc | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 1ec50099b58..ffcc92fc7ca 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -130,11 +130,6 @@ T peelObject(git_repository * repo, git_object * obj, git_object_t type)
     return obj2;
 }
 
-int statusCallbackTrampoline(const char * path, unsigned int statusFlags, void * payload)
-{
-    return (*((std::function<int(const char * path, unsigned int statusFlags)> *) payload))(path, statusFlags);
-}
-
 struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 {
     CanonPath path;
@@ -255,6 +250,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return result;
     }
 
+    // Helper for statusCallback below.
+    static int statusCallbackTrampoline(const char * path, unsigned int statusFlags, void * payload)
+    {
+        return (*((std::function<int(const char * path, unsigned int statusFlags)> *) payload))(path, statusFlags);
+    }
+
     WorkdirInfo getWorkdirInfo() override
     {
         WorkdirInfo info;

From 38b07d63479ebdd4f43145264a026a22a72d940b Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 13:38:03 +0100
Subject: [PATCH 10/20] src/libfetchers/git.cc: Apply suggestion

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 src/libfetchers/git.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 9c2a7df169e..12233ed0a40 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -332,7 +332,7 @@ struct GitInputScheme : InputScheme
            whether the working directory is dirty compared to HEAD. */
         GitRepo::WorkdirInfo workdirInfo;
 
-        /* URL of the repo, or its path if isLocal. */
+        /* URL of the repo, or its path if isLocal. Never a `file` URL. */
         std::string url;
 
         void warnDirty() const

From 25cf8f107125eda79e7faece90e7e05093a39e65 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 13:57:24 +0100
Subject: [PATCH 11/20] src/libfetchers/union-input-accessor.cc: Apply
 suggestion

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
---
 src/libfetchers/union-input-accessor.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/libfetchers/union-input-accessor.cc b/src/libfetchers/union-input-accessor.cc
index ae942cb410d..f9472efa791 100644
--- a/src/libfetchers/union-input-accessor.cc
+++ b/src/libfetchers/union-input-accessor.cc
@@ -12,8 +12,7 @@ struct UnionInputAccessor : InputAccessor
         // Currently we require a root filesystem. This could be relaxed.
         assert(mounts.contains(CanonPath::root));
 
-        // FIXME: should check that every mount point exists. Or we
-        // could return dummy parent directories automatically.
+        // FIXME: return dummy parent directories automatically?
     }
 
     std::string readFile(const CanonPath & path) override

From 4329bdf6a30fadad66384f0b8c835d7dba9f87b3 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 13:58:27 +0100
Subject: [PATCH 12/20] Move comment

---
 src/libfetchers/cache.hh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh
index b517d496ed3..c8d3248bc71 100644
--- a/src/libfetchers/cache.hh
+++ b/src/libfetchers/cache.hh
@@ -6,13 +6,14 @@
 
 namespace nix::fetchers {
 
+/*
+ * A cache for arbitrary `Attrs` -> `Attrs` mappings with a timestamp
+ * for expiration.
+ */
 struct Cache
 {
     virtual ~Cache() { }
 
-    /* A cache for arbitrary Attrs -> Attrs mappings with a timestamp
-       for expiration. */
-
     /*
      * Add a value to the cache. The cache is an arbitrary mapping of
      * Attrs to Attrs.

From 21140c987b7a301c01498864efbc3d92be04aced Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 13:59:00 +0100
Subject: [PATCH 13/20] Fix doxygen comments

---
 src/libfetchers/cache.hh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh
index c8d3248bc71..f70589267f7 100644
--- a/src/libfetchers/cache.hh
+++ b/src/libfetchers/cache.hh
@@ -6,7 +6,7 @@
 
 namespace nix::fetchers {
 
-/*
+/**
  * A cache for arbitrary `Attrs` -> `Attrs` mappings with a timestamp
  * for expiration.
  */
@@ -14,7 +14,7 @@ struct Cache
 {
     virtual ~Cache() { }
 
-    /*
+    /**
      * Add a value to the cache. The cache is an arbitrary mapping of
      * Attrs to Attrs.
      */
@@ -22,13 +22,13 @@ struct Cache
         const Attrs & inAttrs,
         const Attrs & infoAttrs) = 0;
 
-    /*
+    /**
      * Look up a key with infinite TTL.
      */
     virtual std::optional<Attrs> lookup(
         const Attrs & inAttrs) = 0;
 
-    /*
+    /**
      * Look up a key. Return nothing if its TTL has exceeded
      * `settings.tarballTTL`.
      */
@@ -41,7 +41,7 @@ struct Cache
         Attrs infoAttrs;
     };
 
-    /*
+    /**
      * Look up a key. Return a bool denoting whether its TTL has
      * exceeded `settings.tarballTTL`.
      */

From 7f576f5dfe11c3f6b0e69179de95c921caddda18 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 14:01:38 +0100
Subject: [PATCH 14/20] Rename UnionInputAccessor to MountedInputAccessor

---
 src/libfetchers/git.cc                                 | 10 +++++-----
 ...ion-input-accessor.cc => mounted-input-accessor.cc} | 10 +++++-----
 src/libfetchers/mounted-input-accessor.hh              |  9 +++++++++
 src/libfetchers/union-input-accessor.hh                |  9 ---------
 4 files changed, 19 insertions(+), 19 deletions(-)
 rename src/libfetchers/{union-input-accessor.cc => mounted-input-accessor.cc} (86%)
 create mode 100644 src/libfetchers/mounted-input-accessor.hh
 delete mode 100644 src/libfetchers/union-input-accessor.hh

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 12233ed0a40..90c6ad531b7 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -9,7 +9,7 @@
 #include "processes.hh"
 #include "git.hh"
 #include "fs-input-accessor.hh"
-#include "union-input-accessor.hh"
+#include "mounted-input-accessor.hh"
 #include "git-utils.hh"
 
 #include "fetch-settings.hh"
@@ -587,7 +587,7 @@ struct GitInputScheme : InputScheme
 
         auto accessor = repo->getAccessor(rev);
 
-        /* If the repo has submodules, fetch them and return a union
+        /* If the repo has submodules, fetch them and return a mounted
            input accessor consisting of the accessor for the top-level
            repo and the accessors for the submodules. */
         if (repoInfo.submodules) {
@@ -611,7 +611,7 @@ struct GitInputScheme : InputScheme
 
             if (!mounts.empty()) {
                 mounts.insert_or_assign(CanonPath::root, accessor);
-                accessor = makeUnionInputAccessor(std::move(mounts));
+                accessor = makeMountedInputAccessor(std::move(mounts));
             }
         }
 
@@ -636,7 +636,7 @@ struct GitInputScheme : InputScheme
         ref<InputAccessor> accessor =
             makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url));
 
-        /* If the repo has submodules, return a union input accessor
+        /* If the repo has submodules, return a mounted input accessor
            consisting of the accessor for the top-level repo and the
            accessors for the submodule workdirs. */
         if (repoInfo.submodules && !repoInfo.workdirInfo.submodules.empty()) {
@@ -660,7 +660,7 @@ struct GitInputScheme : InputScheme
             }
 
             mounts.insert_or_assign(CanonPath::root, accessor);
-            accessor = makeUnionInputAccessor(std::move(mounts));
+            accessor = makeMountedInputAccessor(std::move(mounts));
         }
 
         if (!repoInfo.workdirInfo.isDirty) {
diff --git a/src/libfetchers/union-input-accessor.cc b/src/libfetchers/mounted-input-accessor.cc
similarity index 86%
rename from src/libfetchers/union-input-accessor.cc
rename to src/libfetchers/mounted-input-accessor.cc
index f9472efa791..49917f6e531 100644
--- a/src/libfetchers/union-input-accessor.cc
+++ b/src/libfetchers/mounted-input-accessor.cc
@@ -1,12 +1,12 @@
-#include "union-input-accessor.hh"
+#include "mounted-input-accessor.hh"
 
 namespace nix {
 
-struct UnionInputAccessor : InputAccessor
+struct MountedInputAccessor : InputAccessor
 {
     std::map<CanonPath, ref<InputAccessor>> mounts;
 
-    UnionInputAccessor(std::map<CanonPath, ref<InputAccessor>> _mounts)
+    MountedInputAccessor(std::map<CanonPath, ref<InputAccessor>> _mounts)
         : mounts(std::move(_mounts))
     {
         // Currently we require a root filesystem. This could be relaxed.
@@ -71,9 +71,9 @@ struct UnionInputAccessor : InputAccessor
     }
 };
 
-ref<InputAccessor> makeUnionInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts)
+ref<InputAccessor> makeMountedInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts)
 {
-    return make_ref<UnionInputAccessor>(std::move(mounts));
+    return make_ref<MountedInputAccessor>(std::move(mounts));
 }
 
 }
diff --git a/src/libfetchers/mounted-input-accessor.hh b/src/libfetchers/mounted-input-accessor.hh
new file mode 100644
index 00000000000..b557c5dad7f
--- /dev/null
+++ b/src/libfetchers/mounted-input-accessor.hh
@@ -0,0 +1,9 @@
+#pragma once
+
+#include "input-accessor.hh"
+
+namespace nix {
+
+ref<InputAccessor> makeMountedInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts);
+
+}
diff --git a/src/libfetchers/union-input-accessor.hh b/src/libfetchers/union-input-accessor.hh
deleted file mode 100644
index 6a1649c1dea..00000000000
--- a/src/libfetchers/union-input-accessor.hh
+++ /dev/null
@@ -1,9 +0,0 @@
-#pragma once
-
-#include "input-accessor.hh"
-
-namespace nix {
-
-ref<InputAccessor> makeUnionInputAccessor(std::map<CanonPath, ref<InputAccessor>> mounts);
-
-}

From c257c824475c92cdfda5daa027db334b6a0137f8 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 14:47:17 +0100
Subject: [PATCH 15/20] Cleanup

---
 src/libfetchers/mounted-input-accessor.cc | 10 ++++------
 src/libutil/canon-path.cc                 |  7 +++++++
 src/libutil/canon-path.hh                 |  6 ++++++
 3 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/libfetchers/mounted-input-accessor.cc b/src/libfetchers/mounted-input-accessor.cc
index 49917f6e531..6f397eb1740 100644
--- a/src/libfetchers/mounted-input-accessor.cc
+++ b/src/libfetchers/mounted-input-accessor.cc
@@ -54,18 +54,16 @@ struct MountedInputAccessor : InputAccessor
     std::pair<ref<InputAccessor>, CanonPath> resolve(CanonPath path)
     {
         // Find the nearest parent of `path` that is a mount point.
-        std::vector<std::string> ss;
+        std::vector<std::string> subpath;
         while (true) {
             auto i = mounts.find(path);
             if (i != mounts.end()) {
-                auto subpath = CanonPath::root;
-                for (auto j = ss.rbegin(); j != ss.rend(); ++j)
-                    subpath.push(*j);
-                return {i->second, std::move(subpath)};
+                std::reverse(subpath.begin(), subpath.end());
+                return {i->second, CanonPath(subpath)};
             }
 
             assert(!path.isRoot());
-            ss.push_back(std::string(*path.baseName()));
+            subpath.push_back(std::string(*path.baseName()));
             path.pop();
         }
     }
diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc
index f678fae9476..1e465f1f699 100644
--- a/src/libutil/canon-path.cc
+++ b/src/libutil/canon-path.cc
@@ -13,6 +13,13 @@ CanonPath::CanonPath(std::string_view raw, const CanonPath & root)
     : path(absPath((Path) raw, root.abs()))
 { }
 
+CanonPath::CanonPath(const std::vector<std::string> & elems)
+    : path("/")
+{
+    for (auto & s : elems)
+        push(s);
+}
+
 CanonPath CanonPath::fromCwd(std::string_view path)
 {
     return CanonPath(unchecked_t(), absPath((Path) path));
diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh
index eefe05ed587..6d0519f4f16 100644
--- a/src/libutil/canon-path.hh
+++ b/src/libutil/canon-path.hh
@@ -6,6 +6,7 @@
 #include <cassert>
 #include <iostream>
 #include <set>
+#include <vector>
 
 namespace nix {
 
@@ -46,6 +47,11 @@ public:
         : path(std::move(path))
     { }
 
+    /**
+     * Construct a canon path from a vector of elements.
+     */
+    CanonPath(const std::vector<std::string> & elems);
+
     static CanonPath fromCwd(std::string_view path = ".");
 
     static CanonPath root;

From 6ec6b8aa363f566a8da0d6959753efa452b152cc Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 15:52:18 +0100
Subject: [PATCH 16/20] Improve git submodule error reporting

---
 src/libfetchers/fetchers.cc | 10 ++++++++++
 src/libfetchers/fetchers.hh |  2 ++
 src/libfetchers/git.cc      |  4 ++--
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 19e089aa802..c2513e0767d 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -220,6 +220,16 @@ std::pair<StorePath, Input> Input::fetch(ref<Store> store) const
     return {std::move(storePath), input};
 }
 
+std::pair<ref<InputAccessor>, Input> Input::getAccessor(ref<Store> store) const
+{
+    try {
+        return scheme->getAccessor(store, *this);
+    } catch (Error & e) {
+        e.addTrace({}, "while fetching the input '%s'", to_string());
+        throw;
+    }
+}
+
 Input Input::applyOverrides(
     std::optional<std::string> ref,
     std::optional<Hash> rev) const
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index 6db1615f2a5..ce5aa4c69da 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -83,6 +83,8 @@ public:
      */
     std::pair<StorePath, Input> fetch(ref<Store> store) const;
 
+    std::pair<ref<InputAccessor>, Input> getAccessor(ref<Store> store) const;
+
     Input applyOverrides(
         std::optional<std::string> ref,
         std::optional<Hash> rev) const;
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 90c6ad531b7..71ae74ddeb8 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -605,7 +605,7 @@ struct GitInputScheme : InputScheme
                 attrs.insert_or_assign("rev", submoduleRev.gitRev());
                 auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
                 auto [submoduleAccessor, submoduleInput2] =
-                    submoduleInput.scheme->getAccessor(store, submoduleInput);
+                    submoduleInput.getAccessor(store);
                 mounts.insert_or_assign(submodule.path, submoduleAccessor);
             }
 
@@ -649,7 +649,7 @@ struct GitInputScheme : InputScheme
                 attrs.insert_or_assign("url", submodulePath.abs());
                 auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
                 auto [submoduleAccessor, submoduleInput2] =
-                    submoduleInput.scheme->getAccessor(store, submoduleInput);
+                    submoduleInput.getAccessor(store);
 
                 /* If the submodule is dirty, mark this repo dirty as
                    well. */

From 2964a9f562748cc698ee1f6ecf1e0da4e63211b9 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Tue, 14 Nov 2023 16:00:21 +0100
Subject: [PATCH 17/20] Fix relative submodule handling

Tested on

  nix flake prefetch 'git+https://github.com/blender/blender.git?rev=4ed8a360e956daf2591add4d3c9ec0719e2628fe&submodules=1'
---
 src/libfetchers/git-utils.cc | 12 ++++++++++--
 src/libfetchers/git-utils.hh |  4 +++-
 src/libfetchers/git.cc       |  2 +-
 src/libutil/url.cc           |  8 ++++++++
 src/libutil/url.hh           |  5 +++++
 5 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index ffcc92fc7ca..1edafbf33ce 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -308,13 +308,21 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
     std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev) override;
 
-    std::string resolveSubmoduleUrl(const std::string & url) override
+    std::string resolveSubmoduleUrl(
+        const std::string & url,
+        const std::string & base) override
     {
         git_buf buf = GIT_BUF_INIT;
         if (git_submodule_resolve_url(&buf, *this, url.c_str()))
             throw Error("resolving Git submodule URL '%s'", url);
         Finally cleanup = [&]() { git_buf_dispose(&buf); };
-        return buf.ptr;
+
+        std::string res(buf.ptr);
+
+        if (!hasPrefix(res, "/") && res.find("://") == res.npos)
+            res = parseURL(base + "/" + res).canonicalise().to_string();
+
+        return res;
     }
 
     bool hasObject(const Hash & oid_) override
diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
index 7efbdedce07..e0cb2c34fec 100644
--- a/src/libfetchers/git-utils.hh
+++ b/src/libfetchers/git-utils.hh
@@ -59,7 +59,9 @@ struct GitRepo
      */
     virtual std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev) = 0;
 
-    virtual std::string resolveSubmoduleUrl(const std::string & url) = 0;
+    virtual std::string resolveSubmoduleUrl(
+        const std::string & url,
+        const std::string & base) = 0;
 
     struct TarballInfo
     {
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 71ae74ddeb8..177c8b66eaa 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -594,7 +594,7 @@ struct GitInputScheme : InputScheme
             std::map<CanonPath, nix::ref<InputAccessor>> mounts;
 
             for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev)) {
-                auto resolved = repo->resolveSubmoduleUrl(submodule.url);
+                auto resolved = repo->resolveSubmoduleUrl(submodule.url, repoInfo.url);
                 debug("Git submodule %s: %s %s %s -> %s",
                     submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved);
                 fetchers::Attrs attrs;
diff --git a/src/libutil/url.cc b/src/libutil/url.cc
index 9b438e6cd23..57b64d6074c 100644
--- a/src/libutil/url.cc
+++ b/src/libutil/url.cc
@@ -2,6 +2,7 @@
 #include "url-parts.hh"
 #include "util.hh"
 #include "split.hh"
+#include "canon-path.hh"
 
 namespace nix {
 
@@ -141,6 +142,13 @@ bool ParsedURL::operator ==(const ParsedURL & other) const
         && fragment == other.fragment;
 }
 
+ParsedURL ParsedURL::canonicalise()
+{
+    ParsedURL res(*this);
+    res.path = CanonPath(res.path).abs();
+    return res;
+}
+
 /**
  * Parse a URL scheme of the form '(applicationScheme\+)?transportScheme'
  * into a tuple '(applicationScheme, transportScheme)'
diff --git a/src/libutil/url.hh b/src/libutil/url.hh
index 26c2dcc28bc..833f546787b 100644
--- a/src/libutil/url.hh
+++ b/src/libutil/url.hh
@@ -19,6 +19,11 @@ struct ParsedURL
     std::string to_string() const;
 
     bool operator ==(const ParsedURL & other) const;
+
+    /**
+     * Remove `.` and `..` path elements.
+     */
+    ParsedURL canonicalise();
 };
 
 MakeError(BadURL, Error);

From 28909999116781e194e2eb1646f3ccec005e774f Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 Nov 2023 13:57:20 +0100
Subject: [PATCH 18/20] Show Git fetch progress

---
 src/libfetchers/git-utils.cc | 30 +++++++++++++++++++++++++++++-
 src/libfetchers/git.cc       |  2 --
 src/libmain/progress-bar.cc  |  8 ++++++++
 src/libutil/logging.hh       |  2 ++
 4 files changed, 39 insertions(+), 3 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index 1edafbf33ce..b7ef05c1026 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -3,6 +3,7 @@
 #include "cache.hh"
 #include "finally.hh"
 #include "processes.hh"
+#include "signals.hh"
 
 #include <boost/core/span.hpp>
 
@@ -341,10 +342,32 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
     ref<InputAccessor> getAccessor(const Hash & rev) override;
 
+    static int sidebandProgressCallback(const char * str, int len, void * payload)
+    {
+        auto act = (Activity *) payload;
+        act->result(resFetchStatus, trim(std::string_view(str, len)));
+        return _isInterrupted ? -1 : 0;
+    }
+
+    static int transferProgressCallback(const git_indexer_progress * stats, void * payload)
+    {
+        auto act = (Activity *) payload;
+        act->result(resFetchStatus,
+            fmt("%d/%d objects received, %d/%d deltas indexed, %.1f MiB",
+                stats->received_objects,
+                stats->total_objects,
+                stats->indexed_deltas,
+                stats->total_deltas,
+                stats->received_bytes / (1024.0 * 1024.0)));
+        return _isInterrupted ? -1 : 0;
+    }
+
     void fetch(
         const std::string & url,
         const std::string & refspec) override
     {
+        Activity act(*logger, lvlTalkative, actFetchTree, fmt("fetching Git repository '%s'", url));
+
         Remote remote;
 
         if (git_remote_create_anonymous(Setter(remote), *this, url.c_str()))
@@ -356,7 +379,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
             .count = 1
         };
 
-        if (git_remote_fetch(remote.get(), &refspecs2, nullptr, nullptr))
+        git_fetch_options opts = GIT_FETCH_OPTIONS_INIT;
+        opts.callbacks.payload = &act;
+        opts.callbacks.sideband_progress = sidebandProgressCallback;
+        opts.callbacks.transfer_progress = transferProgressCallback;
+
+        if (git_remote_fetch(remote.get(), &refspecs2, &opts, nullptr))
             throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message);
     }
 
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 177c8b66eaa..3e7dcd8dec6 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -520,8 +520,6 @@ struct GitInputScheme : InputScheme
             }
 
             if (doFetch) {
-                Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", repoInfo.url));
-
                 try {
                     auto fetchRef = repoInfo.allRefs
                         ? "refs/*"
diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc
index a7aee47c3ac..3aa012ee1cc 100644
--- a/src/libmain/progress-bar.cc
+++ b/src/libmain/progress-bar.cc
@@ -340,6 +340,14 @@ class ProgressBar : public Logger
             state->activitiesByType[type].expected += j;
             update(*state);
         }
+
+        else if (type == resFetchStatus) {
+            auto i = state->its.find(act);
+            assert(i != state->its.end());
+            ActInfo & actInfo = *i->second;
+            actInfo.lastLine = getS(fields, 0);
+            update(*state);
+        }
     }
 
     void update(State & state)
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 5aa6bee956a..183f2d8e13e 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -23,6 +23,7 @@ typedef enum {
     actQueryPathInfo = 109,
     actPostBuildHook = 110,
     actBuildWaiting = 111,
+    actFetchTree = 112,
 } ActivityType;
 
 typedef enum {
@@ -34,6 +35,7 @@ typedef enum {
     resProgress = 105,
     resSetExpected = 106,
     resPostBuildLogLine = 107,
+    resFetchStatus = 108,
 } ResultType;
 
 typedef uint64_t ActivityId;

From 5dd4ae86877cedaf70ea70d80b89c66b850bdc5a Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 Nov 2023 14:08:34 +0100
Subject: [PATCH 19/20] Remove unused cacheType field

---
 src/libfetchers/git.cc | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index 3e7dcd8dec6..b066b384c87 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -323,8 +323,6 @@ struct GitInputScheme : InputScheme
         bool submodules = false;
         bool allRefs = false;
 
-        std::string cacheType;
-
         /* Whether this is a local, non-bare repository. */
         bool isLocal = false;
 
@@ -371,11 +369,6 @@ struct GitInputScheme : InputScheme
             .allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false)
         };
 
-        repoInfo.cacheType = "git";
-        if (repoInfo.shallow) repoInfo.cacheType += "-shallow";
-        if (repoInfo.submodules) repoInfo.cacheType += "-submodules";
-        if (repoInfo.allRefs) repoInfo.cacheType += "-all-refs";
-
         // file:// URIs are normally not cloned (but otherwise treated the
         // same as remote URIs, i.e. we don't use the working tree or
         // HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git

From 7ab91e72387b96d1926f1b9c95b919020d4ba962 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Wed, 15 Nov 2023 14:43:30 +0100
Subject: [PATCH 20/20] Implement shallow fetching

---
 src/libfetchers/git-utils.cc |  4 ++-
 src/libfetchers/git-utils.hh |  3 ++-
 src/libfetchers/git.cc       | 48 +++++++++++++++++++-----------------
 3 files changed, 31 insertions(+), 24 deletions(-)

diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc
index b7ef05c1026..f554dcc5f35 100644
--- a/src/libfetchers/git-utils.cc
+++ b/src/libfetchers/git-utils.cc
@@ -364,7 +364,8 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
     void fetch(
         const std::string & url,
-        const std::string & refspec) override
+        const std::string & refspec,
+        bool shallow) override
     {
         Activity act(*logger, lvlTalkative, actFetchTree, fmt("fetching Git repository '%s'", url));
 
@@ -380,6 +381,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         };
 
         git_fetch_options opts = GIT_FETCH_OPTIONS_INIT;
+        opts.depth = shallow ? 1 : GIT_FETCH_DEPTH_FULL;
         opts.callbacks.payload = &act;
         opts.callbacks.sideband_progress = sidebandProgressCallback;
         opts.callbacks.transfer_progress = transferProgressCallback;
diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh
index e0cb2c34fec..1def82071ef 100644
--- a/src/libfetchers/git-utils.hh
+++ b/src/libfetchers/git-utils.hh
@@ -75,7 +75,8 @@ struct GitRepo
 
     virtual void fetch(
         const std::string & url,
-        const std::string & refspec) = 0;
+        const std::string & refspec,
+        bool shallow) = 0;
 
     /**
      * Verify that commit `rev` is signed by one of the keys in
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index b066b384c87..7208a0b6dbe 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -219,9 +219,6 @@ struct GitInputScheme : InputScheme
                 || name == "publicKeys")
                 experimentalFeatureSettings.require(Xp::VerifiedFetches);
 
-        maybeGetBoolAttr(attrs, "shallow");
-        maybeGetBoolAttr(attrs, "submodules");
-        maybeGetBoolAttr(attrs, "allRefs");
         maybeGetBoolAttr(attrs, "verifyCommit");
 
         if (auto ref = maybeGetStrAttr(attrs, "ref")) {
@@ -234,6 +231,9 @@ struct GitInputScheme : InputScheme
         auto url = fixGitURL(getStrAttr(attrs, "url"));
         parseURL(url);
         input.attrs["url"] = url;
+        getShallowAttr(input);
+        getSubmodulesAttr(input);
+        getAllRefsAttr(input);
         return input;
     }
 
@@ -243,8 +243,10 @@ struct GitInputScheme : InputScheme
         if (url.scheme != "git") url.scheme = "git+" + url.scheme;
         if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev());
         if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
-        if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false))
+        if (getShallowAttr(input))
             url.query.insert_or_assign("shallow", "1");
+        if (getSubmodulesAttr(input))
+            url.query.insert_or_assign("submodules", "1");
         if (maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(false))
             url.query.insert_or_assign("verifyCommit", "1");
         auto publicKeys = getPublicKeys(input.attrs);
@@ -319,10 +321,6 @@ struct GitInputScheme : InputScheme
 
     struct RepoInfo
     {
-        bool shallow = false;
-        bool submodules = false;
-        bool allRefs = false;
-
         /* Whether this is a local, non-bare repository. */
         bool isLocal = false;
 
@@ -347,11 +345,21 @@ struct GitInputScheme : InputScheme
         std::string gitDir = ".git";
     };
 
+    bool getShallowAttr(const Input & input) const
+    {
+        return maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
+    }
+
     bool getSubmodulesAttr(const Input & input) const
     {
         return maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
     }
 
+    bool getAllRefsAttr(const Input & input) const
+    {
+        return maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
+    }
+
     RepoInfo getRepoInfo(const Input & input) const
     {
         auto checkHashType = [&](const std::optional<Hash> & hash)
@@ -363,11 +371,7 @@ struct GitInputScheme : InputScheme
         if (auto rev = input.getRev())
             checkHashType(rev);
 
-        RepoInfo repoInfo {
-            .shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false),
-            .submodules = getSubmodulesAttr(input),
-            .allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false)
-        };
+        RepoInfo repoInfo;
 
         // file:// URIs are normally not cloned (but otherwise treated the
         // same as remote URIs, i.e. we don't use the working tree or
@@ -501,7 +505,7 @@ struct GitInputScheme : InputScheme
             if (auto rev = input.getRev()) {
                 doFetch = !repo->hasObject(*rev);
             } else {
-                if (repoInfo.allRefs) {
+                if (getAllRefsAttr(input)) {
                     doFetch = true;
                 } else {
                     /* If the local ref is older than ‘tarball-ttl’ seconds, do a
@@ -514,7 +518,7 @@ struct GitInputScheme : InputScheme
 
             if (doFetch) {
                 try {
-                    auto fetchRef = repoInfo.allRefs
+                    auto fetchRef = getAllRefsAttr(input)
                         ? "refs/*"
                         : ref.compare(0, 5, "refs/") == 0
                         ? ref
@@ -522,7 +526,7 @@ struct GitInputScheme : InputScheme
                         ? ref
                         : "refs/heads/" + ref;
 
-                    repo->fetch(repoInfo.url, fmt("%s:%s", fetchRef, fetchRef));
+                    repo->fetch(repoInfo.url, fmt("%s:%s", fetchRef, fetchRef), getShallowAttr(input));
                 } catch (Error & e) {
                     if (!pathExists(localRefFile)) throw;
                     logError(e.info());
@@ -556,7 +560,7 @@ struct GitInputScheme : InputScheme
 
         auto isShallow = repo->isShallow();
 
-        if (isShallow && !repoInfo.shallow)
+        if (isShallow && !getShallowAttr(input))
             throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", repoInfo.url);
 
         // FIXME: check whether rev is an ancestor of ref?
@@ -568,7 +572,7 @@ struct GitInputScheme : InputScheme
             {"lastModified", getLastModified(repoInfo, repoDir, rev)},
         });
 
-        if (!repoInfo.shallow)
+        if (!getShallowAttr(input))
             infoAttrs.insert_or_assign("revCount",
                 getRevCount(repoInfo, repoDir, rev));
 
@@ -581,7 +585,7 @@ struct GitInputScheme : InputScheme
         /* If the repo has submodules, fetch them and return a mounted
            input accessor consisting of the accessor for the top-level
            repo and the accessors for the submodules. */
-        if (repoInfo.submodules) {
+        if (getSubmodulesAttr(input)) {
             std::map<CanonPath, nix::ref<InputAccessor>> mounts;
 
             for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev)) {
@@ -607,7 +611,7 @@ struct GitInputScheme : InputScheme
         }
 
         assert(!origRev || origRev == rev);
-        if (!repoInfo.shallow)
+        if (!getShallowAttr(input))
             input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
         input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
 
@@ -619,7 +623,7 @@ struct GitInputScheme : InputScheme
         RepoInfo & repoInfo,
         Input && input) const
     {
-        if (repoInfo.submodules)
+        if (getSubmodulesAttr(input))
             /* Create mountpoints for the submodules. */
             for (auto & submodule : repoInfo.workdirInfo.submodules)
                 repoInfo.workdirInfo.files.insert(submodule.path);
@@ -630,7 +634,7 @@ struct GitInputScheme : InputScheme
         /* If the repo has submodules, return a mounted input accessor
            consisting of the accessor for the top-level repo and the
            accessors for the submodule workdirs. */
-        if (repoInfo.submodules && !repoInfo.workdirInfo.submodules.empty()) {
+        if (getSubmodulesAttr(input) && !repoInfo.workdirInfo.submodules.empty()) {
             std::map<CanonPath, nix::ref<InputAccessor>> mounts;
 
             for (auto & submodule : repoInfo.workdirInfo.submodules) {