diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 980a5a4d7da..42bf42de689 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -315,8 +315,6 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return std::nullopt; } - TarballInfo importTarball(Source & source) override; - std::vector> getSubmodules(const Hash & rev, bool exportIgnore) override; std::string resolveSubmoduleUrl( @@ -947,88 +945,4 @@ std::vector> GitRepoImpl::getSubmodules return result; } -ref getTarballCache() -{ - static auto repoDir = std::filesystem::path(getCacheDir()) / "nix" / "tarball-cache"; - - return make_ref(repoDir, true, true); -} - -} - -#include "tarfile.hh" -#include - -namespace nix { - -GitRepo::TarballInfo GitRepoImpl::importTarball(Source & source) -{ - TarArchive archive { source }; - - auto parseSink = getFileSystemObjectSink(); - - time_t lastModified = 0; - - for (;;) { - // FIXME: merge with extract_archive - struct archive_entry * entry; - int r = archive_read_next_header(archive.archive, &entry); - if (r == ARCHIVE_EOF) break; - auto path = archive_entry_pathname(entry); - if (!path) - throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); - if (r == ARCHIVE_WARN) - warn(archive_error_string(archive.archive)); - else - archive.check(r); - - lastModified = std::max(lastModified, archive_entry_mtime(entry)); - - switch (archive_entry_filetype(entry)) { - - case AE_IFDIR: - parseSink->createDirectory(path); - break; - - case AE_IFREG: { - parseSink->createRegularFile(path, [&](auto & crf) { - if (archive_entry_mode(entry) & S_IXUSR) - crf.isExecutable(); - - while (true) { - std::vector buf(128 * 1024); - auto n = archive_read_data(archive.archive, buf.data(), buf.size()); - if (n < 0) - throw Error("cannot read file '%s' from tarball", path); - if (n == 0) break; - crf(std::string_view { - (const char *) buf.data(), - (size_t) n, - }); - } - }); - - break; - } - - case AE_IFLNK: { - auto target = archive_entry_symlink(entry); - - parseSink->createSymlink(path, target); - - break; - } - - default: - throw Error("file '%s' in tarball has unsupported file type", path); - } - } - - return TarballInfo { - .treeHash = parseSink->sync(), - .lastModified = lastModified - }; -} - - } diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index f82f62fc878..029d39741d2 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -73,12 +73,6 @@ struct GitRepo const std::string & url, const std::string & base) = 0; - struct TarballInfo - { - Hash treeHash; - time_t lastModified; - }; - virtual bool hasObject(const Hash & oid) = 0; virtual ref getAccessor(const Hash & rev, bool exportIgnore) = 0; @@ -99,10 +93,6 @@ struct GitRepo virtual void verifyCommit( const Hash & rev, const std::vector & publicKeys) = 0; - - virtual TarballInfo importTarball(Source & source) = 0; }; -ref getTarballCache(); - } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 1cfc142a590..8b3e6ff202b 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -8,7 +8,9 @@ #include "fetchers.hh" #include "fetch-settings.hh" #include "tarball.hh" +#include "tarfile.hh" #include "git-utils.hh" +#include "tarball-cache.hh" #include #include @@ -191,7 +193,7 @@ struct GitArchiveInputScheme : InputScheme virtual DownloadUrl getDownloadUrl(const Input & input) const = 0; - std::pair downloadArchive(ref store, Input input) const + std::pair downloadArchive(ref store, Input input) const { if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "HEAD"); @@ -218,7 +220,7 @@ struct GitArchiveInputScheme : InputScheme auto treeHash = getRevAttr(*treeHashAttrs, "treeHash"); auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified"); if (getTarballCache()->hasObject(treeHash)) - return {std::move(input), GitRepo::TarballInfo { .treeHash = treeHash, .lastModified = (time_t) lastModified }}; + return {std::move(input), TarballInfo { .treeHash = treeHash, .lastModified = (time_t) lastModified }}; else debug("Git tree with hash '%s' has disappeared from the cache, refetching...", treeHash.gitRev()); } @@ -233,7 +235,14 @@ struct GitArchiveInputScheme : InputScheme getFileTransfer()->download(std::move(req), sink); }); - auto tarballInfo = getTarballCache()->importTarball(*source); + TarArchive archive { *source }; + auto parseSink = getTarballCache()->getFileSystemObjectSink(); + auto lastModified = unpackTarfileToSink(archive, *parseSink); + + TarballInfo tarballInfo { + .treeHash = parseSink->sync(), + .lastModified = lastModified + }; cache->upsert(treeHashKey, Attrs{{"treeHash", tarballInfo.treeHash.gitRev()}}); cache->upsert(lastModifiedKey, Attrs{{"lastModified", (uint64_t) tarballInfo.lastModified}}); diff --git a/src/libfetchers/tarball-cache.cc b/src/libfetchers/tarball-cache.cc new file mode 100644 index 00000000000..bb2c519733d --- /dev/null +++ b/src/libfetchers/tarball-cache.cc @@ -0,0 +1,13 @@ +#include "tarball-cache.hh" +#include "users.hh" + +namespace nix::fetchers { + +ref getTarballCache() +{ + static auto repoDir = std::filesystem::path(getCacheDir()) / "nix" / "tarball-cache"; + + return GitRepo::openRepo(repoDir, true, true); +} + +} diff --git a/src/libfetchers/tarball-cache.hh b/src/libfetchers/tarball-cache.hh new file mode 100644 index 00000000000..e1517038b87 --- /dev/null +++ b/src/libfetchers/tarball-cache.hh @@ -0,0 +1,17 @@ +#pragma once +///@file + +#include "ref.hh" +#include "git-utils.hh" + +namespace nix::fetchers { + +struct TarballInfo +{ + Hash treeHash; + time_t lastModified; +}; + +ref getTarballCache(); + +} diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 187b3e94841..3bb6694f88e 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -132,4 +132,66 @@ void unpackTarfile(const Path & tarFile, const Path & destDir) extract_archive(archive, destDir); } +time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSink) +{ + time_t lastModified = 0; + + for (;;) { + // FIXME: merge with extract_archive + struct archive_entry * entry; + int r = archive_read_next_header(archive.archive, &entry); + if (r == ARCHIVE_EOF) break; + auto path = archive_entry_pathname(entry); + if (!path) + throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); + if (r == ARCHIVE_WARN) + warn(archive_error_string(archive.archive)); + else + archive.check(r); + + lastModified = std::max(lastModified, archive_entry_mtime(entry)); + + switch (archive_entry_filetype(entry)) { + + case AE_IFDIR: + parseSink.createDirectory(path); + break; + + case AE_IFREG: { + parseSink.createRegularFile(path, [&](auto & crf) { + if (archive_entry_mode(entry) & S_IXUSR) + crf.isExecutable(); + + while (true) { + std::vector buf(128 * 1024); + auto n = archive_read_data(archive.archive, buf.data(), buf.size()); + if (n < 0) + throw Error("cannot read file '%s' from tarball", path); + if (n == 0) break; + crf(std::string_view { + (const char *) buf.data(), + (size_t) n, + }); + } + }); + + break; + } + + case AE_IFLNK: { + auto target = archive_entry_symlink(entry); + + parseSink.createSymlink(path, target); + + break; + } + + default: + throw Error("file '%s' in tarball has unsupported file type", path); + } + } + + return lastModified; +} + } diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index 237d18c31a8..6a9c42149b4 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -2,6 +2,7 @@ ///@file #include "serialise.hh" +#include "fs-sink.hh" #include namespace nix { @@ -29,4 +30,6 @@ void unpackTarfile(Source & source, const Path & destDir); void unpackTarfile(const Path & tarFile, const Path & destDir); +time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSink); + }