From 19c43c5d787c113053ef651c3d22fdfde61075e2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 25 Jul 2023 11:44:39 -0400 Subject: [PATCH 1/3] Write test for deleting objects referenced from below Currently fails, as expected. --- tests/hermetic.nix | 4 +-- tests/overlay-local-store/delete-inner.sh | 39 +++++++++++++++++++++++ tests/overlay-local-store/delete.sh | 5 +++ tests/overlay-local-store/local.mk | 1 + 4 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 tests/overlay-local-store/delete-inner.sh create mode 100755 tests/overlay-local-store/delete.sh diff --git a/tests/hermetic.nix b/tests/hermetic.nix index 4c9d7a51f54..c4fbbfa14c1 100644 --- a/tests/hermetic.nix +++ b/tests/hermetic.nix @@ -37,7 +37,7 @@ let buildCommand = '' echo hi-input3 read x < ${input2} - echo $x BAZ > $out + echo ${input2} $x BAZ > $out ''; }; @@ -51,6 +51,6 @@ in '' read x < ${input1} read y < ${input3} - echo "$x $y" > $out + echo ${input1} ${input3} "$x $y" > $out ''; } diff --git a/tests/overlay-local-store/delete-inner.sh b/tests/overlay-local-store/delete-inner.sh new file mode 100644 index 00000000000..0702d1693c7 --- /dev/null +++ b/tests/overlay-local-store/delete-inner.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +storeDirs + +initLowerStore + +mountOverlayfs + +export NIX_REMOTE="$storeB" +stateB="$storeBRoot/nix/var/nix" +hermetic=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2) +input1=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2 -A passthru.input1 -j0) +input2=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2 -A passthru.input2 -j0) +input3=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2 -A passthru.input3 -j0) + +# Can't delete because referenced +expectStderr 1 nix-store --delete $input1 | grepQuiet "Cannot delete path" +expectStderr 1 nix-store --delete $input2 | grepQuiet "Cannot delete path" +expectStderr 1 nix-store --delete $input3 | grepQuiet "Cannot delete path" + +# These same paths are referenced in the lower layer (by the seed 1 +# build done in `initLowerStore`). +expectStderr 1 nix-store --store "$storeA" --delete $input2 | grepQuiet "Cannot delete path" +expectStderr 1 nix-store --store "$storeA" --delete $input3 | grepQuiet "Cannot delete path" + +# Can delete +nix-store --delete $hermetic + +# Now unreferenced in upper layer, can delete +nix-store --delete $input3 +nix-store --delete $input2 diff --git a/tests/overlay-local-store/delete.sh b/tests/overlay-local-store/delete.sh new file mode 100755 index 00000000000..79d67da7182 --- /dev/null +++ b/tests/overlay-local-store/delete.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./delete-inner.sh diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index 1f8de8f271a..c9cb0b7f275 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -4,6 +4,7 @@ overlay-local-store-tests := \ $(d)/build.sh \ $(d)/bad-uris.sh \ $(d)/add-lower.sh \ + $(d)/delete.sh \ $(d)/gc.sh \ $(d)/verify.sh \ $(d)/optimise.sh From 07b34edc449f25e9b4062d7f6d90dad1f6c71760 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 25 Jul 2023 17:09:23 -0400 Subject: [PATCH 2/3] Fix deletion test Lower layer references are ignored for deleting just in the upper layer. --- src/libstore/gc.cc | 2 +- src/libstore/local-overlay-store.cc | 8 ++++++++ src/libstore/local-overlay-store.hh | 9 +++++++++ src/libstore/local-store.hh | 12 ++++++++++++ 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ea2868c5897..8f5ce099488 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -741,7 +741,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) auto i = referrersCache.find(*path); if (i == referrersCache.end()) { StorePathSet referrers; - queryReferrers(*path, referrers); + queryGCReferrers(*path, referrers); referrersCache.emplace(*path, std::move(referrers)); i = referrersCache.find(*path); } diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 47d09dc752b..e8ca8074ecc 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -138,6 +138,12 @@ void LocalOverlayStore::queryReferrers(const StorePath & path, StorePathSet & re } +void LocalOverlayStore::queryGCReferrers(const StorePath & path, StorePathSet & referrers) +{ + LocalStore::queryReferrers(path, referrers); +} + + StorePathSet LocalOverlayStore::queryValidDerivers(const StorePath & path) { auto res = LocalStore::queryValidDerivers(path); @@ -188,6 +194,7 @@ void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed) } } + void LocalOverlayStore::optimiseStore() { Activity act(*logger, actOptimiseStore); @@ -216,6 +223,7 @@ Path LocalOverlayStore::toRealPathForHardLink(const StorePath & path) : Store::toRealPath(path); } + static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 64e2ef488a2..21658c6ab2f 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -115,7 +115,16 @@ private: void optimiseStore() override; + /** + * For lower-store paths, we used the lower store location. This avoids the + * wasteful "copying up" that would otherwise happen. + */ Path toRealPathForHardLink(const StorePath & storePath) override; + + /** + * Deletion only effects the upper layer, so we ignore lower-layer referrers. + */ + void queryGCReferrers(const StorePath & path, StorePathSet & referrers) override; }; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 9a44722d45a..c049de02e3f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -230,6 +230,18 @@ public: void collectGarbage(const GCOptions & options, GCResults & results) override; + /** + * Called by `collectGarbage` to trace in reverse. + * + * Using this rather than `queryReferrers` directly allows us to + * fine-tune which referrers we consider for garbage collection; + * some store implementations take advantage of this. + */ + virtual void queryGCReferrers(const StorePath & path, StorePathSet & referrers) + { + return queryReferrers(path, referrers); + } + /** * Called by `collectGarbage` to recursively delete a path. * The default implementation simply calls `deletePath`, but it can be From b0877ad3c90a806e70cac3e7f6e5c26151239bb8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 26 Jul 2023 09:29:04 -0400 Subject: [PATCH 3/3] Give test a more specific name --- .../{delete-inner.sh => delete-refs-inner.sh} | 0 tests/overlay-local-store/{delete.sh => delete-refs.sh} | 2 +- tests/overlay-local-store/local.mk | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename tests/overlay-local-store/{delete-inner.sh => delete-refs-inner.sh} (100%) rename tests/overlay-local-store/{delete.sh => delete-refs.sh} (58%) diff --git a/tests/overlay-local-store/delete-inner.sh b/tests/overlay-local-store/delete-refs-inner.sh similarity index 100% rename from tests/overlay-local-store/delete-inner.sh rename to tests/overlay-local-store/delete-refs-inner.sh diff --git a/tests/overlay-local-store/delete.sh b/tests/overlay-local-store/delete-refs.sh similarity index 58% rename from tests/overlay-local-store/delete.sh rename to tests/overlay-local-store/delete-refs.sh index 79d67da7182..942d7fbdc1c 100755 --- a/tests/overlay-local-store/delete.sh +++ b/tests/overlay-local-store/delete-refs.sh @@ -2,4 +2,4 @@ source common.sh requireEnvironment setupConfig -execUnshare ./delete-inner.sh +execUnshare ./delete-refs-inner.sh diff --git a/tests/overlay-local-store/local.mk b/tests/overlay-local-store/local.mk index c9cb0b7f275..8bfb22ae7ae 100644 --- a/tests/overlay-local-store/local.mk +++ b/tests/overlay-local-store/local.mk @@ -4,7 +4,7 @@ overlay-local-store-tests := \ $(d)/build.sh \ $(d)/bad-uris.sh \ $(d)/add-lower.sh \ - $(d)/delete.sh \ + $(d)/delete-refs.sh \ $(d)/gc.sh \ $(d)/verify.sh \ $(d)/optimise.sh