Merge pull request NixOS#8 from NixLayeredStore/overlayfs-store-more-…

…tests

Implement deduplication and add more test cases

src/libstore/local-overlay-store.cc

@@ -188,6 +188,34 @@ void LocalOverlayStore::deleteGCPath(const Path & path, uint64_t & bytesFreed)
     }
 }
 
+void LocalOverlayStore::optimiseStore()
+{
+    Activity act(*logger, actOptimiseStore);
+
+    // Note for LocalOverlayStore, queryAllValidPaths only returns paths in upper layer
+    auto paths = queryAllValidPaths();
+
+    act.progress(0, paths.size());
+
+    uint64_t done = 0;
+
+    for (auto & path : paths) {
+        if (lowerStore->isValidPath(path)) {
+            // Deduplicate store path
+            deletePath(toUpperPath(path));
+        }
+        done++;
+        act.progress(done, paths.size());
+    }
+}
+
+bool LocalOverlayStore::verifyStore(bool checkContents, RepairFlag repair)
+{
+    if (repair)
+        warn("local-overlay: store does not support --verify --repair");
+    return LocalStore::verifyStore(checkContents, NoRepair);
+}
+
 static RegisterStoreImplementation<LocalOverlayStore, LocalOverlayStoreConfig> regLocalOverlayStore;
 
 }

src/libstore/local-overlay-store.hh

@@ -112,6 +112,10 @@ private:
         Callback<std::shared_ptr<const Realisation>> callback) noexcept override;
 
     void deleteGCPath(const Path & path, uint64_t & bytesFreed) override;
+
+    void optimiseStore() override;
+
+    bool verifyStore(bool checkContents, RepairFlag repair) override;
 };
 
 }

tests/build-remote-trustless-should-fail-0.sh

@@ -2,7 +2,6 @@ source common.sh
 
 enableFeatures "daemon-trust-override"
 
-requireSandboxSupport
 restartDaemon
 
 [[ $busybox =~ busybox ]] || skipTest "no busybox"

tests/common/vars-and-functions.sh.in

@@ -141,7 +141,7 @@ restartDaemon() {
     startDaemon
 }
 
-if [[ -z "${_NIX_TEST_NO_SANDBOX:-}" ]] && [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then
+if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then
     _canUseSandbox=1
 fi
 

tests/overlay-local-store/add-lower-inner.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+set -x
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+storeDirs
+
+initLowerStore
+
+mountOverlayfs
+
+# Add something to the overlay store
+overlayPath=$(addTextToStore "$storeB" "overlay-file" "Add to overlay store")
+stat "$storeBRoot/$overlayPath"
+
+# Now add something to the lower store
+lowerPath=$(addTextToStore "$storeA" "lower-file" "Add to lower store")
+stat "$storeVolume/store-a/$lowerPath"
+
+# Remount overlayfs to ensure synchronization
+remountOverlayfs
+
+# Path should be accessible via overlay store
+stat "$storeBRoot/$lowerPath"

tests/overlay-local-store/add-lower.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./add-lower-inner.sh

tests/overlay-local-store/bad-uris.sh

@@ -7,8 +7,8 @@ storeDirs
 mkdir -p $TEST_ROOT/bad_test
 badTestRoot=$TEST_ROOT/bad_test
 storeBadRoot="local-overlay?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop"
-storeBadLower="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$badTestRoot&upper-layer=$storeBTop"
-storeBadUpper="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$badTestRoot"
+storeBadLower="local-overlay?root=$storeBRoot&lower-store=$badTestRoot&upper-layer=$storeBTop"
+storeBadUpper="local-overlay?root=$storeBRoot&lower-store=$storeA&upper-layer=$badTestRoot"
 
 declare -a storesBad=(
     "$storeBadRoot" "$storeBadLower" "$storeBadUpper"

tests/overlay-local-store/check-post-init-inner.sh

@@ -25,7 +25,7 @@ stat $(toRealPath "$storeA/nix/store" "$path")
 expect 1 stat $(toRealPath "$storeBTop" "$path")
 
 # Checking for path in overlay store matching lower layer
-diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$TEST_ROOT/merged-store/nix/store" "$path")
+diff $(toRealPath "$storeA/nix/store" "$path") $(toRealPath "$storeBRoot/nix/store" "$path")
 
 # Checking requisites query agreement
 [[ \
@@ -62,7 +62,7 @@ nix-store --verify-path --store "$storeA" "$path"
 # Verifying path in merged-store
 nix-store --verify-path --store "$storeB" "$path"
 
-hashPart=$(echo $path | sed "s^$NIX_STORE_DIR/^^" | sed 's/-.*//')
+hashPart=$(echo $path | sed "s^${NIX_STORE_DIR:-/nix/store}/^^" | sed 's/-.*//')
 
 # Lower store can find from hash part
 [[ $(nix store --store $storeA path-from-hash-part $hashPart) == $path ]]

tests/overlay-local-store/common.sh

@@ -8,38 +8,52 @@ requireEnvironment () {
 }
 
 setupConfig () {
-  echo "drop-supplementary-groups = false" >> "$NIX_CONF_DIR"/nix.conf
+  echo "require-drop-supplementary-groups = false" >> "$NIX_CONF_DIR"/nix.conf
   echo "build-users-group = " >> "$NIX_CONF_DIR"/nix.conf
 }
 
+
+
 storeDirs () {
-  storeA="$TEST_ROOT/store-a"
-  storeBTop="$TEST_ROOT/store-b"
-  storeB="local-overlay?root=$TEST_ROOT/merged-store&lower-store=$storeA&upper-layer=$storeBTop"
+  # Attempt to create store dirs on tmpfs volume.
+  # This ensures lowerdir, upperdir and workdir will be on
+  # a consistent filesystem that fully supports OverlayFS.
+  storeVolume="$TEST_ROOT/stores"
+  mkdir -p "$storeVolume"
+  mount -t tmpfs tmpfs "$storeVolume" || true  # But continue anyway if that fails.
+
+  storeA="$storeVolume/store-a"
+  storeBTop="$storeVolume/store-b"
+  storeBRoot="$storeVolume/merged-store"
+  storeB="local-overlay?root=$storeBRoot&lower-store=$storeA&upper-layer=$storeBTop"
   # Creating testing directories
-  mkdir -p "$TEST_ROOT"/{store-a/nix/store,store-b,merged-store/nix/store,workdir}
+  mkdir -p "$storeVolume"/{store-a/nix/store,store-b,merged-store/nix/store,workdir}
 }
 
 # Mounting Overlay Store
 mountOverlayfs () {
   mount -t overlay overlay \
     -o lowerdir="$storeA/nix/store" \
     -o upperdir="$storeBTop" \
-    -o workdir="$TEST_ROOT/workdir" \
-    "$TEST_ROOT/merged-store/nix/store" \
+    -o workdir="$storeVolume/workdir" \
+    "$storeBRoot/nix/store" \
     || skipTest "overlayfs is not supported"
 
   cleanupOverlay () {
-    umount "$TEST_ROOT/merged-store/nix/store"
-    rm -r $TEST_ROOT/workdir
+    umount "$storeBRoot/nix/store"
+    rm -r $storeVolume/workdir
   }
   trap cleanupOverlay EXIT
 }
 
+remountOverlayfs () {
+  mount -o remount "$storeBRoot/nix/store"
+}
+
 toRealPath () {
   storeDir=$1; shift
   storePath=$1; shift
-  echo $storeDir$(echo $storePath | sed "s^$NIX_STORE_DIR^^")
+  echo $storeDir$(echo $storePath | sed "s^${NIX_STORE_DIR:-/nix/store}^^")
 }
 
 initLowerStore () {
@@ -52,5 +66,14 @@ initLowerStore () {
 }
 
 execUnshare () {
-  exec unshare --mount --map-root-user "$@"
+  exec unshare --mount --map-root-user "$SHELL" "$@"
+}
+
+addTextToStore() {
+  storeDir=$1; shift
+  filename=$1; shift
+  content=$1; shift
+  filePath="$TEST_HOME/$filename"
+  echo "$content" > "$filePath"
+  nix-store --store "$storeDir" --add "$filePath"
 }

tests/overlay-local-store/local.mk

@@ -2,6 +2,9 @@ overlay-local-store-tests := \
   $(d)/check-post-init.sh \
   $(d)/redundant-add.sh \
   $(d)/build.sh \
-  $(d)/bad-uris.sh
+  $(d)/bad-uris.sh \
+  $(d)/add-lower.sh \
+  $(d)/verify.sh \
+  $(d)/optimise.sh
 
 install-tests-groups += overlay-local-store

tests/overlay-local-store/optimise-inner.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+set -x
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+storeDirs
+
+initLowerStore
+
+mountOverlayfs
+
+# Create a file to add to store
+dupFilePath="$TEST_ROOT/dup-file"
+echo Duplicate > "$dupFilePath"
+
+# Add it to the overlay store (it will be written to the upper layer)
+dupFileStorePath=$(nix-store --store "$storeB" --add "$dupFilePath")
+
+# Now add it to the lower store so the store path is duplicated
+nix-store --store "$storeA" --add "$dupFilePath"
+
+# Ensure overlayfs and layers and synchronised
+remountOverlayfs
+
+dupFilename="${dupFileStorePath#/nix/store}"
+lowerPath="$storeA/$dupFileStorePath"
+upperPath="$storeBTop/$dupFilename"
+overlayPath="$storeBRoot/nix/store/$dupFilename"
+
+# Check store path exists in both layers and overlay
+lowerInode=$(stat -c %i "$lowerPath")
+upperInode=$(stat -c %i "$upperPath")
+overlayInode=$(stat -c %i "$overlayPath")
+[[ $upperInode == $overlayInode ]]
+[[ $upperInode != $lowerInode ]]
+
+# Run optimise to deduplicate store paths
+nix-store --store "$storeB" --optimise
+remountOverlayfs
+
+# Check path only exists in lower store
+stat "$lowerPath"
+stat "$overlayPath"
+expect 1 stat "$upperPath"

tests/overlay-local-store/optimise.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./optimise-inner.sh

tests/overlay-local-store/redundant-add-inner.sh

@@ -27,4 +27,4 @@ path=$(nix-store --store "$storeB" --add ../dummy)
 stat $(toRealPath "$storeA/nix/store" "$path")
 
 # upper layer should still not have it (no redundant copy)
-expect 1 stat $(toRealPath "$storeB/nix/store" "$path")
+expect 1 stat $(toRealPath "$storeBTop" "$path")

tests/overlay-local-store/verify-inner.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+set -x
+
+source common.sh
+
+# Avoid store dir being inside sandbox build-dir
+unset NIX_STORE_DIR
+unset NIX_STATE_DIR
+
+storeDirs
+
+initLowerStore
+
+mountOverlayfs
+
+# Realise a derivation from the lower store to propagate paths to overlay DB
+nix-store --store "$storeB" --realise $drvPath
+
+# Also ensure dummy file exists in overlay DB
+dummyPath=$(nix-store --store "$storeB" --add ../dummy)
+
+# Verify should be successful at this point
+nix-store --store "$storeB" --verify --check-contents
+
+# Now delete one of the derivation inputs in the lower store
+inputDrvFullPath=$(find "$storeA" -name "*-hermetic-input-1.drv")
+inputDrvPath=${inputDrvFullPath/*\/nix\/store\///nix/store/}
+rm -v "$inputDrvFullPath"
+
+# And truncate the contents of dummy file in lower store
+find "$storeA" -name "*-dummy" -exec truncate -s 0 {} \;
+
+# Verify should fail with the messages about missing input and modified dummy file
+verifyOutput=$(expectStderr 1 nix-store --store "$storeB" --verify --check-contents --repair)
+<<<"$verifyOutput" grepQuiet "path '$inputDrvPath' disappeared, but it still has valid referrers!"
+<<<"$verifyOutput" grepQuiet "path '$dummyPath' was modified! expected hash"
+<<<"$verifyOutput" grepQuiet "store does not support --verify --repair"

tests/overlay-local-store/verify.sh

@@ -0,0 +1,5 @@
+source common.sh
+
+requireEnvironment
+setupConfig
+execUnshare ./verify-inner.sh