nixos/overlayfs: add a switch to disable prefixing with /sysroot for initrd mounts

nixos/modules/tasks/filesystems/overlayfs.nix

@@ -1,17 +1,29 @@
-{ config, lib, pkgs, utils, ... }:
+{
+  config,
+  lib,
+  pkgs,
+  utils,
+  ...
+}:
 
 let
   # The scripted initrd contains some magic to add the prefix to the
   # paths just in time, so we don't add it here.
-  sysrootPrefix = fs:
-    if config.boot.initrd.systemd.enable && (utils.fsNeededForBoot fs) then
+  sysrootPrefix =
+    fs:
+    if
+      config.boot.initrd.systemd.enable
+      && fs.overlay.useStage1BaseDirectories
+      && (utils.fsNeededForBoot fs)
+    then
       "/sysroot"
     else
       "";
 
   # Returns a service that creates the required directories before the mount is
   # created.
-  preMountService = _name: fs:
+  preMountService =
+    _name: fs:
     let
       prefix = sysrootPrefix fs;
 
@@ -21,89 +33,111 @@ let
       upperdir = prefix + fs.overlay.upperdir;
       workdir = prefix + fs.overlay.workdir;
     in
-    lib.mkIf (fs.overlay.upperdir != null)
-      {
-        "rw-${escapedMountpoint}" = {
-          requiredBy = [ mountUnit ];
-          before = [ mountUnit ];
-          unitConfig = {
-            DefaultDependencies = false;
-            RequiresMountsFor = "${upperdir} ${workdir}";
-          };
-          serviceConfig = {
-            Type = "oneshot";
-            ExecStart = "${pkgs.coreutils}/bin/mkdir -p -m 0755 ${upperdir} ${workdir}";
-          };
+    lib.mkIf (fs.overlay.upperdir != null) {
+      "rw-${escapedMountpoint}" = {
+        requiredBy = [ mountUnit ];
+        before = [ mountUnit ];
+        unitConfig = {
+          DefaultDependencies = false;
+          RequiresMountsFor = "${upperdir} ${workdir}";
+        };
+        serviceConfig = {
+          Type = "oneshot";
+          ExecStart = "${pkgs.coreutils}/bin/mkdir -p -m 0755 ${upperdir} ${workdir}";
         };
       };
+    };
 
-  overlayOpts = { config, ... }: {
+  overlayOpts =
+    {
+      config,
+      ...
+    }:
+    {
+      options.overlay = {
+        lowerdir = lib.mkOption {
+          type = with lib.types; nullOr (nonEmptyListOf (either str pathInStore));
+          default = null;
+          description = ''
+            The list of path(s) to the lowerdir(s).
+
+            To create a writable overlay, you MUST provide an `upperdir` and a
+            `workdir`.
+
+            You can create a read-only overlay when you provide multiple (at
+            least 2!) lowerdirs and neither an `upperdir` nor a `workdir`.
+          '';
+        };
 
-    options.overlay = {
+        upperdir = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          default = null;
+          description = ''
+            The path to the upperdir.
 
-      lowerdir = lib.mkOption {
-        type = with lib.types; nullOr (nonEmptyListOf (either str pathInStore));
-        default = null;
-        description = ''
-          The list of path(s) to the lowerdir(s).
+            If this is null, a read-only overlay is created using the lowerdir.
 
-          To create a writable overlay, you MUST provide an upperdir and a
-          workdir.
+            If the filesystem is `neededForBoot`, this will be prefixed with `/sysroot`,
+            unless `useStage1BaseDirectories` is set to `true`.
 
-          You can create a read-only overlay when you provide multiple (at
-          least 2!) lowerdirs and neither an upperdir nor a workdir.
-        '';
-      };
+            If you set this to some value you MUST also set `workdir`.
+          '';
+        };
 
-      upperdir = lib.mkOption {
-        type = lib.types.nullOr lib.types.str;
-        default = null;
-        description = ''
-          The path to the upperdir.
+        workdir = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          default = null;
+          description = ''
+            The path to the workdir.
 
-          If this is null, a read-only overlay is created using the lowerdir.
+            If the filesystem is `neededForBoot`, this will be prefixed with `/sysroot`,
+            unless `useStage1BaseDirectories` is set to `true`.
 
-          If you set this to some value you MUST also set `workdir`.
-        '';
-      };
+            This MUST be set if you set `upperdir`.
+          '';
+        };
 
-      workdir = lib.mkOption {
-        type = lib.types.nullOr lib.types.str;
-        default = null;
-        description = ''
-          The path to the workdir.
+        useStage1BaseDirectories = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = ''
+            If enabled, `lowerdir`, `upperdir` and `workdir` will be prefixed with `/sysroot`.
 
-          This MUST be set if you set `upperdir`.
-        '';
-      };
+            Disabling this can be useful to create an overlay over directories which aren't on the real root.
 
-    };
+            Disabling this does not work with the scripted (i.e. non-systemd) initrd.
+          '';
+        };
+      };
 
-    config = lib.mkIf (config.overlay.lowerdir != null) {
-      fsType = "overlay";
-      device = lib.mkDefault "overlay";
-      depends = map (x: "${x}") (config.overlay.lowerdir ++ lib.optionals (config.overlay.upperdir != null) [
-        config.overlay.upperdir
-        config.overlay.workdir
-      ]);
-
-      options =
-        let
-          prefix = sysrootPrefix config;
-
-          lowerdir = map (s: prefix + s) config.overlay.lowerdir;
-          upperdir = prefix + config.overlay.upperdir;
-          workdir = prefix + config.overlay.workdir;
-        in
-        [
-          "lowerdir=${lib.concatStringsSep ":" lowerdir}"
-        ] ++ lib.optionals (config.overlay.upperdir != null) [
-          "upperdir=${upperdir}"
-          "workdir=${workdir}"
-        ];
+      config = lib.mkIf (config.overlay.lowerdir != null) {
+        fsType = "overlay";
+        device = lib.mkDefault "overlay";
+        depends = map (x: "${x}") (
+          config.overlay.lowerdir
+          ++ lib.optionals (config.overlay.upperdir != null) [
+            config.overlay.upperdir
+            config.overlay.workdir
+          ]
+        );
+
+        options =
+          let
+            prefix = sysrootPrefix config;
+
+            lowerdir = map (s: prefix + s) config.overlay.lowerdir;
+            upperdir = prefix + config.overlay.upperdir;
+            workdir = prefix + config.overlay.workdir;
+          in
+          [
+            "lowerdir=${lib.concatStringsSep ":" lowerdir}"
+          ]
+          ++ lib.optionals (config.overlay.upperdir != null) [
+            "upperdir=${upperdir}"
+            "workdir=${workdir}"
+          ];
+      };
     };
-
-  };
 in
 
 {
@@ -140,6 +174,13 @@ in
                 -> (lib.length fs.overlay.lowerdir) >= 2;
               message = "A read-only overlay (without an `upperdir`) requires at least 2 `lowerdir`s: ${fs.mountPoint}";
             }
+            {
+              assertion = !fs.overlay.useStage1BaseDirectories -> config.boot.initrd.systemd.enable;
+              message = ''
+                Stage 1 overlay file system ${fs.mountPoint} has 'useStage1BaseDirectories' set to false,
+                which is not supported with scripted initrd. Please enable 'boot.initrd.systemd.enable'.
+              '';
+            }
           ]) overlayFileSystems
         )
         ++ lib.mapAttrsToList (_: fs: {

nixos/tests/filesystems-overlayfs.nix

@@ -24,40 +24,50 @@ in
 
   meta.maintainers = with lib.maintainers; [ nikstur ];
 
-  nodes.machine = { config, pkgs, ... }: {
-    boot.initrd.systemd.enable = true;
+  nodes.machine =
+    { config, pkgs, ... }:
+    {
+      boot.initrd.systemd.enable = true;
 
-    virtualisation.fileSystems = {
-      "/initrd-overlay" = {
-        overlay = {
-          lowerdir = [ initrdLowerdir ];
-          upperdir = "/.rw-initrd-overlay/upper";
-          workdir = "/.rw-initrd-overlay/work";
+      virtualisation.fileSystems = {
+        "/initrd-overlay" = {
+          overlay = {
+            lowerdir = [ initrdLowerdir ];
+            upperdir = "/.rw-initrd-overlay/upper";
+            workdir = "/.rw-initrd-overlay/work";
+          };
+          neededForBoot = true;
         };
-        neededForBoot = true;
-      };
-      "/userspace-overlay" = {
-        overlay = {
-          lowerdir = [ userspaceLowerdir ];
-          upperdir = "/.rw-userspace-overlay/upper";
-          workdir = "/.rw-userspace-overlay/work";
+        "/initrd-real-root-overlay" = {
+          overlay = {
+            lowerdir = [ userspaceLowerdir ];
+            upperdir = "/run/upper"; # from initrd
+            workdir = "/run/work"; # from initrd
+            useStage1BaseDirectories = false;
+          };
+        };
+        "/userspace-overlay" = {
+          overlay = {
+            lowerdir = [ userspaceLowerdir ];
+            upperdir = "/.rw-userspace-overlay/upper";
+            workdir = "/.rw-userspace-overlay/work";
+          };
+        };
+        "/ro-initrd-overlay" = {
+          overlay.lowerdir = [
+            initrdLowerdir
+            initrdLowerdir2
+          ];
+          neededForBoot = true;
+        };
+        "/ro-userspace-overlay" = {
+          overlay.lowerdir = [
+            userspaceLowerdir
+            userspaceLowerdir2
+          ];
         };
-      };
-      "/ro-initrd-overlay" = {
-        overlay.lowerdir = [
-          initrdLowerdir
-          initrdLowerdir2
-        ];
-        neededForBoot = true;
-      };
-      "/ro-userspace-overlay" = {
-        overlay.lowerdir = [
-          userspaceLowerdir
-          userspaceLowerdir2
-        ];
       };
     };
-  };
 
   testScript = ''
     machine.wait_for_unit("default.target")
@@ -67,6 +77,11 @@ in
       machine.succeed("touch /initrd-overlay/writable.txt")
       machine.succeed("findmnt --kernel --types overlay /initrd-overlay")
 
+    with subtest("Userspace overlay with upper/workdir in initrd"):
+      machine.wait_for_file("/initrd-real-root-overlay/userspace.txt", 5)
+      machine.succeed("touch /initrd-real-root-overlay/writable.txt")
+      machine.succeed("findmnt --kernel --types overlay /initrd-real-root-overlay")
+
     with subtest("Userspace overlay"):
       machine.wait_for_file("/userspace-overlay/userspace.txt", 5)
       machine.succeed("touch /userspace-overlay/writable.txt")