[WIP] devShellTools.buildShellEnv

pkgs/build-support/dev-shell-tools/default.nix

@@ -1,5 +1,8 @@
 {
   lib,
+  runtimeShell,
+  bashInteractive,
+  stdenv,
   writeTextFile,
 }:
 let
@@ -62,4 +65,92 @@ rec {
     # https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/primops.cc#L1253
     lib.genAttrs outputList (output: builtins.unsafeDiscardStringContext outputMap.${output}.outPath);
 
+  toBashEnv = { env }:
+    lib.concatStringsSep
+      "\n"
+      (lib.mapAttrsToList
+        ( name: value:
+          if lib.isValidPosixName name
+          then ''export ${name}=${lib.escapeShellArg value}''
+          else ""
+        )
+        env);
+
+  buildShellEnv = { drvAttrs, promptPrefix ? "build shell", promptName ? null }:
+    let
+      name = drvAttrs.pname or drvAttrs.name or "shell";
+      env = unstructuredDerivationInputEnv { inherit drvAttrs; };
+    in
+    stdenv.mkDerivation (finalAttrs: {
+      name = "${name}-env";
+      passAsFile = [ "bashEnv" "bashrc" "runShell" ];
+      bashEnv = toBashEnv { inherit env; };
+      bashrc = ''
+        export NIXPKGS_SHELL_TMP="$(mktemp -d --tmpdir nixpkgs-shell-${name}.XXXXXX)"
+        export TMPDIR="$NIXPKGS_SHELL_TMP"
+        export TEMPDIR="$NIXPKGS_SHELL_TMP"
+        export TMP="$NIXPKGS_SHELL_TMP"
+        export TEMP="$NIXPKGS_SHELL_TMP"
+
+        echo "Using TMPDIR=$TMPDIR"
+
+        source @envbash@
+
+        mkdir -p $TMP/outputs
+        for _output in $outputs; do
+          export "''${_output}=$TMP/outputs/''${_output}"
+        done
+
+        source @stdenv@/setup
+
+        # Set a distinct prompt to make it clear that we are in a build shell
+        case "$PS1" in
+          *"(build shell $name)"*)
+            echo "It looks like your running a build shell inside a build shell."
+            echo "It might work, but this is probably not what you want."
+            echo "You may want to exit your shell before loading a new one."
+            ;;
+        esac
+
+        # Prefix a line to the prompt to indicate that we are in a build shell
+        PS1=$"\n(\[\033[1;33m\]"${lib.escapeShellArg promptPrefix}$": \[\033[1;34m\]"${if promptName != null then lib.escapeShellArg promptName else ''"$name"''}"\[\033[1;33m\]\[\033[0m\]) $PS1"
+
+        runHook shellHook
+      '';
+      buildCommand = ''
+        mkdir -p $out/lib $out/bin
+        bashrc="$out/lib/bashrc"
+        envbash="$out/lib/env.bash"
+
+        mv "$bashEnvPath" "$envbash"
+        substitute "$bashrcPath" "$bashrc" \
+          --replace-fail "@envbash@" "$envbash" \
+          --replace-fail "@stdenv@" "$stdenv" \
+          ;
+
+        substitute ${./run-shell.sh} "$out/bin/run-shell" \
+          --replace-fail "@bashrc@" "$bashrc" \
+          --replace-fail "@runtimeShell@" "${runtimeShell}" \
+          --replace-fail "@bashInteractive@" "${bashInteractive}" \
+          ;
+
+        # NOTE: most other files are script for the source command, and not
+        #       standalone executables, so they should not be made executable.
+        chmod a+x $out/bin/run-shell
+      '';
+      preferLocalBuild = true;
+      passthru = {
+        # Work this as a shell environment, so that commands like `nix-shell`
+        # will be able to check it and use it correctly. This also lets Nix know
+        # to stop when the user requests pkg.devShell explicitly, or a different
+        # attribute containing a shell environment.
+        isShellEnv = true;
+        devShell = throw "You're trying to access the devShell attribute of a shell environment. We appreciate that this is very \"meta\" and interesting, but it's usually just not what you want. Most likely you've selected one `.devShell` to deep in an expression or on the command line. Try removing the last one.";
+      };
+      meta = {
+        description = "An environment similar to the build environment of ${name}";
+        # TODO longDescription
+        mainProgram = "run-shell";
+      };
+    });
 }

pkgs/build-support/dev-shell-tools/tests/default.nix

@@ -1,7 +1,11 @@
 {
+  bash,
+  coreutils,
   devShellTools,
   emptyFile,
+  gnugrep,
   lib,
+  runCommand,
   stdenv,
   hello,
   writeText,
@@ -162,4 +166,50 @@
       # This would break the derivation. Instead, we have a check in the derivation to make sure Nix doesn't set it.
       "args"
     ]);
+
+  buildHelloInShell = derivation {
+    inherit (stdenv.buildPlatform) system;
+    name = "buildHelloInShell";
+    builder = "${bash}/bin/bash";
+    PATH = lib.makeBinPath [ coreutils gnugrep ];
+    args = [
+      "-euo" "pipefail" "-c"
+      ''
+        ${lib.getExe hello.devShell} -c 'genericBuild && ln -s $out "'"$PWD"'/result"'
+        ls -al
+        ./result/bin/hello | grep -i 'hello, world'
+        (set -x; [[ ! -e $out ]])
+        touch $out
+      ''
+    ];
+  };
+
+  various = let
+    hello2 = hello.overrideAttrs (o: {
+      shellHook = ''
+        echo 'shellHook says hi :)'
+      '';
+    });
+    inherit (hello2) devShell;
+  in derivation {
+    inherit (stdenv.buildPlatform) system;
+    name = "various";
+    builder = "${bash}/bin/bash";
+    PATH = lib.makeBinPath [ coreutils gnugrep ];
+    args = [
+      "-euo" "pipefail" "-c"
+      ''
+        ${lib.getExe devShell} -c 'echo "hello, world"' \
+          | grep -F 'hello, world'
+        ${lib.getExe devShell} -c 'echo "hello, world"' \
+          | grep -F 'hello, world'
+
+        ls -al
+        (set -x; [[ ! -e $out ]])
+        touch $out
+      ''
+    ];
+  };
+
+
 }

pkgs/build-support/mkshell/default.nix

@@ -1,4 +1,4 @@
-{ lib, stdenv, buildEnv }:
+{ lib, stdenv, buildEnv, devShellTools }:
 
 # A special kind of derivation that is only meant to be consumed by the
 # nix-shell.
@@ -11,6 +11,7 @@
 , nativeBuildInputs ? [ ]
 , propagatedBuildInputs ? [ ]
 , propagatedNativeBuildInputs ? [ ]
+, passthru ? { }
 , ...
 }@attrs:
 let
@@ -34,7 +35,7 @@ let
   ];
 in
 
-stdenv.mkDerivation ({
+stdenv.mkDerivation (finalAttrs: {
   inherit name;
 
   buildInputs = mergeInputs "buildInputs";
@@ -47,6 +48,7 @@ stdenv.mkDerivation ({
 
   phases = [ "buildPhase" ];
 
+  # TODO: Perhaps mkShell could *be* its own devShell, by setting isDevShell = true;
   buildPhase = ''
     { echo "------------------------------------------------------------";
       echo " WARNING: the existence of this path is not guaranteed.";
@@ -59,4 +61,28 @@ stdenv.mkDerivation ({
   '';
 
   preferLocalBuild = true;
+
+  passthru = {
+    devShell =
+      let
+        inherit (finalAttrs.finalPackage) drvAttrs;
+      in
+      devShellTools.buildShellEnv {
+        inherit drvAttrs;
+
+        # The default prefix is "build shell", but this shell is not derived
+        # directly from a derivation, so we set a more generic title.
+        promptPrefix = "nix";
+
+        # Change the default name
+        promptName =
+          if
+            # name was passed originally
+            attrs?name
+            # or with overrideAttrs
+            || drvAttrs.name != "nix-shell"
+          then null
+          else "mkShell";
+      };
+  } // passthru;
 } // rest)

pkgs/development/haskell-modules/generic-builder.nix

@@ -2,6 +2,7 @@
 , jailbreak-cabal, hscolour, cpphs
 , ghcWithHoogle, ghcWithPackages
 , nodejs
+, devShellTools
 }:
 
 let
@@ -844,6 +845,9 @@ stdenv.mkDerivation ({
 
     env = envFunc { };
 
+    # Specialise the devShell attribute, so we get our improved shell.
+    devShell = env.devShell;
+
   };
 
   meta = { inherit homepage license platforms; }

pkgs/stdenv/generic/default.nix

@@ -53,6 +53,9 @@ argsStdenv@{ name ? "stdenv", preHook ? "", initialPath
 , # The implementation of `mkDerivation`, parameterized with the final stdenv so we can tie the knot.
   # This is convient to have as a parameter so the stdenv "adapters" work better
   mkDerivationFromStdenv ? stdenv: (import ./make-derivation.nix { inherit lib config; } stdenv).mkDerivation
+
+, # callPackage for "passthru" utilities only
+  callPackage ? null
 }:
 
 let
@@ -77,6 +80,11 @@ let
 
   stdenv = (stdenv-overridable argsStdenv);
 
+  devShellTools =
+    if callPackage == null
+    then null
+    else callPackage ../../build-support/dev-shell-tools { inherit stdenv; };
+
   # The stdenv that we are producing.
   in
     derivation (
@@ -164,6 +172,11 @@ let
       # commands and extglob affects the Bash parser, we enable extglob always.
       shellDryRun = "${stdenv.shell} -n -O extglob";
 
+      buildShellEnv =
+        if devShellTools?buildShellEnv
+        then devShellTools.buildShellEnv
+        else _: null;
+
       tests = {
         succeedOnFailure = import ../tests/succeedOnFailure.nix { inherit stdenv; };
       };

pkgs/stdenv/generic/make-derivation.nix

@@ -632,6 +632,7 @@ extendDerivation
 
      inherit passthru overrideAttrs;
      inherit meta;
+     devShell = stdenv.buildShellEnv { drvAttrs = derivationArg; };
    } //
    # Pass through extra attributes that are not inputs, but
    # should be made available to Nix expressions using the

pkgs/stdenv/linux/default.nix

@@ -155,7 +155,7 @@ let
   # the bootstrap.  In all stages, we build an stdenv and the package
   # set that can be built with that stdenv.
   stageFun = prevStage:
-    { name, overrides ? (self: super: {}), extraNativeBuildInputs ? [] }:
+    { name, overrides ? (self: super: {}), extraNativeBuildInputs ? [], callPackage ? null }:
 
     let
 
@@ -205,6 +205,7 @@ let
         });
 
         overrides = self: super: (overrides self super) // { fetchurl = thisStdenv.fetchurlBoot; };
+        inherit callPackage;
       };
 
     in {

pkgs/top-level/stage.nix

@@ -141,7 +141,7 @@ let
     pkgs = self.pkgsHostTarget;
     targetPackages = self.pkgsTargetTarget;
 
-    inherit stdenv;
+    stdenv = stdenv.override (o: { callPackage = self.callPackage; });
   };
 
   splice = self: super: import ./splice.nix lib self (adjacentPackages != null);

shell.nix

@@ -4,24 +4,33 @@
 # because every time you change any file and do another `nix develop`,
 # it would create another copy of the entire ~500MB tree in the store.
 # See https://github.com/NixOS/nix/pull/6530 for the future
-{
-  system ? builtins.currentSystem,
-}:
+#
+# Note: We use a pinned Nixpkgs so that the tools are readily available even
+# when making changes that would otherwise require a new build of those tools.
+# If you'd like to test out changes to the tools themselves, you can pass
+#
+#     nix-shell --arg nixpkgs ./.
+#
 let
   pinnedNixpkgs = builtins.fromJSON (builtins.readFile ci/pinned-nixpkgs.json);
+in
+{
+  system ? builtins.currentSystem,
 
-  nixpkgs = fetchTarball {
+  nixpkgs ? fetchTarball {
     url = "https://github.com/NixOS/nixpkgs/archive/${pinnedNixpkgs.rev}.tar.gz";
     sha256 = pinnedNixpkgs.sha256;
-  };
-
+  },
+}:
+let
   pkgs = import nixpkgs {
     inherit system;
     config = {};
     overlays = [];
   };
 in
 pkgs.mkShellNoCC {
+  name = "nixpkgs";
   packages = [
     # The default formatter for Nix code
     # https://github.com/NixOS/nixfmt