From 62de17150fbf49dc33f1de5d77cbbe8f4bc7b935 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Sat, 28 Aug 2021 16:26:53 -0400 Subject: [PATCH 01/17] Shellbang support with flakes Enables shebang usage of nix shell. All arguments with `#! nix` get added to the nix invocation. This implementation does NOT set any additional arguments other than placing the script path itself as the first argument such that the interpreter can utilize it. Example below: ``` #!/usr/bin/env nix #! nix shell --quiet #! nix nixpkgs#bash #! nix nixpkgs#shellcheck #! nix nixpkgs#hello #! nix --ignore-environment --command bash # shellcheck shell=bash set -eu shellcheck "$0" || exit 1 function main { hello echo 0:"$0" 1:"$1" 2:"$2" } "$@" ``` fix: include programName usage EDIT: For posterity I've changed shellwords to shellwords2 in order not to interfere with other changes during a rebase. shellwords2 is removed in a later commit. -- roberth --- src/libutil/args.cc | 37 +++++++++++++++++++++++++ src/libutil/args.hh | 8 +++++- src/libutil/util.cc | 45 +++++++++++++++++++++++++++++++ src/libutil/util.hh | 11 +++++--- src/nix/main.cc | 2 +- tests/functional/flakes/common.sh | 5 ++-- tests/functional/flakes/flakes.sh | 15 ++++++++++- 7 files changed, 114 insertions(+), 9 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 0b65519a3d8..7106491fda8 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -6,6 +6,7 @@ #include "users.hh" #include "json-utils.hh" +#include #include namespace nix { @@ -78,6 +79,12 @@ std::optional RootArgs::needsCompletion(std::string_view s) } void RootArgs::parseCmdline(const Strings & _cmdline) +{ + // Default via 5.1.2.2.1 in C standard + Args::parseCmdline("", _cmdline); +} + +void Args::parseCmdline(const std::string & programName, const Strings & _cmdline) { Strings pendingArgs; bool dashDash = false; @@ -93,6 +100,36 @@ void RootArgs::parseCmdline(const Strings & _cmdline) } bool argsSeen = false; + + // Heuristic to see if we're invoked as a shebang script, namely, + // if we have at least one argument, it's the name of an + // executable file, and it starts with "#!". + Strings savedArgs; + auto isNixCommand = std::regex_search(programName, std::regex("nix$")); + if (isNixCommand && cmdline.size() > 0) { + auto script = *cmdline.begin(); + try { + auto lines = tokenizeString(readFile(script), "\n"); + if (std::regex_search(lines.front(), std::regex("^#!"))) { + lines.pop_front(); + for (auto pos = std::next(cmdline.begin()); pos != cmdline.end();pos++) + savedArgs.push_back(*pos); + cmdline.clear(); + + for (auto line : lines) { + line = chomp(line); + + std::smatch match; + if (std::regex_match(line, match, std::regex("^#!\\s*nix\\s(.*)$"))) + for (const auto & word : shellwords(match[1].str())) + cmdline.push_back(word); + } + cmdline.push_back(script); + for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++) + cmdline.push_back(*pos); + } + } catch (SysError &) { } + } for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { auto arg = *pos; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 45fd678e7d4..1d056678de9 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -27,8 +27,14 @@ class Args public: /** - * Return a short one-line description of the command. + * Parse the command line with argv0, throwing a UsageError if something + goes wrong. */ + void parseCmdline(const std::string & argv0, const Strings & cmdline); + + /** + * Return a short one-line description of the command. + */ virtual std::string description() { return ""; } virtual bool forceImpureByDefault() { return false; } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index ee7a2284986..6ca1dbd7ad6 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -5,6 +5,8 @@ #include #include #include +#include + namespace nix { @@ -136,6 +138,49 @@ std::string shellEscape(const std::string_view s) return r; } +/* Recreate the effect of the perl shellwords function, breaking up a + * string into arguments like a shell word, including escapes + */ +std::vector shellwords2(const std::string & s) +{ + std::regex whitespace("^(\\s+).*"); + auto begin = s.cbegin(); + std::vector res; + std::string cur; + enum state { + sBegin, + sQuote + }; + state st = sBegin; + auto it = begin; + for (; it != s.cend(); ++it) { + if (st == sBegin) { + std::smatch match; + if (regex_search(it, s.cend(), match, whitespace)) { + cur.append(begin, it); + res.push_back(cur); + cur.clear(); + it = match[1].second; + begin = it; + } + } + switch (*it) { + case '"': + cur.append(begin, it); + begin = it + 1; + st = st == sBegin ? sQuote : sBegin; + break; + case '\\': + /* perl shellwords mostly just treats the next char as part of the string with no special processing */ + cur.append(begin, it); + begin = ++it; + break; + } + } + cur.append(begin, it); + if (!cur.empty()) res.push_back(cur); + return res; +} void ignoreException(Verbosity lvl) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 5f730eaf6be..bcd0c176970 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -189,10 +189,13 @@ std::string toLower(const std::string & s); std::string shellEscape(const std::string_view s); -/** - * Exception handling in destructors: print an error message, then - * ignore the exception. - */ +/* Recreate the effect of the perl shellwords function, breaking up a + * string into arguments like a shell word, including escapes */ +std::vector shellwords2(const std::string & s); + + +/* Exception handling in destructors: print an error message, then + ignore the exception. */ void ignoreException(Verbosity lvl = lvlError); diff --git a/src/nix/main.cc b/src/nix/main.cc index b582fc16640..16fb50806d7 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -428,7 +428,7 @@ void mainWrapped(int argc, char * * argv) }); try { - args.parseCmdline(argvToStrings(argc, argv)); + args.parseCmdline(programName, argvToStrings(argc, argv)); } catch (UsageError &) { if (!args.helpRequested && !args.completions) throw; } diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh index 8aed296e64c..fc45cf7bfa3 100644 --- a/tests/functional/flakes/common.sh +++ b/tests/functional/flakes/common.sh @@ -11,6 +11,7 @@ writeSimpleFlake() { outputs = inputs: rec { packages.$system = rec { foo = import ./simple.nix; + fooScript = (import ./shell.nix {}).foo; default = foo; }; packages.someOtherSystem = rec { @@ -24,13 +25,13 @@ writeSimpleFlake() { } EOF - cp ../simple.nix ../simple.builder.sh ../config.nix $flakeDir/ + cp ../simple.nix ../shell.nix ../simple.builder.sh ../config.nix $flakeDir/ } createSimpleGitFlake() { local flakeDir="$1" writeSimpleFlake $flakeDir - git -C $flakeDir add flake.nix simple.nix simple.builder.sh config.nix + git -C $flakeDir add flake.nix simple.nix shell.nix simple.builder.sh config.nix git -C $flakeDir commit -m 'Initial' } diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index b0038935c84..c4b18a21bc0 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -66,7 +66,17 @@ cat > "$nonFlakeDir/README.md" < "$nonFlakeDir/shebang.sh" < Date: Mon, 14 Nov 2022 17:04:19 +0100 Subject: [PATCH 02/17] src/libutil/util.hh: Formatting --- src/libutil/util.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index bcd0c176970..b7d3ac50453 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -190,7 +190,7 @@ std::string shellEscape(const std::string_view s); /* Recreate the effect of the perl shellwords function, breaking up a - * string into arguments like a shell word, including escapes */ + string into arguments like a shell word, including escapes. */ std::vector shellwords2(const std::string & s); From 6cefab739384e12d8a6c53aae4ce1ccb6198b73a Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Mon, 14 Nov 2022 19:40:01 -0500 Subject: [PATCH 03/17] fix: test to ensure arguments are passed --- tests/functional/flakes/flakes.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index c4b18a21bc0..e7dffde07fa 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -71,8 +71,9 @@ cat > "$nonFlakeDir/shebang.sh" < Date: Mon, 14 Nov 2022 23:58:58 -0500 Subject: [PATCH 04/17] doc: shebang release notes, docs, tests fix: release notes --- doc/manual/src/release-notes/rl-next.md | 44 +++++++++ src/nix/shell.md | 117 ++++++++++++++++++++++++ tests/functional/flakes/flakes.sh | 13 +++ 3 files changed, 174 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 8cd69f8fd70..b4b12ca938e 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,5 +1,49 @@ # Release X.Y (202?-??-??) +- The experimental nix command is now a `#!-interpreter` by appending the + contents of any `#! nix` lines and the script's location to a single call. + Some examples: + ``` + #!/usr/bin/env nix + #! nix shell --file "" hello --command bash + + hello | cowsay + ``` + or with flakes: + ``` + #!/usr/bin/env nix + #! nix shell nixpkgs#bash nixpkgs#hello nixpkgs#cowsay --command bash + + hello | cowsay + ``` + or + ```bash + #! /usr/bin/env nix + #! nix shell --impure --expr + #! nix "with (import (builtins.getFlake ''nixpkgs'') {}); terraform.withPlugins (plugins: [ plugins.openstack ])" + #! nix --command bash + + terraform "$@" + ``` + or + ``` + #!/usr/bin/env nix + //! ```cargo + //! [dependencies] + //! time = "0.1.25" + //! ``` + /* + #!nix shell nixpkgs#rustc nixpkgs#rust-script nixpkgs#cargo --command rust-script + */ + fn main() { + for argument in std::env::args().skip(1) { + println!("{}", argument); + }; + println!("{}", std::env::var("HOME").expect("")); + println!("{}", time::now().rfc822z()); + } + // vim: ft=rust + ``` - [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. - [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). diff --git a/src/nix/shell.md b/src/nix/shell.md index f3691957575..b0bfa160997 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -51,4 +51,121 @@ R""( provides the specified [*installables*](./nix.md#installable). If no command is specified, it starts the default shell of your user account specified by `$SHELL`. +# Use as a `#!`-interpreter + +You can use `nix` as a script interpreter to allow scripts written +in arbitrary languages to obtain their own dependencies via Nix. This is +done by starting the script with the following lines: + +```bash +#! /usr/bin/env nix +#! nix shell installables --command real-interpreter +``` + +where *real-interpreter* is the “real” script interpreter that will be +invoked by `nix shell` after it has obtained the dependencies and +initialised the environment, and *installables* are the attribute names of +the dependencies in Nixpkgs. + +The lines starting with `#! nix` specify options (see above). Note that you +cannot write `#! /usr/bin/env nix shell -i ...` because many operating systems +only allow one argument in `#!` lines. + +For example, here is a Python script that depends on Python and the +`prettytable` package: + +```python +#! /usr/bin/env nix +#! nix shell github:tomberek/-#python3With.prettytable --command python + +import prettytable + +# Print a simple table. +t = prettytable.PrettyTable(["N", "N^2"]) +for n in range(1, 10): t.add_row([n, n * n]) +print t +``` + +Similarly, the following is a Perl script that specifies that it +requires Perl and the `HTML::TokeParser::Simple` and `LWP` packages: + +```perl +#! /usr/bin/env nix +#! nix shell github:tomberek/-#perlWith.HTMLTokeParserSimple.LWP --command perl -x + +use HTML::TokeParser::Simple; + +# Fetch nixos.org and print all hrefs. +my $p = HTML::TokeParser::Simple->new(url => 'http://nixos.org/'); + +while (my $token = $p->get_tag("a")) { + my $href = $token->get_attr("href"); + print "$href\n" if $href; +} +``` + +Sometimes you need to pass a simple Nix expression to customize a +package like Terraform: + +```bash +#! /usr/bin/env nix +#! nix shell --impure --expr +#! nix "with (import (builtins.getFlake ''nixpkgs'') {}); terraform.withPlugins (plugins: [ plugins.openstack ])" +#! nix --command bash + +terraform "$@" +``` + +> **Note** +> +> You must use double quotes (`"`) when passing a simple Nix expression +> in a nix shell shebang. + +Finally, using the merging of multiple nix shell shebangs the following +Haskell script uses a specific branch of Nixpkgs/NixOS (the 21.11 stable +branch): + +```haskell +#!/usr/bin/env nix +#!nix shell --override-input nixpkgs github:NixOS/nixpkgs/nixos-21.11 +#!nix github:tomberek/-#haskellWith.download-curl.tagsoup --command runghc + +import Network.Curl.Download +import Text.HTML.TagSoup +import Data.Either +import Data.ByteString.Char8 (unpack) + +-- Fetch nixos.org and print all hrefs. +main = do + resp <- openURI "https://nixos.org/" + let tags = filter (isTagOpenName "a") $ parseTags $ unpack $ fromRight undefined resp + let tags' = map (fromAttrib "href") tags + mapM_ putStrLn $ filter (/= "") tags' +``` + +If you want to be even more precise, you can specify a specific revision +of Nixpkgs: + + #!nix shell --override-input nixpkgs github:NixOS/nixpkgs/eabc38219184cc3e04a974fe31857d8e0eac098d + +The examples above all used `-p` to get dependencies from Nixpkgs. You +can also use a Nix expression to build your own dependencies. For +example, the Python example could have been written as: + +```python +#! /usr/bin/env nix +#! nix shell --impure --file deps.nix -i python +``` + +where the file `deps.nix` in the same directory as the `#!`-script +contains: + +```nix +with import {}; +python3.withPackages (ps: with ps; [ prettytable ]) +``` + + + + )"" diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index e7dffde07fa..f2792549339 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -80,6 +80,18 @@ chmod +x "$nonFlakeDir/shebang.sh" git -C "$nonFlakeDir" add README.md shebang.sh git -C "$nonFlakeDir" commit -m 'Initial' +cat > $nonFlakeDir/shebang-perl.sh < Date: Wed, 1 Sep 2021 02:19:51 -0400 Subject: [PATCH 05/17] Read file incrementally --- src/libutil/args.cc | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 7106491fda8..80216e7ad72 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -6,6 +6,8 @@ #include "users.hh" #include "json-utils.hh" +#include +#include #include #include @@ -109,14 +111,17 @@ void Args::parseCmdline(const std::string & programName, const Strings & _cmdlin if (isNixCommand && cmdline.size() > 0) { auto script = *cmdline.begin(); try { - auto lines = tokenizeString(readFile(script), "\n"); - if (std::regex_search(lines.front(), std::regex("^#!"))) { - lines.pop_front(); + std::ifstream stream(script); + char shebang[3]={0,0,0}; + stream.get(shebang,3); + if (strncmp(shebang,"#!",2) == 0){ for (auto pos = std::next(cmdline.begin()); pos != cmdline.end();pos++) savedArgs.push_back(*pos); cmdline.clear(); - for (auto line : lines) { + std::string line; + std::getline(stream,line); + while (std::getline(stream,line) && !line.empty()){ line = chomp(line); std::smatch match; From 4a9a0076cd0f1a1cf88e3c1e7230f2a1934dbb3f Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Sat, 26 Nov 2022 09:06:39 -0500 Subject: [PATCH 06/17] feat: break out of shebang processing for non-comments --- src/libutil/args.cc | 3 ++- tests/functional/flakes/flakes.sh | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 80216e7ad72..d90374adc48 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -121,7 +121,8 @@ void Args::parseCmdline(const std::string & programName, const Strings & _cmdlin std::string line; std::getline(stream,line); - while (std::getline(stream,line) && !line.empty()){ + std::string commentChars("#/\\%@*-"); + while (std::getline(stream,line) && !line.empty() && commentChars.find(line[0]) != std::string::npos){ line = chomp(line); std::smatch match; diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index f2792549339..28b5e4e0ffd 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -80,17 +80,17 @@ chmod +x "$nonFlakeDir/shebang.sh" git -C "$nonFlakeDir" add README.md shebang.sh git -C "$nonFlakeDir" commit -m 'Initial' -cat > $nonFlakeDir/shebang-perl.sh < $nonFlakeDir/shebang-comments.sh < Date: Tue, 3 Jan 2023 05:55:06 -0500 Subject: [PATCH 07/17] doc: remove reference to nix-shell --- src/nix/shell.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/nix/shell.md b/src/nix/shell.md index b0bfa160997..7e0e5f213ba 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -148,9 +148,8 @@ of Nixpkgs: #!nix shell --override-input nixpkgs github:NixOS/nixpkgs/eabc38219184cc3e04a974fe31857d8e0eac098d -The examples above all used `-p` to get dependencies from Nixpkgs. You -can also use a Nix expression to build your own dependencies. For -example, the Python example could have been written as: +You can also use a Nix expression to build your own dependencies. For example, +the Python example could have been written as: ```python #! /usr/bin/env nix @@ -166,6 +165,4 @@ python3.withPackages (ps: with ps; [ prettytable ]) ``` - - )"" From 771ac96f673b69c40ada77cd124bb089e77c6468 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 12 May 2023 07:44:25 -0400 Subject: [PATCH 08/17] fix: refactor parseCmdline interface --- src/libutil/args.cc | 9 ++++----- src/libutil/args.hh | 2 +- src/nix/main.cc | 5 ++++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index d90374adc48..481ed33ff8c 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -83,10 +83,10 @@ std::optional RootArgs::needsCompletion(std::string_view s) void RootArgs::parseCmdline(const Strings & _cmdline) { // Default via 5.1.2.2.1 in C standard - Args::parseCmdline("", _cmdline); + Args::parseCmdline(_cmdline, false); } -void Args::parseCmdline(const std::string & programName, const Strings & _cmdline) +void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) { Strings pendingArgs; bool dashDash = false; @@ -107,8 +107,7 @@ void Args::parseCmdline(const std::string & programName, const Strings & _cmdlin // if we have at least one argument, it's the name of an // executable file, and it starts with "#!". Strings savedArgs; - auto isNixCommand = std::regex_search(programName, std::regex("nix$")); - if (isNixCommand && cmdline.size() > 0) { + if (allowShebang){ auto script = *cmdline.begin(); try { std::ifstream stream(script); @@ -121,7 +120,7 @@ void Args::parseCmdline(const std::string & programName, const Strings & _cmdlin std::string line; std::getline(stream,line); - std::string commentChars("#/\\%@*-"); + static const std::string commentChars("#/\\%@*-"); while (std::getline(stream,line) && !line.empty() && commentChars.find(line[0]) != std::string::npos){ line = chomp(line); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 1d056678de9..e753dcaf60d 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -30,7 +30,7 @@ public: * Parse the command line with argv0, throwing a UsageError if something goes wrong. */ - void parseCmdline(const std::string & argv0, const Strings & cmdline); + void parseCmdline(const Strings & _cmdline, bool allowShebang); /** * Return a short one-line description of the command. diff --git a/src/nix/main.cc b/src/nix/main.cc index 16fb50806d7..73641f6d231 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include @@ -428,7 +429,9 @@ void mainWrapped(int argc, char * * argv) }); try { - args.parseCmdline(programName, argvToStrings(argc, argv)); + auto isNixCommand = std::regex_search(programName, std::regex("nix$")); + auto allowShebang = isNixCommand && argc > 1; + args.parseCmdline(argvToStrings(argc, argv),allowShebang); } catch (UsageError &) { if (!args.helpRequested && !args.completions) throw; } From e4970ac3e4bb13a9b3eeb11b8c5b3362ba426381 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 May 2023 19:42:49 +0200 Subject: [PATCH 09/17] libcmd: lookupFileArg(): add baseDir This will allow a different base directory to be used, matching a shebang script location instead of the working directory. --- src/libcmd/common-eval-args.cc | 4 ++-- src/libcmd/common-eval-args.hh | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 91fa881b12d..401acc38e96 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -164,7 +164,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) return res.finish(); } -SourcePath lookupFileArg(EvalState & state, std::string_view s) +SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir) { if (EvalSettings::isPseudoUrl(s)) { auto storePath = fetchers::downloadTarball( @@ -185,7 +185,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s) } else - return state.rootPath(CanonPath::fromCwd(s)); + return state.rootPath(CanonPath(s, baseDir)); } } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 6359b2579ca..4b403d93674 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -2,6 +2,7 @@ ///@file #include "args.hh" +#include "canon-path.hh" #include "common-args.hh" #include "search-path.hh" @@ -28,6 +29,6 @@ private: std::map autoArgs; }; -SourcePath lookupFileArg(EvalState & state, std::string_view s); +SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir = CanonPath::fromCwd()); } From 7db185eb1c3263f872f5b891b73742b6963043f7 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 May 2023 19:46:37 +0200 Subject: [PATCH 10/17] nix: Reserve shebang line syntax and only parse double backtick quotes Being restrictive about syntax leaves opportunity to improve the syntax and functionality later. --- doc/manual/src/release-notes/rl-next.md | 11 +- src/libutil/args.cc | 152 +++++++++++++++++++++++- src/libutil/util.cc | 43 ------- src/libutil/util.hh | 5 - src/nix/shell.md | 8 +- tests/functional/flakes/flakes.sh | 16 ++- 6 files changed, 177 insertions(+), 58 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index b4b12ca938e..23704f72d37 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -2,10 +2,13 @@ - The experimental nix command is now a `#!-interpreter` by appending the contents of any `#! nix` lines and the script's location to a single call. + + Verbatim strings may be passed in double backtick (```` `` ````) quotes. + Some examples: ``` #!/usr/bin/env nix - #! nix shell --file "" hello --command bash + #! nix shell --file ```` hello --command bash hello | cowsay ``` @@ -19,8 +22,10 @@ or ```bash #! /usr/bin/env nix - #! nix shell --impure --expr - #! nix "with (import (builtins.getFlake ''nixpkgs'') {}); terraform.withPlugins (plugins: [ plugins.openstack ])" + #! nix shell --impure --expr `` + #! nix with (import (builtins.getFlake "nixpkgs") {}); + #! nix terraform.withPlugins (plugins: [ plugins.openstack ]) + #! nix `` #! nix --command bash terraform "$@" diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 481ed33ff8c..ab6e0e266f1 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -86,6 +86,147 @@ void RootArgs::parseCmdline(const Strings & _cmdline) Args::parseCmdline(_cmdline, false); } +/** + * Basically this is `typedef std::optional Parser(std::string_view s, Strings & r);` + * + * Except we can't recursively reference the Parser typedef, so we have to write a class. + */ +struct Parser { + std::string_view remaining; + + /** + * @brief Parse the next character(s) + * + * @param r + * @return std::shared_ptr + */ + virtual void operator()(std::shared_ptr & state, Strings & r) = 0; + + Parser(std::string_view s) : remaining(s) {}; +}; + +struct ParseQuoted : public Parser { + /** + * @brief Accumulated string + * + * Parsed argument up to this point. + */ + std::string acc; + + ParseQuoted(std::string_view s) : Parser(s) {}; + + virtual void operator()(std::shared_ptr & state, Strings & r) override; +}; + + +struct ParseUnquoted : public Parser { + /** + * @brief Accumulated string + * + * Parsed argument up to this point. Empty string is not representable in + * unquoted syntax, so we use it for the initial state. + */ + std::string acc; + + ParseUnquoted(std::string_view s) : Parser(s) {}; + + virtual void operator()(std::shared_ptr & state, Strings & r) override { + if (remaining.empty()) { + if (!acc.empty()) + r.push_back(acc); + state = nullptr; // done + return; + } + switch (remaining[0]) { + case ' ': case '\t': case '\n': case '\r': + if (!acc.empty()) + r.push_back(acc); + state = std::make_shared(ParseUnquoted(remaining.substr(1))); + return; + case '`': + if (remaining.size() > 1 && remaining[1] == '`') { + state = std::make_shared(ParseQuoted(remaining.substr(2))); + return; + } + else + throw Error("single backtick is not a supported syntax in the nix shebang."); + + // reserved characters + // meaning to be determined, or may be reserved indefinitely so that + // #!nix syntax looks unambiguous + case '$': + case '*': + case '~': + case '<': + case '>': + case '|': + case ';': + case '(': + case ')': + case '[': + case ']': + case '{': + case '}': + case '\'': + case '"': + case '\\': + throw Error("unsupported unquoted character in nix shebang: " + std::string(1, remaining[0]) + ". Use double backticks to escape?"); + + case '#': + if (acc.empty()) { + throw Error ("unquoted nix shebang argument cannot start with #. Use double backticks to escape?"); + } else { + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + + default: + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + assert(false); + } +}; + +void ParseQuoted::operator()(std::shared_ptr &state, Strings & r) { + if (remaining.empty()) { + throw Error("unterminated quoted string in nix shebang"); + } + switch (remaining[0]) { + case '`': + if (remaining.size() > 1 && remaining[1] == '`') { + state = std::make_shared(ParseUnquoted(remaining.substr(2))); + r.push_back(acc); + return; + } + else { + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + default: + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + assert(false); +} + +static Strings parseShebangContent(std::string_view s) { + Strings result; + std::shared_ptr parserState(std::make_shared(ParseUnquoted(s))); + + // trampoline == iterated strategy pattern + while (parserState) { + auto currentState = parserState; + (*currentState)(parserState, result); + } + + return result; +} + void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) { Strings pendingArgs; @@ -121,13 +262,18 @@ void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) std::string line; std::getline(stream,line); static const std::string commentChars("#/\\%@*-"); + std::string shebangContent; while (std::getline(stream,line) && !line.empty() && commentChars.find(line[0]) != std::string::npos){ line = chomp(line); std::smatch match; - if (std::regex_match(line, match, std::regex("^#!\\s*nix\\s(.*)$"))) - for (const auto & word : shellwords(match[1].str())) - cmdline.push_back(word); + // We match one space after `nix` so that we preserve indentation. + // No space is necessary for an empty line. An empty line has basically no effect. + if (std::regex_match(line, match, std::regex("^#!\\s*nix(:? |$)(.*)$"))) + shebangContent += match[2].str() + "\n"; + } + for (const auto & word : parseShebangContent(shebangContent)) { + cmdline.push_back(word); } cmdline.push_back(script); for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 6ca1dbd7ad6..5bb3f374ba7 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -138,49 +138,6 @@ std::string shellEscape(const std::string_view s) return r; } -/* Recreate the effect of the perl shellwords function, breaking up a - * string into arguments like a shell word, including escapes - */ -std::vector shellwords2(const std::string & s) -{ - std::regex whitespace("^(\\s+).*"); - auto begin = s.cbegin(); - std::vector res; - std::string cur; - enum state { - sBegin, - sQuote - }; - state st = sBegin; - auto it = begin; - for (; it != s.cend(); ++it) { - if (st == sBegin) { - std::smatch match; - if (regex_search(it, s.cend(), match, whitespace)) { - cur.append(begin, it); - res.push_back(cur); - cur.clear(); - it = match[1].second; - begin = it; - } - } - switch (*it) { - case '"': - cur.append(begin, it); - begin = it + 1; - st = st == sBegin ? sQuote : sBegin; - break; - case '\\': - /* perl shellwords mostly just treats the next char as part of the string with no special processing */ - cur.append(begin, it); - begin = ++it; - break; - } - } - cur.append(begin, it); - if (!cur.empty()) res.push_back(cur); - return res; -} void ignoreException(Verbosity lvl) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index b7d3ac50453..27faa4d6d22 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -189,11 +189,6 @@ std::string toLower(const std::string & s); std::string shellEscape(const std::string_view s); -/* Recreate the effect of the perl shellwords function, breaking up a - string into arguments like a shell word, including escapes. */ -std::vector shellwords2(const std::string & s); - - /* Exception handling in destructors: print an error message, then ignore the exception. */ void ignoreException(Verbosity lvl = lvlError); diff --git a/src/nix/shell.md b/src/nix/shell.md index 7e0e5f213ba..7c315fb3f02 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -109,8 +109,10 @@ package like Terraform: ```bash #! /usr/bin/env nix -#! nix shell --impure --expr -#! nix "with (import (builtins.getFlake ''nixpkgs'') {}); terraform.withPlugins (plugins: [ plugins.openstack ])" +#! nix shell --impure --expr `` +#! nix with (import (builtins.getFlake ''nixpkgs'') {}); +#! nix terraform.withPlugins (plugins: [ plugins.openstack ]) +#! nix `` #! nix --command bash terraform "$@" @@ -118,7 +120,7 @@ terraform "$@" > **Note** > -> You must use double quotes (`"`) when passing a simple Nix expression +> You must use double backticks (```` `` ````) when passing a simple Nix expression > in a nix shell shebang. Finally, using the merging of multiple nix shell shebangs the following diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 28b5e4e0ffd..a0a34ffa9e1 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -80,6 +80,7 @@ chmod +x "$nonFlakeDir/shebang.sh" git -C "$nonFlakeDir" add README.md shebang.sh git -C "$nonFlakeDir" commit -m 'Initial' +# this also tests a fairly trivial double backtick quoted string, ``--command`` cat > $nonFlakeDir/shebang-comments.sh < $nonFlakeDir/shebang-comments.sh < $nonFlakeDir/shebang-reject.sh <&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' From 80e9bad3ee289963587e9167d7f55133c0cbe517 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 May 2023 19:54:54 +0200 Subject: [PATCH 11/17] nix: Add command baseDir to parse --expr relative to shebang script --- doc/manual/src/release-notes/rl-next.md | 1 + src/libcmd/installables.cc | 3 ++- src/libutil/args.cc | 9 +++++++++ src/libutil/args.hh | 20 ++++++++++++++++++++ tests/functional/flakes/flakes.sh | 19 +++++++++++++++++++ 5 files changed, 51 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 23704f72d37..091199c3339 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -4,6 +4,7 @@ contents of any `#! nix` lines and the script's location to a single call. Verbatim strings may be passed in double backtick (```` `` ````) quotes. + `--expr` resolves relative paths based on the shebang script location. Some examples: ``` diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index e7f58556ffb..528643dc591 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -445,7 +445,8 @@ Installables SourceExprCommand::parseInstallables( else if (file) state->evalFile(lookupFileArg(*state, *file), *vFile); else { - auto e = state->parseExprFromString(*expr, state->rootPath(CanonPath::fromCwd())); + CanonPath dir(CanonPath::fromCwd(getCommandBaseDir())); + auto e = state->parseExprFromString(*expr, state->rootPath(dir)); state->eval(e, *vFile); } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index ab6e0e266f1..0012b3f47c4 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -276,6 +276,7 @@ void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) cmdline.push_back(word); } cmdline.push_back(script); + commandBaseDir = dirOf(script); for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++) cmdline.push_back(*pos); } @@ -336,6 +337,14 @@ void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) d.completer(*completions, d.n, d.prefix); } +Path Args::getCommandBaseDir() const +{ + if (parent) + return parent->getCommandBaseDir(); + else + return commandBaseDir; +} + bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) { assert(pos != end); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index e753dcaf60d..9c942606e3a 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -24,6 +24,16 @@ class AddCompletions; class Args { + /** + * @brief The command's "working directory", but only set when top level. + * + * Use getCommandBaseDir() to get the directory regardless of whether this + * is a top-level command or subcommand. + * + * @see getCommandBaseDir() + */ + Path commandBaseDir = "."; + public: /** @@ -44,6 +54,16 @@ public: */ virtual std::string doc() { return ""; } + /** + * @brief Get the base directory for the command. + * + * @return Generally the working directory, but in case of a shebang + * interpreter, returns the directory of the script. + * + * This only returns the correct value after parseCmdline() has run. + */ + Path getCommandBaseDir() const; + protected: /** diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index a0a34ffa9e1..76f3495dd5d 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -105,6 +105,24 @@ foo EOF chmod +x $nonFlakeDir/shebang-reject.sh +cat > $nonFlakeDir/shebang-inline-expr.sh <> $nonFlakeDir/shebang-inline-expr.sh <<"EOF" +#! nix --offline shell +#! nix --impure --expr `` +#! nix let flake = (builtins.getFlake (toString ../flake1)).packages; +#! nix fooScript = flake.${builtins.currentSystem}.fooScript; +#! nix /* just a comment !@#$%^&*()__+ # */ +#! nix in fooScript +#! nix `` +#! nix --no-write-lock-file --command bash +set -ex +foo +echo "$@" +EOF +chmod +x $nonFlakeDir/shebang-inline-expr.sh + # Construct a custom registry, additionally test the --registry flag nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" nix registry add --registry "$registry" flake2 "git+file://$percentEncodedFlake2Dir" @@ -552,4 +570,5 @@ expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock [[ $($nonFlakeDir/shebang.sh) = "foo" ]] [[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]] [[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]] +[[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' From 4344d4c1b84969d51555cffbd32ce0296de0f3fb Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 May 2023 19:56:04 +0200 Subject: [PATCH 12/17] nix: Parse --file relative to shebang script --- doc/manual/src/release-notes/rl-next.md | 2 +- src/libcmd/installables.cc | 5 +++-- tests/functional/flakes/flakes.sh | 20 ++++++++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 091199c3339..ab83d222c24 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -4,7 +4,7 @@ contents of any `#! nix` lines and the script's location to a single call. Verbatim strings may be passed in double backtick (```` `` ````) quotes. - `--expr` resolves relative paths based on the shebang script location. + `--file` and `--expr` resolve relative paths based on the script location. Some examples: ``` diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 528643dc591..d897a01c446 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -442,8 +442,9 @@ Installables SourceExprCommand::parseInstallables( auto e = state->parseStdin(); state->eval(e, *vFile); } - else if (file) - state->evalFile(lookupFileArg(*state, *file), *vFile); + else if (file) { + state->evalFile(lookupFileArg(*state, *file, CanonPath::fromCwd(getCommandBaseDir())), *vFile); + } else { CanonPath dir(CanonPath::fromCwd(getCommandBaseDir())); auto e = state->parseExprFromString(*expr, state->rootPath(dir)); diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 76f3495dd5d..ccf1699f9f0 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -123,6 +123,25 @@ echo "$@" EOF chmod +x $nonFlakeDir/shebang-inline-expr.sh +cat > $nonFlakeDir/fooScript.nix <<"EOF" +let flake = (builtins.getFlake (toString ../flake1)).packages; + fooScript = flake.${builtins.currentSystem}.fooScript; + in fooScript +EOF + +cat > $nonFlakeDir/shebang-file.sh <> $nonFlakeDir/shebang-file.sh <<"EOF" +#! nix --offline shell +#! nix --impure --file ./fooScript.nix +#! nix --no-write-lock-file --command bash +set -ex +foo +echo "$@" +EOF +chmod +x $nonFlakeDir/shebang-file.sh + # Construct a custom registry, additionally test the --registry flag nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" nix registry add --registry "$registry" flake2 "git+file://$percentEncodedFlake2Dir" @@ -571,4 +590,5 @@ expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock [[ $($nonFlakeDir/shebang.sh "bar") = "foo"$'\n'"bar" ]] [[ $($nonFlakeDir/shebang-comments.sh ) = "foo" ]] [[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] +[[ $($nonFlakeDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]] expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' From 9fcceaeb58aebf6f2052d548c61a34828dee28f1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 12 May 2023 19:57:36 +0200 Subject: [PATCH 13/17] nix/installables.cc: Use getCommandBaseDir() where possible These usages of the working directory are perhaps unlikely to interact with shebangs, but the code is more consistent this way, and we're less likely to miss usages that do interact. --- src/libcmd/installables.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index d897a01c446..f840865d215 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -88,7 +88,7 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath("."), true)); + parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true)); }}, .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { if (n == 0) { @@ -130,7 +130,7 @@ MixFlakeOptions::MixFlakeOptions() auto evalState = getEvalState(); auto flake = flake::lockFlake( *evalState, - parseFlakeRef(flakeRef, absPath(".")), + parseFlakeRef(flakeRef, absPath(getCommandBaseDir())), { .writeLockFile = false }); for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes @@ -294,7 +294,7 @@ void completeFlakeRefWithFragment( prefixRoot = "."; } auto flakeRefS = std::string(prefix.substr(0, hash)); - auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); + auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(getCommandBaseDir())); auto evalCache = openEvalCache(*evalState, std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); @@ -482,7 +482,7 @@ Installables SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(".")); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir())); result.push_back(make_ref( this, getEvalState(), @@ -756,7 +756,7 @@ std::vector RawInstallablesCommand::getFlakeRefsForCompletion() for (auto i : rawInstallables) res.push_back(parseFlakeRefWithFragment( expandTilde(i), - absPath(".")).first); + absPath(getCommandBaseDir())).first); return res; } @@ -778,7 +778,7 @@ std::vector InstallableCommand::getFlakeRefsForCompletion() return { parseFlakeRefWithFragment( expandTilde(_installable), - absPath(".")).first + absPath(getCommandBaseDir())).first }; } From e73466fec0cc5e0f7476c66f153df3905596279c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 23 Oct 2023 16:16:51 +0200 Subject: [PATCH 14/17] Move shebang docs from rl-next to nix.md --- doc/manual/src/release-notes/rl-next.md | 48 ------------------- src/nix/nix.md | 61 +++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 48 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index ab83d222c24..b7faa528785 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -2,54 +2,6 @@ - The experimental nix command is now a `#!-interpreter` by appending the contents of any `#! nix` lines and the script's location to a single call. - - Verbatim strings may be passed in double backtick (```` `` ````) quotes. - `--file` and `--expr` resolve relative paths based on the script location. - - Some examples: - ``` - #!/usr/bin/env nix - #! nix shell --file ```` hello --command bash - - hello | cowsay - ``` - or with flakes: - ``` - #!/usr/bin/env nix - #! nix shell nixpkgs#bash nixpkgs#hello nixpkgs#cowsay --command bash - - hello | cowsay - ``` - or - ```bash - #! /usr/bin/env nix - #! nix shell --impure --expr `` - #! nix with (import (builtins.getFlake "nixpkgs") {}); - #! nix terraform.withPlugins (plugins: [ plugins.openstack ]) - #! nix `` - #! nix --command bash - - terraform "$@" - ``` - or - ``` - #!/usr/bin/env nix - //! ```cargo - //! [dependencies] - //! time = "0.1.25" - //! ``` - /* - #!nix shell nixpkgs#rustc nixpkgs#rust-script nixpkgs#cargo --command rust-script - */ - fn main() { - for argument in std::env::args().skip(1) { - println!("{}", argument); - }; - println!("{}", std::env::var("HOME").expect("")); - println!("{}", time::now().rfc822z()); - } - // vim: ft=rust - ``` - [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. - [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). diff --git a/src/nix/nix.md b/src/nix/nix.md index 6e7e8a64935..5bf82a8bf23 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -238,4 +238,65 @@ operate are determined as follows: Most `nix` subcommands operate on a *Nix store*. These are documented in [`nix help-stores`](./nix3-help-stores.md). +# Shebang interpreter + +The `nix` command can be used as a `#!` interpreter. +Arguments to Nix can be passed on subsequent lines in the script. + +Verbatim strings may be passed in double backtick (```` `` ````) quotes. + +`--file` and `--expr` resolve relative paths based on the script location. + +Examples: + +``` +#!/usr/bin/env nix +#! nix shell --file ```` hello cowsay --command bash + +hello | cowsay +``` + +or with **flakes**: + +``` +#!/usr/bin/env nix +#! nix shell nixpkgs#bash nixpkgs#hello nixpkgs#cowsay --command bash + +hello | cowsay +``` + +or with an **expression**: + +```bash +#! /usr/bin/env nix +#! nix shell --impure --expr `` +#! nix with (import (builtins.getFlake "nixpkgs") {}); +#! nix terraform.withPlugins (plugins: [ plugins.openstack ]) +#! nix `` +#! nix --command bash + +terraform "$@" +``` + +or with cascading interpreters. Note that the `#! nix` lines don't need to follow after the first line, to accomodate other interpreters. + +``` +#!/usr/bin/env nix +//! ```cargo +//! [dependencies] +//! time = "0.1.25" +//! ``` +/* +#!nix shell nixpkgs#rustc nixpkgs#rust-script nixpkgs#cargo --command rust-script +*/ +fn main() { + for argument in std::env::args().skip(1) { + println!("{}", argument); + }; + println!("{}", std::env::var("HOME").expect("")); + println!("{}", time::now().rfc822z()); +} +// vim: ft=rust +``` + )"" From d6559467088e025cee798a72712c63089ea65581 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 23 Oct 2023 18:38:54 +0200 Subject: [PATCH 15/17] Fix nix shebang interaction with #8131 overhaul completions --- src/libcmd/installables.cc | 4 +++- src/libutil/args.cc | 19 ++++++++----------- src/libutil/args.hh | 17 +---------------- src/libutil/args/root.hh | 14 +++++++++++++- 4 files changed, 25 insertions(+), 29 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index f840865d215..1c6103020f7 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -294,7 +294,9 @@ void completeFlakeRefWithFragment( prefixRoot = "."; } auto flakeRefS = std::string(prefix.substr(0, hash)); - auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(getCommandBaseDir())); + + // TODO: ideally this would use the command base directory instead of assuming ".". + auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 0012b3f47c4..5ba1e5c5528 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -80,12 +80,6 @@ std::optional RootArgs::needsCompletion(std::string_view s) return {}; } -void RootArgs::parseCmdline(const Strings & _cmdline) -{ - // Default via 5.1.2.2.1 in C standard - Args::parseCmdline(_cmdline, false); -} - /** * Basically this is `typedef std::optional Parser(std::string_view s, Strings & r);` * @@ -227,7 +221,7 @@ static Strings parseShebangContent(std::string_view s) { return result; } -void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) +void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) { Strings pendingArgs; bool dashDash = false; @@ -339,10 +333,13 @@ void Args::parseCmdline(const Strings & _cmdline, bool allowShebang) Path Args::getCommandBaseDir() const { - if (parent) - return parent->getCommandBaseDir(); - else - return commandBaseDir; + assert(parent); + return parent->getCommandBaseDir(); +} + +Path RootArgs::getCommandBaseDir() const +{ + return commandBaseDir; } bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 9c942606e3a..30a44cd1036 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -24,24 +24,9 @@ class AddCompletions; class Args { - /** - * @brief The command's "working directory", but only set when top level. - * - * Use getCommandBaseDir() to get the directory regardless of whether this - * is a top-level command or subcommand. - * - * @see getCommandBaseDir() - */ - Path commandBaseDir = "."; public: - /** - * Parse the command line with argv0, throwing a UsageError if something - goes wrong. - */ - void parseCmdline(const Strings & _cmdline, bool allowShebang); - /** * Return a short one-line description of the command. */ @@ -62,7 +47,7 @@ public: * * This only returns the correct value after parseCmdline() has run. */ - Path getCommandBaseDir() const; + virtual Path getCommandBaseDir() const; protected: diff --git a/src/libutil/args/root.hh b/src/libutil/args/root.hh index bb98732a15d..5c55c37a55d 100644 --- a/src/libutil/args/root.hh +++ b/src/libutil/args/root.hh @@ -29,14 +29,26 @@ struct Completions final : AddCompletions */ class RootArgs : virtual public Args { + /** + * @brief The command's "working directory", but only set when top level. + * + * Use getCommandBaseDir() to get the directory regardless of whether this + * is a top-level command or subcommand. + * + * @see getCommandBaseDir() + */ + Path commandBaseDir = "."; + public: /** Parse the command line, throwing a UsageError if something goes * wrong. */ - void parseCmdline(const Strings & cmdline); + void parseCmdline(const Strings & cmdline, bool allowShebang = false); std::shared_ptr completions; + Path getCommandBaseDir() const override; + protected: friend class Args; From 32f75a8d7504c7bd47656064433c94515d295b00 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 6 Nov 2023 18:19:14 +0100 Subject: [PATCH 16/17] parseShebangs: Make strings with backtick sequences representable --- src/libutil/args.cc | 32 ++++++++++++- src/libutil/args.hh | 2 + src/libutil/tests/args.cc | 94 +++++++++++++++++++++++++++++++++++++++ src/nix/nix.md | 4 +- 4 files changed, 129 insertions(+), 3 deletions(-) create mode 100644 src/libutil/tests/args.cc diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 5ba1e5c5528..4359c5e8ede 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -189,12 +189,40 @@ void ParseQuoted::operator()(std::shared_ptr &state, Strings & r) { throw Error("unterminated quoted string in nix shebang"); } switch (remaining[0]) { + case ' ': + if ((remaining.size() == 3 && remaining[1] == '`' && remaining[2] == '`') + || (remaining.size() > 3 && remaining[1] == '`' && remaining[2] == '`' && remaining[3] != '`')) { + // exactly two backticks mark the end of a quoted string, but a preceding space is ignored if present. + state = std::make_shared(ParseUnquoted(remaining.substr(3))); + r.push_back(acc); + return; + } + else { + // just a normal space + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } case '`': - if (remaining.size() > 1 && remaining[1] == '`') { + // exactly two backticks mark the end of a quoted string + if ((remaining.size() == 2 && remaining[1] == '`') + || (remaining.size() > 2 && remaining[1] == '`' && remaining[2] != '`')) { state = std::make_shared(ParseUnquoted(remaining.substr(2))); r.push_back(acc); return; } + + // a sequence of at least 3 backticks is one escape-backtick which is ignored, followed by any number of backticks, which are verbatim + else if (remaining.size() >= 3 && remaining[1] == '`' && remaining[2] == '`') { + // ignore "escape" backtick + remaining = remaining.substr(1); + // add the rest + while (remaining.size() > 0 && remaining[0] == '`') { + acc += '`'; + remaining = remaining.substr(1); + } + return; + } else { acc += remaining[0]; remaining = remaining.substr(1); @@ -208,7 +236,7 @@ void ParseQuoted::operator()(std::shared_ptr &state, Strings & r) { assert(false); } -static Strings parseShebangContent(std::string_view s) { +Strings parseShebangContent(std::string_view s) { Strings result; std::shared_ptr parserState(std::make_shared(ParseUnquoted(s))); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 30a44cd1036..7af82b17884 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -409,4 +409,6 @@ public: virtual void add(std::string completion, std::string description = "") = 0; }; +Strings parseShebangContent(std::string_view s); + } diff --git a/src/libutil/tests/args.cc b/src/libutil/tests/args.cc new file mode 100644 index 00000000000..e7a16b0be46 --- /dev/null +++ b/src/libutil/tests/args.cc @@ -0,0 +1,94 @@ +#include "../args.hh" +#include + +#include + +namespace nix { + + TEST(parseShebangContent, basic) { + std::list r = parseShebangContent("hi there"); + ASSERT_EQ(r.size(), 2); + auto i = r.begin(); + ASSERT_EQ(*i++, "hi"); + ASSERT_EQ(*i++, "there"); + } + + TEST(parseShebangContent, empty) { + std::list r = parseShebangContent(""); + ASSERT_EQ(r.size(), 0); + } + + TEST(parseShebangContent, doubleBacktick) { + std::list r = parseShebangContent("``\"ain't that nice\"``"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, "\"ain't that nice\""); + } + + TEST(parseShebangContent, doubleBacktickEmpty) { + std::list r = parseShebangContent("````"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, ""); + } + + TEST(parseShebangContent, doubleBacktickMarkdownInlineCode) { + std::list r = parseShebangContent("``# I'm markdown section about `coolFunction` ``"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, "# I'm markdown section about `coolFunction`"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlockNaive) { + std::list r = parseShebangContent("``Example 1\n```nix\na: a\n``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n``nix\na: a\n``"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlockCorrect) { + std::list r = parseShebangContent("``Example 1\n````nix\na: a\n```` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n```nix\na: a\n```"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlock2) { + std::list r = parseShebangContent("``Example 1\n````nix\na: a\n````\nExample 2\n````nix\na: a\n```` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n```nix\na: a\n```\nExample 2\n```nix\na: a\n```"); + } + + TEST(parseShebangContent, singleBacktickInDoubleBacktickQuotes) { + std::list r = parseShebangContent("``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "`"); + } + + TEST(parseShebangContent, singleBacktickAndSpaceInDoubleBacktickQuotes) { + std::list r = parseShebangContent("``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "` "); + } + + TEST(parseShebangContent, doubleBacktickInDoubleBacktickQuotes) { + std::list r = parseShebangContent("````` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "``"); + } + + TEST(parseShebangContent, increasingQuotes) { + std::list r = parseShebangContent("```` ``` `` ````` `` `````` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 4); + ASSERT_EQ(*i++, ""); + ASSERT_EQ(*i++, "`"); + ASSERT_EQ(*i++, "``"); + ASSERT_EQ(*i++, "```"); + } + +} \ No newline at end of file diff --git a/src/nix/nix.md b/src/nix/nix.md index 5bf82a8bf23..eb150f03b41 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -243,7 +243,9 @@ in [`nix help-stores`](./nix3-help-stores.md). The `nix` command can be used as a `#!` interpreter. Arguments to Nix can be passed on subsequent lines in the script. -Verbatim strings may be passed in double backtick (```` `` ````) quotes. +Verbatim strings may be passed in double backtick (```` `` ````) quotes. +Sequences of _n_ backticks of 3 or longer are parsed as _n-1_ literal backticks. +A single space before the closing ```` `` ```` is ignored if present. `--file` and `--expr` resolve relative paths based on the script location. From 8387c43c2618a4d8f954072bfc8d36b8c6e76a50 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 6 Nov 2023 19:15:36 +0100 Subject: [PATCH 17/17] Test parseShebangContent round trip --- src/libutil/tests/args.cc | 74 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/src/libutil/tests/args.cc b/src/libutil/tests/args.cc index e7a16b0be46..bea74a8c839 100644 --- a/src/libutil/tests/args.cc +++ b/src/libutil/tests/args.cc @@ -1,7 +1,9 @@ #include "../args.hh" +#include "libutil/fs-sink.hh" #include #include +#include namespace nix { @@ -91,4 +93,76 @@ namespace nix { ASSERT_EQ(*i++, "```"); } + +#ifndef COVERAGE + +// quick and dirty +static inline std::string escape(std::string_view s_) { + + std::string_view s = s_; + std::string r = "``"; + + // make a guess to allocate ahead of time + r.reserve( + // plain chars + s.size() + // quotes + + 5 + // some "escape" backticks + + s.size() / 8); + + while (!s.empty()) { + if (s[0] == '`' && s.size() >= 2 && s[1] == '`') { + // escape it + r += "`"; + while (!s.empty() && s[0] == '`') { + r += "`"; + s = s.substr(1); + } + } else { + r += s[0]; + s = s.substr(1); + } + } + + if (!r.empty() + && ( + r[r.size() - 1] == '`' + || r[r.size() - 1] == ' ' + )) { + r += " "; + } + + r += "``"; + + return r; +}; + +RC_GTEST_PROP( + parseShebangContent, + prop_round_trip_single, + (const std::string & orig)) +{ + auto escaped = escape(orig); + // RC_LOG() << "escaped: <[[" << escaped << "]]>" << std::endl; + auto ss = parseShebangContent(escaped); + RC_ASSERT(ss.size() == 1); + RC_ASSERT(*ss.begin() == orig); +} + +RC_GTEST_PROP( + parseShebangContent, + prop_round_trip_two, + (const std::string & one, const std::string & two)) +{ + auto ss = parseShebangContent(escape(one) + " " + escape(two)); + RC_ASSERT(ss.size() == 2); + auto i = ss.begin(); + RC_ASSERT(*i++ == one); + RC_ASSERT(*i++ == two); +} + + +#endif + } \ No newline at end of file