From 156b0be76c398e56ee7b7783593cbc0d5fe64a58 Mon Sep 17 00:00:00 2001 From: Markus Kuhn Date: Fri, 9 Jul 2021 19:52:20 +0100 Subject: [PATCH] Distributed: add C shell support (fixes #32690) (#41485) The Unix C Shell (csh, tcsh) remains in common use as a login shell in some communities. It is not compatible with POSIX shells (bash, dash, ksh, zsh, etc.) and requires different syntax and escaping. This patch adds the function `Base.shell_escape_csh()`, a C shell equivalent of `Base.shell_escape_posixly()`, along with a new option `:csh` for the `shell` argument of `Distributed.addprocs()`, for use if the login shell on worker accounts is a C shell. --- base/cmd.jl | 2 ++ base/shell.jl | 47 +++++++++++++++++++++++++++ stdlib/Distributed/src/Distributed.jl | 3 +- stdlib/Distributed/src/managers.jl | 21 +++++++++++- test/spawn.jl | 14 ++++++++ 5 files changed, 85 insertions(+), 2 deletions(-) diff --git a/base/cmd.jl b/base/cmd.jl index 0d66cb932a04a..ff52191fc51ff 100644 --- a/base/cmd.jl +++ b/base/cmd.jl @@ -103,6 +103,8 @@ shell_escape(cmd::Cmd; special::AbstractString="") = shell_escape(cmd.exec..., special=special) shell_escape_posixly(cmd::Cmd) = shell_escape_posixly(cmd.exec...) +shell_escape_csh(cmd::Cmd) = + shell_escape_csh(cmd.exec...) escape_microsoft_c_args(cmd::Cmd) = escape_microsoft_c_args(cmd.exec...) escape_microsoft_c_args(io::IO, cmd::Cmd) = diff --git a/base/shell.jl b/base/shell.jl index 2efe61f4bbfdb..bcece48681e5c 100644 --- a/base/shell.jl +++ b/base/shell.jl @@ -261,6 +261,53 @@ julia> Base.shell_escape_posixly("echo", "this", "&&", "that") shell_escape_posixly(args::AbstractString...) = sprint(print_shell_escaped_posixly, args...) +""" + shell_escape_csh(args::Union{Cmd,AbstractString...}) + shell_escape_csh(io::IO, args::Union{Cmd,AbstractString...}) + +This function quotes any metacharacters in the string arguments such +that the string returned can be inserted into a command-line for +interpretation by the Unix C shell (csh, tcsh), where each string +argument will form one word. + +In contrast to a POSIX shell, csh does not support the use of the +backslash as a general escape character in double-quoted strings. +Therefore, this function wraps strings that might contain +metacharacters in single quotes, except for parts that contain single +quotes, which it wraps in double quotes instead. It switches between +these types of quotes as needed. Linefeed characters are escaped with +a backslash. + +This function should also work for a POSIX shell, except if the input +string contains a linefeed (`"\\n"`) character. + +See also: [`shell_escape_posixly`](@ref) +""" +function shell_escape_csh(io::IO, args::AbstractString...) + first = true + for arg in args + first || write(io, ' ') + first = false + i = 1 + while true + for (r,e) = (r"^[A-Za-z0-9/\._-]+\z" => "", + r"^[^']*\z" => "'", r"^[^\$\`\"]*\z" => "\"", + r"^[^']+" => "'", r"^[^\$\`\"]+" => "\"") + if ((m = match(r, SubString(arg, i))) !== nothing) + write(io, e) + write(io, replace(m.match, '\n' => "\\\n")) + write(io, e) + i += ncodeunits(m.match) + break + end + end + i <= lastindex(arg) || break + end + end +end +shell_escape_csh(args::AbstractString...) = + sprint(shell_escape_csh, args...; + sizehint = sum(sizeof.(args)) + length(args) * 3) """ shell_escape_wincmd(s::AbstractString) diff --git a/stdlib/Distributed/src/Distributed.jl b/stdlib/Distributed/src/Distributed.jl index 50108f05eed26..dd9101fa1b4ce 100644 --- a/stdlib/Distributed/src/Distributed.jl +++ b/stdlib/Distributed/src/Distributed.jl @@ -13,7 +13,8 @@ import Base: getindex, wait, put!, take!, fetch, isready, push!, length, using Base: Process, Semaphore, JLOptions, buffer_writes, @sync_add, VERSION_STRING, binding_module, atexit, julia_exename, julia_cmd, AsyncGenerator, acquire, release, invokelatest, - shell_escape_posixly, shell_escape_wincmd, escape_microsoft_c_args, + shell_escape_posixly, shell_escape_csh, + shell_escape_wincmd, escape_microsoft_c_args, uv_error, something, notnothing, isbuffered, mapany using Base.Threads: Event diff --git a/stdlib/Distributed/src/managers.jl b/stdlib/Distributed/src/managers.jl index ce99d85801e17..08686fc2a0b87 100644 --- a/stdlib/Distributed/src/managers.jl +++ b/stdlib/Distributed/src/managers.jl @@ -82,7 +82,10 @@ Keyword arguments: * `shell`: specifies the type of shell to which ssh connects on the workers. - + `shell=:posix`: a POSIX-compatible Unix/Linux shell (bash, sh, etc.). The default. + + `shell=:posix`: a POSIX-compatible Unix/Linux shell + (sh, ksh, bash, dash, zsh, etc.). The default. + + + `shell=:csh`: a Unix C shell (csh, tcsh). + `shell=:wincmd`: Microsoft Windows `cmd.exe`. @@ -281,6 +284,22 @@ function launch_on_machine(manager::SSHManager, machine::AbstractString, cnt, pa # shell login (-l) with string command (-c) to launch julia process remotecmd = shell_escape_posixly(`sh -l -c $cmds`) + elseif shell == :csh + # ssh connects to (t)csh + + remotecmd = "$(shell_escape_csh(exename)) $(shell_escape_csh(exeflags))" + + # set environment variables + for (var, val) in env + occursin(r"^[a-zA-Z_][a-zA-Z_0-9]*\z", var) || + throw(ArgumentError("invalid env key $var")) + remotecmd = "setenv $(var) $(shell_escape_csh(val))\n$remotecmd" + end + # change working directory + if dir !== nothing && dir != "" + remotecmd = "cd $(shell_escape_csh(dir))\n$remotecmd" + end + elseif shell == :wincmd # ssh connects to Windows cmd.exe diff --git a/test/spawn.jl b/test/spawn.jl index fe6912faf0447..95915a5ad804b 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -812,6 +812,20 @@ if Sys.iswindows() end +# test (t)csh escaping if tcsh is installed +cshcmd = "/bin/tcsh" +if isfile(cshcmd) + csh_echo(s) = chop(read(Cmd([cshcmd, "-c", + "echo " * Base.shell_escape_csh(s)]), String)) + csh_test(s) = csh_echo(s) == s + @testset "shell_escape_csh" begin + for s in ["", "-a/b", "'", "'£\"", join(' ':'~') ^ 2, + "\t", "\n", "'\n", "\"\n", "'\n\n\""] + @test csh_test(s) + end + end +end + @testset "shell escaping on Windows" begin # Note argument A can be parsed both as A or "A". # We do not test that the parsing satisfies either of these conditions.