From fdfba8501697e50b2fb3dd6b7d62944029705227 Mon Sep 17 00:00:00 2001 From: Stephane Thiell Date: Mon, 25 Sep 2023 23:20:14 -0700 Subject: [PATCH] clush: fix --[r]copy dest when --dest is omitted (#525) When multiple files or directories are specified as arguments with --[r]copy, and --dest is omitted, use each argument's dirnname for each destination, instead of the dirname of the first argument only. Fixes #525. --- doc/man/man1/clush.1 | 61 ++++++++++++++++++----------------- doc/sphinx/tools/clush.rst | 17 ++++++---- doc/txt/clush.txt | 11 ++++--- lib/ClusterShell/CLI/Clush.py | 54 ++++++++++++++++++------------- tests/CLIClushTest.py | 54 +++++++++++++++++++++++++------ 5 files changed, 126 insertions(+), 71 deletions(-) diff --git a/doc/man/man1/clush.1 b/doc/man/man1/clush.1 index 5419d834..278c0698 100644 --- a/doc/man/man1/clush.1 +++ b/doc/man/man1/clush.1 @@ -1,5 +1,8 @@ .\" Man page generated from reStructuredText. . +.TH CLUSH 1 "2023-02-09" "1.9.1" "ClusterShell User Manual" +.SH NAME +clush \- execute shell commands on a cluster . .nr rst2man-indent-level 0 . @@ -27,9 +30,6 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "CLUSH" 1 "2023-02-09" "1.9.1" "ClusterShell User Manual" -.SH NAME -clush \- execute shell commands on a cluster .SH SYNOPSIS .sp \fBclush\fP \fB\-a\fP | \fB\-g\fP \fIgroup\fP | \fB\-w\fP \fInodes\fP [OPTIONS] @@ -138,32 +138,35 @@ command for each node. \fB%h\fP or \fB%host\fP will be replaced by node name and .TP .B File copying mode ( \fB\-\-copy\fP ) When \fBclush\fP is started with the \fB\-c\fP or \fB\-\-copy\fP option, it will -attempt to copy specified \fIfile\fP and/or \fIdir\fP to the provided target -cluster nodes. If the \fB\-\-dest\fP option is specified, it will put the -copied files there. +attempt to copy specified \fIfiles\fP and/or \fIdirectories\fP to the provided +cluster nodes. The \fB\-\-dest\fP option can be used to specify a single +path where all the file(s) should be copied to on the target nodes. +In the absence of \fB\-\-dest\fP, \fBclush\fP will attempt to copy each file or +directory found in the command line to their same location on the +target nodes. .TP .B Reverse file copying mode ( \fB\-\-rcopy\fP ) When \fBclush\fP is started with the \fB\-\-rcopy\fP option, it will attempt to retrieve specified \fIfile\fP and/or \fIdir\fP from provided cluster nodes. If the \fB\-\-dest\fP option is specified, it must be a directory path where the files will be stored with their hostname appended. If the destination path is not -specified, it will take the first \fIfile\fP or \fIdir\fP basename directory as the +specified, it will take each \fIfile\fP or \fIdirectory\fP\(aqs parent directory as the local destination. .UNINDENT .SH OPTIONS .INDENT 0.0 .TP -.B \-\-version +.B \-\-version show \fBclush\fP version number and exit .TP .BI \-s \ GROUPSOURCE\fR,\fB \ \-\-groupsource\fB= GROUPSOURCE optional \fBgroups.conf\fP(5) group source to use .TP -.B \-n\fP,\fB \-\-nostdin +.B \-n\fP,\fB \-\-nostdin do not watch for possible input from stdin; this should be used when \fBclush\fP is run in the background (or in scripts). .TP -.B \-\-sudo -enable sudo password prompt: a prompt will ask for your sudo password and sudo will be used to run your commands on the target nodes. The password must be the same on all target nodes. The actual sudo command used by \fBclush\fP can be changed in \fBclush.conf\fP(5) or in command line using \fB\-O sudo_command=\(dq...\(dq\fP\&. The configured \fBsudo_command\fP must be able to read a password on stdin followed by a new line (which is what \fBsudo \-S\fP does). +.B \-\-sudo +enable sudo password prompt: a prompt will ask for your sudo password and sudo will be used to run your commands on the target nodes. The password must be the same on all target nodes. The actual sudo command used by \fBclush\fP can be changed in \fBclush.conf\fP(5) or in command line using \fB\-O sudo_command="..."\fP\&. The configured \fBsudo_command\fP must be able to read a password on stdin followed by a new line (which is what \fBsudo \-S\fP does). .TP .BI \-\-groupsconf\fB= FILE use alternate config file for groups.conf(5) @@ -185,7 +188,7 @@ nodes where to run the command .BI \-x \ NODES exclude nodes from the node list .TP -.B \-a\fP,\fB \-\-all +.B \-a\fP,\fB \-\-all run command on all nodes .TP .BI \-g \ GROUP\fR,\fB \ \-\-group\fB= GROUP @@ -207,43 +210,43 @@ pick N node(s) at random in nodeset .B Output behaviour: .INDENT 7.0 .TP -.B \-q\fP,\fB \-\-quiet +.B \-q\fP,\fB \-\-quiet be quiet, print essential output only .TP -.B \-v\fP,\fB \-\-verbose +.B \-v\fP,\fB \-\-verbose be verbose, print informative messages .TP -.B \-d\fP,\fB \-\-debug +.B \-d\fP,\fB \-\-debug output more messages for debugging purpose .TP -.B \-G\fP,\fB \-\-groupbase +.B \-G\fP,\fB \-\-groupbase do not display group source prefix .TP -.B \-L -disable header block and order output by nodes; if \-b/\-B is not specified, \fBclush\fP will wait for all commands to finish and then display aggregated output of commands with same return codes, ordered by node name; alternatively, when used in conjunction with \-b/\-B (eg. \-bL), \fBclush\fP will enable a \(dqlife gathering\(dq of results by line, such as the next line is displayed as soon as possible (eg. when all nodes have sent the line) +.B \-L +disable header block and order output by nodes; if \-b/\-B is not specified, \fBclush\fP will wait for all commands to finish and then display aggregated output of commands with same return codes, ordered by node name; alternatively, when used in conjunction with \-b/\-B (eg. \-bL), \fBclush\fP will enable a "life gathering" of results by line, such as the next line is displayed as soon as possible (eg. when all nodes have sent the line) .TP -.B \-N +.B \-N disable labeling of command line .TP -.B \-P\fP,\fB \-\-progress +.B \-P\fP,\fB \-\-progress show progress during command execution; if writing is performed to standard input, the live progress indicator will display the global bandwidth of data written to the target nodes .TP -.B \-b\fP,\fB \-\-dshbak +.B \-b\fP,\fB \-\-dshbak display gathered results in a dshbak\-like way (note: it will only try to aggregate the output of commands with same return codes) .TP -.B \-B +.B \-B like \-b but including standard error .TP -.B \-r\fP,\fB \-\-regroup +.B \-r\fP,\fB \-\-regroup fold nodeset using node groups .TP -.B \-S\fP,\fB \-\-maxrc +.B \-S\fP,\fB \-\-maxrc return the largest of command return codes .TP .BI \-\-color\fB= WHENCOLOR \fBclush\fP can use NO_COLOR, CLICOLOR and CLICOLOR_FORCE environment variables. NO_COLOR takes precedence over CLICOLOR_FORCE which takes precedence over CLICOLOR. When \fB\-\-color\fP option is used these environment variables are not taken into account. \fB\-\-color\fP tells whether to use ANSI colors to surround node or nodeset prefix/header with escape sequences to display them in color on the terminal. \fIWHENCOLOR\fP is \fBnever\fP, \fBalways\fP or \fBauto\fP (which use color if standard output/error refer to a terminal). Colors are set to [34m (blue foreground text) for stdout and [31m (red foreground text) for stderr, and cannot be modified. .TP -.B \-\-diff +.B \-\-diff show diff between common outputs (find the best reference output by focusing on largest nodeset and also smaller command return code) .TP .BI \-\-outdir\fB= OUTDIR @@ -256,10 +259,10 @@ output directory for stderr files (OPTIONAL) .B File copying: .INDENT 7.0 .TP -.B \-c\fP,\fB \-\-copy +.B \-c\fP,\fB \-\-copy copy local file or directory to remote nodes .TP -.B \-\-rcopy +.B \-\-rcopy copy file or directory from remote nodes .TP .BI \-\-dest\fB= DEST_PATH @@ -267,7 +270,7 @@ destination file or directory on the nodes (optional: use the first source directory path when not specified) .TP -.B \-p +.B \-p preserve modification times and modes .UNINDENT .TP @@ -281,7 +284,7 @@ do not execute more than FANOUT commands at the same time, useful to limit resou execute remote command as user .TP .BI \-o \ OPTIONS\fR,\fB \ \-\-options\fB= OPTIONS -can be used to give ssh options, eg. \fB\-o \(dq\-p 2022 \-i ~/.ssh/myidrsa\(dq\fP; these options are added first to ssh and override default ones +can be used to give ssh options, eg. \fB\-o "\-p 2022 \-i ~/.ssh/myidrsa"\fP; these options are added first to ssh and override default ones .TP .BI \-t \ CONNECT_TIMEOUT\fR,\fB \ \-\-connect_timeout\fB= CONNECT_TIMEOUT limit time to connect to a node diff --git a/doc/sphinx/tools/clush.rst b/doc/sphinx/tools/clush.rst index 04535d70..b00fbc60 100644 --- a/doc/sphinx/tools/clush.rst +++ b/doc/sphinx/tools/clush.rst @@ -562,9 +562,12 @@ File copying mode ^^^^^^^^^^^^^^^^^ When *clush* is started with the ``-c`` or ``--copy`` option, it will -attempt to copy specified file and/or directory to the provided target cluster -nodes. If the ``--dest`` option is specified, it will put the copied files -or directory there. +attempt to copy specified files and/or directories to the provided cluster +nodes. The ``--dest`` option can be used to specify a single path where all +the file(s) should be copied to on the target nodes. +In the absence of ``--dest``, *clush* will attempt to copy each file or +directory found in the command line to their same location on the target +nodes. Here are some examples of file copying with *clush*:: @@ -572,8 +575,8 @@ Here are some examples of file copying with *clush*:: `/tmp/foo' -> node[11-12]:`/tmp' $ clush -v -w node[11-12] --copy /tmp/foo /tmp/bar - `/tmp/bar' -> aury[11-12]:`/tmp' - `/tmp/foo' -> aury[11-12]:`/tmp' + `/tmp/bar' -> node[11-12]:`/tmp' + `/tmp/foo' -> node[11-12]:`/tmp' $ clush -v -w node[11-12] --copy /tmp/foo --dest /var/tmp/ `/tmp/foo' -> node[11-12]:`/var/tmp/' @@ -588,8 +591,8 @@ When *clush* is started with the ``--rcopy`` option, it will attempt to retrieve specified file and/or directory from provided cluster nodes. If the ``--dest`` option is specified, it must be a directory path where the files will be stored with their hostname appended. If the destination path is not -specified, it will take the first file or dir basename directory as the local -destination, example:: +specified, it will take each file or directory's parent directory as the +local destination, for example:: $ clush -v -w node[11-12] --rcopy /tmp/foo node[11-12]:`/tmp/foo' -> `/tmp' diff --git a/doc/txt/clush.txt b/doc/txt/clush.txt index 6c355041..5f31e26e 100644 --- a/doc/txt/clush.txt +++ b/doc/txt/clush.txt @@ -110,16 +110,19 @@ Local execution ( ``--worker=exec`` or ``-R exec`` ) File copying mode ( ``--copy`` ) When ``clush`` is started with the ``-c`` or ``--copy`` option, it will - attempt to copy specified *file* and/or *dir* to the provided target - cluster nodes. If the ``--dest`` option is specified, it will put the - copied files there. + attempt to copy specified *files* and/or *directories* to the provided + cluster nodes. The ``--dest`` option can be used to specify a single + path where all the file(s) should be copied to on the target nodes. + In the absence of ``--dest``, ``clush`` will attempt to copy each file or + directory found in the command line to their same location on the + target nodes. Reverse file copying mode ( ``--rcopy`` ) When ``clush`` is started with the ``--rcopy`` option, it will attempt to retrieve specified *file* and/or *dir* from provided cluster nodes. If the ``--dest`` option is specified, it must be a directory path where the files will be stored with their hostname appended. If the destination path is not - specified, it will take the first *file* or *dir* basename directory as the + specified, it will take each *file* or *directory*'s parent directory as the local destination. diff --git a/lib/ClusterShell/CLI/Clush.py b/lib/ClusterShell/CLI/Clush.py index 59c4dbb4..51a6d339 100755 --- a/lib/ClusterShell/CLI/Clush.py +++ b/lib/ClusterShell/CLI/Clush.py @@ -733,7 +733,7 @@ def run_command(task, cmd, ns, timeout, display, remote, trytree): worker.set_write_eof() # we only enabled stdin to send the password task.resume() -def run_copy(task, sources, dest, ns, timeout, preserve_flag, display): +def run_copy(task, sources, dests, ns, timeout, preserve_flag, display): """run copy command""" task.set_default("USER_running", True) task.set_default("USER_copies", len(sources)) @@ -748,31 +748,32 @@ def run_copy(task, sources, dest, ns, timeout, preserve_flag, display): display.vprint_err(VERB_QUIET, 'ERROR: file "%s" not found' % source) clush_exit(1, task) - task.copy(source, dest, ns, handler=copyhandler, timeout=timeout, - preserve=preserve_flag) + task.copy(source, dests.pop(0), ns, handler=copyhandler, + timeout=timeout, preserve=preserve_flag) task.resume() -def run_rcopy(task, sources, dest, ns, timeout, preserve_flag, display): +def run_rcopy(task, sources, dests, ns, timeout, preserve_flag, display): """run reverse copy command""" task.set_default("USER_running", True) task.set_default("USER_copies", len(sources)) # Sanity checks - if not exists(dest): - display.vprint_err(VERB_QUIET, - 'ERROR: directory "%s" not found' % dest) - clush_exit(1, task) - if not isdir(dest): - display.vprint_err(VERB_QUIET, - 'ERROR: destination "%s" is not a directory' % dest) - clush_exit(1, task) + for dest in dests: + if not exists(dest): + display.vprint_err(VERB_QUIET, + 'ERROR: directory "%s" not found' % dest) + clush_exit(1, task) + if not isdir(dest): + display.vprint_err(VERB_QUIET, + 'ERROR: destination "%s" is not a directory' % dest) + clush_exit(1, task) copyhandler = CopyOutputHandler(display, True) if display.verbosity == VERB_STD or display.verbosity == VERB_VERB: copyhandler.runtimer_init(task, len(ns) * len(sources)) for source in sources: - task.rcopy(source, dest, ns, handler=copyhandler, timeout=timeout, - stderr=True, preserve=preserve_flag) + task.rcopy(source, dests.pop(0), ns, handler=copyhandler, + timeout=timeout, stderr=True, preserve=preserve_flag) task.resume() def set_fdlimit(fd_max, display): @@ -1122,15 +1123,24 @@ def main(): if (options.copy or options.rcopy) and not args: parser.error("--[r]copy option requires at least one argument") + dest_paths = [] if options.copy: - if not options.dest_path: + if options.dest_path: + for arg in args: + dest_paths.append(options.dest_path) + else: # append '/' to clearly indicate a directory for tree mode - options.dest_path = join(dirname(abspath(args[0])), '') - op = "copy sources=%s dest=%s" % (args, options.dest_path) + for arg in args: + dest_paths.append(join(dirname(abspath(arg)), '')) + op = "copy sources=%s dest=%s" % (args, dest_paths) elif options.rcopy: - if not options.dest_path: - options.dest_path = dirname(abspath(args[0])) - op = "rcopy sources=%s dest=%s" % (args, options.dest_path) + if options.dest_path: + for arg in args: + dest_paths.append(options.dest_path) + else: + for arg in args: + dest_paths.append(dirname(abspath(arg))) + op = "rcopy sources=%s dest=%s" % (args, dest_paths) else: op = "command=\"%s\"" % ' '.join(args) @@ -1147,10 +1157,10 @@ def main(): print(Display.COLOR_RESULT_FMT % task.topology, end='') print(Display.COLOR_RESULT_FMT % '-' * 15) if options.copy: - run_copy(task, args, options.dest_path, nodeset_base, timeout, + run_copy(task, args, dest_paths, nodeset_base, timeout, options.preserve_flag, display) elif options.rcopy: - run_rcopy(task, args, options.dest_path, nodeset_base, timeout, + run_rcopy(task, args, dest_paths, nodeset_base, timeout, options.preserve_flag, display) else: run_command(task, ' '.join(args), nodeset_base, timeout, display, diff --git a/tests/CLIClushTest.py b/tests/CLIClushTest.py index 2d731946..01fb08b2 100644 --- a/tests/CLIClushTest.py +++ b/tests/CLIClushTest.py @@ -6,7 +6,7 @@ import codecs import errno import os -from os.path import basename +from os.path import basename, dirname import pwd import re import resource @@ -183,28 +183,64 @@ def test_008_file_copy(self): """test clush (file copy)""" content = "%f" % time.time() content = content.encode() - f = make_temp_file(content) - self._clush_t(["-w", HOSTNAME, "-c", f.name], None, b"") - f.seek(0) - self.assertEqual(f.read(), content) + sf = make_temp_file(content) + self._clush_t(["-w", HOSTNAME, "-c", sf.name], None, b"") + sf.seek(0) + self.assertEqual(sf.read(), content) # test --dest option f2 = tempfile.NamedTemporaryFile() - self._clush_t(["-w", HOSTNAME, "-c", f.name, "--dest", f2.name], None, + self._clush_t(["-w", HOSTNAME, "-c", sf.name, "--dest", f2.name], None, b"") f2.seek(0) self.assertEqual(f2.read(), content) + # test multi --dest (manual) + tdir = make_temp_dir() + sf2 = make_temp_file(b'second') + try: + f2 = tempfile.NamedTemporaryFile() + self._clush_t(["-w", HOSTNAME, "-c", sf.name, sf2.name, "--dest", + tdir.name], + None, b"") + with open(os.path.join(tdir.name, basename(sf.name)), 'rb') as chkf: + self.assertEqual(chkf.read(), content) + with open(os.path.join(tdir.name, basename(sf2.name)), 'rb') as chkf: + self.assertEqual(chkf.read(), b'second') + finally: + sf2.close() + tdir.cleanup() + # test multi --dest (auto) + tdir = make_temp_dir() + sf2 = make_temp_file(b'second', dir=tdir.name) + try: + f2 = tempfile.NamedTemporaryFile() + self._clush_t(["-w", HOSTNAME, "-c", sf.name, sf2.name], None, + b"") + sf.seek(0) + sf2.seek(0) + self.assertEqual(sf.read(), content) + self.assertEqual(sf2.read(), b'second') + finally: + sf2.close() + tdir.cleanup() # test --user option f2 = tempfile.NamedTemporaryFile() self._clush_t(["--user", pwd.getpwuid(os.getuid())[0], "-w", HOSTNAME, - "--copy", f.name, "--dest", f2.name], None, b"") + "--copy", sf.name, "--dest", f2.name], None, b"") f2.seek(0) self.assertEqual(f2.read(), content) # test --rcopy self._clush_t(["--user", pwd.getpwuid(os.getuid())[0], "-w", HOSTNAME, - "--rcopy", f.name, "--dest", os.path.dirname(f.name)], + "--rcopy", sf.name, "--dest", dirname(sf.name)], + None, b"") + f2.seek(0) + self.assertEqual(open("%s.%s" % (sf.name, HOSTNAME), 'rb').read(), + content) + # test --rcopy with implicit --dest + self._clush_t(["--user", pwd.getpwuid(os.getuid())[0], "-w", HOSTNAME, + "--rcopy", sf.name], None, b"") f2.seek(0) - self.assertEqual(open("%s.%s" % (f.name, HOSTNAME), 'rb').read(), + self.assertEqual(open("%s.%s" % (sf.name, HOSTNAME), 'rb').read(), content) def test_009_file_copy_tty(self):