From 3177c80d7727b7c2c82b25164d929e4f3a7205cf Mon Sep 17 00:00:00 2001 From: Robert Smith Date: Thu, 15 Oct 2020 14:28:20 +1100 Subject: [PATCH 1/5] Updates for latest pylint checks Note that these changes are not compatible with Python2. --- bin/dwi2response | 8 ++++---- bin/dwicat | 4 ++-- bin/dwifslpreproc | 8 ++++---- bin/for_each | 4 ++-- bin/population_template | 12 ++++++------ build | 8 ++++---- configure | 2 +- lib/mrtrix3/image.py | 4 ++-- lib/mrtrix3/phaseencoding.py | 4 ++-- lib/mrtrix3/run.py | 8 ++++---- 10 files changed, 31 insertions(+), 31 deletions(-) diff --git a/bin/dwi2response b/bin/dwi2response index f037aef9b6..c33070307d 100755 --- a/bin/dwi2response +++ b/bin/dwi2response @@ -62,16 +62,16 @@ def execute(): #pylint: disable=unused-variable lmax = [ int(x) for x in app.ARGS.lmax.split(',') ] if any([lmax_value%2 for lmax_value in lmax]): raise MRtrixError('Value of lmax must be even') - except: - raise MRtrixError('Parameter lmax must be a number') + except ValueError as exception: + raise MRtrixError('Parameter lmax must be a number') from exception if alg.needs_single_shell() and not len(lmax) == 1: raise MRtrixError('Can only specify a single lmax value for single-shell algorithms') shells_option = '' if app.ARGS.shells: try: shells_values = [ int(round(float(x))) for x in app.ARGS.shells.split(',') ] - except: - raise MRtrixError('-shells option should provide a comma-separated list of b-values') + except ValueError as exception: + raise MRtrixError('-shells option should provide a comma-separated list of b-values') from exception if alg.needs_single_shell() and not len(shells_values) == 1: raise MRtrixError('Can only specify a single b-value shell for single-shell algorithms') shells_option = ' -shells ' + app.ARGS.shells diff --git a/bin/dwicat b/bin/dwicat index 979289b74b..9cb21ad870 100755 --- a/bin/dwicat +++ b/bin/dwicat @@ -109,8 +109,8 @@ def execute(): #pylint: disable=unused-variable if 'Estimated scale factor is' in line: try: scaling_factor = float(line.split()[-1]) - except ValueError: - raise MRtrixError('Unable to convert scaling factor from mrhistmatch output to floating-point number') + except ValueError as exception: + raise MRtrixError('Unable to convert scaling factor from mrhistmatch output to floating-point number') from exception break if scaling_factor is None: raise MRtrixError('Unable to extract scaling factor from mrhistmatch output') diff --git a/bin/dwifslpreproc b/bin/dwifslpreproc index df1c04085a..47eb655202 100755 --- a/bin/dwifslpreproc +++ b/bin/dwifslpreproc @@ -306,10 +306,10 @@ def execute(): #pylint: disable=unused-variable 'contents of file \"' + app.ARGS.eddy_slspec + '\" appears to instead be slice timings; ' 'these data have been imported and will be converted to the appropriate format') if len(slice_timing) != dwi_num_slices: - raise MRtrixError('Cannot use slice timing information from file \"' + app.ARGS.eddy_slspec + '\" for slice-to-volume correction: ' + \ + raise MRtrixError('Cannot use slice timing information from file \"' + app.ARGS.eddy_slspec + '\" for slice-to-volume correction: ' # pylint: disable=raise-missing-from 'number of entries (' + str(len(slice_timing)) + ') does not match number of slices (' + str(dwi_num_slices) + ')') except ValueError: - raise MRtrixError('Error parsing eddy \"slspec\" file \"' + app.ARGS.eddy_slspec + '\" ' + raise MRtrixError('Error parsing eddy \"slspec\" file \"' + app.ARGS.eddy_slspec + '\" ' # pylint: disable=raise-missing-from '(please see FSL eddy help page, specifically the --slspec option)') else: if 'SliceTiming' not in dwi_header.keyval(): @@ -341,9 +341,9 @@ def execute(): #pylint: disable=unused-variable if not all( [ isinstance(entry, float) for entry in slice_timing ] ): try: slice_timing = [ float(entry) for entry in slice_timing ] - except ValueError: + except ValueError as exception: raise MRtrixError('Cannot use slice timing information in image header for slice-to-volume correction: ' - 'data are not numeric') + 'data are not numeric') from exception app.debug('Re-formatted slice timing contents from header: ' + str(slice_timing)) if len(slice_timing) != dwi_num_slices: raise MRtrixError('Cannot use slice timing information in image header for slice-to-volume correction: ' diff --git a/bin/for_each b/bin/for_each index b44f5a0963..c134e0ae89 100755 --- a/bin/for_each +++ b/bin/for_each @@ -154,8 +154,8 @@ def execute(): #pylint: disable=unused-variable try: next(entry for entry in CMDSPLIT if any(key for key in KEYLIST if key in entry)) - except StopIteration: - raise MRtrixError('None of the unique for_each keys ' + str(KEYLIST) + ' appear in command string "' + app.ARGS.command + '"; no substitution can occur') + except StopIteration as exception: + raise MRtrixError('None of the unique for_each keys ' + str(KEYLIST) + ' appear in command string "' + app.ARGS.command + '"; no substitution can occur') from exception class Entry(object): def __init__(self, input_text): diff --git a/bin/population_template b/bin/population_template index 590d4872a3..fb4a6d6215 100755 --- a/bin/population_template +++ b/bin/population_template @@ -92,13 +92,13 @@ except NameError: pass -def abspath(*arg): - return os.path.abspath(os.path.join(*arg)) +def abspath(first, *arg): + return os.path.abspath(os.path.join(first, *arg)) -def relpath(*arg): +def relpath(first, *arg): from mrtrix3 import app #pylint: disable=no-name-in-module, import-outside-toplevel - return os.path.relpath(os.path.join(*arg), app.WORKING_DIR) + return os.path.relpath(os.path.join(first, *arg), app.WORKING_DIR) def copy(src, dst, follow_symlinks=True): """Copy data but do not set mode bits. Return the file's destination. @@ -626,8 +626,8 @@ def execute(): #pylint: disable=unused-variable if len(voxel_size) != 3: raise ValueError [float(v) for v in voxel_size] #pylint: disable=expression-not-assigned - except ValueError: - raise MRtrixError('voxel size needs to be a single or three comma-separated floating point numbers; received: ' + str(app.ARGS.voxel_size)) + except ValueError as exception: + raise MRtrixError('voxel size needs to be a single or three comma-separated floating point numbers; received: ' + str(app.ARGS.voxel_size)) from exception agg_measure = 'mean' if app.ARGS.aggregate is not None: diff --git a/build b/build index 6ae36bf5fd..b8e52962bc 100755 --- a/build +++ b/build @@ -1157,9 +1157,9 @@ def version_from_git (folder): commit = subprocess.check_output ([ 'git', 'describe', '--abbrev=8', '--dirty', '--always' ], cwd=folder, stderr=devnull).decode(errors='ignore').strip() #pylint: disable=unexpected-keyword-arg log (tag + ', ' + commit + '\n') return [ tag, commit ] - except Exception: + except Exception as e: log ('not found\n') - raise LookupError + raise LookupError from e @@ -1175,9 +1175,9 @@ def version_from_file (version_file, version_macro): return tag if not tag: raise NameError - except Exception: + except Exception as e: log ('not found\n') - raise LookupError + raise LookupError from e return None diff --git a/configure b/configure index 990cbeb82e..28e119ea5c 100755 --- a/configure +++ b/configure @@ -505,7 +505,7 @@ def execute (cmd, exception, raise_on_non_zero_exit_code = True, cwd = None): except OSError as e: log ('error invoking command "' + cmd[0] + '": ' + e.strerror + '\n>>\n\n') - raise exception + raise exception from e except Exception as excp: error ('unexpected exception of type ' + type(excp).__name__ + ': ' + str(excp) + configure_log_hint) else: diff --git a/lib/mrtrix3/image.py b/lib/mrtrix3/image.py index eb92dea3ad..519f4dfa28 100644 --- a/lib/mrtrix3/image.py +++ b/lib/mrtrix3/image.py @@ -63,8 +63,8 @@ def __init__(self, image_path): self._keyval = { } else: self._keyval = data['keyval'] - except: - raise MRtrixError('Error in reading header information from file \'' + image_path + '\'') + except Exception as exception: + raise MRtrixError('Error in reading header information from file \'' + image_path + '\'') from exception app.debug(str(vars(self))) def name(self): diff --git a/lib/mrtrix3/phaseencoding.py b/lib/mrtrix3/phaseencoding.py index 8956b79718..918335999e 100644 --- a/lib/mrtrix3/phaseencoding.py +++ b/lib/mrtrix3/phaseencoding.py @@ -36,7 +36,7 @@ def direction(string): #pylint: disable=unused-variable pe_dir[pe_axis] = -1 else: pe_dir[pe_axis] = 1 - except: + except Exception as exception: string = string.lower() if string == 'lr': pe_dir = [1,0,0] @@ -63,7 +63,7 @@ def direction(string): #pylint: disable=unused-variable elif string == 'k-': pe_dir = [0,0,-1] else: - raise MRtrixError('Unrecognized phase encode direction specifier: ' + string) + raise MRtrixError('Unrecognized phase encode direction specifier: ' + string) from exception app.debug(string + ' -> ' + str(pe_dir)) return pe_dir diff --git a/lib/mrtrix3/run.py b/lib/mrtrix3/run.py index 566b6a8d30..9f2e363f30 100644 --- a/lib/mrtrix3/run.py +++ b/lib/mrtrix3/run.py @@ -184,7 +184,7 @@ def terminate(self, signum): #pylint: disable=unused-variable class MRtrixCmdError(MRtrixBaseError): def __init__(self, cmd, code, stdout, stderr): - super(MRtrixCmdError, self).__init__('Command failed') + super().__init__('Command failed') self.command = cmd self.returncode = code self.stdout = stdout @@ -194,7 +194,7 @@ def __str__(self): class MRtrixFnError(MRtrixBaseError): def __init__(self, fn, text): - super(MRtrixFnError, self).__init__('Function failed') + super().__init__('Function failed') self.function = fn self.errortext = text def __str__(self): @@ -372,7 +372,7 @@ def quote_nonpipe(item): this_process_list.append(shared.Process(to_execute, this_stdin, this_stdout, this_stderr, **subprocess_kwargs)) # FileNotFoundError not defined in Python 2.7 except OSError as exception: - raise MRtrixCmdError(cmdstring, 1, '', str(exception)) + raise MRtrixCmdError(cmdstring, 1, '', str(exception)) from exception # End branching based on shell=True/False @@ -497,7 +497,7 @@ def function(fn_to_execute, *args, **kwargs): #pylint: disable=unused-variable else: result = fn_to_execute(*args) except Exception as exception: # pylint: disable=broad-except - raise MRtrixFnError(fnstring, str(exception)) + raise MRtrixFnError(fnstring, str(exception)) from exception # Only now do we append to the script log, since the function has completed successfully if shared.get_scratch_dir(): From 3a856ce1c2d6e128e080c69bf12ab275498bd338 Mon Sep 17 00:00:00 2001 From: Robert Smith Date: Wed, 28 Oct 2020 22:48:22 +1100 Subject: [PATCH 2/5] Remove support for Python2 - All executable Python scripts directly invoke /usr/bin/python3. - Separate testing of configure and build scripts using Python2 and Python3 no longer required. - Replace distutils.spawn.find_executable() with shutil.which(). - Activate more stringent pylint tests, with some limited corresponding code changes. - configure: Test presence of shlex.quote() and shutil.which(). - Remove mrtrix3.utils.STRING_TYPES, which is no longer required for testing the types of provided function arguments. - population_template: No longer need to bind Python2's raw_input() to input(). - mrtrix3.path: Remove quote() function: shlex.quote() should be used directly. - run_pylint: Remove capability for forcing Python version via environment variable. - Binary test data: Update vectorstats generation / testing scripts to use Python3. --- .github/workflows/checks.yml | 25 ++---------- bin/5ttgen | 12 +----- bin/blend | 2 +- bin/convert_bruker | 16 ++++---- bin/dwi2response | 7 +--- bin/dwibiascorrect | 6 +-- bin/dwicat | 4 +- bin/dwifslpreproc | 9 ++--- bin/dwigradcheck | 4 +- bin/dwinormalise | 6 +-- bin/dwishellmath | 4 +- bin/for_each | 10 ++--- bin/labelsgmfix | 4 +- bin/mrtrix3.py | 1 - bin/mrtrix_cleanup | 4 +- bin/population_template | 15 +++----- bin/responsemean | 4 +- build | 18 +++++---- configure | 31 ++++++++++----- docs/concepts/fixels_dixels.rst | 9 ++++- docs/format_config_options | 2 +- docs/format_environment_variables | 2 +- docs/installation/build_from_source.rst | 19 +++------- docs/tips_and_tricks/external_modules.rst | 2 +- generate_bash_completion.py | 3 +- lib/mrtrix3/_5ttgen/fsl.py | 7 ++-- lib/mrtrix3/_5ttgen/hsvs.py | 5 +-- lib/mrtrix3/__init__.py | 8 +--- lib/mrtrix3/app.py | 46 ++++++++++++----------- lib/mrtrix3/dwi2response/dhollander.py | 4 +- lib/mrtrix3/dwi2response/msmt_5tt.py | 6 +-- lib/mrtrix3/dwibiascorrect/ants.py | 4 +- lib/mrtrix3/dwinormalise/group.py | 8 ++-- lib/mrtrix3/fsl.py | 13 +++---- lib/mrtrix3/image.py | 9 ++--- lib/mrtrix3/matrix.py | 5 +-- lib/mrtrix3/path.py | 24 ++++-------- lib/mrtrix3/phaseencoding.py | 11 +++--- lib/mrtrix3/run.py | 46 +++++++++++------------ lib/mrtrix3/utils.py | 21 +++-------- run_pylint | 13 ++----- set_path | 4 +- src/connectome/enhance.cpp | 3 +- testing/binaries/data | 2 +- testing/pylint.rc | 2 +- update_copyright | 2 +- 46 files changed, 199 insertions(+), 263 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 3b0a0c9496..2f8ba80b45 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -160,22 +160,17 @@ jobs: - name: install dependencies run: | sudo apt-get update - sudo apt-get install pylint pylint3 python3-sphinx sphinx-rtd-theme-common python3-recommonmark python3-sphinx-rtd-theme python3-pip + sudo apt-get install pylint3 python3-sphinx sphinx-rtd-theme-common python3-recommonmark python3-sphinx-rtd-theme python3-pip pip3 install sphinx-notfound-page - name: check syntax run: ./check_syntax || { cat syntax.log; false; } - - name: pylint (Python 2) + - name: pylint run: | echo "__version__ = 'pylint testing' #pylint: disable=unused-variable" > ./lib/mrtrix3/_version.py - PYTHON=python2 ./run_pylint || { cat pylint.log; false; } - - - name: pylint (Python 3) - run: | - echo "__version__ = 'pylint testing' #pylint: disable=unused-variable" > ./lib/mrtrix3/_version.py - PYTHON=python3 ./run_pylint || { cat pylint.log; false; } + ./run_pylint || { cat pylint.log; false; } - name: check copyright headers run: ./update_copyright && git diff --exit-code @@ -183,17 +178,3 @@ jobs: - name: check building of documentation run: python3 -m sphinx -n -N -W -w sphinx.log docs/ tmp/ - - name: install build dependencies - run: sudo apt-get install clang-8 libqt5opengl5-dev libqt5svg5-dev libglvnd-dev libeigen3-dev zlib1g-dev libfftw3-dev - - - name: check configure with Python 2 - run: python2 ./configure || { cat configure.log; false; } - - - name: check build with Python 2 - run: python2 ./build -dryrun || { cat build.log; false; } - - - name: check configure with Python 3 - run: python3 ./configure || { cat configure.log; false; } - - - name: check build with Python 3 - run: python3 ./build -dryrun || { cat build.log; false; } diff --git a/bin/5ttgen b/bin/5ttgen index 08da0e42ff..acb84b5dc3 100755 --- a/bin/5ttgen +++ b/bin/5ttgen @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # @@ -15,14 +15,6 @@ # # For more details, see http://www.mrtrix.org/. -# Script that generates a five-tissue-type (5TT) segmented image: the format appropriate for ACT -# -# In this script, major stages of processing can be performed in one of two ways: -# - Using FSL tools: BET for brain extraction, FAST for tissue segmentation, FIRST for sub-cortical grey matter segmentation -# - Using segmentations from FreeSurfer -# Alternative algorithms for performing this conversion can be added by creating a new file in lib/mrtrix3/_5ttgen/ and -# defining the appropriate functions; 5ttgen will automatically make that algorithm available at the command-line - def usage(cmdline): #pylint: disable=unused-variable @@ -66,5 +58,5 @@ def execute(): #pylint: disable=unused-variable # Execute the script -import mrtrix3 +import mrtrix3 #pylint: disable=wrong-import-position mrtrix3.execute() #pylint: disable=no-member diff --git a/bin/blend b/bin/blend index dcc5206ca6..bf915f3f0f 100755 --- a/bin/blend +++ b/bin/blend @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # diff --git a/bin/convert_bruker b/bin/convert_bruker index fbb954326a..74d617167c 100755 --- a/bin/convert_bruker +++ b/bin/convert_bruker @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # @@ -83,13 +83,13 @@ def main(): try: nslices #pylint: disable=pointless-statement file_out.write (',' + str(nslices)) - except: + except NameError: pass try: nacq #pylint: disable=pointless-statement file_out.write (',' + str(nacq)) - except: + except NameError: pass file_out.write ('\nvox: ' + str(res[0]) + ',' + str(res[1])) @@ -99,12 +99,12 @@ def main(): try: slicethick #pylint: disable=pointless-statement file_out.write (',' + str(slicethick)) - except: + except NameError: pass try: nacq #pylint: disable=pointless-statement file_out.write (',') - except: + except NameError: pass file_out.write ('\ndatatype: ') @@ -122,12 +122,12 @@ def main(): try: nslices #pylint: disable=pointless-statement file_out.write (',+2') - except: + except NameError: pass try: nacq #pylint: disable=pointless-statement file_out.write (',+3') - except: + except NameError: pass file_out.write ('\nfile: ' + sys.argv[1] + '\n') @@ -137,7 +137,7 @@ def main(): bvec = [ bvec[n:n+3] for n in range(0,len(bvec),3) ] for direction, value in zip(bvec, bval): file_out.write ('dw_scheme: ' + direction[0] + ',' + direction[1] + ',' + str(-float(direction[2])) + ',' + value + '\n') - except: + except AssertionError: pass main() diff --git a/bin/dwi2response b/bin/dwi2response index c33070307d..40dc50989d 100755 --- a/bin/dwi2response +++ b/bin/dwi2response @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # @@ -15,9 +15,6 @@ # # For more details, see http://www.mrtrix.org/. -# Script for estimating response functions for spherical deconvolution -# A number of different approaches are available within this script for performing response function estimation. - def usage(cmdline): #pylint: disable=unused-variable @@ -123,5 +120,5 @@ def execute(): #pylint: disable=unused-variable # Execute the script -import mrtrix3 +import mrtrix3 #pylint: disable=wrong-import-position mrtrix3.execute() #pylint: disable=no-member diff --git a/bin/dwibiascorrect b/bin/dwibiascorrect index d084ca9c46..3bba224a7a 100755 --- a/bin/dwibiascorrect +++ b/bin/dwibiascorrect @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # @@ -15,8 +15,6 @@ # # For more details, see http://www.mrtrix.org/. -# Script that performs B1 field inhomogeneity correction for a DWI volume series -# Bias field is typically estimated using the mean b=0 image, and subsequently used to correct all volumes def usage(cmdline): #pylint: disable=unused-variable @@ -79,5 +77,5 @@ def execute(): #pylint: disable=unused-variable # Execute the script -import mrtrix3 +import mrtrix3 #pylint: disable=wrong-import-position mrtrix3.execute() #pylint: disable=no-member diff --git a/bin/dwicat b/bin/dwicat index 9cb21ad870..b34804a897 100755 --- a/bin/dwicat +++ b/bin/dwicat @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # @@ -137,5 +137,5 @@ def execute(): #pylint: disable=unused-variable # Execute the script -import mrtrix3 +import mrtrix3 #pylint: disable=wrong-import-position mrtrix3.execute() #pylint: disable=no-member diff --git a/bin/dwifslpreproc b/bin/dwifslpreproc index 47eb655202..127ea5d461 100755 --- a/bin/dwifslpreproc +++ b/bin/dwifslpreproc @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # @@ -18,7 +18,6 @@ import glob, itertools, json, math, os, shutil, sys -from distutils.spawn import find_executable @@ -323,7 +322,7 @@ def execute(): #pylint: disable=unused-variable # Fudges necessary to maniupulate nature of slice timing data in cases where # bad JSON formatting has led to the data not being simply a list of floats # (whether from MRtrix3 DICOM conversion or from anything else) - if isinstance(slice_timing, utils.STRING_TYPES): + if isinstance(slice_timing, str): slice_timing = slice_timing.split() if not isinstance(slice_timing, list): raise MRtrixError('Cannot use slice timing information in image header for slice-to-volume correction: ' @@ -1040,7 +1039,7 @@ def execute(): #pylint: disable=unused-variable # Run eddy qc tool QUAD if installed and one of -eddyqc_text or -eddyqc_all is specified if eddyqc_path: - if find_executable('eddy_quad'): + if shutil.which('eddy_quad'): eddyqc_options = ' -idx eddy_indices.txt -par eddy_config.txt -m eddy_mask.nii -b bvals' if os.path.isfile('dwi_post_eddy.eddy_residuals'): eddyqc_options += ' -g ' + bvecs_path @@ -1309,5 +1308,5 @@ def execute(): #pylint: disable=unused-variable # Execute the script -import mrtrix3 +import mrtrix3 #pylint: disable=wrong-import-position mrtrix3.execute() #pylint: disable=no-member diff --git a/bin/dwigradcheck b/bin/dwigradcheck index dbf20ca18d..4cc8b4360d 100755 --- a/bin/dwigradcheck +++ b/bin/dwigradcheck @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # @@ -202,5 +202,5 @@ def execute(): #pylint: disable=unused-variable # Execute the script -import mrtrix3 +import mrtrix3 #pylint: disable=wrong-import-position mrtrix3.execute() #pylint: disable=no-member diff --git a/bin/dwinormalise b/bin/dwinormalise index a9d8b9a66e..3ad95158d7 100755 --- a/bin/dwinormalise +++ b/bin/dwinormalise @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # @@ -15,8 +15,6 @@ # # For more details, see http://www.mrtrix.org/. -# Script that performs intensity normalisation of DWIs in various ways - def usage(cmdline): #pylint: disable=unused-variable from mrtrix3 import algorithm #pylint: disable=no-name-in-module, import-outside-toplevel @@ -46,5 +44,5 @@ def execute(): #pylint: disable=unused-variable # Execute the script -import mrtrix3 +import mrtrix3 #pylint: disable=wrong-import-position mrtrix3.execute() #pylint: disable=no-member diff --git a/bin/dwishellmath b/bin/dwishellmath index 81c085543f..9f5cd81728 100755 --- a/bin/dwishellmath +++ b/bin/dwishellmath @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # @@ -61,5 +61,5 @@ def execute(): #pylint: disable=unused-variable # Execute the script -import mrtrix3 +import mrtrix3 #pylint: disable=wrong-import-position mrtrix3.execute() #pylint: disable=no-member diff --git a/bin/for_each b/bin/for_each index c134e0ae89..fb616e134f 100755 --- a/bin/for_each +++ b/bin/for_each @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # @@ -81,7 +81,7 @@ def usage(cmdline): #pylint: disable=unused-variable # These need to be globals in order to be accessible from execute_parallel() -class Shared(object): +class Shared: def __init__(self): self._job_index = 0 self.lock = threading.Lock() @@ -132,7 +132,7 @@ def execute(): #pylint: disable=unused-variable to_exclude.extend(regex_hits) else: exclude_unmatched.append(exclude) - except Exception: + except re.error: exclude_unmatched.append(exclude) if exclude_unmatched: app.warn('Item' + ('s' if len(exclude_unmatched) > 1 else '') + ' specified via -exclude did not appear in input list: ' + str('\'' + exclude_unmatched[0] + '\'' if len(exclude_unmatched) == 1 else exclude_unmatched)) @@ -157,7 +157,7 @@ def execute(): #pylint: disable=unused-variable except StopIteration as exception: raise MRtrixError('None of the unique for_each keys ' + str(KEYLIST) + ' appear in command string "' + app.ARGS.command + '"; no substitution can occur') from exception - class Entry(object): + class Entry: def __init__(self, input_text): self.input_text = input_text self.sub_in = input_text @@ -297,5 +297,5 @@ def execute(): #pylint: disable=unused-variable # Execute the script -import mrtrix3 +import mrtrix3 #pylint: disable=wrong-import-position mrtrix3.execute() #pylint: disable=no-member diff --git a/bin/labelsgmfix b/bin/labelsgmfix index 27f353ad36..e460729554 100755 --- a/bin/labelsgmfix +++ b/bin/labelsgmfix @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # @@ -171,5 +171,5 @@ def execute(): #pylint: disable=unused-variable # Execute the script -import mrtrix3 +import mrtrix3 #pylint: disable=wrong-import-position mrtrix3.execute() #pylint: disable=no-member diff --git a/bin/mrtrix3.py b/bin/mrtrix3.py index ea8967d532..c697f1a62d 100644 --- a/bin/mrtrix3.py +++ b/bin/mrtrix3.py @@ -15,7 +15,6 @@ # For more details, see http://www.mrtrix.org/. import imp, os, sys -from distutils.spawn import find_executable def imported(lib_path): success = False diff --git a/bin/mrtrix_cleanup b/bin/mrtrix_cleanup index 3403883e97..3920c0c7bf 100755 --- a/bin/mrtrix_cleanup +++ b/bin/mrtrix_cleanup @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # @@ -136,5 +136,5 @@ def execute(): #pylint: disable=unused-variable # Execute the script -import mrtrix3 +import mrtrix3 #pylint: disable=wrong-import-position mrtrix3.execute() #pylint: disable=no-member diff --git a/bin/population_template b/bin/population_template index fb4a6d6215..82f01178db 100755 --- a/bin/population_template +++ b/bin/population_template @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # @@ -84,12 +84,6 @@ def usage(cmdline): #pylint: disable=unused-variable # ENH: add option to initialise warps / transformations -# Binds raw_input() to input() in Python2, so that input() can be used -# and the code will work on both Python 2 and 3 -try: - input = raw_input #pylint: disable=redefined-builtin, invalid-name -except NameError: - pass def abspath(first, *arg): @@ -243,7 +237,7 @@ def get_common_prefix(file_list): return os.path.commonprefix(file_list) -class Contrasts(object): +class Contrasts: """ Class that parses arguments and holds information specific to each image contrast @@ -333,7 +327,7 @@ class Contrasts(object): return text -class Input(object): +class Input: """ Class that holds input information specific to a single image (multiple contrasts) @@ -1431,6 +1425,7 @@ def execute(): #pylint: disable=unused-variable mrconvert_keyval='NULL', force=app.FORCE_OVERWRITE) + # Execute the script -import mrtrix3 +import mrtrix3 #pylint: disable=wrong-import-position mrtrix3.execute() #pylint: disable=no-member diff --git a/bin/responsemean b/bin/responsemean index 39fff52c45..a41e412bc9 100755 --- a/bin/responsemean +++ b/bin/responsemean @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # @@ -81,5 +81,5 @@ def execute(): #pylint: disable=unused-variable # Execute the script -import mrtrix3 +import mrtrix3 #pylint: disable=wrong-import-position mrtrix3.execute() #pylint: disable=no-member diff --git a/build b/build index b8e52962bc..4ae82702df 100755 --- a/build +++ b/build @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # @@ -15,9 +15,12 @@ # # For more details, see http://www.mrtrix.org/. -# pylint: disable=redefined-outer-name,invalid-name +# pylint: disable=redefined-outer-name,invalid-name,broad-except -usage_string = ''' +import atexit, codecs, copy, glob, os, platform, re, shutil, subprocess, sys, tempfile, time, threading +from timeit import default_timer as timer + +USAGE_STRING = ''' USAGE ./build [-verbose] [-showdep[=target|all]] [target ...] @@ -108,12 +111,11 @@ OPTIONS # COMMON DEFINITIONS # ############################################################################ -import atexit, codecs, copy, glob, os, platform, re, shutil, subprocess, sys, tempfile, time, threading -from timeit import default_timer as timer + # on Windows, need to use MSYS2 version of python - not MinGW version: if sys.executable[0].isalpha() and sys.executable[1] == ':': - python_cmd = subprocess.check_output ([ 'cygpath.exe', '-w', '/usr/bin/python' ]).decode(errors='ignore').splitlines()[0].strip() + python_cmd = subprocess.check_output ([ 'cygpath.exe', '-w', '/usr/bin/python3' ]).decode(errors='ignore').splitlines()[0].strip() sys.exit (subprocess.call ([ python_cmd ] + sys.argv)) @@ -359,7 +361,7 @@ bash_completion = False for arg in sys.argv[1:]: if '-help'.startswith(arg): - sys.stdout.write (usage_string) + sys.stdout.write (USAGE_STRING) sys.exit (0) elif '-verbose'.startswith(arg): verbose = True @@ -694,7 +696,7 @@ for filepath in list_unexpected_bin_files(exe_suffix): class TargetException (Exception): pass -class Entry(object): +class Entry: def __init__ (self, name): global todo name = os.path.normpath (name) diff --git a/configure b/configure index 28e119ea5c..8c97eca865 100755 --- a/configure +++ b/configure @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # @@ -15,9 +15,11 @@ # # For more details, see http://www.mrtrix.org/. -# pylint: disable=invalid-name +# pylint: disable=invalid-name,broad-except -usage_string = ''' +import subprocess, sys, os, platform, tempfile, shlex, re, copy + +USAGE_STRING = ''' USAGE [ENV] ./configure [-debug] [-assert] [-profile] [-nogui] [-noshared] @@ -165,12 +167,11 @@ ENVIRONMENT VARIABLES specified correctly at configure time. ''' -import subprocess, sys, os, platform, tempfile, shlex, re, copy system = platform.system().lower() # on Windows, need to use MSYS2 version of python - not MinGW version: if sys.executable[0].isalpha() and sys.executable[1] == ':': - python_cmd = subprocess.check_output ([ 'cygpath.exe', '-w', '/usr/bin/python' ]).decode(errors='ignore').splitlines()[0].strip() + python_cmd = subprocess.check_output ([ 'cygpath.exe', '-w', '/usr/bin/python3' ]).decode(errors='ignore').splitlines()[0].strip() sys.exit (subprocess.call ([ python_cmd ] + sys.argv)) @@ -219,7 +220,7 @@ for arg in sys.argv[1:]: elif '-conda'.startswith (arg): conda = True else: - sys.stdout.write (usage_string) + sys.stdout.write (USAGE_STRING) sys.exit (1) @@ -298,6 +299,16 @@ log ('PATH set to: ' + path) +# Test Python version +try: + from shlex import quote # pylint: disable=unused-variable, unused-import + from shutil import which # pylint: disable=unused-variable, unused-import + report ('Python verion: ' + platform.python_version()) +except ImportError: + error ('Incompatible Python version detected: ' + platform.python_version() + '(at least 3.3 is required)') + + + cpp = ld = None @@ -316,7 +327,7 @@ if static: ld_lib_args = 'OBJECTS LINKLIB_FLAGS -o LIB'.split() -class TempFile(object): +class TempFile: def __init__ (self, suffix): self.fid = None self.name = None @@ -334,7 +345,7 @@ class TempFile(object): -class DeleteAfter(object): +class DeleteAfter: def __init__ (self, name): self.name = name @@ -348,7 +359,7 @@ class DeleteAfter(object): log ('error deleting temporary file "' + self.name + '": ' + excp.strerror) -class TempDir(object): +class TempDir: def __init__ (self): self.name = tempfile.mkdtemp() @@ -1430,7 +1441,7 @@ sys.stdout.write ('\nwriting configuration to file \'' + cache_filename + '\': ' cache = open (cache_filename, 'w') -cache.write ("""#!/usr/bin/python +cache.write ("""#!/usr/bin/python3 # # autogenerated by MRtrix configure script # diff --git a/docs/concepts/fixels_dixels.rst b/docs/concepts/fixels_dixels.rst index f29a18e322..07cb6d34d5 100644 --- a/docs/concepts/fixels_dixels.rst +++ b/docs/concepts/fixels_dixels.rst @@ -1,7 +1,7 @@ .. _fixels_dixels: -"Fixels" (and "Dixels") -======================= +"Fixels" (and "Dixels" and "Peaks") +=================================== Internally we have created a couple of new terms that we find invaluable when discussing diffusion MRI processing methods and @@ -141,6 +141,11 @@ Some observations / contexts in which the term 'dixel' may be useful: thought of as converting the FOD from a continuous SH representation, to a dixel representation, then finally to a fixel representation. +'Peak': *Local ODF maxima* +-------------------------- + + + .. NOTE:: During the development of many of the aforementioned methods, diff --git a/docs/format_config_options b/docs/format_config_options index e60124590e..19cef6c919 100755 --- a/docs/format_config_options +++ b/docs/format_config_options @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # diff --git a/docs/format_environment_variables b/docs/format_environment_variables index f6acc28174..38638f74ff 100755 --- a/docs/format_environment_variables +++ b/docs/format_environment_variables @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # diff --git a/docs/installation/build_from_source.rst b/docs/installation/build_from_source.rst index f95cafbad5..bec48b5a99 100644 --- a/docs/installation/build_from_source.rst +++ b/docs/installation/build_from_source.rst @@ -31,8 +31,7 @@ Required dependencies: - a `C++11 `__ compliant compiler (GCC version >= 5, clang); -- `Python `__ version >= 2.7 (>= 3 strongly recommended - due to `deprecation of Python2 `__); +- `Python `__ version >= 3.3; - The `zlib `__ compression library; - `Eigen `__ version >= 3.2 (>= 3.3 recommended); - `Qt `__ version >= 5.5 *[GUI components only]*; @@ -76,21 +75,15 @@ some of the most common Linux distributions below. - Ubuntu Linux (and derivatives, e.g. Linux Mint):: - sudo apt-get install git g++ python libeigen3-dev zlib1g-dev libqt5opengl5-dev libqt5svg5-dev libgl1-mesa-dev libfftw3-dev libtiff5-dev libpng-dev - - .. NOTE:: - - On Ubuntu 20.04 and newer, you'll to replace ``python`` in the line - above with ``python-is-python3`` (or ``python-is-python2`` if you're - still using version 2.7, which is now *very* deprecated). + sudo apt-get install git g++ python3 libeigen3-dev zlib1g-dev libqt5opengl5-dev libqt5svg5-dev libgl1-mesa-dev libfftw3-dev libtiff5-dev libpng-dev - RPM-based distros (Fedora, CentOS):: - sudo yum install git g++ python eigen3-devel zlib-devel libqt5-devel libgl1-mesa-dev fftw-devel libtiff-devel libpng-devel + sudo yum install git g++ python3 eigen3-devel zlib-devel libqt5-devel libgl1-mesa-dev fftw-devel libtiff-devel libpng-devel On Fedora 24, this is reported to work:: - sudo yum install git gcc-c++ python eigen3-devel zlib-devel qt-devel mesa-libGL-devel fftw-devel libtiff-devel libpng-devel + sudo yum install git gcc-c++ python3 eigen3-devel zlib-devel qt-devel mesa-libGL-devel fftw-devel libtiff-devel libpng-devel - Arch Linux:: @@ -106,7 +99,7 @@ packages: - an appropriate C++ compiler (e.g. GCC 5 or above, or clang); -- Python version >= 2.7 (version >= 3.0 strongly recommended); +- Python version >= 3.3; - the ``zlib`` compression library and its corresponding development header/include files; @@ -272,7 +265,7 @@ All of these dependencies are installed below by the MSYS2 package manager. 4. From the **'MinGW-w64 Win64 Shell'** run:: - pacman -S git python pkg-config mingw-w64-x86_64-gcc mingw-w64-x86_64-eigen3 mingw-w64-x86_64-qt5 mingw-w64-x86_64-fftw mingw-w64-x86_64-libtiff mingw-w64-x86_64-libpng + pacman -S git python3 pkg-config mingw-w64-x86_64-gcc mingw-w64-x86_64-eigen3 mingw-w64-x86_64-qt5 mingw-w64-x86_64-fftw mingw-w64-x86_64-libtiff mingw-w64-x86_64-libpng Sometimes ``pacman`` may fail to find a particular package from any of the available mirrors. If this occurs, you can download the relevant diff --git a/docs/tips_and_tricks/external_modules.rst b/docs/tips_and_tricks/external_modules.rst index 823e34b8ee..0d05e00f9d 100644 --- a/docs/tips_and_tricks/external_modules.rst +++ b/docs/tips_and_tricks/external_modules.rst @@ -200,7 +200,7 @@ New code can be added to this new module as follows: - **Stand-alone Python file**: A stand-alone Python script designed to make use of the *MRtrix3* Python APIs will typically not have any file extension, and - will have its first line set to ``#!/usr/bin/env python``. Such files should be + will have its first line set to ``#!/usr/bin/python3``. Such files should be placed directly into the ``bin/`` directory. It will also typically be necessary to mark the file as executable before the system will allow it to run:: diff --git a/generate_bash_completion.py b/generate_bash_completion.py index 3a10ba43a4..0945058a20 100755 --- a/generate_bash_completion.py +++ b/generate_bash_completion.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # @@ -23,7 +23,6 @@ # -from __future__ import print_function import getopt, sys, os, re, subprocess, locale def usage(): diff --git a/lib/mrtrix3/_5ttgen/fsl.py b/lib/mrtrix3/_5ttgen/fsl.py index ecf3aa4812..236761a22f 100644 --- a/lib/mrtrix3/_5ttgen/fsl.py +++ b/lib/mrtrix3/_5ttgen/fsl.py @@ -13,8 +13,7 @@ # # For more details, see http://www.mrtrix.org/. -import math, os -from distutils.spawn import find_executable +import math, os, shutil from mrtrix3 import MRtrixError from mrtrix3 import app, fsl, image, path, run, utils @@ -74,7 +73,7 @@ def execute(): #pylint: disable=unused-variable fsl_suffix = fsl.suffix() - if not app.ARGS.mask and not app.ARGS.premasked and not find_executable('dc'): + if not app.ARGS.mask and not app.ARGS.premasked and not shutil.which('dc'): app.warn('Unix command "dc" not found; FSL script "standard_space_roi" may fail') sgm_structures = [ 'L_Accu', 'R_Accu', 'L_Caud', 'R_Caud', 'L_Pall', 'R_Pall', 'L_Puta', 'R_Puta', 'L_Thal', 'R_Thal' ] @@ -149,7 +148,7 @@ def execute(): #pylint: disable=unused-variable pre_bet_image = fsl.find_image('T1_preBET') except MRtrixError: app.warn('FSL script \'standard_space_roi\' did not complete successfully' + \ - ('' if find_executable('dc') else ' (possibly due to program \'dc\' not being installed') + '; ' + \ + ('' if shutil.which('dc') else ' (possibly due to program \'dc\' not being installed') + '; ' + \ 'attempting to continue by providing un-cropped image to BET') pre_bet_image = 'T1.nii' diff --git a/lib/mrtrix3/_5ttgen/hsvs.py b/lib/mrtrix3/_5ttgen/hsvs.py index 5977b04112..9f2e5cce72 100644 --- a/lib/mrtrix3/_5ttgen/hsvs.py +++ b/lib/mrtrix3/_5ttgen/hsvs.py @@ -15,8 +15,7 @@ -import glob, os, re -from distutils.spawn import find_executable +import glob, os, re, shutil from mrtrix3 import MRtrixError from mrtrix3 import app, fsl, image, path, run @@ -201,7 +200,7 @@ def execute(): #pylint: disable=unused-variable app.warn('Environment variable FSLDIR is not set; script will run without FSL components') acpc_string = 'anterior ' + ('& posterior commissures' if ATTEMPT_PC else 'commissure') - have_acpcdetect = bool(find_executable('acpcdetect')) and 'ARTHOME' in os.environ + have_acpcdetect = bool(shutil.which('acpcdetect')) and 'ARTHOME' in os.environ if have_acpcdetect: if have_fast: app.console('ACPCdetect and FSL FAST will be used for explicit segmentation of ' + acpc_string) diff --git a/lib/mrtrix3/__init__.py b/lib/mrtrix3/__init__.py index e3e493cca4..721ec5e517 100644 --- a/lib/mrtrix3/__init__.py +++ b/lib/mrtrix3/__init__.py @@ -13,12 +13,8 @@ # # For more details, see http://www.mrtrix.org/. -import inspect, os, sys +import inspect, os, shlex, sys from collections import namedtuple -try: - from shlex import quote -except ImportError: - from pipes import quote from mrtrix3._version import __version__ @@ -36,7 +32,7 @@ class MRtrixError(MRtrixBaseError): #pylint: disable=unused-variable if sys.argv: COMMAND_HISTORY_STRING = sys.argv[0] for arg in sys.argv[1:]: - COMMAND_HISTORY_STRING += ' ' + quote(arg) # Use quotation marks only if required + COMMAND_HISTORY_STRING += ' ' + shlex.quote(arg) # Use quotation marks only if required COMMAND_HISTORY_STRING += ' (version=' + __version__ + ')' diff --git a/lib/mrtrix3/app.py b/lib/mrtrix3/app.py index 8342d28a53..60984d3156 100644 --- a/lib/mrtrix3/app.py +++ b/lib/mrtrix3/app.py @@ -115,7 +115,7 @@ def _execute(module): #pylint: disable=unused-variable for sig in _SIGNALS: try: signal.signal(getattr(signal, sig), handler) - except: + except AttributeError: pass CMDLINE = Parser() @@ -206,7 +206,7 @@ def _execute(module): #pylint: disable=unused-variable try: filename = exception_frame.filename lineno = exception_frame.lineno - except: # Prior to Python 3.5 + except AttributeError: # Prior to Python 3.5 filename = exception_frame[1] lineno = exception_frame[2] sys.stderr.write('\n') @@ -454,7 +454,7 @@ def warn(text): #pylint: disable=unused-variable # A class that can be used to display a progress bar on the terminal, # mimicing the behaviour of MRtrix3 binary commands -class ProgressBar(object): #pylint: disable=unused-variable +class ProgressBar: #pylint: disable=unused-variable BUSY = [ '. ', ' . ', @@ -470,7 +470,7 @@ class ProgressBar(object): #pylint: disable=unused-variable def __init__(self, msg, target=0): from mrtrix3 import run #pylint: disable=import-outside-toplevel global EXEC_NAME, VERBOSITY - if not (isinstance(msg, utils.STRING_TYPES) or callable(msg)): + if not (isinstance(msg, str) or callable(msg)): raise TypeError('app.ProgressBar must be constructed using either a string or a function') self.counter = 0 self.isatty = sys.stderr.isatty() @@ -630,19 +630,19 @@ def set_copyright(self, text): #pylint: disable=unused-variable # Mutually exclusive options need to be added before the command-line input is parsed def flag_mutually_exclusive_options(self, options, required=False): #pylint: disable=unused-variable - if not isinstance(options, list) or not isinstance(options[0], utils.STRING_TYPES): + if not isinstance(options, list) or not isinstance(options[0], str): raise Exception('Parser.flagMutuallyExclusiveOptions() only accepts a list of strings') self._mutually_exclusive_option_groups.append( (options, required) ) - def parse_args(self): + def parse_args(self, args=None, namespace=None): if not self._author: raise Exception('Script author MUST be set in script\'s usage() function') if not self._synopsis: raise Exception('Script synopsis MUST be set in script\'s usage() function') - if '-version' in sys.argv: + if '-version' in args if args else '-version' in sys.argv[1:]: self.print_version() sys.exit(0) - result = argparse.ArgumentParser.parse_args(self) + result = super().parse_args(args, namespace) self._check_mutex_options(result) if self._subparsers: for alg in self._subparsers._group_actions[0].choices: @@ -669,7 +669,7 @@ def print_citation_warning(self): console('') # Overloads argparse.ArgumentParser function to give a better error message on failed parsing - def error(self, text): + def error(self, message): for entry in sys.argv: if '-help'.startswith(entry): self.print_help() @@ -683,11 +683,11 @@ def error(self, text): if alg == sys.argv[1]: usage = self._subparsers._group_actions[0].choices[alg].format_usage() continue - sys.stderr.write('\nError: %s\n' % text) + sys.stderr.write('\nError: %s\n' % message) sys.stderr.write('Usage: ' + usage + '\n') sys.stderr.write(' (Run ' + self.prog + ' -help for more information)\n\n') sys.stderr.flush() - sys.exit(1) + sys.exit(2) def _check_mutex_options(self, args_in): for group in self._mutually_exclusive_option_groups: @@ -726,7 +726,7 @@ def format_usage(self): argument_list.append(arg.dest) return self.prog + ' ' + ' '.join(argument_list) + ' [ options ]' + trailing_ellipsis - def print_help(self): + def print_help(self, file=None): def bold(text): return ''.join( c + chr(0x08) + c for c in text) @@ -856,17 +856,21 @@ def print_group_options(group): text += wrapper_other.fill(entry[1]) + '\n' text += '\n' text += wrapper_other.fill(_MRTRIX3_CORE_REFERENCE) + '\n\n' - command = CONFIG.get('HelpCommand', 'less -X') - if command: - try: - process = subprocess.Popen(command.split(' '), stdin=subprocess.PIPE) - process.communicate(text.encode()) - except: + if file: + file.write(text) + file.flush() + else: + command = CONFIG.get('HelpCommand', 'less -X') + if command: + try: + process = subprocess.Popen(command.split(' '), stdin=subprocess.PIPE) + process.communicate(text.encode()) + except (subprocess.CalledProcessError, FileNotFoundError): + sys.stdout.write(text) + sys.stdout.flush() + else: sys.stdout.write(text) sys.stdout.flush() - else: - sys.stdout.write(text) - sys.stdout.flush() def print_full_usage(self): sys.stdout.write(self._synopsis + '\n') diff --git a/lib/mrtrix3/dwi2response/dhollander.py b/lib/mrtrix3/dwi2response/dhollander.py index b7d17f2dcc..8a10d24018 100644 --- a/lib/mrtrix3/dwi2response/dhollander.py +++ b/lib/mrtrix3/dwi2response/dhollander.py @@ -13,7 +13,7 @@ # # For more details, see http://www.mrtrix.org/. -import math, shutil +import math, shlex, shutil from mrtrix3 import CONFIG, MRtrixError from mrtrix3 import app, image, path, run @@ -240,7 +240,7 @@ def execute(): #pylint: disable=unused-variable app.warn('Single-fibre WM response function selection algorithm "tax" will not honour requested WM voxel percentage') run.command('dwi2response ' + app.ARGS.wm_algo + ' dwi.mif _respsfwmss.txt -mask refined_wm.mif -voxels voxels_sfwm.mif' + ('' if app.ARGS.wm_algo == 'tax' else (' -number ' + str(voxsfwmcount))) - + ' -scratch ' + path.quote(app.SCRATCH_DIR) + + ' -scratch ' + shlex.quote(app.SCRATCH_DIR) + recursive_cleanup_option, show=False) else: diff --git a/lib/mrtrix3/dwi2response/msmt_5tt.py b/lib/mrtrix3/dwi2response/msmt_5tt.py index 9ca669c4a6..942454c13c 100644 --- a/lib/mrtrix3/dwi2response/msmt_5tt.py +++ b/lib/mrtrix3/dwi2response/msmt_5tt.py @@ -13,7 +13,7 @@ # # For more details, see http://www.mrtrix.org/. -import os, shutil +import os, shlex, shutil from mrtrix3 import MRtrixError from mrtrix3 import app, image, path, run @@ -110,10 +110,10 @@ def execute(): #pylint: disable=unused-variable recursive_cleanup_option = ' -nocleanup' if not app.ARGS.sfwm_fa_threshold: app.console('Selecting WM single-fibre voxels using \'' + app.ARGS.wm_algo + '\' algorithm') - run.command('dwi2response ' + app.ARGS.wm_algo + ' dwi.mif wm_ss_response.txt -mask wm_mask.mif -voxels wm_sf_mask.mif -scratch ' + path.quote(app.SCRATCH_DIR) + recursive_cleanup_option) + run.command('dwi2response ' + app.ARGS.wm_algo + ' dwi.mif wm_ss_response.txt -mask wm_mask.mif -voxels wm_sf_mask.mif -scratch ' + shlex.quote(app.SCRATCH_DIR) + recursive_cleanup_option) else: app.console('Selecting WM single-fibre voxels using \'fa\' algorithm with a hard FA threshold of ' + str(app.ARGS.sfwm_fa_threshold)) - run.command('dwi2response fa dwi.mif wm_ss_response.txt -mask wm_mask.mif -threshold ' + str(app.ARGS.sfwm_fa_threshold) + ' -voxels wm_sf_mask.mif -scratch ' + path.quote(app.SCRATCH_DIR) + recursive_cleanup_option) + run.command('dwi2response fa dwi.mif wm_ss_response.txt -mask wm_mask.mif -threshold ' + str(app.ARGS.sfwm_fa_threshold) + ' -voxels wm_sf_mask.mif -scratch ' + shlex.quote(app.SCRATCH_DIR) + recursive_cleanup_option) # Check for empty masks wm_voxels = image.statistics('wm_sf_mask.mif', mask='wm_sf_mask.mif').count diff --git a/lib/mrtrix3/dwibiascorrect/ants.py b/lib/mrtrix3/dwibiascorrect/ants.py index f427b9b41a..3014dec1d9 100644 --- a/lib/mrtrix3/dwibiascorrect/ants.py +++ b/lib/mrtrix3/dwibiascorrect/ants.py @@ -13,7 +13,7 @@ # # For more details, see http://www.mrtrix.org/. -from distutils.spawn import find_executable +import shutil from mrtrix3 import MRtrixError from mrtrix3 import app, path, run @@ -50,7 +50,7 @@ def get_inputs(): #pylint: disable=unused-variable def execute(): #pylint: disable=unused-variable - if not find_executable('N4BiasFieldCorrection'): + if not shutil.which('N4BiasFieldCorrection'): raise MRtrixError('Could not find ANTS program N4BiasFieldCorrection; please check installation') for key in sorted(OPT_N4_BIAS_FIELD_CORRECTION): diff --git a/lib/mrtrix3/dwinormalise/group.py b/lib/mrtrix3/dwinormalise/group.py index 6250881212..f37497d548 100644 --- a/lib/mrtrix3/dwinormalise/group.py +++ b/lib/mrtrix3/dwinormalise/group.py @@ -13,7 +13,7 @@ # # For more details, see http://www.mrtrix.org/. -import os +import os, shlex from mrtrix3 import MRtrixError from mrtrix3 import app, image, path, run @@ -43,7 +43,7 @@ def check_output_paths(): #pylint: disable=unused-variable def execute(): #pylint: disable=unused-variable - class Input(object): + class Input: def __init__(self, filename, prefix, mask_filename = ''): self.filename = filename self.prefix = prefix @@ -86,7 +86,7 @@ def __init__(self, filename, prefix, mask_filename = ''): path.make_dir('fa') progress = app.ProgressBar('Computing FA images', len(input_list)) for i in input_list: - run.command('dwi2tensor ' + path.quote(os.path.join(input_dir, i.filename)) + ' -mask ' + path.quote(os.path.join(mask_dir, i.mask_filename)) + ' - | tensor2metric - -fa ' + os.path.join('fa', i.prefix + '.mif')) + run.command('dwi2tensor ' + shlex.quote(os.path.join(input_dir, i.filename)) + ' -mask ' + shlex.quote(os.path.join(mask_dir, i.mask_filename)) + ' - | tensor2metric - -fa ' + os.path.join('fa', i.prefix + '.mif')) progress.increment() progress.done() @@ -111,7 +111,7 @@ def __init__(self, filename, prefix, mask_filename = ''): path.make_dir('wm_mask_warped') for i in input_list: run.command('mrtransform template_wm_mask.mif -interp nearest -warp_full ' + os.path.join('warps', i.prefix + '.mif') + ' ' + os.path.join('wm_mask_warped', i.prefix + '.mif') + ' -from 2 -template ' + os.path.join('fa', i.prefix + '.mif')) - run.command('dwinormalise individual ' + path.quote(os.path.join(input_dir, i.filename)) + ' ' + os.path.join('wm_mask_warped', i.prefix + '.mif') + ' temp.mif') + run.command('dwinormalise individual ' + shlex.quote(os.path.join(input_dir, i.filename)) + ' ' + os.path.join('wm_mask_warped', i.prefix + '.mif') + ' temp.mif') run.command('mrconvert temp.mif ' + path.from_user(os.path.join(app.ARGS.output_dir, i.filename)), mrconvert_keyval=path.from_user(os.path.join(input_dir, i.filename), False), force=app.FORCE_OVERWRITE) os.remove('temp.mif') progress.increment() diff --git a/lib/mrtrix3/fsl.py b/lib/mrtrix3/fsl.py index 47e7a1863a..5310088de2 100644 --- a/lib/mrtrix3/fsl.py +++ b/lib/mrtrix3/fsl.py @@ -13,8 +13,7 @@ # # For more details, see http://www.mrtrix.org/. -import os -from distutils.spawn import find_executable +import os, shutil from mrtrix3 import MRtrixError @@ -51,7 +50,7 @@ def check_first(prefix, structures): #pylint: disable=unused-variable def eddy_binary(cuda): #pylint: disable=unused-variable from mrtrix3 import app #pylint: disable=import-outside-toplevel if cuda: - if find_executable('eddy_cuda'): + if shutil.which('eddy_cuda'): app.debug('Selected soft-linked CUDA version (\'eddy_cuda\')') return 'eddy_cuda' # Cuda versions are now provided with a CUDA trailing version number @@ -73,7 +72,7 @@ def eddy_binary(cuda): #pylint: disable=unused-variable if version > max_version: max_version = version exe_path = entry - except: + except TypeError: pass if exe_path: app.debug('CUDA version ' + str(max_version) + ': ' + exe_path) @@ -81,7 +80,7 @@ def eddy_binary(cuda): #pylint: disable=unused-variable app.debug('No CUDA version of eddy found') return '' for candidate in [ 'eddy_openmp', 'eddy_cpu', 'eddy', 'fsl5.0-eddy' ]: - if find_executable(candidate): + if shutil.which(candidate): app.debug(candidate) return candidate app.debug('No CPU version of eddy found') @@ -95,9 +94,9 @@ def eddy_binary(cuda): #pylint: disable=unused-variable # function will select the version 5 executable. def exe_name(name): #pylint: disable=unused-variable from mrtrix3 import app #pylint: disable=import-outside-toplevel - if find_executable(name): + if shutil.which(name): output = name - elif find_executable('fsl5.0-' + name): + elif shutil.which('fsl5.0-' + name): output = 'fsl5.0-' + name app.warn('Using FSL binary \"' + output + '\" rather than \"' + name + '\"; suggest checking FSL installation') else: diff --git a/lib/mrtrix3/image.py b/lib/mrtrix3/image.py index 519f4dfa28..afc01752f4 100644 --- a/lib/mrtrix3/image.py +++ b/lib/mrtrix3/image.py @@ -22,12 +22,11 @@ import json, math, os, subprocess from collections import namedtuple from mrtrix3 import MRtrixError -from mrtrix3.utils import STRING_TYPES # Class for importing header information from an image file for reading -class Header(object): +class Header: def __init__(self, image_path): from mrtrix3 import app, path, run #pylint: disable=import-outside-toplevel filename = path.name_temporary('json') @@ -129,7 +128,7 @@ def axis2dir(string): #pylint: disable=unused-variable def check_3d_nonunity(image_in): #pylint: disable=unused-variable from mrtrix3 import app #pylint: disable=import-outside-toplevel if not isinstance(image_in, Header): - if not isinstance(image_in, STRING_TYPES): + if not isinstance(image_in, str): raise MRtrixError('Error trying to test \'' + str(image_in) + '\': Not an image header or file path') image_in = Header(image_in) if len(image_in.size()) < 3: @@ -170,11 +169,11 @@ def match(image_one, image_two, **kwargs): #pylint: disable=unused-variable, too if kwargs: raise TypeError('Unsupported keyword arguments passed to image.match(): ' + str(kwargs)) if not isinstance(image_one, Header): - if not isinstance(image_one, STRING_TYPES): + if not isinstance(image_one, str): raise MRtrixError('Error trying to test \'' + str(image_one) + '\': Not an image header or file path') image_one = Header(image_one) if not isinstance(image_two, Header): - if not isinstance(image_two, STRING_TYPES): + if not isinstance(image_two, str): raise MRtrixError('Error trying to test \'' + str(image_two) + '\': Not an image header or file path') image_two = Header(image_two) debug_prefix = '\'' + image_one.name() + '\' \'' + image_two.name() + '\'' diff --git a/lib/mrtrix3/matrix.py b/lib/mrtrix3/matrix.py index f7a40b0edb..28ba0c80e4 100644 --- a/lib/mrtrix3/matrix.py +++ b/lib/mrtrix3/matrix.py @@ -19,7 +19,6 @@ import io, itertools, os, re from mrtrix3 import COMMAND_HISTORY_STRING, MRtrixError -from mrtrix3.utils import STRING_TYPES _TRANSFORM_LAST_ROW = [ 0.0, 0.0, 0.0, 1.0 ] @@ -180,7 +179,7 @@ def save_numeric(filename, data, **kwargs): encode_args['encoding'] = encoding if header: - if isinstance(header, STRING_TYPES): + if isinstance(header, str): header = { 'comments' : header } elif isinstance(header, list): header = { 'comments' : '\n'.join(str(entry) for entry in header) } @@ -198,7 +197,7 @@ def save_numeric(filename, data, **kwargs): header['command_history'] = COMMAND_HISTORY_STRING if footer: - if isinstance(footer, STRING_TYPES): + if isinstance(footer, str): footer = { 'comments' : footer } elif isinstance(footer, list): footer = { 'comments' : '\n'.join(str(entry) for entry in footer) } diff --git a/lib/mrtrix3/path.py b/lib/mrtrix3/path.py index 42f6235b12..925b458f10 100644 --- a/lib/mrtrix3/path.py +++ b/lib/mrtrix3/path.py @@ -17,16 +17,8 @@ -import ctypes, errno, inspect, os, random, string, subprocess, time -from distutils.spawn import find_executable -# Function can be used in isolation if potentially needing to place quotation marks around a -# filesystem path that is to be included as part of a command string -try: - from shlex import quote -except ImportError: - from pipes import quote +import ctypes, errno, inspect, os, random, shlex, shutil, string, subprocess, time from mrtrix3 import CONFIG -from mrtrix3.utils import STRING_TYPES @@ -66,7 +58,7 @@ def from_user(filename, escape=True): #pylint: disable=unused-variable from mrtrix3 import app #pylint: disable=import-outside-toplevel fullpath = os.path.abspath(os.path.join(app.WORKING_DIR, filename)) if escape: - fullpath = quote(fullpath) + fullpath = shlex.quote(fullpath) app.debug(filename + ' -> ' + fullpath) return fullpath @@ -132,7 +124,7 @@ def script_subdir_name(): #pylint: disable=unused-variable frameinfo = inspect.stack()[-1] try: frame = frameinfo.frame - except: # Prior to Version 3.5 + except AttributeError: # Prior to Version 3.5 frame = frameinfo[0] # If the script has been run through a softlink, we need the name of the original # script in order to locate the additional data @@ -164,7 +156,7 @@ def to_scratch(filename, escape=True): #pylint: disable=unused-variable from mrtrix3 import app #pylint: disable=import-outside-toplevel fullpath = os.path.abspath(os.path.join(app.SCRATCH_DIR, filename)) if escape: - fullpath = quote(fullpath) + fullpath = shlex.quote(fullpath) app.debug(filename + ' -> ' + fullpath) return fullpath @@ -200,9 +192,9 @@ def in_use(path): with open(path, 'rb+') as dummy_f: pass return False - except: + except IOError: return True - if not find_executable('fuser'): + if not shutil.which('fuser'): return None # fuser returns zero if there IS at least one process accessing the file # A fatal error will result in a non-zero code -> in_use() = False, so wait_for() can return @@ -230,12 +222,12 @@ def num_in_use(data): # Make sure the data we're dealing with is a list of strings; # or make it a list of strings if it's just a single entry - if isinstance(paths, STRING_TYPES): + if isinstance(paths, str): paths = [ paths ] else: assert isinstance(paths, list) for entry in paths: - assert isinstance(entry, STRING_TYPES) + assert isinstance(entry, str) app.debug(str(paths)) diff --git a/lib/mrtrix3/phaseencoding.py b/lib/mrtrix3/phaseencoding.py index 918335999e..df771b8c35 100644 --- a/lib/mrtrix3/phaseencoding.py +++ b/lib/mrtrix3/phaseencoding.py @@ -18,7 +18,6 @@ from mrtrix3 import COMMAND_HISTORY_STRING, MRtrixError -from mrtrix3.utils import STRING_TYPES @@ -29,14 +28,14 @@ def direction(string): #pylint: disable=unused-variable try: pe_axis = abs(int(string)) if pe_axis > 2: - raise MRtrixError('When specified as a number, phase encode axis must be either 0, 1 or 2 (positive or negative)') + raise MRtrixError('When specified as a number, phase encoding axis must be either 0, 1 or 2 (positive or negative)') reverse = (string.contains('-')) # Allow -0 pe_dir = [0,0,0] if reverse: pe_dir[pe_axis] = -1 else: pe_dir[pe_axis] = 1 - except Exception as exception: + except TypeError: string = string.lower() if string == 'lr': pe_dir = [1,0,0] @@ -63,7 +62,7 @@ def direction(string): #pylint: disable=unused-variable elif string == 'k-': pe_dir = [0,0,-1] else: - raise MRtrixError('Unrecognized phase encode direction specifier: ' + string) from exception + raise MRtrixError('Unrecognized phase encode direction specifier: ' + string) # pylint: disable=raise-missing-from app.debug(string + ' -> ' + str(pe_dir)) return pe_dir @@ -74,7 +73,7 @@ def direction(string): #pylint: disable=unused-variable def get_scheme(arg): #pylint: disable=unused-variable from mrtrix3 import app, image #pylint: disable=import-outside-toplevel if not isinstance(arg, image.Header): - if not isinstance(arg, STRING_TYPES): + if not isinstance(arg, str): raise TypeError('Error trying to derive phase-encoding scheme from \'' + str(arg) + '\': Not an image header or file path') arg = image.Header(arg) if 'pe_scheme' in arg.keyval(): @@ -109,7 +108,7 @@ def save(filename, scheme, **kwargs): #pylint: disable=unused-variable '(contains ' + str(len(scheme[0])) + ' columns rather than 4)') if header: - if isinstance(header, STRING_TYPES): + if isinstance(header, str): header = { 'comments' : header } elif isinstance(header, list): header = { 'comments' : '\n'.join(str(entry) for entry in header) } diff --git a/lib/mrtrix3/run.py b/lib/mrtrix3/run.py index 9f2e363f30..bc3be16c53 100644 --- a/lib/mrtrix3/run.py +++ b/lib/mrtrix3/run.py @@ -13,16 +13,14 @@ # # For more details, see http://www.mrtrix.org/. -import collections, itertools, os, shlex, signal, string, subprocess, sys, tempfile, threading -from distutils.spawn import find_executable +import collections, itertools, os, shlex, shutil, signal, string, subprocess, sys, tempfile, threading from mrtrix3 import ANSI, BIN_PATH, COMMAND_HISTORY_STRING, EXE_LIST, MRtrixBaseError, MRtrixError -from mrtrix3.utils import STRING_TYPES IOStream = collections.namedtuple('IOStream', 'handle filename') -class Shared(object): +class Shared: # A class for storing all information related to a running process # This includes: @@ -207,11 +205,11 @@ def __str__(self): def command(cmd, **kwargs): #pylint: disable=unused-variable - from mrtrix3 import app, path #pylint: disable=import-outside-toplevel + from mrtrix3 import app #pylint: disable=import-outside-toplevel global shared #pylint: disable=invalid-name def quote_nonpipe(item): - return item if item == '|' else path.quote(item) + return item if item == '|' else shlex.quote(item) shell = kwargs.pop('shell', False) show = kwargs.pop('show', True) @@ -237,11 +235,11 @@ def quote_nonpipe(item): cmdstring = '' cmdsplit = [] for entry in cmd: - if isinstance(entry, STRING_TYPES): + if isinstance(entry, str): cmdstring += (' ' if cmdstring else '') + quote_nonpipe(entry) cmdsplit.append(entry) elif isinstance(entry, list): - assert all([ isinstance(item, STRING_TYPES) for item in entry ]) + assert all([ isinstance(item, str) for item in entry ]) if len(entry) > 1: common_prefix = os.path.commonprefix(entry) common_suffix = os.path.commonprefix([i[::-1] for i in entry])[::-1] @@ -254,7 +252,7 @@ def quote_nonpipe(item): cmdsplit.extend(entry) else: raise TypeError('When run.command() is provided with a list as input, entries in the list must be either strings or lists of strings') - elif isinstance(cmd, STRING_TYPES): + elif isinstance(cmd, str): cmdstring = cmd # Split the command string by spaces, preserving anything encased within quotation marks if os.sep == '/': # Cheap POSIX compliance check @@ -343,7 +341,7 @@ def quote_nonpipe(item): # If a shebang is found, and this call is therefore invoking an # interpreter, can't rely on the interpreter finding the script # from PATH; need to find the full path ourselves. - line[0] = find_executable(line[0]) + line[0] = shutil.which(line[0]) for item in reversed(shebang): line.insert(0, item) @@ -370,8 +368,7 @@ def quote_nonpipe(item): # Set off the process try: this_process_list.append(shared.Process(to_execute, this_stdin, this_stdout, this_stderr, **subprocess_kwargs)) - # FileNotFoundError not defined in Python 2.7 - except OSError as exception: + except FileNotFoundError as exception: raise MRtrixCmdError(cmdstring, 1, '', str(exception)) from exception # End branching based on shell=True/False @@ -474,7 +471,7 @@ def function(fn_to_execute, *args, **kwargs): #pylint: disable=unused-variable show = kwargs.pop('show', True) fnstring = fn_to_execute.__module__ + '.' + fn_to_execute.__name__ + \ - '(' + ', '.join(['\'' + str(a) + '\'' if isinstance(a, STRING_TYPES) else str(a) for a in args]) + \ + '(' + ', '.join(['\'' + str(a) + '\'' if isinstance(a, str) else str(a) for a in args]) + \ (', ' if (args and kwargs) else '') + \ ', '.join([key+'='+str(value) for key, value in kwargs.items()]) + ')' @@ -521,11 +518,11 @@ def exe_name(item): path = item elif os.path.isfile(os.path.join(BIN_PATH, item + '.exe')): path = item + '.exe' - elif find_executable(item) is not None: + elif shutil.which(item) is not None: path = item - elif find_executable(item + '.exe') is not None: + elif shutil.which(item + '.exe') is not None: path = item + '.exe' - # If it can't be found, return the item as-is; find_executable() fails to identify Python scripts + # If it can't be found, return the item as-is else: path = item app.debug(item + ' -> ' + path) @@ -547,7 +544,7 @@ def version_match(item): if os.path.isfile(exe_path_manual): app.debug('Version-matched executable for ' + item + ': ' + exe_path_manual) return exe_path_manual - exe_path_sys = find_executable(exe_name(item)) + exe_path_sys = shutil.which(exe_name(item)) if exe_path_sys and os.path.isfile(exe_path_sys): app.debug('Using non-version-matched executable for ' + item + ': ' + exe_path_sys) return exe_path_sys @@ -565,7 +562,7 @@ def _shebang(item): else: path = version_match(item) if path == item: - path = find_executable(exe_name(item)) + path = shutil.which(exe_name(item)) if not path: app.debug('File \"' + item + '\": Could not find file to query') return [] @@ -577,30 +574,31 @@ def _shebang(item): # Are there any non-text characters? If so, it's a binary file, so no need to looking for a shebang try: line = str(line.decode('utf-8')) - except: + except UnicodeDecodeError: app.debug('File \"' + item + '\": Not a text file') return [] line = line.strip() if len(line) > 2 and line[0:2] == '#!': # Need to strip first in case there's a gap between the shebang symbol and the interpreter path shebang = line[2:].strip().split(' ') - # On Windows, /usr/bin/env can't be easily found, and any direct interpreter path will have a similar issue. + # On Windows, /usr/bin/env can't be easily found # Instead, manually find the right interpreter to call using distutils - # Also if script is written in Python, try to execute it using the same interpreter as that currently running + # Also if script is written in Python, try to execute it using the same interpreter as that currently running, + # as long as Python2 is not explicitly requested if os.path.basename(shebang[0]) == 'env': if len(shebang) < 2: app.warn('Invalid shebang in script file \"' + item + '\" (missing interpreter after \"env\")') return [] - if shebang[1] == 'python': + if shebang[1] == 'python' or shebang[1] == 'python3': if not sys.executable: app.warn('Unable to self-identify Python interpreter; file \"' + item + '\" not guaranteed to execute on same version') return [] shebang = [ sys.executable ] + shebang[2:] app.debug('File \"' + item + '\": Using current Python interpreter') elif utils.is_windows(): - shebang = [ os.path.abspath(find_executable(exe_name(shebang[1]))) ] + shebang[2:] + shebang = [ os.path.abspath(shutil.which(exe_name(shebang[1]))) ] + shebang[2:] elif utils.is_windows(): - shebang = [ os.path.abspath(find_executable(exe_name(os.path.basename(shebang[0])))) ] + shebang[1:] + shebang = [ os.path.abspath(shutil.which(exe_name(os.path.basename(shebang[0])))) ] + shebang[1:] app.debug('File \"' + item + '\": string \"' + line + '\": ' + str(shebang)) return shebang app.debug('File \"' + item + '\": No shebang found') diff --git a/lib/mrtrix3/utils.py b/lib/mrtrix3/utils.py index afcb2064d6..c62730fcf3 100644 --- a/lib/mrtrix3/utils.py +++ b/lib/mrtrix3/utils.py @@ -17,18 +17,7 @@ -import platform, re, sys - - - - -# For identifying function input arguments as strings on -# both Python 2 and 3 -if sys.version_info[0] == 2: - STRING_TYPES = (basestring,) #pylint: disable=undefined-variable -else: - STRING_TYPES = (str,) - +import platform, re @@ -40,7 +29,7 @@ # use the corresponding functions in the mrtrix3.run module # - Construct using a progress bar message, and a list of command strings to run; # all commands within the list will be executed sequentially within the constructor -class RunList(object): #pylint: disable=unused-variable +class RunList: #pylint: disable=unused-variable def __init__(self, message, value): from mrtrix3 import app, run #pylint: disable=import-outside-toplevel if isinstance(value, int): @@ -49,7 +38,7 @@ def __init__(self, message, value): self.counter = 0 self.valid = True elif isinstance(value, list): - assert all(isinstance(entry, STRING_TYPES) for entry in value) + assert all(isinstance(entry, str) for entry in value) self.progress = app.ProgressBar(message, len(value)) for entry in value: run.command(entry) @@ -60,10 +49,10 @@ def __init__(self, message, value): raise TypeError('Construction of RunList class expects either an ' 'integer (number of commands/functions to run), or a ' 'list of command strings to execute') - def command(self, cmd): + def command(self, cmd, **kwargs): from mrtrix3 import run #pylint: disable=import-outside-toplevel assert self.valid - run.command(cmd) + run.command(cmd, **kwargs) self._increment() def function(self, func, *args, **kwargs): from mrtrix3 import run #pylint: disable=import-outside-toplevel diff --git a/run_pylint b/run_pylint index 5271605324..756bb6425f 100755 --- a/run_pylint +++ b/run_pylint @@ -33,7 +33,7 @@ if [ $# == 0 ]; then fi done for bin_path in bin/*; do - if [ -f ${bin_path} ] && $(head -n1 ${bin_path} | grep -q "#!/usr/bin/env python"); then + if [ -f ${bin_path} ] && $(head -n1 ${bin_path} | grep -q "#!\/usr\/bin\/python3"); then tests="$tests $bin_path" fi done @@ -44,13 +44,6 @@ fi success=0 ntests=0 -if [ -z $PYTHON ]; then - forcepyversion="" -else - echo "Using pylint for "${PYTHON}" (from PYTHON environment variable)" - forcepyversion="$PYTHON -m " -fi - cat > $LOGFILE < $LOGFILE <> $LOGFILE +python3 -m pylint --version >> $LOGFILE cat >> $LOGFILE < .__tmp.log 2>&1 + python3 -m pylint --rcfile=testing/pylint.rc ${file} > .__tmp.log 2>&1 if [[ $? -ne 0 ]]; then echo 'ERROR' diff --git a/set_path b/set_path index 9c3b331ff2..69fb0329d1 100755 --- a/set_path +++ b/set_path @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # @@ -36,7 +36,7 @@ import sys, os, platform, subprocess # on Windows, need to use MSYS2 version of python - not MinGW version: if sys.executable[0].isalpha() and sys.executable[1] == ':': - python_cmd = subprocess.check_output ([ 'cygpath.exe', '-w', '/usr/bin/python' ]).decode(errors='ignore').splitlines()[0].strip() + python_cmd = subprocess.check_output ([ 'cygpath.exe', '-w', '/usr/bin/python3' ]).decode(errors='ignore').splitlines()[0].strip() sys.exit (subprocess.call ([ python_cmd ] + sys.argv)) # check whether we are in the right location: diff --git a/src/connectome/enhance.cpp b/src/connectome/enhance.cpp index f1514cd0f4..eba6251e55 100644 --- a/src/connectome/enhance.cpp +++ b/src/connectome/enhance.cpp @@ -38,11 +38,12 @@ namespace MR { void NBS::operator() (in_column_type in, const value_type T, out_column_type out) const { out.setZero(); + BitSet visited (in.size()); for (ssize_t seed = 0; seed != in.size(); ++seed) { if (std::isfinite (in[seed]) && in[seed] >= T && !out[seed]) { - BitSet visited (in.size()); + visited.clear(); visited[seed] = true; vector to_expand (1, seed); size_t cluster_size = 0; diff --git a/testing/binaries/data b/testing/binaries/data index e97aaf1d51..a568c005c7 160000 --- a/testing/binaries/data +++ b/testing/binaries/data @@ -1 +1 @@ -Subproject commit e97aaf1d514bfa0dc3c94b854ece143ef60de62e +Subproject commit a568c005c7d4f8c9b4fce709387ac4d958c09c2a diff --git a/testing/pylint.rc b/testing/pylint.rc index 863517c534..5e0189a7a4 100644 --- a/testing/pylint.rc +++ b/testing/pylint.rc @@ -48,7 +48,7 @@ confidence= # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" #disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call -disable=line-too-long,bad-whitespace,multiple-imports,wrong-import-position,missing-docstring,global-statement,bare-except,too-many-branches,arguments-differ,too-many-statements,too-many-nested-blocks,too-many-locals,too-many-instance-attributes,too-few-public-methods,too-many-arguments,too-many-lines,useless-object-inheritance,broad-except,bad-option-value +disable=line-too-long,multiple-imports,missing-docstring,global-statement,too-many-branches,too-many-statements,too-many-nested-blocks,too-many-locals,too-many-instance-attributes,too-few-public-methods,too-many-arguments,too-many-lines # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where diff --git a/update_copyright b/update_copyright index 40c84b9ad0..eba9259f65 100755 --- a/update_copyright +++ b/update_copyright @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python3 # Copyright (c) 2008-2020 the MRtrix3 contributors. # From b4560ca47ceb598742d03ed9c1d1c210468cc884 Mon Sep 17 00:00:00 2001 From: Robert Smith Date: Thu, 29 Oct 2020 18:49:24 +1100 Subject: [PATCH 3/5] Fixes to Python2 support removal --- bin/population_template | 6 +++--- build | 2 +- lib/mrtrix3/fsl.py | 2 +- lib/mrtrix3/phaseencoding.py | 2 +- testing/pylint.rc | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bin/population_template b/bin/population_template index 82f01178db..c80a041362 100755 --- a/bin/population_template +++ b/bin/population_template @@ -17,7 +17,7 @@ # Generates an unbiased group-average template via image registration of images to a midway space. -import collections, json, math, os, re, shutil, sys +import collections, json, math, os, re, shlex, shutil, sys DEFAULT_RIGID_SCALES = [0.3,0.4,0.6,0.8,1.0,1.0] DEFAULT_RIGID_LMAX = [2,2,2,4,4,4] @@ -1388,8 +1388,8 @@ def execute(): #pylint: disable=unused-variable with open(json_path, 'w') as json_file: json.dump(keyval, json_file) run.command('mrconvert ' + os.path.join('warps', inp.uid + '.mif') + ' ' + - path.quote(os.path.join(warp_path, xcontrast_xsubject_pre_postfix[0] + - inp.uid + xcontrast_xsubject_pre_postfix[1] + '.mif')), + shlex.quote(os.path.join(warp_path, xcontrast_xsubject_pre_postfix[0] + + inp.uid + xcontrast_xsubject_pre_postfix[1] + '.mif')), mrconvert_keyval=json_path, force=app.FORCE_OVERWRITE) progress.increment() progress.done() diff --git a/build b/build index 4ae82702df..b771a65967 100755 --- a/build +++ b/build @@ -1456,7 +1456,7 @@ log ('TODO list contains ' + str(len(todo)) + ''' items # entry.display() try: num_processors = int(os.environ['NUMBER_OF_PROCESSORS']) -except (KeyError, TypeError): +except (KeyError, ValueError): try: num_processors = os.sysconf('SC_NPROCESSORS_ONLN') except ValueError: diff --git a/lib/mrtrix3/fsl.py b/lib/mrtrix3/fsl.py index 5310088de2..b45a96c6aa 100644 --- a/lib/mrtrix3/fsl.py +++ b/lib/mrtrix3/fsl.py @@ -72,7 +72,7 @@ def eddy_binary(cuda): #pylint: disable=unused-variable if version > max_version: max_version = version exe_path = entry - except TypeError: + except ValueError: pass if exe_path: app.debug('CUDA version ' + str(max_version) + ': ' + exe_path) diff --git a/lib/mrtrix3/phaseencoding.py b/lib/mrtrix3/phaseencoding.py index df771b8c35..fcfe1641c8 100644 --- a/lib/mrtrix3/phaseencoding.py +++ b/lib/mrtrix3/phaseencoding.py @@ -35,7 +35,7 @@ def direction(string): #pylint: disable=unused-variable pe_dir[pe_axis] = -1 else: pe_dir[pe_axis] = 1 - except TypeError: + except ValueError: string = string.lower() if string == 'lr': pe_dir = [1,0,0] diff --git a/testing/pylint.rc b/testing/pylint.rc index 5e0189a7a4..c1ebadf0d2 100644 --- a/testing/pylint.rc +++ b/testing/pylint.rc @@ -48,7 +48,7 @@ confidence= # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" #disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call -disable=line-too-long,multiple-imports,missing-docstring,global-statement,too-many-branches,too-many-statements,too-many-nested-blocks,too-many-locals,too-many-instance-attributes,too-few-public-methods,too-many-arguments,too-many-lines +disable=line-too-long,multiple-imports,missing-docstring,bad-whitespace,global-statement,too-many-branches,too-many-statements,too-many-nested-blocks,too-many-locals,too-many-instance-attributes,too-few-public-methods,too-many-arguments,too-many-lines # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option # multiple time (only on the command line, not in the configuration file where From f15651164ea9030589d5c6754abd37e020a72ffb Mon Sep 17 00:00:00 2001 From: Robert Smith Date: Wed, 18 Nov 2020 09:58:17 +1100 Subject: [PATCH 4/5] CI: Install pylint via pip rather than apt-get --- .github/workflows/checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 2f8ba80b45..e5188cf47e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -160,8 +160,8 @@ jobs: - name: install dependencies run: | sudo apt-get update - sudo apt-get install pylint3 python3-sphinx sphinx-rtd-theme-common python3-recommonmark python3-sphinx-rtd-theme python3-pip - pip3 install sphinx-notfound-page + sudo apt-get install python3-sphinx sphinx-rtd-theme-common python3-recommonmark python3-sphinx-rtd-theme python3-pip + pip3 install pylint sphinx-notfound-page - name: check syntax From 4ad93e9c7e7dc7c66621bccd679de7312ac31483 Mon Sep 17 00:00:00 2001 From: Robert Smith Date: Wed, 18 Nov 2020 10:01:27 +1100 Subject: [PATCH 5/5] CI: Install Python3 setuptools prior to invoking pip --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index e5188cf47e..87e2d4ceed 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -160,7 +160,7 @@ jobs: - name: install dependencies run: | sudo apt-get update - sudo apt-get install python3-sphinx sphinx-rtd-theme-common python3-recommonmark python3-sphinx-rtd-theme python3-pip + sudo apt-get install python3-sphinx sphinx-rtd-theme-common python3-recommonmark python3-sphinx-rtd-theme python3-pip python3-setuptools pip3 install pylint sphinx-notfound-page