From a74175d90c42040502010780ccd064fe48e056b9 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Fri, 31 May 2024 08:05:09 -0400 Subject: [PATCH] migrate linting from flake8, isort, autoflake to ruff (same as atomate2, jobflow now) --- .pre-commit-config.yaml | 48 ++++----------- doc/source/conf.py | 4 +- pyproject.toml | 75 ++++++++---------------- src/jobflow_remote/cli/__init__.py | 1 + src/jobflow_remote/cli/runner.py | 2 +- src/jobflow_remote/remote/host/remote.py | 1 - src/jobflow_remote/testing/__init__.py | 2 - src/jobflow_remote/testing/cli.py | 4 +- tests/db/cli/test_flow.py | 3 - tests/db/cli/test_job.py | 15 ----- 10 files changed, 42 insertions(+), 113 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ea61965c..a3c82455 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,46 +1,27 @@ default_language_version: - python: python3.9 -#exclude: '^src/{{ package_name }}/some/directory/' + python: python3 +exclude: ^(.github/|tests/test_data/abinit/) repos: +- repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.4.5 + hooks: + - id: ruff + args: [--fix] + - id: ruff-format - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-yaml - id: fix-encoding-pragma args: [--remove] - id: end-of-file-fixer - id: trailing-whitespace -- repo: https://github.com/myint/autoflake - rev: v2.2.1 - hooks: - - id: autoflake -- repo: https://github.com/psf/black - rev: 24.1.1 - hooks: - - id: black - repo: https://github.com/asottile/blacken-docs rev: 1.16.0 hooks: - id: blacken-docs additional_dependencies: [black] exclude: README.md -- repo: https://github.com/pycqa/isort - rev: 5.13.2 - hooks: - - id: isort -- repo: https://github.com/pycqa/flake8 - rev: 6.1.0 - hooks: - - id: flake8 - entry: pflake8 - files: ^src/ - additional_dependencies: - - pyproject-flake8 - - flake8-bugbear - - flake8-typing-imports - - flake8-docstrings - - flake8-rst-docstrings - - flake8-rst - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 hooks: @@ -49,13 +30,12 @@ repos: - id: rst-directive-colons - id: rst-inline-touching-normal - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.8.0 + rev: v1.10.0 hooks: - id: mypy files: ^src/ additional_dependencies: - tokenize-rt==4.1.0 - - types-pkg_resources==0.1.2 - types-paramiko - pydantic~=2.0 - repo: https://github.com/codespell-project/codespell @@ -63,9 +43,5 @@ repos: hooks: - id: codespell stages: [commit, commit-msg] - args: [--ignore-words-list, 'titel,statics,ba,nd,te,nin', --skip, '*.svg'] -- repo: https://github.com/asottile/pyupgrade - rev: v3.15.0 - hooks: - - id: pyupgrade - args: [--py39-plus] + args: [--ignore-words-list, 'titel,statics,ba,nd,te'] + types_or: [python, rst, markdown] diff --git a/doc/source/conf.py b/doc/source/conf.py index 9a48c698..2954baa1 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -14,10 +14,10 @@ import os import sys -# sys.path.insert(0, os.path.abspath('.')) sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) ) +import jobflow_remote # -- Project information ----------------------------------------------------- @@ -26,8 +26,6 @@ author = "Guido Petretto, Matthew Evans, David Waroquiers" -import jobflow_remote - # The short X.Y version version = jobflow_remote.__version__ # The full version, including alpha/beta/rc tags diff --git a/pyproject.toml b/pyproject.toml index 07006871..3306c2ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,3 @@ -[build-system] -requires = ["setuptools >= 42", "versioningit ~= 1.0", "wheel"] -build-backend = "setuptools.build_meta" - [project] name = "jobflow-remote" description = "Jobflow Remote is a Python package to run jobflow workflows on remote resources" @@ -11,50 +7,44 @@ license = { text = "modified BSD" } authors = [{ name = "Guido Petretto", email = "guido.petretto@matgenix.com" }] dynamic = ["version"] classifiers = [ - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", "Development Status :: 5 - Production/Stable", + "Intended Audience :: Information Technology", "Intended Audience :: Science/Research", "Intended Audience :: System Administrators", - "Intended Audience :: Information Technology", "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.9", "Topic :: Other/Nonlisted Topic", "Topic :: Scientific/Engineering", ] requires-python = ">=3.9" -dependencies =[ +dependencies = [ + "fabric ~= 3.2", + "flufl.lock ~= 8.0", "jobflow >= 0.1.14", + "psutil ~= 5.9", "pydantic ~= 2.4", - "fabric ~= 3.2", - "tomlkit ~= 0.12", "qtoolkit ~= 0.1, >= 0.1.4", - "typer ~= 0.9", "rich ~= 13.7", - "psutil ~= 5.9", - "supervisor ~= 4.2", "ruamel.yaml ~= 0.17", "schedule ~= 1.2", - "flufl.lock ~= 8.0" + "supervisor ~= 4.2", + "tomlkit ~= 0.12", + "typer ~= 0.9", ] [project.optional-dependencies] -dev = [ - "pre-commit>=3.0.0", -] -tests = [ - "pytest ~= 8.0", - "pytest-cov ~= 4.0", - "docker ~= 7.0", -] +dev = ["pre-commit>=3.0.0"] +tests = ["docker ~= 7.0", "pytest ~= 8.0", "pytest-cov ~= 4.0"] docs = [ - "sphinx", - "sphinx_design", + "autodoc_pydantic>=2.0.0", "pydata-sphinx-theme", + "sphinx", "sphinx-copybutton", - "autodoc_pydantic>=2.0.0", - "sphinxcontrib-mermaid" + "sphinx_design", + "sphinxcontrib-mermaid", ] [project.scripts] @@ -69,22 +59,14 @@ changelog = "https://matgenix.github.io/jobflow-remote/changelog" [tool.setuptools.package-data] jobflow_remote = ["py.typed"] +[build-system] +requires = ["setuptools >= 42", "versioningit ~= 1.0", "wheel"] +build-backend = "setuptools.build_meta" + [tool.versioningit.vcs] method = "git" default-tag = "0.0.1" -[tool.isort] -profile = "black" - -[tool.flake8] -max-line-length = 88 -max-doc-length = 88 -select = "C, E, F, W, B" -extend-ignore = "E203, W503, E501, F401, RST21" -min-python-version = "3.9.0" -docstring-convention = "numpy" -rst-roles = "class, func, ref, obj" - [tool.mypy] ignore_missing_imports = true strict_optional = false @@ -104,9 +86,9 @@ warn_required_dynamic_aliases = true [tool.pytest.ini_options] filterwarnings = [ "ignore:.*POTCAR.*:UserWarning", - "ignore:.*magmom.*:UserWarning", - "ignore:.*is not gzipped.*:UserWarning", "ignore:.*input structure.*:UserWarning", + "ignore:.*is not gzipped.*:UserWarning", + "ignore:.*magmom.*:UserWarning", "ignore::DeprecationWarning", ] @@ -122,14 +104,7 @@ source = ["src/"] skip_covered = true show_missing = true exclude_lines = [ + '^\s*@overload( |$)', '^\s*assert False(,|$)', 'if typing.TYPE_CHECKING:', - '^\s*@overload( |$)', ] - -[tool.autoflake] -in-place = true -remove-all-unused-imports = true -remove-unused-variables = true -ignore-init-module-imports = true -expand-star-imports = true diff --git a/src/jobflow_remote/cli/__init__.py b/src/jobflow_remote/cli/__init__.py index ffb5fa72..fcc97905 100644 --- a/src/jobflow_remote/cli/__init__.py +++ b/src/jobflow_remote/cli/__init__.py @@ -1,3 +1,4 @@ +# ruff: noqa: F401 # Import the submodules with a local app to register them to the main app import jobflow_remote.cli.admin import jobflow_remote.cli.execution diff --git a/src/jobflow_remote/cli/runner.py b/src/jobflow_remote/cli/runner.py index 480df6b1..245095c0 100644 --- a/src/jobflow_remote/cli/runner.py +++ b/src/jobflow_remote/cli/runner.py @@ -181,7 +181,7 @@ def stop( "Wait until the daemon has stopped. NOTE: this may take a while if a large file is being transferred!" ), ), - ] = False + ] = False, ): """ Send a stop signal to the Runner processes. diff --git a/src/jobflow_remote/remote/host/remote.py b/src/jobflow_remote/remote/host/remote.py index fe308f17..f420762e 100644 --- a/src/jobflow_remote/remote/host/remote.py +++ b/src/jobflow_remote/remote/host/remote.py @@ -79,7 +79,6 @@ def _create_connection(self): # if the authentication is ssh-key + OTP paramiko already # handles it. Don't use the alternative strategy. if not self._connection.connect_kwargs.get("key_filename"): - if not config: config = Config() config.authentication.strategy_class = InteractiveAuthStrategy diff --git a/src/jobflow_remote/testing/__init__.py b/src/jobflow_remote/testing/__init__.py index dc418f0f..679e8024 100644 --- a/src/jobflow_remote/testing/__init__.py +++ b/src/jobflow_remote/testing/__init__.py @@ -88,8 +88,6 @@ def self_replace(n: int): """ Create a replace Job with the same job n times. """ - from jobflow import Flow - if n > 0: return Response(replace=self_replace(n - 1)) diff --git a/src/jobflow_remote/testing/cli.py b/src/jobflow_remote/testing/cli.py index 48d775df..c985d0c7 100644 --- a/src/jobflow_remote/testing/cli.py +++ b/src/jobflow_remote/testing/cli.py @@ -35,8 +35,8 @@ def run_check_cli( ) # note that stderr is not captured separately - assert error == ( - result.exit_code != 0 + assert ( + error == (result.exit_code != 0) ), f"cli should have {'not ' if not error else ''}failed. exit code: {result.exit_code}. stdout: {result.stdout}" if required_out: diff --git a/tests/db/cli/test_flow.py b/tests/db/cli/test_flow.py index 58b1dcf0..6d8bb981 100644 --- a/tests/db/cli/test_flow.py +++ b/tests/db/cli/test_flow.py @@ -1,5 +1,4 @@ def test_flows_list(job_controller, four_jobs): - from jobflow_remote.testing.cli import run_check_cli columns = ["DB id", "Name", "State", "Flow id", "Num Jobs", "Last updated"] @@ -19,7 +18,6 @@ def test_flows_list(job_controller, four_jobs): def test_delete(job_controller, four_jobs): - from jobflow_remote.testing.cli import run_check_cli run_check_cli( @@ -40,7 +38,6 @@ def test_delete(job_controller, four_jobs): def test_flow_info(job_controller, four_jobs): - from jobflow_remote.testing.cli import run_check_cli columns = ["DB id", "Name", "State", "Job id", "(Index)", "Worker"] diff --git a/tests/db/cli/test_job.py b/tests/db/cli/test_job.py index 91717f0b..74ffe56b 100644 --- a/tests/db/cli/test_job.py +++ b/tests/db/cli/test_job.py @@ -1,5 +1,4 @@ def test_jobs_list(job_controller, four_jobs): - from jobflow_remote.jobs.state import JobState from jobflow_remote.testing.cli import run_check_cli @@ -32,7 +31,6 @@ def test_jobs_list(job_controller, four_jobs): def test_job_info(job_controller, four_jobs): - from jobflow_remote.testing.cli import run_check_cli outputs = ["name = 'add1'", "state = 'READY'"] @@ -54,7 +52,6 @@ def test_job_info(job_controller, four_jobs): def test_set_state(job_controller, four_jobs): - from jobflow_remote.jobs.state import JobState from jobflow_remote.testing.cli import run_check_cli @@ -70,7 +67,6 @@ def test_set_state(job_controller, four_jobs): def test_rerun(job_controller, four_jobs): - from jobflow_remote.jobs.state import JobState from jobflow_remote.testing.cli import run_check_cli @@ -86,7 +82,6 @@ def test_rerun(job_controller, four_jobs): def test_retry(job_controller, four_jobs): - from jobflow_remote.jobs.state import JobState from jobflow_remote.testing.cli import run_check_cli @@ -102,7 +97,6 @@ def test_retry(job_controller, four_jobs): def test_play_pause(job_controller, four_jobs): - from jobflow_remote.jobs.state import JobState from jobflow_remote.testing.cli import run_check_cli @@ -122,7 +116,6 @@ def test_play_pause(job_controller, four_jobs): def test_stop(job_controller, four_jobs): - from jobflow_remote.jobs.state import JobState from jobflow_remote.testing.cli import run_check_cli @@ -135,7 +128,6 @@ def test_stop(job_controller, four_jobs): def test_queue_out(job_controller, one_job): - from jobflow_remote.jobs.runner import Runner from jobflow_remote.testing.cli import run_check_cli @@ -157,7 +149,6 @@ def test_queue_out(job_controller, one_job): def test_set_worker(job_controller, one_job): - from jobflow_remote.testing.cli import run_check_cli run_check_cli( @@ -169,7 +160,6 @@ def test_set_worker(job_controller, one_job): def test_set_exec_config(job_controller, one_job): - from jobflow_remote.testing.cli import run_check_cli run_check_cli( @@ -181,7 +171,6 @@ def test_set_exec_config(job_controller, one_job): def test_set_resources(job_controller, one_job): - from jobflow_remote.testing.cli import run_check_cli run_check_cli( @@ -193,7 +182,6 @@ def test_set_resources(job_controller, one_job): def test_job_dump(job_controller, one_job, tmp_dir): - import os from jobflow_remote.testing.cli import run_check_cli @@ -203,7 +191,6 @@ def test_job_dump(job_controller, one_job, tmp_dir): def test_output(job_controller, one_job): - from jobflow_remote.jobs.runner import Runner from jobflow_remote.testing.cli import run_check_cli @@ -216,7 +203,6 @@ def test_output(job_controller, one_job): def test_files_list(job_controller, one_job): - from jobflow_remote.jobs.runner import Runner from jobflow_remote.testing.cli import run_check_cli @@ -238,7 +224,6 @@ def test_files_list(job_controller, one_job): def test_files_get(job_controller, one_job, tmp_dir): - import os from jobflow_remote.jobs.runner import Runner