From da353562464f32c26ff17bd738bdc3f578fa4644 Mon Sep 17 00:00:00 2001 From: Dmitrii Sutiagin Date: Fri, 16 Jun 2023 11:54:17 -0700 Subject: [PATCH 1/7] Avoid cache collision between wheel and editable wheel builds Currently we use the same cache key for both editable wheel and normal wheel, causing whichever happens to build first to be used for both "package = wheel" and "package = editable" configurations, which is a bug. This change attempts to solve that by using different keys for wheel and editable --- .../tox_env/python/virtual_env/package/pyproject.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/tox/tox_env/python/virtual_env/package/pyproject.py b/src/tox/tox_env/python/virtual_env/package/pyproject.py index 7c423434d..b923d251b 100644 --- a/src/tox/tox_env/python/virtual_env/package/pyproject.py +++ b/src/tox/tox_env/python/virtual_env/package/pyproject.py @@ -331,13 +331,9 @@ def __init__(self, root: Path, env: Pep517VirtualEnvPackager) -> None: self._tox_env = env self._backend_executor_: LocalSubProcessPep517Executor | None = None into: dict[str, Any] = {} - pkg_cache = cached( - into, - key=lambda *args, **kwargs: "wheel" if "wheel_directory" in kwargs else "sdist", # noqa: ARG005 - ) - self.build_wheel = pkg_cache(self.build_wheel) # type: ignore[method-assign] - self.build_sdist = pkg_cache(self.build_sdist) # type: ignore[method-assign] - self.build_editable = pkg_cache(self.build_editable) # type: ignore[method-assign] + self.build_wheel = cached(into, key=lambda *args, **kwargs: "wheel")(self.build_wheel) # type: ignore[method-assign] + self.build_sdist = cached(into, key=lambda *args, **kwargs: "sdist")(self.build_sdist) # type: ignore[method-assign] + self.build_editable = cached(into, key=lambda *args, **kwargs: "editable")(self.build_editable) # type: ignore[method-assign] @property def backend_cmd(self) -> Sequence[str]: From c8d80615eafd965fc41cadbf82e46aa993ffe866 Mon Sep 17 00:00:00 2001 From: Dmitrii Sutiagin Date: Fri, 16 Jun 2023 13:46:44 -0700 Subject: [PATCH 2/7] Create 3035.bugfix.rst --- docs/changelog/3035.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changelog/3035.bugfix.rst diff --git a/docs/changelog/3035.bugfix.rst b/docs/changelog/3035.bugfix.rst new file mode 100644 index 000000000..10cb85bcf --- /dev/null +++ b/docs/changelog/3035.bugfix.rst @@ -0,0 +1 @@ +Avoid cache collision between editable wheel build and normal wheel build -- by :user:`f3flight`. From 1df0d4fc8535b4e1bf85092f53437223a34a9fbe Mon Sep 17 00:00:00 2001 From: Dmitrii Sutiagin Date: Fri, 16 Jun 2023 14:11:11 -0700 Subject: [PATCH 3/7] Resolve ruff errors (attempt 01) --- src/tox/tox_env/python/virtual_env/package/pyproject.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/tox/tox_env/python/virtual_env/package/pyproject.py b/src/tox/tox_env/python/virtual_env/package/pyproject.py index b923d251b..088afee91 100644 --- a/src/tox/tox_env/python/virtual_env/package/pyproject.py +++ b/src/tox/tox_env/python/virtual_env/package/pyproject.py @@ -331,9 +331,10 @@ def __init__(self, root: Path, env: Pep517VirtualEnvPackager) -> None: self._tox_env = env self._backend_executor_: LocalSubProcessPep517Executor | None = None into: dict[str, Any] = {} - self.build_wheel = cached(into, key=lambda *args, **kwargs: "wheel")(self.build_wheel) # type: ignore[method-assign] - self.build_sdist = cached(into, key=lambda *args, **kwargs: "sdist")(self.build_sdist) # type: ignore[method-assign] - self.build_editable = cached(into, key=lambda *args, **kwargs: "editable")(self.build_editable) # type: ignore[method-assign] + for build_type in ["editable", "sdist", "wheel"]: + build_method = getattr(self, f'build_{build_type}') + key = lambda *args, **kwargs: build_type #noqa: ARG005 + setattr(self, f'build_{build_type}', cached(into, key)(build_method)) @property def backend_cmd(self) -> Sequence[str]: From 81845d3948b3faf8b73494458d17493335f93d8e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Jun 2023 21:11:52 +0000 Subject: [PATCH 4/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/tox/tox_env/python/virtual_env/package/pyproject.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/tox/tox_env/python/virtual_env/package/pyproject.py b/src/tox/tox_env/python/virtual_env/package/pyproject.py index 088afee91..f43d81c44 100644 --- a/src/tox/tox_env/python/virtual_env/package/pyproject.py +++ b/src/tox/tox_env/python/virtual_env/package/pyproject.py @@ -332,9 +332,12 @@ def __init__(self, root: Path, env: Pep517VirtualEnvPackager) -> None: self._backend_executor_: LocalSubProcessPep517Executor | None = None into: dict[str, Any] = {} for build_type in ["editable", "sdist", "wheel"]: - build_method = getattr(self, f'build_{build_type}') - key = lambda *args, **kwargs: build_type #noqa: ARG005 - setattr(self, f'build_{build_type}', cached(into, key)(build_method)) + build_method = getattr(self, f"build_{build_type}") + + def key(*args, **kwargs): + return build_type # noqa: ARG005 + + setattr(self, f"build_{build_type}", cached(into, key)(build_method)) @property def backend_cmd(self) -> Sequence[str]: From ecabf051f14472b348c470a862bb1752eae3643a Mon Sep 17 00:00:00 2001 From: Dmitrii Sutiagin Date: Fri, 16 Jun 2023 14:34:39 -0700 Subject: [PATCH 5/7] Resolve ruff errors (attempt 02) --- .../python/virtual_env/package/pyproject.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/tox/tox_env/python/virtual_env/package/pyproject.py b/src/tox/tox_env/python/virtual_env/package/pyproject.py index f43d81c44..1c626404b 100644 --- a/src/tox/tox_env/python/virtual_env/package/pyproject.py +++ b/src/tox/tox_env/python/virtual_env/package/pyproject.py @@ -331,13 +331,17 @@ def __init__(self, root: Path, env: Pep517VirtualEnvPackager) -> None: self._tox_env = env self._backend_executor_: LocalSubProcessPep517Executor | None = None into: dict[str, Any] = {} + # wrap build methods in a cache wrapper for build_type in ["editable", "sdist", "wheel"]: - build_method = getattr(self, f"build_{build_type}") - - def key(*args, **kwargs): - return build_type # noqa: ARG005 - - setattr(self, f"build_{build_type}", cached(into, key)(build_method)) + method_name = f"build_{build_type}" + setattr( + self, + method_name, + cached( + into, + key=lambda *args, bound_return=build_type, **kwargs: bound_return # noqa: ARG005 + )(getattr(self, method_name)) + ) @property def backend_cmd(self) -> Sequence[str]: From 5c56adb7dadefeaac8bebebd577fe423294a2b9d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 16 Jun 2023 21:34:51 +0000 Subject: [PATCH 6/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/tox/tox_env/python/virtual_env/package/pyproject.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tox/tox_env/python/virtual_env/package/pyproject.py b/src/tox/tox_env/python/virtual_env/package/pyproject.py index 1c626404b..908fb6cb8 100644 --- a/src/tox/tox_env/python/virtual_env/package/pyproject.py +++ b/src/tox/tox_env/python/virtual_env/package/pyproject.py @@ -339,8 +339,8 @@ def __init__(self, root: Path, env: Pep517VirtualEnvPackager) -> None: method_name, cached( into, - key=lambda *args, bound_return=build_type, **kwargs: bound_return # noqa: ARG005 - )(getattr(self, method_name)) + key=lambda *args, bound_return=build_type, **kwargs: bound_return, # noqa: ARG005 + )(getattr(self, method_name)), ) @property From 7d2c508f6c9c679ae0fb8dadb9f2590176bcf285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Fri, 16 Jun 2023 17:52:18 -0700 Subject: [PATCH 7/7] Add test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bernát Gábor --- .pre-commit-config.yaml | 18 +++--- pyproject.toml | 64 +++++++++---------- .../python/virtual_env/package/pyproject.py | 18 ++---- tests/demo_pkg_inline/build.py | 10 +++ .../package/test_package_pyproject.py | 31 +++++++++ 5 files changed, 89 insertions(+), 52 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b123844bd..da45b1506 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,35 +4,35 @@ repos: hooks: - id: end-of-file-fixer - id: trailing-whitespace - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.0.272" - hooks: - - id: ruff - args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/psf/black rev: 23.3.0 hooks: - id: black - repo: https://github.com/tox-dev/tox-ini-fmt - rev: "1.3.0" + rev: "1.3.1" hooks: - id: tox-ini-fmt args: ["-p", "fix"] - repo: https://github.com/tox-dev/pyproject-fmt - rev: "0.11.2" + rev: "0.12.0" hooks: - id: pyproject-fmt - additional_dependencies: ["tox>=4.6"] + additional_dependencies: ["tox>=4.6.1"] - repo: https://github.com/pre-commit/mirrors-prettier rev: "v3.0.0-alpha.9-for-vscode" hooks: - id: prettier args: ["--print-width=120", "--prose-wrap=always"] - repo: https://github.com/asottile/blacken-docs - rev: 1.13.0 + rev: 1.14.0 hooks: - id: blacken-docs additional_dependencies: [black==23.3] + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.0.272" + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 hooks: diff --git a/pyproject.toml b/pyproject.toml index 1d1d3bb0f..427a0137d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,16 +56,16 @@ dependencies = [ "packaging>=23.1", "platformdirs>=3.5.3", "pluggy>=1", - "pyproject-api>=1.5.1", + "pyproject-api>=1.5.2", 'tomli>=2.0.1; python_version < "3.11"', 'typing-extensions>=4.6.3; python_version < "3.8"', - "virtualenv>=20.23", + "virtualenv>=20.23.1", ] optional-dependencies.docs = [ "furo>=2023.5.20", "sphinx>=7.0.1", - "sphinx-argparse-cli>=1.11", - "sphinx-autodoc-typehints!=1.23.4,>=1.23", + "sphinx-argparse-cli>=1.11.1", + "sphinx-autodoc-typehints!=1.23.4,>=1.23.2", "sphinx-copybutton>=0.5.2", "sphinx-inline-tabs>=2023.4.21", "sphinxcontrib-towncrier>=0.2.1a0", @@ -75,7 +75,7 @@ optional-dependencies.testing = [ "build[virtualenv]>=0.10", "covdefaults>=2.3", "detect-test-pollution>=1.1.1", - "devpi-process>=0.3", + "devpi-process>=0.3.1", "diff-cover>=7.6", "distlib>=0.3.6", "flaky>=3.7", @@ -84,10 +84,10 @@ optional-dependencies.testing = [ "psutil>=5.9.5", "pytest>=7.3.2", "pytest-cov>=4.1", - "pytest-mock>=3.10", + "pytest-mock>=3.11.1", "pytest-xdist>=3.3.1", "re-assert>=1.1", - 'time-machine>=2.9; implementation_name != "pypy"', + 'time-machine>=2.10; implementation_name != "pypy"', "wheel>=0.40", ] urls.Documentation = "https://tox.wiki" @@ -106,6 +106,31 @@ version.source = "vcs" [tool.black] line-length = 120 +[tool.ruff] +select = ["ALL"] +line-length = 120 +target-version = "py37" +isort = {known-first-party = ["tox", "tests"], required-imports = ["from __future__ import annotations"]} +ignore = [ + "INP001", # no implicit namespaces here + "D", # ignore documentation for now + "ANN401", # Dynamically typed expressions (typing.Any) are disallowed in `arg`" + "ANN101", # Missing type annotation for `self` in method + "ANN102", # Missing type annotation for `cls` in classmethod" + "D203", # `one-blank-line-before-class` (D203) and `no-blank-line-before-class` (D211) are incompatible + "D212", # `multi-line-summary-first-line` (D212) and `multi-line-summary-second-line` (D213) are incompatible + "S104", # Possible binding to all interface +] +[tool.ruff.per-file-ignores] +"tests/**/*.py" = [ + "S101", # asserts allowed in tests... + "FBT", # don"t care about booleans as positional arguments in tests + "INP001", # no implicit namespace + "D", # don"t care about documentation in tests + "S603", # `subprocess` call: check for execution of untrusted input + "PLR2004", # Magic value used in comparison, consider replacing with a constant variable +] + [tool.pytest.ini_options] testpaths = ["tests"] addopts = "--tb=auto -ra --showlocals --no-success-flaky-report" @@ -152,28 +177,3 @@ title_format = false issue_format = ":issue:`{issue}`" template = "docs/changelog/template.jinja2" # possible types, all default: feature, bugfix, doc, removal, misc - -[tool.ruff] -select = ["ALL"] -line-length = 120 -target-version = "py37" -isort = {known-first-party = ["tox", "tests"], required-imports = ["from __future__ import annotations"]} -ignore = [ - "INP001", # no implicit namespaces here - "D", # ignore documentation for now - "ANN401", # Dynamically typed expressions (typing.Any) are disallowed in `arg`" - "ANN101", # Missing type annotation for `self` in method - "ANN102", # Missing type annotation for `cls` in classmethod" - "D203", # `one-blank-line-before-class` (D203) and `no-blank-line-before-class` (D211) are incompatible - "D212", # `multi-line-summary-first-line` (D212) and `multi-line-summary-second-line` (D213) are incompatible - "S104", # Possible binding to all interface -] -[tool.ruff.per-file-ignores] -"tests/**/*.py" = [ - "S101", # asserts allowed in tests... - "FBT", # don"t care about booleans as positional arguments in tests - "INP001", # no implicit namespace - "D", # don"t care about documentation in tests - "S603", # `subprocess` call: check for execution of untrusted input - "PLR2004", # Magic value used in comparison, consider replacing with a constant variable -] diff --git a/src/tox/tox_env/python/virtual_env/package/pyproject.py b/src/tox/tox_env/python/virtual_env/package/pyproject.py index 908fb6cb8..158707324 100644 --- a/src/tox/tox_env/python/virtual_env/package/pyproject.py +++ b/src/tox/tox_env/python/virtual_env/package/pyproject.py @@ -331,17 +331,13 @@ def __init__(self, root: Path, env: Pep517VirtualEnvPackager) -> None: self._tox_env = env self._backend_executor_: LocalSubProcessPep517Executor | None = None into: dict[str, Any] = {} - # wrap build methods in a cache wrapper - for build_type in ["editable", "sdist", "wheel"]: - method_name = f"build_{build_type}" - setattr( - self, - method_name, - cached( - into, - key=lambda *args, bound_return=build_type, **kwargs: bound_return, # noqa: ARG005 - )(getattr(self, method_name)), - ) + + for build_type in ("editable", "sdist", "wheel"): # wrap build methods in a cache wrapper + + def key(*args: Any, bound_return: str = build_type, **kwargs: Any) -> str: # noqa: ARG001 + return bound_return + + setattr(self, f"build_{build_type}", cached(into, key=key)(getattr(self, f"build_{build_type}"))) @property def backend_cmd(self) -> Sequence[str]: diff --git a/tests/demo_pkg_inline/build.py b/tests/demo_pkg_inline/build.py index 28ffea5ec..eedf6d797 100644 --- a/tests/demo_pkg_inline/build.py +++ b/tests/demo_pkg_inline/build.py @@ -117,6 +117,16 @@ def get_requires_for_build_wheel(config_settings: dict[str, str] | None = None) return [] # pragma: no cover # only executed in non-host pythons +if os.environ.get("BACKEND_HAS_EDITABLE"): + + def build_editable( + wheel_directory: str, + config_settings: dict[str, str] | None = None, + metadata_directory: str | None = None, + ) -> str: + return build_wheel(wheel_directory, config_settings, metadata_directory) + + def build_sdist(sdist_directory: str, config_settings: dict[str, str] | None = None) -> str: # noqa: ARG001 result = f"{name}-{version}.tar.gz" # pragma: win32 cover with tarfile.open(str(Path(sdist_directory) / result), "w:gz") as tar: # pragma: win32 cover diff --git a/tests/tox_env/python/virtual_env/package/test_package_pyproject.py b/tests/tox_env/python/virtual_env/package/test_package_pyproject.py index bf5e24ada..9ccacf24b 100644 --- a/tests/tox_env/python/virtual_env/package/test_package_pyproject.py +++ b/tests/tox_env/python/virtual_env/package/test_package_pyproject.py @@ -263,3 +263,34 @@ def test_project_package_with_deps(tox_project: ToxProjectCreator, demo_pkg_setu else: assert found_calls[0] == (".pkg", "install_requires") assert found_calls[1] == (".pkg", "install_deps") + + +def test_pyproject_build_editable_and_wheel(tox_project: ToxProjectCreator, demo_pkg_inline: Path) -> None: + # test that build wheel and build editable are cached separately + + ini = """ + [testenv:.pkg] + set_env= BACKEND_HAS_EDITABLE=1 + [testenv:a,b] + package = editable + [testenv:c,d] + package = wheel + """ + proj = tox_project({"tox.ini": ini}, base=demo_pkg_inline) + execute_calls = proj.patch_execute(lambda r: 0 if "install" in r.run_id else None) + + result = proj.run("r", "-e", "a,b,c,d", "--notest", "--workdir", str(proj.path / ".tox")) + + result.assert_success() + found_calls = [(i[0][0].conf.name, i[0][3].run_id) for i in execute_calls.call_args_list] + assert found_calls == [ + (".pkg", "_optional_hooks"), + (".pkg", "get_requires_for_build_wheel"), + (".pkg", "build_editable"), + ("a", "install_package"), + ("b", "install_package"), + (".pkg", "build_wheel"), + ("c", "install_package"), + ("d", "install_package"), + (".pkg", "_exit"), + ]