From a605177bceb2751b3b49b2fea175899fc5929a67 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Tue, 19 Dec 2023 22:51:23 +0100 Subject: [PATCH 1/4] Add extended-length prefix support Fixes #588. --- .github/workflows/tests.yaml | 15 +++- conda-store-server/conda_store_server/app.py | 6 ++ conda-store-server/conda_store_server/orm.py | 16 ++++- conda-store-server/tests/test_server.py | 72 +++++++++++++++++++ docusaurus-docs/conda-store/references/faq.md | 53 +++++++++++--- 5 files changed, 148 insertions(+), 14 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 13df6db6e..d5b9ead5d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -62,7 +62,20 @@ jobs: - name: "Unit tests ✅" run: | - pytest tests + pytest -m "not extended_prefix" tests + + # https://github.com/actions/runner-images/issues/1052 + - name: "Windows extended prefix unit tests ✅" + shell: pwsh + run: | + Set-ItemProperty "HKLM:\System\CurrentControlSet\Control\FileSystem" ` + -Name "LongPathsEnabled" ` + -Value 0 ` + -Type DWord + (Get-ItemProperty "HKLM:System\CurrentControlSet\Control\FileSystem").LongPathsEnabled + pytest -m "extended_prefix" tests + if: matrix.os == 'windows' + integration-test-conda-store-server: name: "integration-test conda-store-server" diff --git a/conda-store-server/conda_store_server/app.py b/conda-store-server/conda_store_server/app.py index dc8685e7f..a4ba24b51 100644 --- a/conda-store-server/conda_store_server/app.py +++ b/conda-store-server/conda_store_server/app.py @@ -120,6 +120,12 @@ def _check_build_key_version(self, proposal): except Exception as e: raise TraitError(f"c.CondaStore.build_key_version: {e}") + win_extended_length_prefix = Bool( + False, + help="Use the extended-length prefix '\\\\?\\' (Windows-only), default: False", + config=True, + ) + conda_command = Unicode( "mamba", help="conda executable to use for solves", diff --git a/conda-store-server/conda_store_server/orm.py b/conda-store-server/conda_store_server/orm.py index 0275f21f5..304ac9574 100644 --- a/conda-store-server/conda_store_server/orm.py +++ b/conda-store-server/conda_store_server/orm.py @@ -4,6 +4,7 @@ import pathlib import re import shutil +import sys from conda_store_server import conda_utils, schema, utils from conda_store_server.environment import validate_environment @@ -254,7 +255,12 @@ def build_path(self, conda_store): # https://github.com/conda-incubator/conda-store/issues/649 if len(str(res)) > 255: raise BuildPathError("build_path too long: must be <= 255 characters") - return res + # Note: cannot use the '/' operator to prepend the extended-length + # prefix + if sys.platform == "win32" and conda_store.win_extended_length_prefix: + return pathlib.Path(f"\\\\?\\{res}") + else: + return res def environment_path(self, conda_store): """Environment path is the path for the symlink to the build @@ -264,11 +270,17 @@ def environment_path(self, conda_store): store_directory = os.path.abspath(conda_store.store_directory) namespace = self.environment.namespace.name name = self.specification.name - return pathlib.Path( + res = pathlib.Path( conda_store.environment_directory.format( store_directory=store_directory, namespace=namespace, name=name ) ) + # Note: cannot use the '/' operator to prepend the extended-length + # prefix + if sys.platform == "win32" and conda_store.win_extended_length_prefix: + return pathlib.Path(f"\\\\?\\{res}") + else: + return res @property def build_key(self): diff --git a/conda-store-server/tests/test_server.py b/conda-store-server/tests/test_server.py index a20d0ff5a..ea7d553d8 100644 --- a/conda-store-server/tests/test_server.py +++ b/conda-store-server/tests/test_server.py @@ -1,4 +1,5 @@ import json +import sys import time import pytest @@ -396,6 +397,77 @@ def test_create_specification_auth_env_name_too_long(testclient, celery_worker, assert False, f"failed to update status" +@pytest.fixture +def win_extended_length_prefix(request): + # Overrides the attribute before other fixtures are called + import conda_store_server + conda_store_server.app.CondaStore.win_extended_length_prefix = request.param + yield request.param + + +@pytest.mark.skipif(sys.platform != "win32", reason="tests a Windows issue") +@pytest.mark.parametrize('win_extended_length_prefix', [True, False], indirect=True) +@pytest.mark.extended_prefix +def test_create_specification_auth_extended_prefix(win_extended_length_prefix, testclient, celery_worker, authenticate): + # Adds padding to cause an error if the extended prefix is not enabled + namespace = "default" + 'A' * 10 + environment_name = "pytest" + + # The debugpy 1.8.0 package was deliberately chosen because it has long + # paths internally, which causes issues on Windows due to the path length + # limit + response = testclient.post( + "api/v1/specification", + json={ + "namespace": namespace, + "specification": json.dumps({ + "name": environment_name, + "channels": ["conda-forge"], + "dependencies": ["debugpy==1.8.0"], + "variables": None, + "prefix": None, + "description": "test" + }), + }, + timeout=30, + ) + response.raise_for_status() + + r = schema.APIPostSpecification.parse_obj(response.json()) + assert r.status == schema.APIStatus.OK + build_id = r.data.build_id + + # Try checking that the status is 'FAILED' + is_updated = False + for _ in range(30): + time.sleep(5) + + # check for the given build + response = testclient.get(f"api/v1/build/{build_id}", timeout=30) + response.raise_for_status() + + r = schema.APIGetBuild.parse_obj(response.json()) + assert r.status == schema.APIStatus.OK + assert r.data.specification.name == environment_name + if r.data.status in ["QUEUED", "BUILDING"]: + continue # checked too fast, try again + + if win_extended_length_prefix: + assert r.data.status == "COMPLETED" + else: + assert r.data.status == "FAILED" + response = testclient.get(f"api/v1/build/{build_id}/logs", timeout=30) + response.raise_for_status() + assert "[WinError 206] The filename or extension is too long" in response.text + + is_updated = True + break + + # If we're here, the task didn't update the status on failure + if not is_updated: + assert False, f"failed to get status" + + def test_create_specification_auth(testclient, celery_worker, authenticate): namespace = "default" environment_name = "pytest" diff --git a/docusaurus-docs/conda-store/references/faq.md b/docusaurus-docs/conda-store/references/faq.md index a0e0488c7..356d13548 100644 --- a/docusaurus-docs/conda-store/references/faq.md +++ b/docusaurus-docs/conda-store/references/faq.md @@ -115,29 +115,60 @@ c.CondaStore.build_key_version = 2 ## Long paths on Windows conda-store supports Windows in standalone mode. However, when creating -environments with certain packages, you may see errors like +environments with certain packages, you may see errors like: ```bash ERROR:root:[WinError 206] The filename or extension is too long: 'C:\\...' ``` This error is due to the fact that Windows has a limitation that file paths -cannot be more than 260 characters. The fix is to set the registry key +cannot be more than 260 characters. + +See [conda-store issue #588][max-path-issue] for more details. + +### Solution 1: Extended-length path prefix (`\\?\`) + +If you *don't have administrator privileges*, try using the following config +option: + +```python +c.CondaStore.win_extended_length_prefix = True +``` + +This adds the extended-length path prefix (`\\?\`) to conda-store `build_path` +and `environment_path` methods, which should allow for a maximum total path +length of 32,767 characters when building packages. + +See [this Microsoft support article][max-path] for more details on the +extended-length path prefix. + +### Solution 2: `LongPathsEnabled` + +If you *have administrator privileges*, set the registry key `Computer\HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem\LongPathsEnabled -(Type: REG_DWORD)` to `1`, which removes this MAX_PATH limitation. See [this -Microsoft support -article](https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation) -for more details on how to set this registry key. +(Type: REG_DWORD)` to `1`, which removes this `MAX_PATH` limitation. + +See [this Microsoft support article][max-path] for more details on how to set +this registry key. + +### Solution 3: `store_directory` -If it is not possible to set this registry key, for instance, because you do -not have access to administrator privileges, you should configure the +If it is not possible to set the registry key, for instance, because you *do +not have access to administrator privileges*, you should configure the conda-store `CondaStore.store_directory` to be as close to the filesystem root as possible, so that the total length of the paths of package files is minimized. -See [conda-store issue -#588](https://github.com/conda-incubator/conda-store/issues/588) for more -details. +### Solution 4: `build_key_version` + +Use the short build key version as explained [above](#build-key-versions): + +```python +c.CondaStore.build_key_version = 2 +``` + +[max-path-issue]: https://github.com/conda-incubator/conda-store/issues/588 +[max-path]: https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation ## What are the resource requirements for `conda-store-server` From 4155b54d72a7e0621116072eeb25f2a62842873f Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Mon, 25 Dec 2023 01:33:28 +0100 Subject: [PATCH 2/4] Fixups --- conda-store-server/tests/test_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda-store-server/tests/test_server.py b/conda-store-server/tests/test_server.py index ea7d553d8..ee2e80a5d 100644 --- a/conda-store-server/tests/test_server.py +++ b/conda-store-server/tests/test_server.py @@ -449,7 +449,7 @@ def test_create_specification_auth_extended_prefix(win_extended_length_prefix, t r = schema.APIGetBuild.parse_obj(response.json()) assert r.status == schema.APIStatus.OK assert r.data.specification.name == environment_name - if r.data.status in ["QUEUED", "BUILDING"]: + if r.data.status in ("QUEUED", "BUILDING"): continue # checked too fast, try again if win_extended_length_prefix: @@ -465,7 +465,7 @@ def test_create_specification_auth_extended_prefix(win_extended_length_prefix, t # If we're here, the task didn't update the status on failure if not is_updated: - assert False, f"failed to get status" + assert False, "failed to update status" def test_create_specification_auth(testclient, celery_worker, authenticate): From fd931c9b41c805368276d4d457e1c142bf31fee7 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Thu, 18 Jan 2024 18:27:04 +0100 Subject: [PATCH 3/4] Update fixture --- conda-store-server/tests/test_server.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/conda-store-server/tests/test_server.py b/conda-store-server/tests/test_server.py index ee2e80a5d..af8a4a7c6 100644 --- a/conda-store-server/tests/test_server.py +++ b/conda-store-server/tests/test_server.py @@ -3,6 +3,7 @@ import time import pytest +import traitlets import yaml from conda_store_server import __version__, schema @@ -400,9 +401,12 @@ def test_create_specification_auth_env_name_too_long(testclient, celery_worker, @pytest.fixture def win_extended_length_prefix(request): # Overrides the attribute before other fixtures are called - import conda_store_server - conda_store_server.app.CondaStore.win_extended_length_prefix = request.param + from conda_store_server.app import CondaStore + assert type(CondaStore.win_extended_length_prefix) is traitlets.Bool + old_prefix = CondaStore.win_extended_length_prefix.default_value + CondaStore.win_extended_length_prefix.default_value = request.param yield request.param + CondaStore.win_extended_length_prefix.default_value = old_prefix @pytest.mark.skipif(sys.platform != "win32", reason="tests a Windows issue") From 5ce660be38a567847ebfe8dc4b3b72ba4197f4f1 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Thu, 18 Jan 2024 20:06:41 +0100 Subject: [PATCH 4/4] Update fixture --- conda-store-server/tests/test_server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conda-store-server/tests/test_server.py b/conda-store-server/tests/test_server.py index af8a4a7c6..fb262fc75 100644 --- a/conda-store-server/tests/test_server.py +++ b/conda-store-server/tests/test_server.py @@ -403,10 +403,10 @@ def win_extended_length_prefix(request): # Overrides the attribute before other fixtures are called from conda_store_server.app import CondaStore assert type(CondaStore.win_extended_length_prefix) is traitlets.Bool - old_prefix = CondaStore.win_extended_length_prefix.default_value - CondaStore.win_extended_length_prefix.default_value = request.param + old_prefix = CondaStore.win_extended_length_prefix + CondaStore.win_extended_length_prefix = request.param yield request.param - CondaStore.win_extended_length_prefix.default_value = old_prefix + CondaStore.win_extended_length_prefix = old_prefix @pytest.mark.skipif(sys.platform != "win32", reason="tests a Windows issue")