diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 13df6db6e..d5b9ead5d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -62,7 +62,20 @@ jobs: - name: "Unit tests ✅" run: | - pytest tests + pytest -m "not extended_prefix" tests + + # https://github.com/actions/runner-images/issues/1052 + - name: "Windows extended prefix unit tests ✅" + shell: pwsh + run: | + Set-ItemProperty "HKLM:\System\CurrentControlSet\Control\FileSystem" ` + -Name "LongPathsEnabled" ` + -Value 0 ` + -Type DWord + (Get-ItemProperty "HKLM:System\CurrentControlSet\Control\FileSystem").LongPathsEnabled + pytest -m "extended_prefix" tests + if: matrix.os == 'windows' + integration-test-conda-store-server: name: "integration-test conda-store-server" diff --git a/conda-store-server/conda_store_server/app.py b/conda-store-server/conda_store_server/app.py index dc8685e7f..a4ba24b51 100644 --- a/conda-store-server/conda_store_server/app.py +++ b/conda-store-server/conda_store_server/app.py @@ -120,6 +120,12 @@ def _check_build_key_version(self, proposal): except Exception as e: raise TraitError(f"c.CondaStore.build_key_version: {e}") + win_extended_length_prefix = Bool( + False, + help="Use the extended-length prefix '\\\\?\\' (Windows-only), default: False", + config=True, + ) + conda_command = Unicode( "mamba", help="conda executable to use for solves", diff --git a/conda-store-server/conda_store_server/orm.py b/conda-store-server/conda_store_server/orm.py index 0275f21f5..304ac9574 100644 --- a/conda-store-server/conda_store_server/orm.py +++ b/conda-store-server/conda_store_server/orm.py @@ -4,6 +4,7 @@ import pathlib import re import shutil +import sys from conda_store_server import conda_utils, schema, utils from conda_store_server.environment import validate_environment @@ -254,7 +255,12 @@ def build_path(self, conda_store): # https://github.com/conda-incubator/conda-store/issues/649 if len(str(res)) > 255: raise BuildPathError("build_path too long: must be <= 255 characters") - return res + # Note: cannot use the '/' operator to prepend the extended-length + # prefix + if sys.platform == "win32" and conda_store.win_extended_length_prefix: + return pathlib.Path(f"\\\\?\\{res}") + else: + return res def environment_path(self, conda_store): """Environment path is the path for the symlink to the build @@ -264,11 +270,17 @@ def environment_path(self, conda_store): store_directory = os.path.abspath(conda_store.store_directory) namespace = self.environment.namespace.name name = self.specification.name - return pathlib.Path( + res = pathlib.Path( conda_store.environment_directory.format( store_directory=store_directory, namespace=namespace, name=name ) ) + # Note: cannot use the '/' operator to prepend the extended-length + # prefix + if sys.platform == "win32" and conda_store.win_extended_length_prefix: + return pathlib.Path(f"\\\\?\\{res}") + else: + return res @property def build_key(self): diff --git a/conda-store-server/tests/test_server.py b/conda-store-server/tests/test_server.py index a20d0ff5a..fb262fc75 100644 --- a/conda-store-server/tests/test_server.py +++ b/conda-store-server/tests/test_server.py @@ -1,7 +1,9 @@ import json +import sys import time import pytest +import traitlets import yaml from conda_store_server import __version__, schema @@ -396,6 +398,80 @@ def test_create_specification_auth_env_name_too_long(testclient, celery_worker, assert False, f"failed to update status" +@pytest.fixture +def win_extended_length_prefix(request): + # Overrides the attribute before other fixtures are called + from conda_store_server.app import CondaStore + assert type(CondaStore.win_extended_length_prefix) is traitlets.Bool + old_prefix = CondaStore.win_extended_length_prefix + CondaStore.win_extended_length_prefix = request.param + yield request.param + CondaStore.win_extended_length_prefix = old_prefix + + +@pytest.mark.skipif(sys.platform != "win32", reason="tests a Windows issue") +@pytest.mark.parametrize('win_extended_length_prefix', [True, False], indirect=True) +@pytest.mark.extended_prefix +def test_create_specification_auth_extended_prefix(win_extended_length_prefix, testclient, celery_worker, authenticate): + # Adds padding to cause an error if the extended prefix is not enabled + namespace = "default" + 'A' * 10 + environment_name = "pytest" + + # The debugpy 1.8.0 package was deliberately chosen because it has long + # paths internally, which causes issues on Windows due to the path length + # limit + response = testclient.post( + "api/v1/specification", + json={ + "namespace": namespace, + "specification": json.dumps({ + "name": environment_name, + "channels": ["conda-forge"], + "dependencies": ["debugpy==1.8.0"], + "variables": None, + "prefix": None, + "description": "test" + }), + }, + timeout=30, + ) + response.raise_for_status() + + r = schema.APIPostSpecification.parse_obj(response.json()) + assert r.status == schema.APIStatus.OK + build_id = r.data.build_id + + # Try checking that the status is 'FAILED' + is_updated = False + for _ in range(30): + time.sleep(5) + + # check for the given build + response = testclient.get(f"api/v1/build/{build_id}", timeout=30) + response.raise_for_status() + + r = schema.APIGetBuild.parse_obj(response.json()) + assert r.status == schema.APIStatus.OK + assert r.data.specification.name == environment_name + if r.data.status in ("QUEUED", "BUILDING"): + continue # checked too fast, try again + + if win_extended_length_prefix: + assert r.data.status == "COMPLETED" + else: + assert r.data.status == "FAILED" + response = testclient.get(f"api/v1/build/{build_id}/logs", timeout=30) + response.raise_for_status() + assert "[WinError 206] The filename or extension is too long" in response.text + + is_updated = True + break + + # If we're here, the task didn't update the status on failure + if not is_updated: + assert False, "failed to update status" + + def test_create_specification_auth(testclient, celery_worker, authenticate): namespace = "default" environment_name = "pytest" diff --git a/docusaurus-docs/conda-store/references/faq.md b/docusaurus-docs/conda-store/references/faq.md index a0e0488c7..356d13548 100644 --- a/docusaurus-docs/conda-store/references/faq.md +++ b/docusaurus-docs/conda-store/references/faq.md @@ -115,29 +115,60 @@ c.CondaStore.build_key_version = 2 ## Long paths on Windows conda-store supports Windows in standalone mode. However, when creating -environments with certain packages, you may see errors like +environments with certain packages, you may see errors like: ```bash ERROR:root:[WinError 206] The filename or extension is too long: 'C:\\...' ``` This error is due to the fact that Windows has a limitation that file paths -cannot be more than 260 characters. The fix is to set the registry key +cannot be more than 260 characters. + +See [conda-store issue #588][max-path-issue] for more details. + +### Solution 1: Extended-length path prefix (`\\?\`) + +If you *don't have administrator privileges*, try using the following config +option: + +```python +c.CondaStore.win_extended_length_prefix = True +``` + +This adds the extended-length path prefix (`\\?\`) to conda-store `build_path` +and `environment_path` methods, which should allow for a maximum total path +length of 32,767 characters when building packages. + +See [this Microsoft support article][max-path] for more details on the +extended-length path prefix. + +### Solution 2: `LongPathsEnabled` + +If you *have administrator privileges*, set the registry key `Computer\HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem\LongPathsEnabled -(Type: REG_DWORD)` to `1`, which removes this MAX_PATH limitation. See [this -Microsoft support -article](https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation) -for more details on how to set this registry key. +(Type: REG_DWORD)` to `1`, which removes this `MAX_PATH` limitation. + +See [this Microsoft support article][max-path] for more details on how to set +this registry key. + +### Solution 3: `store_directory` -If it is not possible to set this registry key, for instance, because you do -not have access to administrator privileges, you should configure the +If it is not possible to set the registry key, for instance, because you *do +not have access to administrator privileges*, you should configure the conda-store `CondaStore.store_directory` to be as close to the filesystem root as possible, so that the total length of the paths of package files is minimized. -See [conda-store issue -#588](https://github.com/conda-incubator/conda-store/issues/588) for more -details. +### Solution 4: `build_key_version` + +Use the short build key version as explained [above](#build-key-versions): + +```python +c.CondaStore.build_key_version = 2 +``` + +[max-path-issue]: https://github.com/conda-incubator/conda-store/issues/588 +[max-path]: https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation ## What are the resource requirements for `conda-store-server`