Add extended-length prefix support

.github/workflows/tests.yaml

@@ -62,7 +62,20 @@ jobs:
 
       - name: "Unit tests ✅"
         run: |
-          pytest tests
+          pytest -m "not extended_prefix" tests
+
+      # https://github.com/actions/runner-images/issues/1052
+      - name: "Windows extended prefix unit tests ✅"
+        shell: pwsh
+        run: |
+          Set-ItemProperty "HKLM:\System\CurrentControlSet\Control\FileSystem" `
+            -Name "LongPathsEnabled" `
+            -Value 0 `
+            -Type DWord
+          (Get-ItemProperty "HKLM:System\CurrentControlSet\Control\FileSystem").LongPathsEnabled
+          pytest -m "extended_prefix" tests
+        if: matrix.os == 'windows'
+
 
   integration-test-conda-store-server:
     name: "integration-test conda-store-server"

conda-store-server/conda_store_server/app.py

@@ -120,6 +120,12 @@ def _check_build_key_version(self, proposal):
         except Exception as e:
             raise TraitError(f"c.CondaStore.build_key_version: {e}")
 
+    win_extended_length_prefix = Bool(
+        False,
+        help="Use the extended-length prefix '\\\\?\\' (Windows-only), default: False",
+        config=True,
+    )
+
     conda_command = Unicode(
         "mamba",
         help="conda executable to use for solves",

conda-store-server/conda_store_server/orm.py

@@ -4,6 +4,7 @@
 import pathlib
 import re
 import shutil
+import sys
 
 from conda_store_server import conda_utils, schema, utils
 from conda_store_server.environment import validate_environment
@@ -254,7 +255,12 @@ def build_path(self, conda_store):
         # https://github.com/conda-incubator/conda-store/issues/649
         if len(str(res)) > 255:
             raise BuildPathError("build_path too long: must be <= 255 characters")
-        return res
+        # Note: cannot use the '/' operator to prepend the extended-length
+        # prefix
+        if sys.platform == "win32" and conda_store.win_extended_length_prefix:
+            return pathlib.Path(f"\\\\?\\{res}")
+        else:
+            return res
 
     def environment_path(self, conda_store):
         """Environment path is the path for the symlink to the build
@@ -264,11 +270,17 @@ def environment_path(self, conda_store):
         store_directory = os.path.abspath(conda_store.store_directory)
         namespace = self.environment.namespace.name
         name = self.specification.name
-        return pathlib.Path(
+        res = pathlib.Path(
             conda_store.environment_directory.format(
                 store_directory=store_directory, namespace=namespace, name=name
             )
         )
+        # Note: cannot use the '/' operator to prepend the extended-length
+        # prefix
+        if sys.platform == "win32" and conda_store.win_extended_length_prefix:
+            return pathlib.Path(f"\\\\?\\{res}")
+        else:
+            return res
 
     @property
     def build_key(self):

conda-store-server/tests/test_server.py

@@ -1,7 +1,9 @@
 import json
+import sys
 import time
 
 import pytest
+import traitlets
 import yaml
 from conda_store_server import __version__, schema
 
@@ -396,6 +398,80 @@ def test_create_specification_auth_env_name_too_long(testclient, celery_worker,
         assert False, f"failed to update status"
 
 
+@pytest.fixture
+def win_extended_length_prefix(request):
+    # Overrides the attribute before other fixtures are called
+    from conda_store_server.app import CondaStore
+    assert type(CondaStore.win_extended_length_prefix) is traitlets.Bool
+    old_prefix = CondaStore.win_extended_length_prefix
+    CondaStore.win_extended_length_prefix = request.param
+    yield request.param
+    CondaStore.win_extended_length_prefix = old_prefix
+
+
+@pytest.mark.skipif(sys.platform != "win32", reason="tests a Windows issue")
+@pytest.mark.parametrize('win_extended_length_prefix', [True, False], indirect=True)
+@pytest.mark.extended_prefix
+def test_create_specification_auth_extended_prefix(win_extended_length_prefix, testclient, celery_worker, authenticate):
+    # Adds padding to cause an error if the extended prefix is not enabled
+    namespace = "default" + 'A' * 10
+    environment_name = "pytest"
+
+    # The debugpy 1.8.0 package was deliberately chosen because it has long
+    # paths internally, which causes issues on Windows due to the path length
+    # limit
+    response = testclient.post(
+        "api/v1/specification",
+        json={
+            "namespace": namespace,
+            "specification": json.dumps({
+                "name": environment_name,
+                "channels": ["conda-forge"],
+                "dependencies": ["debugpy==1.8.0"],
+                "variables": None,
+                "prefix": None,
+                "description": "test"
+            }),
+        },
+        timeout=30,
+    )
+    response.raise_for_status()
+
+    r = schema.APIPostSpecification.parse_obj(response.json())
+    assert r.status == schema.APIStatus.OK
+    build_id = r.data.build_id
+
+    # Try checking that the status is 'FAILED'
+    is_updated = False
+    for _ in range(30):
+        time.sleep(5)
+
+        # check for the given build
+        response = testclient.get(f"api/v1/build/{build_id}", timeout=30)
+        response.raise_for_status()
+
+        r = schema.APIGetBuild.parse_obj(response.json())
+        assert r.status == schema.APIStatus.OK
+        assert r.data.specification.name == environment_name
+        if r.data.status in ("QUEUED", "BUILDING"):
+            continue  # checked too fast, try again
+
+        if win_extended_length_prefix:
+            assert r.data.status == "COMPLETED"
+        else:
+            assert r.data.status == "FAILED"
+            response = testclient.get(f"api/v1/build/{build_id}/logs", timeout=30)
+            response.raise_for_status()
+            assert "[WinError 206] The filename or extension is too long" in response.text
+
+        is_updated = True
+        break
+
+    # If we're here, the task didn't update the status on failure
+    if not is_updated:
+        assert False, "failed to update status"
+
+
 def test_create_specification_auth(testclient, celery_worker, authenticate):
     namespace = "default"
     environment_name = "pytest"

docusaurus-docs/conda-store/references/faq.md

@@ -115,29 +115,60 @@ c.CondaStore.build_key_version = 2
 ## Long paths on Windows
 
 conda-store supports Windows in standalone mode. However, when creating
-environments with certain packages, you may see errors like
+environments with certain packages, you may see errors like:
 
 ```bash
 ERROR:root:[WinError 206] The filename or extension is too long: 'C:\\...'
 ```
 
 This error is due to the fact that Windows has a limitation that file paths
-cannot be more than 260 characters. The fix is to set the registry key
+cannot be more than 260 characters.
+
+See [conda-store issue #588][max-path-issue] for more details.
+
+### Solution 1: Extended-length path prefix (`\\?\`)
+
+If you *don't have administrator privileges*, try using the following config
+option:
+
+```python
+c.CondaStore.win_extended_length_prefix = True
+```
+
+This adds the extended-length path prefix (`\\?\`) to conda-store `build_path`
+and `environment_path` methods, which should allow for a maximum total path
+length of 32,767 characters when building packages.
+
+See [this Microsoft support article][max-path] for more details on the
+extended-length path prefix.
+
+### Solution 2: `LongPathsEnabled`
+
+If you *have administrator privileges*, set the registry key
 `Computer\HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem\LongPathsEnabled
-(Type: REG_DWORD)` to `1`, which removes this MAX_PATH limitation. See [this
-Microsoft support
-article](https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation)
-for more details on how to set this registry key.
+(Type: REG_DWORD)` to `1`, which removes this `MAX_PATH` limitation.
+
+See [this Microsoft support article][max-path] for more details on how to set
+this registry key.
+
+### Solution 3: `store_directory`
 
-If it is not possible to set this registry key, for instance, because you do
-not have access to administrator privileges, you should configure the
+If it is not possible to set the registry key, for instance, because you *do
+not have access to administrator privileges*, you should configure the
 conda-store `CondaStore.store_directory` to be as close to the filesystem root
 as possible, so that the total length of the paths of package files is
 minimized.
 
-See [conda-store issue
-#588](https://github.com/conda-incubator/conda-store/issues/588) for more
-details.
+### Solution 4: `build_key_version`
+
+Use the short build key version as explained [above](#build-key-versions):
+
+```python
+c.CondaStore.build_key_version = 2
+```
+
+[max-path-issue]: https://github.com/conda-incubator/conda-store/issues/588
+[max-path]: https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation
 
 ## What are the resource requirements for `conda-store-server`