From 6696a8492a3a2f89350cf92874e232ec5c872015 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Wed, 20 Dec 2023 03:19:24 +0100 Subject: [PATCH 01/37] Generate `constructor` artifacts Fixes #657. --- .../conda_store_server/action/__init__.py | 3 + .../action/generate_constructor_artifacts.py | 98 +++++++++++++++++++ conda-store-server/tests/test_actions.py | 40 ++++++++ 3 files changed, 141 insertions(+) create mode 100644 conda-store-server/conda_store_server/action/generate_constructor_artifacts.py diff --git a/conda-store-server/conda_store_server/action/__init__.py b/conda-store-server/conda_store_server/action/__init__.py index a4e04c754..36eb571e9 100644 --- a/conda-store-server/conda_store_server/action/__init__.py +++ b/conda-store-server/conda_store_server/action/__init__.py @@ -32,3 +32,6 @@ from conda_store_server.action.add_lockfile_packages import ( action_add_lockfile_packages, # noqa ) +from conda_store_server.action.generate_constructor_artifacts import ( + action_generate_constructor_artifacts, # noqa +) diff --git a/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py b/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py new file mode 100644 index 000000000..4d6dbacad --- /dev/null +++ b/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py @@ -0,0 +1,98 @@ +import pathlib +import subprocess +import sys +import tempfile + +import yaml +from conda_store_server import action, schema + + +@action.action +def action_generate_constructor_artifacts( + context, + conda_command: str, + specification: schema.CondaSpecification, + installer_dir: str, +): + # Helpers + def print_cmd(cmd): + context.log.info(f"Running command: {' '.join(cmd)}") + context.log.info( + subprocess.check_output(cmd, stderr=subprocess.STDOUT, encoding="utf-8") + ) + + def write_file(filename, s): + with open(filename, "w") as f: + context.log.info(f"{filename}:\n{s}") + f.write(s) + + # pip dependencies are not directly supported by constructor, they will be + # installed via the post_install script: + # https://github.com/conda/constructor/issues/515 + dependencies = [] + pip_dependencies = [] + for d in specification.dependencies: + if type(d) is schema.CondaSpecificationPip: + pip_dependencies.extend(d.pip) + else: + dependencies.append(d) + + # Creates the construct.yaml file and post_install script + ext = ".exe" if sys.platform == "win32" else ".sh" + installer_dir = pathlib.Path(installer_dir) + installer_filename = (installer_dir / specification.name).with_suffix(ext) + + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_dir = pathlib.Path(tmp_dir) + construct_file = tmp_dir / "construct.yaml" + post_install_file = tmp_dir / "post-install.sh" + env_dir = tmp_dir / "env" + + construct = { + "installer_filename": str(installer_filename), + "post_install": str(post_install_file), + "name": specification.name, + "channels": specification.channels, + "specs": dependencies, + # XXX: This is required: use the env hash and datetime? + "version": 1, + } + + # XXX: Support Windows + post_install = """\ +#!/usr/bin/env bash +set -euxo pipefail +""" + if pip_dependencies: + post_install += f""" +conda run -p "$PREFIX" pip install {' '.join(pip_dependencies)} +""" + + # Writes files to disk + write_file(construct_file, yaml.dump(construct)) + write_file(post_install_file, post_install) + + # Installs constructor + command = [ + conda_command, + "create", + "-y", + "-p", + str(env_dir), + "constructor", + ] + print_cmd(command) + + # Calls constructor + command = [ + conda_command, + "run", + "-p", + str(env_dir), + "--no-capture-output", + "constructor", + str(tmp_dir), + ] + print_cmd(command) + + return installer_filename diff --git a/conda-store-server/tests/test_actions.py b/conda-store-server/tests/test_actions.py index 8d79e3ea4..6d0a63ec6 100644 --- a/conda-store-server/tests/test_actions.py +++ b/conda-store-server/tests/test_actions.py @@ -1,8 +1,11 @@ import asyncio import datetime +import os import pathlib import re +import subprocess import sys +import tempfile import pytest import yarl @@ -110,6 +113,43 @@ def test_solve_lockfile_multiple_platforms(conda_store, specification, request): assert len(context.result["package"]) != 0 +@pytest.mark.parametrize( + "specification_name", + [ + "simple_specification", + "simple_specification_with_pip", + ], +) +def test_generate_constructor_artifacts(conda_store, specification_name, request): + specification = request.getfixturevalue(specification_name) + with tempfile.TemporaryDirectory() as installer_dir: + # Creates the installer + context = action.action_generate_constructor_artifacts( + conda_command=conda_store.conda_command, + specification=specification, + installer_dir=installer_dir, + ) + + # Checks that the installer was created + installer = context.result + assert installer.exists() + + with tempfile.TemporaryDirectory() as tmp_dir: + # Runs the installer + out_dir = pathlib.Path(tmp_dir) / 'out' + subprocess.check_output([installer, '-b', '-p', str(out_dir)]) + + # Checks the output directory + assert out_dir.exists() + lib_dir = out_dir / 'lib' + if specification_name == 'simple_specification': + assert any(str(x).endswith('libz.so') for x in lib_dir.iterdir()) + else: + # Uses rglob to not depend on the version of the python + # directory, which is where site-packages is located + assert any(str(x).endswith('site-packages/flask') for x in lib_dir.rglob('*')) + + def test_fetch_and_extract_conda_packages(tmp_path, simple_conda_lock): context = action.action_fetch_and_extract_conda_packages( conda_lock_spec=simple_conda_lock, From 2fd15c13e6ba4d3722c78996c54ab6e6fcaaa326 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Thu, 21 Dec 2023 10:16:01 +0100 Subject: [PATCH 02/37] Windows support --- .../action/generate_constructor_artifacts.py | 20 ++++-- conda-store-server/tests/test_actions.py | 61 +++++++++++-------- 2 files changed, 50 insertions(+), 31 deletions(-) diff --git a/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py b/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py index 4d6dbacad..30dc44e97 100644 --- a/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py +++ b/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py @@ -1,3 +1,4 @@ +import os import pathlib import subprocess import sys @@ -39,13 +40,16 @@ def write_file(filename, s): # Creates the construct.yaml file and post_install script ext = ".exe" if sys.platform == "win32" else ".sh" + pi_ext = ".bat" if sys.platform == "win32" else ".sh" installer_dir = pathlib.Path(installer_dir) installer_filename = (installer_dir / specification.name).with_suffix(ext) - with tempfile.TemporaryDirectory() as tmp_dir: + os.makedirs(installer_dir) + + with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmp_dir: tmp_dir = pathlib.Path(tmp_dir) construct_file = tmp_dir / "construct.yaml" - post_install_file = tmp_dir / "post-install.sh" + post_install_file = (tmp_dir / "post_install").with_suffix(pi_ext) env_dir = tmp_dir / "env" construct = { @@ -58,14 +62,20 @@ def write_file(filename, s): "version": 1, } - # XXX: Support Windows - post_install = """\ + if sys.platform == "win32": + post_install = """\ +call "%PREFIX%\Scripts\activate.bat +""" + else: + post_install = """\ #!/usr/bin/env bash set -euxo pipefail +source "$PREFIX/etc/profile.d/conda.sh" +conda activate "$PREFIX" """ if pip_dependencies: post_install += f""" -conda run -p "$PREFIX" pip install {' '.join(pip_dependencies)} +pip install {' '.join(pip_dependencies)} """ # Writes files to disk diff --git a/conda-store-server/tests/test_actions.py b/conda-store-server/tests/test_actions.py index 6d0a63ec6..7b0df8eaa 100644 --- a/conda-store-server/tests/test_actions.py +++ b/conda-store-server/tests/test_actions.py @@ -120,34 +120,43 @@ def test_solve_lockfile_multiple_platforms(conda_store, specification, request): "simple_specification_with_pip", ], ) -def test_generate_constructor_artifacts(conda_store, specification_name, request): +def test_generate_constructor_artifacts(conda_store, specification_name, request, tmp_path): specification = request.getfixturevalue(specification_name) - with tempfile.TemporaryDirectory() as installer_dir: - # Creates the installer - context = action.action_generate_constructor_artifacts( - conda_command=conda_store.conda_command, - specification=specification, - installer_dir=installer_dir, - ) + installer_dir = tmp_path / "installer_dir" + + # Creates the installer + context = action.action_generate_constructor_artifacts( + conda_command=conda_store.conda_command, + specification=specification, + installer_dir=installer_dir, + ) - # Checks that the installer was created - installer = context.result - assert installer.exists() - - with tempfile.TemporaryDirectory() as tmp_dir: - # Runs the installer - out_dir = pathlib.Path(tmp_dir) / 'out' - subprocess.check_output([installer, '-b', '-p', str(out_dir)]) - - # Checks the output directory - assert out_dir.exists() - lib_dir = out_dir / 'lib' - if specification_name == 'simple_specification': - assert any(str(x).endswith('libz.so') for x in lib_dir.iterdir()) - else: - # Uses rglob to not depend on the version of the python - # directory, which is where site-packages is located - assert any(str(x).endswith('site-packages/flask') for x in lib_dir.rglob('*')) + # Checks that the installer was created + installer = context.result + assert installer.exists() + + tmp_dir = tmp_path / "tmp" + + # Runs the installer + out_dir = pathlib.Path(tmp_dir) / 'out' + if sys.platform == 'win32': + subprocess.check_output([installer, '/S', f'/D={out_dir}']) + else: + subprocess.check_output([installer, '-b', '-p', str(out_dir)]) + + # Checks the output directory + assert out_dir.exists() + lib_dir = out_dir / 'lib' + if specification_name == 'simple_specification': + if sys.platform == 'win32': + assert any(str(x).endswith('zlib.dll') for x in out_dir.iterdir()) + else: + assert any(str(x).endswith('libz.so') for x in lib_dir.iterdir()) + else: + # Uses rglob to not depend on the version of the python + # directory, which is where site-packages is located + flask = pathlib.Path('site-packages') / 'flask' + assert any(str(x).endswith(str(flask)) for x in out_dir.rglob('*')) def test_fetch_and_extract_conda_packages(tmp_path, simple_conda_lock): From 0ccb2e719a987eefe8d33e12c617dd0c4c561ae1 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Thu, 21 Dec 2023 10:56:47 +0100 Subject: [PATCH 03/37] Fixup --- .../action/generate_constructor_artifacts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py b/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py index 30dc44e97..47108caa7 100644 --- a/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py +++ b/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py @@ -30,7 +30,8 @@ def write_file(filename, s): # pip dependencies are not directly supported by constructor, they will be # installed via the post_install script: # https://github.com/conda/constructor/issues/515 - dependencies = [] + # conda and pip need to be in dependencies for the post_install script + dependencies = ["conda", "pip"] pip_dependencies = [] for d in specification.dependencies: if type(d) is schema.CondaSpecificationPip: From 2f19f462f60331412aa92e3cb441caba1595f087 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Thu, 21 Dec 2023 11:00:09 +0100 Subject: [PATCH 04/37] macOS support --- conda-store-server/tests/test_actions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda-store-server/tests/test_actions.py b/conda-store-server/tests/test_actions.py index 7b0df8eaa..57afe47d0 100644 --- a/conda-store-server/tests/test_actions.py +++ b/conda-store-server/tests/test_actions.py @@ -150,6 +150,8 @@ def test_generate_constructor_artifacts(conda_store, specification_name, request if specification_name == 'simple_specification': if sys.platform == 'win32': assert any(str(x).endswith('zlib.dll') for x in out_dir.iterdir()) + elif sys.platform == 'darwin': + assert any(str(x).endswith('libz.dylib') for x in lib_dir.iterdir()) else: assert any(str(x).endswith('libz.so') for x in lib_dir.iterdir()) else: From aa7e133cdf373bc13f0f9c8eb07e633945556f0a Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Thu, 21 Dec 2023 11:08:32 +0100 Subject: [PATCH 05/37] Fixup --- .../conda_store_server/action/generate_constructor_artifacts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py b/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py index 47108caa7..335a57569 100644 --- a/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py +++ b/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py @@ -65,7 +65,7 @@ def write_file(filename, s): if sys.platform == "win32": post_install = """\ -call "%PREFIX%\Scripts\activate.bat +call "%PREFIX%\Scripts\activate.bat" """ else: post_install = """\ From c5ce16155965db23cad5f52595477fbe49e694ab Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Fri, 22 Dec 2023 00:22:46 +0100 Subject: [PATCH 06/37] Rename `generate_constructor_artifacts` to `generate_constructor_installer` This is better because `constructor` might support other formats in the future. --- conda-store-server/conda_store_server/action/__init__.py | 4 ++-- ...tructor_artifacts.py => generate_constructor_installer.py} | 2 +- conda-store-server/tests/test_actions.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) rename conda-store-server/conda_store_server/action/{generate_constructor_artifacts.py => generate_constructor_installer.py} (98%) diff --git a/conda-store-server/conda_store_server/action/__init__.py b/conda-store-server/conda_store_server/action/__init__.py index 36eb571e9..5cf70c697 100644 --- a/conda-store-server/conda_store_server/action/__init__.py +++ b/conda-store-server/conda_store_server/action/__init__.py @@ -32,6 +32,6 @@ from conda_store_server.action.add_lockfile_packages import ( action_add_lockfile_packages, # noqa ) -from conda_store_server.action.generate_constructor_artifacts import ( - action_generate_constructor_artifacts, # noqa +from conda_store_server.action.generate_constructor_installer import ( + action_generate_constructor_installer, # noqa ) diff --git a/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py b/conda-store-server/conda_store_server/action/generate_constructor_installer.py similarity index 98% rename from conda-store-server/conda_store_server/action/generate_constructor_artifacts.py rename to conda-store-server/conda_store_server/action/generate_constructor_installer.py index 335a57569..9cfcb805e 100644 --- a/conda-store-server/conda_store_server/action/generate_constructor_artifacts.py +++ b/conda-store-server/conda_store_server/action/generate_constructor_installer.py @@ -9,7 +9,7 @@ @action.action -def action_generate_constructor_artifacts( +def action_generate_constructor_installer( context, conda_command: str, specification: schema.CondaSpecification, diff --git a/conda-store-server/tests/test_actions.py b/conda-store-server/tests/test_actions.py index 57afe47d0..78207f737 100644 --- a/conda-store-server/tests/test_actions.py +++ b/conda-store-server/tests/test_actions.py @@ -120,12 +120,12 @@ def test_solve_lockfile_multiple_platforms(conda_store, specification, request): "simple_specification_with_pip", ], ) -def test_generate_constructor_artifacts(conda_store, specification_name, request, tmp_path): +def test_generate_constructor_installer(conda_store, specification_name, request, tmp_path): specification = request.getfixturevalue(specification_name) installer_dir = tmp_path / "installer_dir" # Creates the installer - context = action.action_generate_constructor_artifacts( + context = action.action_generate_constructor_installer( conda_command=conda_store.conda_command, specification=specification, installer_dir=installer_dir, From 1cb6a0fbbc78169af69884828afc5d5b077307db Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Fri, 22 Dec 2023 00:44:14 +0100 Subject: [PATCH 07/37] Fix type hint --- .../action/generate_constructor_installer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/conda-store-server/conda_store_server/action/generate_constructor_installer.py b/conda-store-server/conda_store_server/action/generate_constructor_installer.py index 9cfcb805e..68e13ec4e 100644 --- a/conda-store-server/conda_store_server/action/generate_constructor_installer.py +++ b/conda-store-server/conda_store_server/action/generate_constructor_installer.py @@ -13,7 +13,7 @@ def action_generate_constructor_installer( context, conda_command: str, specification: schema.CondaSpecification, - installer_dir: str, + installer_dir: pathlib.Path, ): # Helpers def print_cmd(cmd): @@ -42,7 +42,6 @@ def write_file(filename, s): # Creates the construct.yaml file and post_install script ext = ".exe" if sys.platform == "win32" else ".sh" pi_ext = ".bat" if sys.platform == "win32" else ".sh" - installer_dir = pathlib.Path(installer_dir) installer_filename = (installer_dir / specification.name).with_suffix(ext) os.makedirs(installer_dir) From 3d8e6f1a27843d67cdaace6441aa630654fe7b05 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Fri, 22 Dec 2023 00:45:49 +0100 Subject: [PATCH 08/37] Add `task_build_constructor_installer` and use it --- conda-store-server/conda_store_server/app.py | 9 +++++ .../conda_store_server/build.py | 39 +++++++++++++++++++ .../conda_store_server/schema.py | 1 + .../conda_store_server/server/views/api.py | 1 + .../conda_store_server/worker/tasks.py | 9 +++++ 5 files changed, 59 insertions(+) diff --git a/conda-store-server/conda_store_server/app.py b/conda-store-server/conda_store_server/app.py index a4ba24b51..ef9ff2c63 100644 --- a/conda-store-server/conda_store_server/app.py +++ b/conda-store-server/conda_store_server/app.py @@ -724,6 +724,15 @@ def create_build(self, db: Session, environment_id: int, specification_sha256: s ) ) + if schema.BuildArtifactType.CONSTRUCTOR_INSTALLER in settings.build_artifacts: + artifact_tasks.append( + tasks.task_build_constructor_installer.subtask( + args=(build.id,), + task_id=f"build-{build.id}-constructor-installer", + immutable=True, + ) + ) + ( tasks.task_update_storage_metrics.si() | tasks.task_build_conda_environment.subtask( diff --git a/conda-store-server/conda_store_server/build.py b/conda-store-server/conda_store_server/build.py index 4d23180e1..063d8c91c 100644 --- a/conda-store-server/conda_store_server/build.py +++ b/conda-store-server/conda_store_server/build.py @@ -382,3 +382,42 @@ def build_conda_docker(db: Session, conda_store, build: orm.Build): conda_store.log.exception(e) append_to_logs(db, conda_store, build, traceback.format_exc()) raise e + + +def build_constructor_installer(db: Session, conda_store, build: orm.Build): + conda_prefix = build.build_path(conda_store) + + settings = conda_store.get_settings( + db=db, + namespace=build.environment.namespace.name, + environment_name=build.environment.name, + ) + + with utils.timer( + conda_store.log, f"creating installer for conda environment={conda_prefix}" + ): + with tempfile.TemporaryDirectory() as tmpdir: + context = action.action_generate_constructor_installer( + conda_command=settings.conda_command, + specification=schema.CondaSpecification.parse_obj( + build.specification.spec + ), + installer_dir=pathlib.Path(tmpdir), + ) + output_filename = context.result + append_to_logs( + db, + conda_store, + build, + "::group::action_generate_constructor_installer\n" + + context.stdout.getvalue() + + "\n::endgroup::\n", + ) + conda_store.storage.fset( + db, + build.id, + build.constructor_installer_key, + output_filename, + content_type="application/octet-stream", + artifact_type=schema.BuildArtifactType.CONSTRUCTOR_INSTALLER, + ) diff --git a/conda-store-server/conda_store_server/schema.py b/conda-store-server/conda_store_server/schema.py index d4b908098..729e54f62 100644 --- a/conda-store-server/conda_store_server/schema.py +++ b/conda-store-server/conda_store_server/schema.py @@ -148,6 +148,7 @@ class BuildArtifactType(enum.Enum): DOCKER_BLOB = "DOCKER_BLOB" DOCKER_MANIFEST = "DOCKER_MANIFEST" CONTAINER_REGISTRY = "CONTAINER_REGISTRY" + CONSTRUCTOR_INSTALLER = "CONSTRUCTOR_INSTALLER" class BuildStatus(enum.Enum): diff --git a/conda-store-server/conda_store_server/server/views/api.py b/conda-store-server/conda_store_server/server/views/api.py index d47345984..ea0bf6c5b 100644 --- a/conda-store-server/conda_store_server/server/views/api.py +++ b/conda-store-server/conda_store_server/server/views/api.py @@ -972,6 +972,7 @@ async def api_put_build_cancel( f"build-{build_id}-conda-env-export", f"build-{build_id}-conda-pack", f"build-{build_id}-docker", + f"build-{build_id}-constructor-installer", f"build-{build_id}-environment", ], terminate=True, diff --git a/conda-store-server/conda_store_server/worker/tasks.py b/conda-store-server/conda_store_server/worker/tasks.py index fa7dd5cee..9d33e5e64 100644 --- a/conda-store-server/conda_store_server/worker/tasks.py +++ b/conda-store-server/conda_store_server/worker/tasks.py @@ -15,6 +15,7 @@ build_conda_env_export, build_conda_environment, build_conda_pack, + build_constructor_installer, solve_conda_environment, ) from conda_store_server.worker.app import CondaStoreWorker @@ -237,6 +238,14 @@ def task_build_conda_docker(self, build_id): build_conda_docker(db, conda_store, build) +@shared_task(base=WorkerTask, name="task_build_constructor_installer", bind=True) +def task_build_constructor_installer(self, build_id): + conda_store = self.worker.conda_store + with conda_store.session_factory() as db: + build = api.get_build(db, build_id) + build_constructor_installer(db, conda_store, build) + + @shared_task(base=WorkerTask, name="task_update_environment_build", bind=True) def task_update_environment_build(self, environment_id): conda_store = self.worker.conda_store From c95e2d2893067c2743d65fa340f7766229dc231d Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Fri, 22 Dec 2023 01:23:11 +0100 Subject: [PATCH 09/37] Add `api_get_build_installer` and test it --- conda-store-server/conda_store_server/orm.py | 11 ++++ .../conda_store_server/server/views/api.py | 28 ++++++++++ conda-store-server/tests/test_actions.py | 52 +++++++++++++++++++ 3 files changed, 91 insertions(+) diff --git a/conda-store-server/conda_store_server/orm.py b/conda-store-server/conda_store_server/orm.py index 3541cc52c..2fba18611 100644 --- a/conda-store-server/conda_store_server/orm.py +++ b/conda-store-server/conda_store_server/orm.py @@ -337,6 +337,10 @@ def conda_pack_key(self): def docker_manifest_key(self): return f"docker/manifest/{self.build_key}" + @property + def constructor_installer_key(self): + return f"installer/{self.build_key}" + def docker_blob_key(self, blob_hash): return f"docker/blobs/{blob_hash}" @@ -368,6 +372,13 @@ def has_docker_manifest(self): for artifact in self.build_artifacts ) + @hybrid_property + def has_constructor_installer(self): + return any( + artifact.artifact_type == schema.BuildArtifactType.CONSTRUCTOR_INSTALLER + for artifact in self.build_artifacts + ) + def __repr__(self): return f"" diff --git a/conda-store-server/conda_store_server/server/views/api.py b/conda-store-server/conda_store_server/server/views/api.py index ea0bf6c5b..6cd3ca7a9 100644 --- a/conda-store-server/conda_store_server/server/views/api.py +++ b/conda-store-server/conda_store_server/server/views/api.py @@ -1280,6 +1280,34 @@ async def api_get_build_docker_image_url( ) +@router_api.get("/build/{build_id}/installer/") +async def api_get_build_installer( + build_id: int, + request: Request, + conda_store=Depends(dependencies.get_conda_store), + auth=Depends(dependencies.get_auth), +): + with conda_store.get_db() as db: + build = api.get_build(db, build_id) + auth.authorize_request( + request, + f"{build.environment.namespace.name}/{build.environment.name}", + {Permissions.ENVIRONMENT_READ}, + require=True, + ) + + if build.has_constructor_installer: + return RedirectResponse( + conda_store.storage.get_url(build.constructor_installer_key) + ) + + else: + raise HTTPException( + status_code=400, + detail=f"Build {build_id} doesn't have an installer", + ) + + @router_api.get( "/setting/", response_model=schema.APIGetSetting, diff --git a/conda-store-server/tests/test_actions.py b/conda-store-server/tests/test_actions.py index 78207f737..b22e14df0 100644 --- a/conda-store-server/tests/test_actions.py +++ b/conda-store-server/tests/test_actions.py @@ -423,6 +423,58 @@ def lockfile_url(build_key): assert res.status_code == 307 +def test_api_get_build_installer( + request, conda_store, db, simple_specification_with_pip, conda_prefix +): + # initializes data needed to get the installer + specification = simple_specification_with_pip + specification.name = "my-env" + namespace = "pytest" + + class MyAuthentication(DummyAuthentication): + # Skips auth (used in api_get_build_installer). Test version of request + # has no state attr, which is returned in the real impl of this method. + # So I have to overwrite the method itself. + def authorize_request(self, *args, **kwargs): + pass + + auth = MyAuthentication() + build_id = conda_store.register_environment( + db, specification=specification, namespace=namespace + ) + db.commit() + + build = api.get_build(db, build_id=build_id) + + # creates build artifacts + build_artifact = orm.BuildArtifact( + build_id=build_id, + build=build, + artifact_type=schema.BuildArtifactType.CONSTRUCTOR_INSTALLER, + key=build.constructor_installer_key, + ) + db.add(build_artifact) + db.commit() + + # gets installer for this build + res = asyncio.run( + server.views.api.api_get_build_installer( + request=request, + conda_store=conda_store, + auth=auth, + build_id=build_id, + )) + + # redirects to installer + def installer_url(build_key): + return f"installer/{build_key}" + + assert type(res) is RedirectResponse + assert build.constructor_installer_key == res.headers['location'] + assert installer_url(build.build_key) == build.constructor_installer_key + assert res.status_code == 307 + + def test_get_channel_url(): conda_main = "https://conda.anaconda.org/main" repo_main = "https://repo.anaconda.com/pkgs/main" From d72c9333b9cad854b50ef4b6cb94001d63316deb Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Fri, 22 Dec 2023 02:13:46 +0100 Subject: [PATCH 10/37] Expose installer in the admin UI --- .../conda_store_server/server/templates/build.html | 3 +++ .../conda_store_server/server/templates/home.html | 3 +++ 2 files changed, 6 insertions(+) diff --git a/conda-store-server/conda_store_server/server/templates/build.html b/conda-store-server/conda_store_server/server/templates/build.html index dc5c76612..6dddf8d0a 100644 --- a/conda-store-server/conda_store_server/server/templates/build.html +++ b/conda-store-server/conda_store_server/server/templates/build.html @@ -79,6 +79,9 @@

Conda Environment Artifacts

{% if build.has_conda_pack %}
  • Conda-Pack archive: environment.tar.gz
    • {% endif %} + {% if build.has_constructor_installer %} +
  • Constructor installer: installer
    • + {% endif %} {% if build.has_docker_manifest %}
  • Docker image registry url: {{ registry_external_url }}/{{ build.environment.namespace.name }}/{{ build.environment.name }}:{{ build.build_key }}
    • {% endif %} diff --git a/conda-store-server/conda_store_server/server/templates/home.html b/conda-store-server/conda_store_server/server/templates/home.html index ae4c66447..6434ff76d 100644 --- a/conda-store-server/conda_store_server/server/templates/home.html +++ b/conda-store-server/conda_store_server/server/templates/home.html @@ -41,6 +41,9 @@
    {% if environment.current_build.has_docker_manifest %} Docker {% endif %} + {% if environment.current_build.has_constructor_installer %} + Installer + {% endif %} {% endfor %} From ef44962ca768c31a3db06cacb397eca68e01ec7a Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Fri, 22 Dec 2023 02:18:15 +0100 Subject: [PATCH 11/37] Add installer to default `build_artifacts` --- conda-store-server/conda_store_server/app.py | 1 + conda-store-server/conda_store_server/schema.py | 1 + 2 files changed, 2 insertions(+) diff --git a/conda-store-server/conda_store_server/app.py b/conda-store-server/conda_store_server/app.py index ef9ff2c63..8d68dd138 100644 --- a/conda-store-server/conda_store_server/app.py +++ b/conda-store-server/conda_store_server/app.py @@ -263,6 +263,7 @@ def _check_redis(self, proposal): schema.BuildArtifactType.LOCKFILE, schema.BuildArtifactType.YAML, schema.BuildArtifactType.CONDA_PACK, + schema.BuildArtifactType.CONSTRUCTOR_INSTALLER, *( [ schema.BuildArtifactType.DOCKER_MANIFEST, diff --git a/conda-store-server/conda_store_server/schema.py b/conda-store-server/conda_store_server/schema.py index 729e54f62..0db0e535a 100644 --- a/conda-store-server/conda_store_server/schema.py +++ b/conda-store-server/conda_store_server/schema.py @@ -343,6 +343,7 @@ class Settings(BaseModel): BuildArtifactType.LOCKFILE, BuildArtifactType.YAML, BuildArtifactType.CONDA_PACK, + BuildArtifactType.CONSTRUCTOR_INSTALLER, *( [ BuildArtifactType.DOCKER_MANIFEST, From 13bc001c8cfe6a95951a3154731322432f274312 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Fri, 22 Dec 2023 02:32:50 +0100 Subject: [PATCH 12/37] Do not error out if `installer_dir` exists --- .../conda_store_server/action/generate_constructor_installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-store-server/conda_store_server/action/generate_constructor_installer.py b/conda-store-server/conda_store_server/action/generate_constructor_installer.py index 68e13ec4e..8d90ee158 100644 --- a/conda-store-server/conda_store_server/action/generate_constructor_installer.py +++ b/conda-store-server/conda_store_server/action/generate_constructor_installer.py @@ -44,7 +44,7 @@ def write_file(filename, s): pi_ext = ".bat" if sys.platform == "win32" else ".sh" installer_filename = (installer_dir / specification.name).with_suffix(ext) - os.makedirs(installer_dir) + os.makedirs(installer_dir, exist_ok=True) with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmp_dir: tmp_dir = pathlib.Path(tmp_dir) From 14f0ec50d61bf9ffaf0e2239b20ba30d8ad08378 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Fri, 22 Dec 2023 04:00:29 +0100 Subject: [PATCH 13/37] Force installer download, add extension --- conda-store-server/conda_store_server/orm.py | 3 ++- .../conda_store_server/server/templates/build.html | 2 +- .../conda_store_server/server/templates/home.html | 2 +- conda-store-server/tests/test_actions.py | 3 ++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/conda-store-server/conda_store_server/orm.py b/conda-store-server/conda_store_server/orm.py index 2fba18611..97acd79e8 100644 --- a/conda-store-server/conda_store_server/orm.py +++ b/conda-store-server/conda_store_server/orm.py @@ -339,7 +339,8 @@ def docker_manifest_key(self): @property def constructor_installer_key(self): - return f"installer/{self.build_key}" + ext = "exe" if sys.platform == "win32" else "sh" + return f"installer/{self.build_key}.{ext}" def docker_blob_key(self, blob_hash): return f"docker/blobs/{blob_hash}" diff --git a/conda-store-server/conda_store_server/server/templates/build.html b/conda-store-server/conda_store_server/server/templates/build.html index 6dddf8d0a..af89eaf37 100644 --- a/conda-store-server/conda_store_server/server/templates/build.html +++ b/conda-store-server/conda_store_server/server/templates/build.html @@ -80,7 +80,7 @@

    Conda Environment Artifacts

  • Conda-Pack archive: environment.tar.gz
    • {% endif %} {% if build.has_constructor_installer %} -
  • Constructor installer: installer
    • +
  • Constructor installer: installer
    • {% endif %} {% if build.has_docker_manifest %}
  • Docker image registry url: {{ registry_external_url }}/{{ build.environment.namespace.name }}/{{ build.environment.name }}:{{ build.build_key }}
    • diff --git a/conda-store-server/conda_store_server/server/templates/home.html b/conda-store-server/conda_store_server/server/templates/home.html index 6434ff76d..91286aec7 100644 --- a/conda-store-server/conda_store_server/server/templates/home.html +++ b/conda-store-server/conda_store_server/server/templates/home.html @@ -42,7 +42,7 @@
    Docker {% endif %} {% if environment.current_build.has_constructor_installer %} - Installer + Installer {% endif %} diff --git a/conda-store-server/tests/test_actions.py b/conda-store-server/tests/test_actions.py index b22e14df0..71d625735 100644 --- a/conda-store-server/tests/test_actions.py +++ b/conda-store-server/tests/test_actions.py @@ -467,7 +467,8 @@ def authorize_request(self, *args, **kwargs): # redirects to installer def installer_url(build_key): - return f"installer/{build_key}" + ext = "exe" if sys.platform == "win32" else "sh" + return f"installer/{build_key}.{ext}" assert type(res) is RedirectResponse assert build.constructor_installer_key == res.headers['location'] From edee0153f20f141fa44c44856e31967dbf116c7e Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Fri, 22 Dec 2023 04:13:12 +0100 Subject: [PATCH 14/37] Add platform to installer link --- .../conda_store_server/server/templates/build.html | 2 +- .../conda_store_server/server/templates/home.html | 2 +- conda-store-server/conda_store_server/server/views/ui.py | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/conda-store-server/conda_store_server/server/templates/build.html b/conda-store-server/conda_store_server/server/templates/build.html index af89eaf37..bd36351e6 100644 --- a/conda-store-server/conda_store_server/server/templates/build.html +++ b/conda-store-server/conda_store_server/server/templates/build.html @@ -80,7 +80,7 @@

    Conda Environment Artifacts

  • Conda-Pack archive: environment.tar.gz
    • {% endif %} {% if build.has_constructor_installer %} -
  • Constructor installer: installer
    • +
  • Constructor installer: installer ({{ platform }})
    • {% endif %} {% if build.has_docker_manifest %}
  • Docker image registry url: {{ registry_external_url }}/{{ build.environment.namespace.name }}/{{ build.environment.name }}:{{ build.build_key }}
    • diff --git a/conda-store-server/conda_store_server/server/templates/home.html b/conda-store-server/conda_store_server/server/templates/home.html index 91286aec7..ab97178bc 100644 --- a/conda-store-server/conda_store_server/server/templates/home.html +++ b/conda-store-server/conda_store_server/server/templates/home.html @@ -42,7 +42,7 @@
    Docker {% endif %} {% if environment.current_build.has_constructor_installer %} - Installer + Installer ({{ platform }}) {% endif %} diff --git a/conda-store-server/conda_store_server/server/views/ui.py b/conda-store-server/conda_store_server/server/views/ui.py index 5cf8c11cd..1ef5ef4eb 100644 --- a/conda-store-server/conda_store_server/server/views/ui.py +++ b/conda-store-server/conda_store_server/server/views/ui.py @@ -1,3 +1,4 @@ +import sys from typing import Optional import yaml @@ -63,6 +64,7 @@ async def ui_list_environments( "environments": orm_environments.all(), "registry_external_url": server.registry_external_url, "entity": entity, + "platform": "Windows" if sys.platform == "win32" else "Linux, macOS", } return templates.TemplateResponse("home.html", context) @@ -208,6 +210,7 @@ async def ui_get_build( "registry_external_url": server.registry_external_url, "entity": entity, "spec": yaml.dump(build.specification.spec), + "platform": "Windows" if sys.platform == "win32" else "Linux, macOS", } return templates.TemplateResponse("build.html", context) From 8b2f000e5833dd126425fb765eec751d3a9fe09d Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Fri, 22 Dec 2023 04:19:20 +0100 Subject: [PATCH 15/37] Make version an argument and set it to build_key --- .../action/generate_constructor_installer.py | 4 ++-- conda-store-server/conda_store_server/build.py | 1 + conda-store-server/tests/test_actions.py | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/conda-store-server/conda_store_server/action/generate_constructor_installer.py b/conda-store-server/conda_store_server/action/generate_constructor_installer.py index 8d90ee158..6bce2e8f6 100644 --- a/conda-store-server/conda_store_server/action/generate_constructor_installer.py +++ b/conda-store-server/conda_store_server/action/generate_constructor_installer.py @@ -14,6 +14,7 @@ def action_generate_constructor_installer( conda_command: str, specification: schema.CondaSpecification, installer_dir: pathlib.Path, + version: str, ): # Helpers def print_cmd(cmd): @@ -58,8 +59,7 @@ def write_file(filename, s): "name": specification.name, "channels": specification.channels, "specs": dependencies, - # XXX: This is required: use the env hash and datetime? - "version": 1, + "version": version, } if sys.platform == "win32": diff --git a/conda-store-server/conda_store_server/build.py b/conda-store-server/conda_store_server/build.py index 063d8c91c..679f6fc8d 100644 --- a/conda-store-server/conda_store_server/build.py +++ b/conda-store-server/conda_store_server/build.py @@ -403,6 +403,7 @@ def build_constructor_installer(db: Session, conda_store, build: orm.Build): build.specification.spec ), installer_dir=pathlib.Path(tmpdir), + version=build.build_key, ) output_filename = context.result append_to_logs( diff --git a/conda-store-server/tests/test_actions.py b/conda-store-server/tests/test_actions.py index 71d625735..df5d91a65 100644 --- a/conda-store-server/tests/test_actions.py +++ b/conda-store-server/tests/test_actions.py @@ -129,6 +129,7 @@ def test_generate_constructor_installer(conda_store, specification_name, request conda_command=conda_store.conda_command, specification=specification, installer_dir=installer_dir, + version="1", ) # Checks that the installer was created From 340852fb564d466fb09f0e19133f72672ccecdb1 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Fri, 22 Dec 2023 21:32:00 +0100 Subject: [PATCH 16/37] Add the installer task to a test --- conda-store-server/tests/test_app_api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conda-store-server/tests/test_app_api.py b/conda-store-server/tests/test_app_api.py index e5e026615..7d4fe2f1e 100644 --- a/conda-store-server/tests/test_app_api.py +++ b/conda-store-server/tests/test_app_api.py @@ -73,6 +73,9 @@ def test_conda_store_register_environment_workflow(db, conda_store, celery_worke task = AsyncResult(f"build-{build.id}-docker") task.wait(timeout=2 * 60) + task = AsyncResult(f"build-{build.id}-constructor-installer") + task.wait(timeout=60) + db.expire_all() assert build.status == schema.BuildStatus.COMPLETED From 8e53732bd09c3f0fae91e92e3381e03fcbe2dbeb Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Sun, 24 Dec 2023 21:48:41 +0100 Subject: [PATCH 17/37] Add docs --- .../conda-store/explanations/artifacts.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/docusaurus-docs/conda-store/explanations/artifacts.md b/docusaurus-docs/conda-store/explanations/artifacts.md index cd50a21ab..9f28479d2 100644 --- a/docusaurus-docs/conda-store/explanations/artifacts.md +++ b/docusaurus-docs/conda-store/explanations/artifacts.md @@ -93,7 +93,7 @@ source my_env/bin/activate conda-unpack ``` -### +### Docker images :::note Docker image creation is currently only supported on Linux. @@ -158,7 +158,24 @@ name. `:/conda-store-dynamic/python.lt.3.8/numpy.gt will then create the following environment and the docker image will download upon the docker image being built. +### Installers + +conda-store uses [constructor] to generate an installer for the current platform +(where the server is running): + +- on Linux and macOS, it generates a `.sh` installer +- on Windows, it generates a `.exe` installer using NSIS. + +conda-store automatically adds `conda` and `pip` to the target environment +because these are required for the installer to work. + +Also note that `constructor` uses a separate dependency solver instead of +utilizing the generated lockfile, so the package versions used by the installer +might be different compared to the environment available in conda-store. There +are plans to address this issue in the future. + [conda-docs]: https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/environments.html [conda-forge-immutability-policy]: https://conda-forge.org/docs/maintainer/updating_pkgs.html#packages-on-conda-forge-are-immutable [conda-lock-github]: https://github.com/conda-incubator/conda-lock +[constructor]: https://github.com/conda/constructor From 478e7058ffc695c76614bd3f8515182a79cb85ac Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Mon, 25 Dec 2023 07:05:21 +0100 Subject: [PATCH 18/37] Use `sys.platform` directly --- conda-store-server/conda_store_server/server/views/ui.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda-store-server/conda_store_server/server/views/ui.py b/conda-store-server/conda_store_server/server/views/ui.py index 1ef5ef4eb..7580a3637 100644 --- a/conda-store-server/conda_store_server/server/views/ui.py +++ b/conda-store-server/conda_store_server/server/views/ui.py @@ -64,7 +64,7 @@ async def ui_list_environments( "environments": orm_environments.all(), "registry_external_url": server.registry_external_url, "entity": entity, - "platform": "Windows" if sys.platform == "win32" else "Linux, macOS", + "platform": sys.platform, } return templates.TemplateResponse("home.html", context) @@ -210,7 +210,7 @@ async def ui_get_build( "registry_external_url": server.registry_external_url, "entity": entity, "spec": yaml.dump(build.specification.spec), - "platform": "Windows" if sys.platform == "win32" else "Linux, macOS", + "platform": sys.platform, } return templates.TemplateResponse("build.html", context) From b6e2fb5238d1b424cdf1dc269343f07ab3a2f810 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Mon, 25 Dec 2023 09:02:26 +0100 Subject: [PATCH 19/37] Escape `\a` --- .../conda_store_server/action/generate_constructor_installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-store-server/conda_store_server/action/generate_constructor_installer.py b/conda-store-server/conda_store_server/action/generate_constructor_installer.py index 6bce2e8f6..4d20fee80 100644 --- a/conda-store-server/conda_store_server/action/generate_constructor_installer.py +++ b/conda-store-server/conda_store_server/action/generate_constructor_installer.py @@ -64,7 +64,7 @@ def write_file(filename, s): if sys.platform == "win32": post_install = """\ -call "%PREFIX%\Scripts\activate.bat" +call "%PREFIX%\Scripts\\activate.bat" """ else: post_install = """\ From fbc5f0e5af3b8fa85584963a414bd18b6f1a02d1 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Mon, 25 Dec 2023 09:03:53 +0100 Subject: [PATCH 20/37] Call pip via python --- .../conda_store_server/action/generate_constructor_installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-store-server/conda_store_server/action/generate_constructor_installer.py b/conda-store-server/conda_store_server/action/generate_constructor_installer.py index 4d20fee80..8efe2f67a 100644 --- a/conda-store-server/conda_store_server/action/generate_constructor_installer.py +++ b/conda-store-server/conda_store_server/action/generate_constructor_installer.py @@ -75,7 +75,7 @@ def write_file(filename, s): """ if pip_dependencies: post_install += f""" -pip install {' '.join(pip_dependencies)} +python -m pip install {' '.join(pip_dependencies)} """ # Writes files to disk From a38639a33a5c0e400bdcb135ea5102228af97d0c Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Mon, 25 Dec 2023 10:54:17 +0100 Subject: [PATCH 21/37] Increase test wait timeout --- conda-store-server/tests/test_app_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-store-server/tests/test_app_api.py b/conda-store-server/tests/test_app_api.py index 7d4fe2f1e..e08d81c67 100644 --- a/conda-store-server/tests/test_app_api.py +++ b/conda-store-server/tests/test_app_api.py @@ -74,7 +74,7 @@ def test_conda_store_register_environment_workflow(db, conda_store, celery_worke task.wait(timeout=2 * 60) task = AsyncResult(f"build-{build.id}-constructor-installer") - task.wait(timeout=60) + task.wait(timeout=5 * 60) db.expire_all() assert build.status == schema.BuildStatus.COMPLETED From f7348c0f22708fcc89da024d78a742cc09a26b2f Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Wed, 3 Jan 2024 13:06:30 +0100 Subject: [PATCH 22/37] Make `constructor` optional --- .../action/generate_constructor_installer.py | 31 +++++++++---------- .../conda_store_server/build.py | 2 ++ 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/conda-store-server/conda_store_server/action/generate_constructor_installer.py b/conda-store-server/conda_store_server/action/generate_constructor_installer.py index 8efe2f67a..2c9f1e032 100644 --- a/conda-store-server/conda_store_server/action/generate_constructor_installer.py +++ b/conda-store-server/conda_store_server/action/generate_constructor_installer.py @@ -3,6 +3,7 @@ import subprocess import sys import tempfile +import warnings import yaml from conda_store_server import action, schema @@ -28,6 +29,19 @@ def write_file(filename, s): context.log.info(f"{filename}:\n{s}") f.write(s) + # Checks if constructor is available + try: + command = [ + "constructor", + "--help", + ] + print_cmd(command) + except FileNotFoundError: + warnings.warn( + "Installer generation requires constructor: https://github.com/conda/constructor" + ) + return + # pip dependencies are not directly supported by constructor, they will be # installed via the post_install script: # https://github.com/conda/constructor/issues/515 @@ -51,7 +65,6 @@ def write_file(filename, s): tmp_dir = pathlib.Path(tmp_dir) construct_file = tmp_dir / "construct.yaml" post_install_file = (tmp_dir / "post_install").with_suffix(pi_ext) - env_dir = tmp_dir / "env" construct = { "installer_filename": str(installer_filename), @@ -82,24 +95,8 @@ def write_file(filename, s): write_file(construct_file, yaml.dump(construct)) write_file(post_install_file, post_install) - # Installs constructor - command = [ - conda_command, - "create", - "-y", - "-p", - str(env_dir), - "constructor", - ] - print_cmd(command) - # Calls constructor command = [ - conda_command, - "run", - "-p", - str(env_dir), - "--no-capture-output", "constructor", str(tmp_dir), ] diff --git a/conda-store-server/conda_store_server/build.py b/conda-store-server/conda_store_server/build.py index 679f6fc8d..d0257be88 100644 --- a/conda-store-server/conda_store_server/build.py +++ b/conda-store-server/conda_store_server/build.py @@ -414,6 +414,8 @@ def build_constructor_installer(db: Session, conda_store, build: orm.Build): + context.stdout.getvalue() + "\n::endgroup::\n", ) + if output_filename is None: + return conda_store.storage.fset( db, build.id, From c72f586c13123adeac7d77ab4a066f22631c49f2 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Wed, 3 Jan 2024 13:45:56 +0100 Subject: [PATCH 23/37] Add `constructor` to env files --- conda-store-server/environment-dev.yaml | 2 ++ conda-store-server/environment-macos-dev.yaml | 2 ++ conda-store-server/environment-windows-dev.yaml | 2 ++ conda-store-server/environment.yaml | 2 ++ 4 files changed, 8 insertions(+) diff --git a/conda-store-server/environment-dev.yaml b/conda-store-server/environment-dev.yaml index 7d30ed06d..fe30479a0 100644 --- a/conda-store-server/environment-dev.yaml +++ b/conda-store-server/environment-dev.yaml @@ -34,6 +34,8 @@ dependencies: - alembic # artifact storage - minio + # installer + - constructor # CLI - typer diff --git a/conda-store-server/environment-macos-dev.yaml b/conda-store-server/environment-macos-dev.yaml index 8bd0c1d45..6ece10f5d 100644 --- a/conda-store-server/environment-macos-dev.yaml +++ b/conda-store-server/environment-macos-dev.yaml @@ -34,6 +34,8 @@ dependencies: - alembic # artifact storage - minio + # installer + - constructor # CLI - typer diff --git a/conda-store-server/environment-windows-dev.yaml b/conda-store-server/environment-windows-dev.yaml index 8bd0c1d45..6ece10f5d 100644 --- a/conda-store-server/environment-windows-dev.yaml +++ b/conda-store-server/environment-windows-dev.yaml @@ -34,6 +34,8 @@ dependencies: - alembic # artifact storage - minio + # installer + - constructor # CLI - typer diff --git a/conda-store-server/environment.yaml b/conda-store-server/environment.yaml index e8cf5be12..f3eaa0614 100644 --- a/conda-store-server/environment.yaml +++ b/conda-store-server/environment.yaml @@ -33,3 +33,5 @@ dependencies: - python-multipart # artifact storage - minio + # installer + - constructor From ba5333afacfe36b1ec5f4fa59369ff5a1ca1f491 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Wed, 3 Jan 2024 18:26:34 +0100 Subject: [PATCH 24/37] Use the same platform name in the UI as with `constructor` --- .../action/generate_constructor_installer.py | 11 +++++++++++ .../conda_store_server/server/views/ui.py | 8 +++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/conda-store-server/conda_store_server/action/generate_constructor_installer.py b/conda-store-server/conda_store_server/action/generate_constructor_installer.py index 2c9f1e032..bc3515931 100644 --- a/conda-store-server/conda_store_server/action/generate_constructor_installer.py +++ b/conda-store-server/conda_store_server/action/generate_constructor_installer.py @@ -9,6 +9,15 @@ from conda_store_server import action, schema +def get_installer_platform(): + # This is how the default platform name is generated internally by + # constructor. For example: osx-arm64, linux-64, win-64. + # https://github.com/conda/constructor/blob/main/CONSTRUCT.md#available-platforms + from conda.base.context import context + + return context.subdir + + @action.action def action_generate_constructor_installer( context, @@ -98,6 +107,8 @@ def write_file(filename, s): # Calls constructor command = [ "constructor", + "--platform", + get_installer_platform(), str(tmp_dir), ] print_cmd(command) diff --git a/conda-store-server/conda_store_server/server/views/ui.py b/conda-store-server/conda_store_server/server/views/ui.py index 7580a3637..522891f3c 100644 --- a/conda-store-server/conda_store_server/server/views/ui.py +++ b/conda-store-server/conda_store_server/server/views/ui.py @@ -1,8 +1,10 @@ -import sys from typing import Optional import yaml from conda_store_server import api +from conda_store_server.action.generate_constructor_installer import ( + get_installer_platform, +) from conda_store_server.schema import Permissions from conda_store_server.server import dependencies from fastapi import APIRouter, Depends, Request @@ -64,7 +66,7 @@ async def ui_list_environments( "environments": orm_environments.all(), "registry_external_url": server.registry_external_url, "entity": entity, - "platform": sys.platform, + "platform": get_installer_platform(), } return templates.TemplateResponse("home.html", context) @@ -210,7 +212,7 @@ async def ui_get_build( "registry_external_url": server.registry_external_url, "entity": entity, "spec": yaml.dump(build.specification.spec), - "platform": sys.platform, + "platform": get_installer_platform(), } return templates.TemplateResponse("build.html", context) From 70d83438720afbfc0af12840909d57291345c025 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Mon, 15 Jan 2024 01:32:24 +0100 Subject: [PATCH 25/37] Pass `--cache-dir` to `constructor` --- .../action/generate_constructor_installer.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/conda-store-server/conda_store_server/action/generate_constructor_installer.py b/conda-store-server/conda_store_server/action/generate_constructor_installer.py index bc3515931..25e17ad20 100644 --- a/conda-store-server/conda_store_server/action/generate_constructor_installer.py +++ b/conda-store-server/conda_store_server/action/generate_constructor_installer.py @@ -72,6 +72,10 @@ def write_file(filename, s): with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmp_dir: tmp_dir = pathlib.Path(tmp_dir) + cache_dir = tmp_dir / "pkgs" + tmp_dir /= "build" + os.makedirs(cache_dir, exist_ok=True) + os.makedirs(tmp_dir, exist_ok=True) construct_file = tmp_dir / "construct.yaml" post_install_file = (tmp_dir / "post_install").with_suffix(pi_ext) @@ -105,8 +109,14 @@ def write_file(filename, s): write_file(post_install_file, post_install) # Calls constructor + # Note: `cache_dir` is the same as the conda `pkgs` directory. It needs + # to be specified here because the default `pkgs` directory is not + # available in Docker, which was causing conda's `create_cache_dir` to + # fail. command = [ "constructor", + "--cache-dir", + str(cache_dir), "--platform", get_installer_platform(), str(tmp_dir), From f3e72b5debd61fcca072e5982ce5d14682f5a6b7 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Mon, 15 Jan 2024 01:34:32 +0100 Subject: [PATCH 26/37] Pass `-v` to `constructor` --- .../conda_store_server/action/generate_constructor_installer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conda-store-server/conda_store_server/action/generate_constructor_installer.py b/conda-store-server/conda_store_server/action/generate_constructor_installer.py index 25e17ad20..2abd89602 100644 --- a/conda-store-server/conda_store_server/action/generate_constructor_installer.py +++ b/conda-store-server/conda_store_server/action/generate_constructor_installer.py @@ -115,6 +115,7 @@ def write_file(filename, s): # fail. command = [ "constructor", + "-v", "--cache-dir", str(cache_dir), "--platform", From bf56d3f4e823eb69602d7d7837d0c1c59c66ed32 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Mon, 15 Jan 2024 00:58:59 +0100 Subject: [PATCH 27/37] Add migration for `build_artifact` --- .../versions/82c1cfe39561_add_installer.py | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py diff --git a/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py b/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py new file mode 100644 index 000000000..db35207e2 --- /dev/null +++ b/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py @@ -0,0 +1,61 @@ +"""add installer + +Revision ID: 82c1cfe39561 +Revises: 771180018e1b +Create Date: 2024-01-15 07:13:06.261592 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "82c1cfe39561" +down_revision = "771180018e1b" +branch_labels = None +depends_on = None + + +def upgrade(): + with op.batch_alter_table( + "build_artifact", schema=None, recreate="always" + ) as batch_op: + batch_op.alter_column( + "artifact_type", + existing_type=sa.VARCHAR(length=18), + type_=sa.Enum( + "DIRECTORY", + "LOCKFILE", + "LOGS", + "YAML", + "CONDA_PACK", + "DOCKER_BLOB", + "DOCKER_MANIFEST", + "CONTAINER_REGISTRY", + "CONSTRUCTOR_INSTALLER", + name="buildartifacttype", + ), + existing_nullable=False, + ) + + +def downgrade(): + with op.batch_alter_table( + "build_artifact", schema=None, recreate="always" + ) as batch_op: + batch_op.alter_column( + "artifact_type", + existing_type=sa.Enum( + "DIRECTORY", + "LOCKFILE", + "LOGS", + "YAML", + "CONDA_PACK", + "DOCKER_BLOB", + "DOCKER_MANIFEST", + "CONTAINER_REGISTRY", + "CONSTRUCTOR_INSTALLER", + name="buildartifacttype", + ), + type_=sa.VARCHAR(length=18), + existing_nullable=False, + ) From f323f265a03ba6f8a226f7cb902f62b4ec3d703e Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Tue, 16 Jan 2024 07:40:04 +0100 Subject: [PATCH 28/37] Update migration --- .../versions/82c1cfe39561_add_installer.py | 29 ++++--------------- 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py b/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py index db35207e2..478f50500 100644 --- a/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py +++ b/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py @@ -22,40 +22,21 @@ def upgrade(): batch_op.alter_column( "artifact_type", existing_type=sa.VARCHAR(length=18), - type_=sa.Enum( - "DIRECTORY", - "LOCKFILE", - "LOGS", - "YAML", - "CONDA_PACK", - "DOCKER_BLOB", - "DOCKER_MANIFEST", - "CONTAINER_REGISTRY", - "CONSTRUCTOR_INSTALLER", - name="buildartifacttype", - ), + type_=sa.VARCHAR(length=21), existing_nullable=False, ) def downgrade(): + op.execute( + 'DELETE FROM build_artifact WHERE artifact_type = "CONSTRUCTOR_INSTALLER"' + ) with op.batch_alter_table( "build_artifact", schema=None, recreate="always" ) as batch_op: batch_op.alter_column( "artifact_type", - existing_type=sa.Enum( - "DIRECTORY", - "LOCKFILE", - "LOGS", - "YAML", - "CONDA_PACK", - "DOCKER_BLOB", - "DOCKER_MANIFEST", - "CONTAINER_REGISTRY", - "CONSTRUCTOR_INSTALLER", - name="buildartifacttype", - ), + existing_type=sa.VARCHAR(length=21), type_=sa.VARCHAR(length=18), existing_nullable=False, ) From 33687c094cce2485a0fb22a529a5d9917598fe56 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Tue, 16 Jan 2024 07:59:57 +0100 Subject: [PATCH 29/37] Update migration --- .../alembic/versions/82c1cfe39561_add_installer.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py b/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py index 478f50500..06510f597 100644 --- a/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py +++ b/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py @@ -17,7 +17,8 @@ def upgrade(): with op.batch_alter_table( - "build_artifact", schema=None, recreate="always" + "build_artifact", + schema=None, ) as batch_op: batch_op.alter_column( "artifact_type", @@ -32,7 +33,8 @@ def downgrade(): 'DELETE FROM build_artifact WHERE artifact_type = "CONSTRUCTOR_INSTALLER"' ) with op.batch_alter_table( - "build_artifact", schema=None, recreate="always" + "build_artifact", + schema=None, ) as batch_op: batch_op.alter_column( "artifact_type", From 9792e8f0a1dbc61797fd373b666e562e1e9ddb40 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Tue, 16 Jan 2024 20:43:59 +0100 Subject: [PATCH 30/37] Update migration --- .../alembic/versions/82c1cfe39561_add_installer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py b/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py index 06510f597..4448fc22b 100644 --- a/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py +++ b/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py @@ -26,6 +26,8 @@ def upgrade(): type_=sa.VARCHAR(length=21), existing_nullable=False, ) + if not str(op.get_bind().engine.url).startswith("sqlite"): + op.execute("DROP TYPE buildartifacttype") def downgrade(): From 0f4043afdc736a55e72007ecfad8753965ddc8f9 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Tue, 16 Jan 2024 21:32:07 +0100 Subject: [PATCH 31/37] Add a note for existing deployments --- docusaurus-docs/conda-store/explanations/artifacts.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docusaurus-docs/conda-store/explanations/artifacts.md b/docusaurus-docs/conda-store/explanations/artifacts.md index 9f28479d2..6ba8630f5 100644 --- a/docusaurus-docs/conda-store/explanations/artifacts.md +++ b/docusaurus-docs/conda-store/explanations/artifacts.md @@ -174,6 +174,15 @@ utilizing the generated lockfile, so the package versions used by the installer might be different compared to the environment available in conda-store. There are plans to address this issue in the future. +#### Existing Deployments + +conda-store saves environment settings and doesn't automatically update them on +startup (see `CondaStore.ensure_settings`). Existing deployments need to +manually enable installer builds via the admin interface. This can be done by +going to `/admin/setting///` (or +clicking on the `Settings` button on the environment page) and adding +`"CONSTRUCTOR_INSTALLER"` to `build_artifacts`. + [conda-docs]: https://docs.conda.io/projects/conda/en/latest/user-guide/concepts/environments.html [conda-forge-immutability-policy]: https://conda-forge.org/docs/maintainer/updating_pkgs.html#packages-on-conda-forge-are-immutable From b9638fbe6b6816d84a43e046eff81569c187b9f2 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Wed, 17 Jan 2024 08:30:56 +0100 Subject: [PATCH 32/37] Update migration --- .../alembic/versions/82c1cfe39561_add_installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py b/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py index 4448fc22b..f90c97444 100644 --- a/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py +++ b/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py @@ -32,7 +32,7 @@ def upgrade(): def downgrade(): op.execute( - 'DELETE FROM build_artifact WHERE artifact_type = "CONSTRUCTOR_INSTALLER"' + "DELETE FROM build_artifact WHERE artifact_type = 'CONSTRUCTOR_INSTALLER'" ) with op.batch_alter_table( "build_artifact", From dc1b457cd717962f04c569f9f70645db503b4acf Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Wed, 17 Jan 2024 08:35:25 +0100 Subject: [PATCH 33/37] Update migration --- .../alembic/versions/82c1cfe39561_add_installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py b/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py index f90c97444..0e563303b 100644 --- a/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py +++ b/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py @@ -27,7 +27,7 @@ def upgrade(): existing_nullable=False, ) if not str(op.get_bind().engine.url).startswith("sqlite"): - op.execute("DROP TYPE buildartifacttype") + op.execute("DROP TYPE IF EXISTS buildartifacttype") def downgrade(): From ae06519488d6aa7b23f19c1fe6c046416dbc9dd3 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Sun, 28 Jan 2024 14:23:21 +0100 Subject: [PATCH 34/37] Update migration hashes --- ..._add_installer.py => 57cd11b949d5_add_installer.py} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename conda-store-server/conda_store_server/alembic/versions/{82c1cfe39561_add_installer.py => 57cd11b949d5_add_installer.py} (87%) diff --git a/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py b/conda-store-server/conda_store_server/alembic/versions/57cd11b949d5_add_installer.py similarity index 87% rename from conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py rename to conda-store-server/conda_store_server/alembic/versions/57cd11b949d5_add_installer.py index 0e563303b..db815fe1d 100644 --- a/conda-store-server/conda_store_server/alembic/versions/82c1cfe39561_add_installer.py +++ b/conda-store-server/conda_store_server/alembic/versions/57cd11b949d5_add_installer.py @@ -1,16 +1,16 @@ """add installer -Revision ID: 82c1cfe39561 -Revises: 771180018e1b -Create Date: 2024-01-15 07:13:06.261592 +Revision ID: 57cd11b949d5 +Revises: 0f7e23ff24ee +Create Date: 2024-01-28 14:31:35.723505 """ import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. -revision = "82c1cfe39561" -down_revision = "771180018e1b" +revision = "57cd11b949d5" +down_revision = "0f7e23ff24ee" branch_labels = None depends_on = None From 6aa7d747a2a196cc41232376a9da37e71a1e026e Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Sun, 28 Jan 2024 17:39:30 +0100 Subject: [PATCH 35/37] Add a timeout when calling `constructor --help` --- .../action/generate_constructor_installer.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/conda-store-server/conda_store_server/action/generate_constructor_installer.py b/conda-store-server/conda_store_server/action/generate_constructor_installer.py index 2abd89602..30dc9aff2 100644 --- a/conda-store-server/conda_store_server/action/generate_constructor_installer.py +++ b/conda-store-server/conda_store_server/action/generate_constructor_installer.py @@ -27,10 +27,12 @@ def action_generate_constructor_installer( version: str, ): # Helpers - def print_cmd(cmd): + def print_cmd(cmd, **kwargs): context.log.info(f"Running command: {' '.join(cmd)}") context.log.info( - subprocess.check_output(cmd, stderr=subprocess.STDOUT, encoding="utf-8") + subprocess.check_output( + cmd, stderr=subprocess.STDOUT, encoding="utf-8", **kwargs + ) ) def write_file(filename, s): @@ -44,7 +46,7 @@ def write_file(filename, s): "constructor", "--help", ] - print_cmd(command) + print_cmd(command, timeout=10) except FileNotFoundError: warnings.warn( "Installer generation requires constructor: https://github.com/conda/constructor" From f79b19832ab55d61128e3cf8f5782f0c411779b2 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Sun, 28 Jan 2024 17:47:07 +0100 Subject: [PATCH 36/37] Rename `print_cmd` to `logged_command`, move it to `utils` --- .../action/generate_constructor_installer.py | 15 +++------------ .../action/generate_lockfile.py | 14 ++++---------- .../conda_store_server/action/utils.py | 10 ++++++++++ 3 files changed, 17 insertions(+), 22 deletions(-) create mode 100644 conda-store-server/conda_store_server/action/utils.py diff --git a/conda-store-server/conda_store_server/action/generate_constructor_installer.py b/conda-store-server/conda_store_server/action/generate_constructor_installer.py index 30dc9aff2..73bf55b75 100644 --- a/conda-store-server/conda_store_server/action/generate_constructor_installer.py +++ b/conda-store-server/conda_store_server/action/generate_constructor_installer.py @@ -1,12 +1,12 @@ import os import pathlib -import subprocess import sys import tempfile import warnings import yaml from conda_store_server import action, schema +from conda_store_server.action.utils import logged_command def get_installer_platform(): @@ -26,15 +26,6 @@ def action_generate_constructor_installer( installer_dir: pathlib.Path, version: str, ): - # Helpers - def print_cmd(cmd, **kwargs): - context.log.info(f"Running command: {' '.join(cmd)}") - context.log.info( - subprocess.check_output( - cmd, stderr=subprocess.STDOUT, encoding="utf-8", **kwargs - ) - ) - def write_file(filename, s): with open(filename, "w") as f: context.log.info(f"{filename}:\n{s}") @@ -46,7 +37,7 @@ def write_file(filename, s): "constructor", "--help", ] - print_cmd(command, timeout=10) + logged_command(context, command, timeout=10) except FileNotFoundError: warnings.warn( "Installer generation requires constructor: https://github.com/conda/constructor" @@ -124,6 +115,6 @@ def write_file(filename, s): get_installer_platform(), str(tmp_dir), ] - print_cmd(command) + logged_command(context, command) return installer_filename diff --git a/conda-store-server/conda_store_server/action/generate_lockfile.py b/conda-store-server/conda_store_server/action/generate_lockfile.py index caf2d9118..550012426 100644 --- a/conda-store-server/conda_store_server/action/generate_lockfile.py +++ b/conda-store-server/conda_store_server/action/generate_lockfile.py @@ -1,12 +1,12 @@ import json import os import pathlib -import subprocess import typing import yaml from conda_lock.conda_lock import run_lock from conda_store_server import action, conda_utils, schema +from conda_store_server.action.utils import logged_command @action.action @@ -25,17 +25,11 @@ def action_solve_lockfile( with environment_filename.open("w") as f: json.dump(specification.dict(), f) - def print_cmd(cmd): - context.log.info(f"Running command: {' '.join(cmd)}") - context.log.info( - subprocess.check_output(cmd, stderr=subprocess.STDOUT, encoding="utf-8") - ) - # The info command can be used with either mamba or conda - print_cmd([conda_command, "info"]) + logged_command(context, [conda_command, "info"]) # The config command is not supported by mamba - print_cmd(["conda", "config", "--show"]) - print_cmd(["conda", "config", "--show-sources"]) + logged_command(context, ["conda", "config", "--show"]) + logged_command(context, ["conda", "config", "--show-sources"]) # conda-lock ignores variables defined in the specification, so this code # gets the value of CONDA_OVERRIDE_CUDA and passes it to conda-lock via diff --git a/conda-store-server/conda_store_server/action/utils.py b/conda-store-server/conda_store_server/action/utils.py new file mode 100644 index 000000000..8c7ba5699 --- /dev/null +++ b/conda-store-server/conda_store_server/action/utils.py @@ -0,0 +1,10 @@ +import subprocess + + +def logged_command(context, command, **kwargs): + context.log.info(f"Running command: {' '.join(command)}") + context.log.info( + subprocess.check_output( + command, stderr=subprocess.STDOUT, encoding="utf-8", **kwargs + ) + ) From 30f2f236e18ea6d1ae72798f0d8b366fbae89837 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Sun, 28 Jan 2024 17:58:55 +0100 Subject: [PATCH 37/37] Add a note about cross-building --- .../conda_store_server/action/generate_constructor_installer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda-store-server/conda_store_server/action/generate_constructor_installer.py b/conda-store-server/conda_store_server/action/generate_constructor_installer.py index 73bf55b75..0fbb6afd2 100644 --- a/conda-store-server/conda_store_server/action/generate_constructor_installer.py +++ b/conda-store-server/conda_store_server/action/generate_constructor_installer.py @@ -13,6 +13,8 @@ def get_installer_platform(): # This is how the default platform name is generated internally by # constructor. For example: osx-arm64, linux-64, win-64. # https://github.com/conda/constructor/blob/main/CONSTRUCT.md#available-platforms + # Note: constructor is cross-friendly, see: + # https://github.com/conda-incubator/conda-store/pull/714#discussion_r1465115323 from conda.base.context import context return context.subdir