From 65fc72e7333a5f11b4315244dd298e20f4bc811c Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Tue, 21 Nov 2023 08:58:47 +0000 Subject: [PATCH 1/6] Skip `test_generate_conda_docker` See #666. --- conda-store-server/tests/test_actions.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conda-store-server/tests/test_actions.py b/conda-store-server/tests/test_actions.py index 3ad926422..60573d2ab 100644 --- a/conda-store-server/tests/test_actions.py +++ b/conda-store-server/tests/test_actions.py @@ -128,7 +128,9 @@ def test_generate_conda_pack(tmp_path, conda_prefix): assert output_filename.exists() -@pytest.mark.skipif(sys.platform != "linux", reason="conda-docker only works on linux") +@pytest.mark.skip(reason=( + "upstream issue in conda-docker, see " + "https://github.com/conda-incubator/conda-store/issues/666")) def test_generate_conda_docker(conda_store, conda_prefix): action.action_generate_conda_docker( conda_prefix=conda_prefix, From bf69e05708a774bdc9e62dfcec09c88ec39d54fe Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Tue, 21 Nov 2023 10:29:11 +0000 Subject: [PATCH 2/6] Disable `action_generate_conda_docker` on all platforms --- .../action/generate_conda_docker.py | 60 +++++++++---------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/conda-store-server/conda_store_server/action/generate_conda_docker.py b/conda-store-server/conda_store_server/action/generate_conda_docker.py index 42d60147f..57177c699 100644 --- a/conda-store-server/conda_store_server/action/generate_conda_docker.py +++ b/conda-store-server/conda_store_server/action/generate_conda_docker.py @@ -1,5 +1,4 @@ import pathlib -import sys from conda_store_server import action @@ -13,35 +12,34 @@ def action_generate_conda_docker( output_image_name: str, output_image_tag: str, ): - if sys.platform != "linux": - raise RuntimeError( - "Generating Docker images is currently only supported on Linux" - ) + # Upstream issue in conda-docker, see + # https://github.com/conda-incubator/conda-store/issues/666 + raise RuntimeError("Generating Docker images is currently not supported") - # Import is inside the function because conda_docker is only available on - # Linux - from conda_docker.conda import ( - build_docker_environment_image, - conda_info, - fetch_precs, - find_user_conda, - precs_from_environment_prefix, - ) + # # Import is inside the function because conda_docker is only available on + # # Linux + # from conda_docker.conda import ( + # build_docker_environment_image, + # conda_info, + # fetch_precs, + # find_user_conda, + # precs_from_environment_prefix, + # ) - user_conda = find_user_conda() - info = conda_info(user_conda) - download_dir = info["pkgs_dirs"][0] - precs = precs_from_environment_prefix(str(conda_prefix), download_dir, user_conda) - records = fetch_precs(download_dir, precs) - base_image = container_registry.pull_image(default_docker_image) - image = build_docker_environment_image( - base_image=base_image, - output_image=f"{output_image_name}:{output_image_tag}", - records=records, - default_prefix=info["env_vars"]["CONDA_ROOT"], - download_dir=download_dir, - user_conda=user_conda, - channels_remap=info.get("channels_remap", []), - layering_strategy="single", - ) - return image + # user_conda = find_user_conda() + # info = conda_info(user_conda) + # download_dir = info["pkgs_dirs"][0] + # precs = precs_from_environment_prefix(str(conda_prefix), download_dir, user_conda) + # records = fetch_precs(download_dir, precs) + # base_image = container_registry.pull_image(default_docker_image) + # image = build_docker_environment_image( + # base_image=base_image, + # output_image=f"{output_image_name}:{output_image_tag}", + # records=records, + # default_prefix=info["env_vars"]["CONDA_ROOT"], + # download_dir=download_dir, + # user_conda=user_conda, + # channels_remap=info.get("channels_remap", []), + # layering_strategy="single", + # ) + # return image From 82d8f13409a061d41d7110a27b2cefc6c554602b Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Tue, 21 Nov 2023 10:43:16 +0000 Subject: [PATCH 3/6] Disable `build_conda_docker` --- .../conda_store_server/build.py | 89 ++++++++++--------- 1 file changed, 46 insertions(+), 43 deletions(-) diff --git a/conda-store-server/conda_store_server/build.py b/conda-store-server/conda_store_server/build.py index 6475c8921..99a58d7d0 100644 --- a/conda-store-server/conda_store_server/build.py +++ b/conda-store-server/conda_store_server/build.py @@ -313,46 +313,49 @@ def build_conda_pack(db: Session, conda_store, build: orm.Build): def build_conda_docker(db: Session, conda_store, build: orm.Build): - conda_prefix = build.build_path(conda_store) - settings = conda_store.get_settings( - db=db, - namespace=build.environment.namespace.name, - environment_name=build.environment.name, - ) - - try: - with utils.timer( - conda_store.log, - f"packaging docker image of conda environment={conda_prefix}", - ): - context = action.action_generate_conda_docker( - conda_prefix=conda_prefix, - default_docker_image=utils.callable_or_value( - settings.default_docker_base_image, None - ), - container_registry=conda_store.container_registry, - output_image_name=build.specification.name, - output_image_tag=build.build_key, - ) - append_to_logs( - db, - conda_store, - build, - "::group::action_generate_conda_docker\n" - + context.stdout.getvalue() - + "\n::endgroup::\n", - ) - - image = context.result - - if schema.BuildArtifactType.DOCKER_MANIFEST in settings.build_artifacts: - conda_store.container_registry.store_image( - db, conda_store, build, image - ) - - if schema.BuildArtifactType.CONTAINER_REGISTRY in settings.build_artifacts: - conda_store.container_registry.push_image(db, build, image) - except Exception as e: - conda_store.log.exception(e) - append_to_logs(db, conda_store, build, traceback.format_exc()) - raise e + # Upstream issue in conda-docker, see + # https://github.com/conda-incubator/conda-store/issues/666 + pass + # conda_prefix = build.build_path(conda_store) + # settings = conda_store.get_settings( + # db=db, + # namespace=build.environment.namespace.name, + # environment_name=build.environment.name, + # ) + + # try: + # with utils.timer( + # conda_store.log, + # f"packaging docker image of conda environment={conda_prefix}", + # ): + # context = action.action_generate_conda_docker( + # conda_prefix=conda_prefix, + # default_docker_image=utils.callable_or_value( + # settings.default_docker_base_image, None + # ), + # container_registry=conda_store.container_registry, + # output_image_name=build.specification.name, + # output_image_tag=build.build_key, + # ) + # append_to_logs( + # db, + # conda_store, + # build, + # "::group::action_generate_conda_docker\n" + # + context.stdout.getvalue() + # + "\n::endgroup::\n", + # ) + + # image = context.result + + # if schema.BuildArtifactType.DOCKER_MANIFEST in settings.build_artifacts: + # conda_store.container_registry.store_image( + # db, conda_store, build, image + # ) + + # if schema.BuildArtifactType.CONTAINER_REGISTRY in settings.build_artifacts: + # conda_store.container_registry.push_image(db, build, image) + # except Exception as e: + # conda_store.log.exception(e) + # append_to_logs(db, conda_store, build, traceback.format_exc()) + # raise e From 69e9b77e23d7997f4cb47f81ffbef9615c1799f8 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Tue, 21 Nov 2023 10:50:13 +0000 Subject: [PATCH 4/6] Mark `test_generate_conda_docker` as xfail --- conda-store-server/tests/test_actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-store-server/tests/test_actions.py b/conda-store-server/tests/test_actions.py index 60573d2ab..1aa35e477 100644 --- a/conda-store-server/tests/test_actions.py +++ b/conda-store-server/tests/test_actions.py @@ -128,7 +128,7 @@ def test_generate_conda_pack(tmp_path, conda_prefix): assert output_filename.exists() -@pytest.mark.skip(reason=( +@pytest.mark.xfail(reason=( "upstream issue in conda-docker, see " "https://github.com/conda-incubator/conda-store/issues/666")) def test_generate_conda_docker(conda_store, conda_prefix): From a560d1fe048f45639ffe6b080b4b1b561ffc753c Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Wed, 22 Nov 2023 10:25:33 +0000 Subject: [PATCH 5/6] Do not comment out code, show a warning --- .../action/generate_conda_docker.py | 54 +++++------ .../conda_store_server/build.py | 92 ++++++++++--------- 2 files changed, 75 insertions(+), 71 deletions(-) diff --git a/conda-store-server/conda_store_server/action/generate_conda_docker.py b/conda-store-server/conda_store_server/action/generate_conda_docker.py index 57177c699..c713ad4c8 100644 --- a/conda-store-server/conda_store_server/action/generate_conda_docker.py +++ b/conda-store-server/conda_store_server/action/generate_conda_docker.py @@ -14,32 +14,32 @@ def action_generate_conda_docker( ): # Upstream issue in conda-docker, see # https://github.com/conda-incubator/conda-store/issues/666 - raise RuntimeError("Generating Docker images is currently not supported") + raise RuntimeError("Generating Docker images is currently not supported, see #666") - # # Import is inside the function because conda_docker is only available on - # # Linux - # from conda_docker.conda import ( - # build_docker_environment_image, - # conda_info, - # fetch_precs, - # find_user_conda, - # precs_from_environment_prefix, - # ) + # Import is inside the function because conda_docker is only available on + # Linux + from conda_docker.conda import ( + build_docker_environment_image, + conda_info, + fetch_precs, + find_user_conda, + precs_from_environment_prefix, + ) - # user_conda = find_user_conda() - # info = conda_info(user_conda) - # download_dir = info["pkgs_dirs"][0] - # precs = precs_from_environment_prefix(str(conda_prefix), download_dir, user_conda) - # records = fetch_precs(download_dir, precs) - # base_image = container_registry.pull_image(default_docker_image) - # image = build_docker_environment_image( - # base_image=base_image, - # output_image=f"{output_image_name}:{output_image_tag}", - # records=records, - # default_prefix=info["env_vars"]["CONDA_ROOT"], - # download_dir=download_dir, - # user_conda=user_conda, - # channels_remap=info.get("channels_remap", []), - # layering_strategy="single", - # ) - # return image + user_conda = find_user_conda() + info = conda_info(user_conda) + download_dir = info["pkgs_dirs"][0] + precs = precs_from_environment_prefix(str(conda_prefix), download_dir, user_conda) + records = fetch_precs(download_dir, precs) + base_image = container_registry.pull_image(default_docker_image) + image = build_docker_environment_image( + base_image=base_image, + output_image=f"{output_image_name}:{output_image_tag}", + records=records, + default_prefix=info["env_vars"]["CONDA_ROOT"], + download_dir=download_dir, + user_conda=user_conda, + channels_remap=info.get("channels_remap", []), + layering_strategy="single", + ) + return image diff --git a/conda-store-server/conda_store_server/build.py b/conda-store-server/conda_store_server/build.py index 99a58d7d0..6843fd21d 100644 --- a/conda-store-server/conda_store_server/build.py +++ b/conda-store-server/conda_store_server/build.py @@ -315,47 +315,51 @@ def build_conda_pack(db: Session, conda_store, build: orm.Build): def build_conda_docker(db: Session, conda_store, build: orm.Build): # Upstream issue in conda-docker, see # https://github.com/conda-incubator/conda-store/issues/666 - pass - # conda_prefix = build.build_path(conda_store) - # settings = conda_store.get_settings( - # db=db, - # namespace=build.environment.namespace.name, - # environment_name=build.environment.name, - # ) - - # try: - # with utils.timer( - # conda_store.log, - # f"packaging docker image of conda environment={conda_prefix}", - # ): - # context = action.action_generate_conda_docker( - # conda_prefix=conda_prefix, - # default_docker_image=utils.callable_or_value( - # settings.default_docker_base_image, None - # ), - # container_registry=conda_store.container_registry, - # output_image_name=build.specification.name, - # output_image_tag=build.build_key, - # ) - # append_to_logs( - # db, - # conda_store, - # build, - # "::group::action_generate_conda_docker\n" - # + context.stdout.getvalue() - # + "\n::endgroup::\n", - # ) - - # image = context.result - - # if schema.BuildArtifactType.DOCKER_MANIFEST in settings.build_artifacts: - # conda_store.container_registry.store_image( - # db, conda_store, build, image - # ) - - # if schema.BuildArtifactType.CONTAINER_REGISTRY in settings.build_artifacts: - # conda_store.container_registry.push_image(db, build, image) - # except Exception as e: - # conda_store.log.exception(e) - # append_to_logs(db, conda_store, build, traceback.format_exc()) - # raise e + import warnings + + warnings.warn("Generating Docker images is currently not supported, see #666") + return + + conda_prefix = build.build_path(conda_store) + settings = conda_store.get_settings( + db=db, + namespace=build.environment.namespace.name, + environment_name=build.environment.name, + ) + + try: + with utils.timer( + conda_store.log, + f"packaging docker image of conda environment={conda_prefix}", + ): + context = action.action_generate_conda_docker( + conda_prefix=conda_prefix, + default_docker_image=utils.callable_or_value( + settings.default_docker_base_image, None + ), + container_registry=conda_store.container_registry, + output_image_name=build.specification.name, + output_image_tag=build.build_key, + ) + append_to_logs( + db, + conda_store, + build, + "::group::action_generate_conda_docker\n" + + context.stdout.getvalue() + + "\n::endgroup::\n", + ) + + image = context.result + + if schema.BuildArtifactType.DOCKER_MANIFEST in settings.build_artifacts: + conda_store.container_registry.store_image( + db, conda_store, build, image + ) + + if schema.BuildArtifactType.CONTAINER_REGISTRY in settings.build_artifacts: + conda_store.container_registry.push_image(db, build, image) + except Exception as e: + conda_store.log.exception(e) + append_to_logs(db, conda_store, build, traceback.format_exc()) + raise e From a566e7d9aa21c77a01e6200d9fcfe439ac7bc331 Mon Sep 17 00:00:00 2001 From: Nikita Karetnikov Date: Wed, 22 Nov 2023 10:57:11 +0000 Subject: [PATCH 6/6] Unify error and warning messages, use a URL --- .../conda_store_server/action/generate_conda_docker.py | 7 ++++--- conda-store-server/conda_store_server/build.py | 7 ++++--- conda-store-server/tests/test_actions.py | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/conda-store-server/conda_store_server/action/generate_conda_docker.py b/conda-store-server/conda_store_server/action/generate_conda_docker.py index c713ad4c8..e8d2735f0 100644 --- a/conda-store-server/conda_store_server/action/generate_conda_docker.py +++ b/conda-store-server/conda_store_server/action/generate_conda_docker.py @@ -12,9 +12,10 @@ def action_generate_conda_docker( output_image_name: str, output_image_tag: str, ): - # Upstream issue in conda-docker, see - # https://github.com/conda-incubator/conda-store/issues/666 - raise RuntimeError("Generating Docker images is currently not supported, see #666") + raise RuntimeError( + "Generating Docker images is currently not supported, see " + "https://github.com/conda-incubator/conda-store/issues/666" + ) # Import is inside the function because conda_docker is only available on # Linux diff --git a/conda-store-server/conda_store_server/build.py b/conda-store-server/conda_store_server/build.py index 6843fd21d..2fc267b01 100644 --- a/conda-store-server/conda_store_server/build.py +++ b/conda-store-server/conda_store_server/build.py @@ -313,11 +313,12 @@ def build_conda_pack(db: Session, conda_store, build: orm.Build): def build_conda_docker(db: Session, conda_store, build: orm.Build): - # Upstream issue in conda-docker, see - # https://github.com/conda-incubator/conda-store/issues/666 import warnings - warnings.warn("Generating Docker images is currently not supported, see #666") + warnings.warn( + "Generating Docker images is currently not supported, see " + "https://github.com/conda-incubator/conda-store/issues/666" + ) return conda_prefix = build.build_path(conda_store) diff --git a/conda-store-server/tests/test_actions.py b/conda-store-server/tests/test_actions.py index 1aa35e477..288808fcf 100644 --- a/conda-store-server/tests/test_actions.py +++ b/conda-store-server/tests/test_actions.py @@ -129,7 +129,7 @@ def test_generate_conda_pack(tmp_path, conda_prefix): @pytest.mark.xfail(reason=( - "upstream issue in conda-docker, see " + "Generating Docker images is currently not supported, see " "https://github.com/conda-incubator/conda-store/issues/666")) def test_generate_conda_docker(conda_store, conda_prefix): action.action_generate_conda_docker(