Merge pull request #714 from nkaretnikov/constructor-artifacts-657

Generate `constructor` artifacts

conda-store-server/conda_store_server/action/__init__.py

@@ -32,3 +32,6 @@
 from conda_store_server.action.add_lockfile_packages import (
     action_add_lockfile_packages,  # noqa
 )
+from conda_store_server.action.generate_constructor_installer import (
+    action_generate_constructor_installer,  # noqa
+)

conda-store-server/conda_store_server/action/generate_constructor_installer.py

@@ -0,0 +1,122 @@
+import os
+import pathlib
+import sys
+import tempfile
+import warnings
+
+import yaml
+from conda_store_server import action, schema
+from conda_store_server.action.utils import logged_command
+
+
+def get_installer_platform():
+    # This is how the default platform name is generated internally by
+    # constructor. For example: osx-arm64, linux-64, win-64.
+    # https://github.com/conda/constructor/blob/main/CONSTRUCT.md#available-platforms
+    # Note: constructor is cross-friendly, see:
+    # https://github.com/conda-incubator/conda-store/pull/714#discussion_r1465115323
+    from conda.base.context import context
+
+    return context.subdir
+
+
+@action.action
+def action_generate_constructor_installer(
+    context,
+    conda_command: str,
+    specification: schema.CondaSpecification,
+    installer_dir: pathlib.Path,
+    version: str,
+):
+    def write_file(filename, s):
+        with open(filename, "w") as f:
+            context.log.info(f"{filename}:\n{s}")
+            f.write(s)
+
+    # Checks if constructor is available
+    try:
+        command = [
+            "constructor",
+            "--help",
+        ]
+        logged_command(context, command, timeout=10)
+    except FileNotFoundError:
+        warnings.warn(
+            "Installer generation requires constructor: https://github.com/conda/constructor"
+        )
+        return
+
+    # pip dependencies are not directly supported by constructor, they will be
+    # installed via the post_install script:
+    # https://github.com/conda/constructor/issues/515
+    # conda and pip need to be in dependencies for the post_install script
+    dependencies = ["conda", "pip"]
+    pip_dependencies = []
+    for d in specification.dependencies:
+        if type(d) is schema.CondaSpecificationPip:
+            pip_dependencies.extend(d.pip)
+        else:
+            dependencies.append(d)
+
+    # Creates the construct.yaml file and post_install script
+    ext = ".exe" if sys.platform == "win32" else ".sh"
+    pi_ext = ".bat" if sys.platform == "win32" else ".sh"
+    installer_filename = (installer_dir / specification.name).with_suffix(ext)
+
+    os.makedirs(installer_dir, exist_ok=True)
+
+    with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmp_dir:
+        tmp_dir = pathlib.Path(tmp_dir)
+        cache_dir = tmp_dir / "pkgs"
+        tmp_dir /= "build"
+        os.makedirs(cache_dir, exist_ok=True)
+        os.makedirs(tmp_dir, exist_ok=True)
+        construct_file = tmp_dir / "construct.yaml"
+        post_install_file = (tmp_dir / "post_install").with_suffix(pi_ext)
+
+        construct = {
+            "installer_filename": str(installer_filename),
+            "post_install": str(post_install_file),
+            "name": specification.name,
+            "channels": specification.channels,
+            "specs": dependencies,
+            "version": version,
+        }
+
+        if sys.platform == "win32":
+            post_install = """\
+call "%PREFIX%\Scripts\\activate.bat"
+"""
+        else:
+            post_install = """\
+#!/usr/bin/env bash
+set -euxo pipefail
+source "$PREFIX/etc/profile.d/conda.sh"
+conda activate "$PREFIX"
+"""
+        if pip_dependencies:
+            post_install += f"""
+python -m pip install {' '.join(pip_dependencies)}
+"""
+
+        # Writes files to disk
+        write_file(construct_file, yaml.dump(construct))
+        write_file(post_install_file, post_install)
+
+        # Calls constructor
+        # Note: `cache_dir` is the same as the conda `pkgs` directory. It needs
+        # to be specified here because the default `pkgs` directory is not
+        # available in Docker, which was causing conda's `create_cache_dir` to
+        # fail.
+        command = [
+            "constructor",
+            "-v",
+            "--cache-dir",
+            str(cache_dir),
+            "--platform",
+            get_installer_platform(),
+            str(tmp_dir),
+        ]
+        logged_command(context, command)
+
+    return installer_filename

conda-store-server/conda_store_server/action/generate_lockfile.py

@@ -1,12 +1,12 @@
 import json
 import os
 import pathlib
-import subprocess
 import typing
 
 import yaml
 from conda_lock.conda_lock import run_lock
 from conda_store_server import action, conda_utils, schema
+from conda_store_server.action.utils import logged_command
 
 
 @action.action
@@ -25,17 +25,11 @@ def action_solve_lockfile(
     with environment_filename.open("w") as f:
         json.dump(specification.dict(), f)
 
-    def print_cmd(cmd):
-        context.log.info(f"Running command: {' '.join(cmd)}")
-        context.log.info(
-            subprocess.check_output(cmd, stderr=subprocess.STDOUT, encoding="utf-8")
-        )
-
     # The info command can be used with either mamba or conda
-    print_cmd([conda_command, "info"])
+    logged_command(context, [conda_command, "info"])
     # The config command is not supported by mamba
-    print_cmd(["conda", "config", "--show"])
-    print_cmd(["conda", "config", "--show-sources"])
+    logged_command(context, ["conda", "config", "--show"])
+    logged_command(context, ["conda", "config", "--show-sources"])
 
     # conda-lock ignores variables defined in the specification, so this code
     # gets the value of CONDA_OVERRIDE_CUDA and passes it to conda-lock via

conda-store-server/conda_store_server/action/utils.py

@@ -0,0 +1,10 @@
+import subprocess
+
+
+def logged_command(context, command, **kwargs):
+    context.log.info(f"Running command: {' '.join(command)}")
+    context.log.info(
+        subprocess.check_output(
+            command, stderr=subprocess.STDOUT, encoding="utf-8", **kwargs
+        )
+    )

conda-store-server/conda_store_server/alembic/versions/57cd11b949d5_add_installer.py

@@ -0,0 +1,46 @@
+"""add installer
+
+Revision ID: 57cd11b949d5
+Revises: 0f7e23ff24ee
+Create Date: 2024-01-28 14:31:35.723505
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "57cd11b949d5"
+down_revision = "0f7e23ff24ee"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table(
+        "build_artifact",
+        schema=None,
+    ) as batch_op:
+        batch_op.alter_column(
+            "artifact_type",
+            existing_type=sa.VARCHAR(length=18),
+            type_=sa.VARCHAR(length=21),
+            existing_nullable=False,
+        )
+    if not str(op.get_bind().engine.url).startswith("sqlite"):
+        op.execute("DROP TYPE IF EXISTS buildartifacttype")
+
+
+def downgrade():
+    op.execute(
+        "DELETE FROM build_artifact WHERE artifact_type = 'CONSTRUCTOR_INSTALLER'"
+    )
+    with op.batch_alter_table(
+        "build_artifact",
+        schema=None,
+    ) as batch_op:
+        batch_op.alter_column(
+            "artifact_type",
+            existing_type=sa.VARCHAR(length=21),
+            type_=sa.VARCHAR(length=18),
+            existing_nullable=False,
+        )

conda-store-server/conda_store_server/app.py

@@ -263,6 +263,7 @@ def _check_redis(self, proposal):
             schema.BuildArtifactType.LOCKFILE,
             schema.BuildArtifactType.YAML,
             schema.BuildArtifactType.CONDA_PACK,
+            schema.BuildArtifactType.CONSTRUCTOR_INSTALLER,
             *(
                 [
                     schema.BuildArtifactType.DOCKER_MANIFEST,
@@ -724,6 +725,15 @@ def create_build(self, db: Session, environment_id: int, specification_sha256: s
                 )
             )
 
+        if schema.BuildArtifactType.CONSTRUCTOR_INSTALLER in settings.build_artifacts:
+            artifact_tasks.append(
+                tasks.task_build_constructor_installer.subtask(
+                    args=(build.id,),
+                    task_id=f"build-{build.id}-constructor-installer",
+                    immutable=True,
+                )
+            )
+
         (
             tasks.task_update_storage_metrics.si()
             | tasks.task_build_conda_environment.subtask(

conda-store-server/conda_store_server/build.py

@@ -382,3 +382,45 @@ def build_conda_docker(db: Session, conda_store, build: orm.Build):
         conda_store.log.exception(e)
         append_to_logs(db, conda_store, build, traceback.format_exc())
         raise e
+
+
+def build_constructor_installer(db: Session, conda_store, build: orm.Build):
+    conda_prefix = build.build_path(conda_store)
+
+    settings = conda_store.get_settings(
+        db=db,
+        namespace=build.environment.namespace.name,
+        environment_name=build.environment.name,
+    )
+
+    with utils.timer(
+        conda_store.log, f"creating installer for conda environment={conda_prefix}"
+    ):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            context = action.action_generate_constructor_installer(
+                conda_command=settings.conda_command,
+                specification=schema.CondaSpecification.parse_obj(
+                    build.specification.spec
+                ),
+                installer_dir=pathlib.Path(tmpdir),
+                version=build.build_key,
+            )
+            output_filename = context.result
+            append_to_logs(
+                db,
+                conda_store,
+                build,
+                "::group::action_generate_constructor_installer\n"
+                + context.stdout.getvalue()
+                + "\n::endgroup::\n",
+            )
+            if output_filename is None:
+                return
+            conda_store.storage.fset(
+                db,
+                build.id,
+                build.constructor_installer_key,
+                output_filename,
+                content_type="application/octet-stream",
+                artifact_type=schema.BuildArtifactType.CONSTRUCTOR_INSTALLER,
+            )

conda-store-server/conda_store_server/orm.py

@@ -337,6 +337,11 @@ def conda_pack_key(self):
     def docker_manifest_key(self):
         return f"docker/manifest/{self.build_key}"
 
+    @property
+    def constructor_installer_key(self):
+        ext = "exe" if sys.platform == "win32" else "sh"
+        return f"installer/{self.build_key}.{ext}"
+
     def docker_blob_key(self, blob_hash):
         return f"docker/blobs/{blob_hash}"
 
@@ -368,6 +373,13 @@ def has_docker_manifest(self):
             for artifact in self.build_artifacts
         )
 
+    @hybrid_property
+    def has_constructor_installer(self):
+        return any(
+            artifact.artifact_type == schema.BuildArtifactType.CONSTRUCTOR_INSTALLER
+            for artifact in self.build_artifacts
+        )
+
     def __repr__(self):
         return f"<Build (id={self.id} status={self.status} nbr package_builds={len(self.package_builds)})>"
 

conda-store-server/conda_store_server/schema.py

@@ -148,6 +148,7 @@ class BuildArtifactType(enum.Enum):
     DOCKER_BLOB = "DOCKER_BLOB"
     DOCKER_MANIFEST = "DOCKER_MANIFEST"
     CONTAINER_REGISTRY = "CONTAINER_REGISTRY"
+    CONSTRUCTOR_INSTALLER = "CONSTRUCTOR_INSTALLER"
 
 
 class BuildStatus(enum.Enum):
@@ -342,6 +343,7 @@ class Settings(BaseModel):
             BuildArtifactType.LOCKFILE,
             BuildArtifactType.YAML,
             BuildArtifactType.CONDA_PACK,
+            BuildArtifactType.CONSTRUCTOR_INSTALLER,
             *(
                 [
                     BuildArtifactType.DOCKER_MANIFEST,

conda-store-server/conda_store_server/server/templates/build.html

@@ -79,6 +79,9 @@ <h3 class="card-title">Conda Environment Artifacts</h3>
         {% if build.has_conda_pack %}
         <li class="list-group-item"><a href="https://github.com/conda/conda-pack">Conda-Pack</a> archive: <a href="{{ url_for('api_get_build_archive', build_id=build.id) }}">environment.tar.gz</a></li>
         {% endif %}
+        {% if build.has_constructor_installer %}
+        <li class="list-group-item"><a href="https://github.com/conda/constructor">Constructor</a> installer: <a href="{{ url_for('api_get_build_installer', build_id=build.id) }}" download>installer ({{ platform }})</a></li>
+        {% endif %}
         {% if build.has_docker_manifest %}
         <li class="list-group-item">Docker image registry url: {{ registry_external_url }}/{{ build.environment.namespace.name }}/{{ build.environment.name }}:{{ build.build_key }}</a></li>
         {% endif %}

conda-store-server/conda_store_server/server/templates/home.html

@@ -41,6 +41,9 @@ <h5 class="card-title">
         {% if environment.current_build.has_docker_manifest %}
         <a class="card-link" onclick="setClipboard('{{ registry_external_url }}/{{ environment.namespace.name }}/{{ environment.name }}:{{ environment.current_build.build_key }}')" role="button" data-toggle="popover" data-trigger="hover focus" data-content="Click to copy!"><ion-icon name="logo-docker"></ion-icon> Docker</a>
         {% endif %}
+        {% if environment.current_build.has_constructor_installer %}
+        <a class="card-link" href="{{ url_for('api_get_build_installer', build_id=environment.current_build_id) }}" download><ion-icon name="archive-outline"></ion-icon> Installer ({{ platform }})</a>
+        {% endif %}
     </div>
 </div>
 {% endfor %}

conda-store-server/conda_store_server/server/views/api.py

@@ -972,6 +972,7 @@ async def api_put_build_cancel(
                 f"build-{build_id}-conda-env-export",
                 f"build-{build_id}-conda-pack",
                 f"build-{build_id}-docker",
+                f"build-{build_id}-constructor-installer",
                 f"build-{build_id}-environment",
             ],
             terminate=True,
@@ -1279,6 +1280,34 @@ async def api_get_build_docker_image_url(
             )
 
 
+@router_api.get("/build/{build_id}/installer/")
+async def api_get_build_installer(
+    build_id: int,
+    request: Request,
+    conda_store=Depends(dependencies.get_conda_store),
+    auth=Depends(dependencies.get_auth),
+):
+    with conda_store.get_db() as db:
+        build = api.get_build(db, build_id)
+        auth.authorize_request(
+            request,
+            f"{build.environment.namespace.name}/{build.environment.name}",
+            {Permissions.ENVIRONMENT_READ},
+            require=True,
+        )
+
+        if build.has_constructor_installer:
+            return RedirectResponse(
+                conda_store.storage.get_url(build.constructor_installer_key)
+            )
+
+        else:
+            raise HTTPException(
+                status_code=400,
+                detail=f"Build {build_id} doesn't have an installer",
+            )
+
+
 @router_api.get(
     "/setting/",
     response_model=schema.APIGetSetting,