From c68a71bf701faadf371ccc4916ce1cf20b8f895d Mon Sep 17 00:00:00 2001 From: Emanuele Bezzi Date: Thu, 16 Mar 2023 15:58:36 -0400 Subject: [PATCH 1/3] more stuff --- Dockerfile | 22 ++++++++++++++++++++++ build-census.yaml | 11 +++++++++++ entrypoint.py | 31 +++++++++++++++++++++++++++++++ entrypoint.sh | 3 +++ tools/cell_census_builder/util.py | 10 +++++++++- 5 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 Dockerfile create mode 100644 build-census.yaml create mode 100755 entrypoint.py create mode 100755 entrypoint.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..3f44f3ae7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM ubuntu:22.04 + +ENV DEBIAN_FRONTEND=noninteractive + +ARG COMMIT_SHA +ENV COMMIT_SHA=${COMMIT_SHA} + + +# RUN apt-get update && \ +# apt-get install -y python3 libhdf5-dev python3-h5py gettext moreutils build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests python3-aiohttp llvm jq && \ +# rm -rf /var/lib/apt/lists/* + +RUN apt update && apt -y install python3.10-venv python3-pip awscli gh jq + +ADD tools/cell_census_builder/ /tools/cell_census_builder +ADD tools/scripts/requirements.txt . +ADD entrypoint.py . +ADD build-census.yaml . + +RUN python3 -m pip install -r requirements.txt + +ENTRYPOINT ["./entrypoint.py"] diff --git a/build-census.yaml b/build-census.yaml new file mode 100644 index 000000000..17678f15e --- /dev/null +++ b/build-census.yaml @@ -0,0 +1,11 @@ +census-builder: + uri: + /data/cell-census-small/ + verbose: + true + commands: + build: + manifest: + /data/manifest-small.csv + test-disable-dirty-git-check: + true diff --git a/entrypoint.py b/entrypoint.py new file mode 100755 index 000000000..d16cb303c --- /dev/null +++ b/entrypoint.py @@ -0,0 +1,31 @@ +#!/bin/python3 + +import yaml +import subprocess + +def add_args(opts, args): + for opt_key, opt_val in opts.items(): + if opt_key == "uri": + args.append(opt_val) + elif opt_key == "commands": + continue + elif isinstance(opt_val, bool): + args.append(f"--{opt_key}") + else: + args.append(f"--{opt_key}") + args.append(opt_val) + + +with open("build-census.yaml") as y: + args = ["python3", "-m", "tools.cell_census_builder"] + config = yaml.safe_load(y) + builder = config["census-builder"] + uri = builder["uri"] + add_args(builder, args) + commands = builder["commands"] + for cmd, opts in commands.items(): + subcommand_args = args.copy() + subcommand_args.append(cmd) + add_args(opts, subcommand_args) + print(subcommand_args) + subprocess.call(subcommand_args) \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 000000000..9e3646273 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +python3 -m tools.cell_census_builder /data/cell-census-small/ -v build --manifest /data/manifest-small.csv --test-disable-dirty-git-check \ No newline at end of file diff --git a/tools/cell_census_builder/util.py b/tools/cell_census_builder/util.py index b9bf7d869..3e5496210 100644 --- a/tools/cell_census_builder/util.py +++ b/tools/cell_census_builder/util.py @@ -1,8 +1,8 @@ +import os import time import urllib.parse from typing import Any, Iterator, Optional, Union -import git import numpy as np import numpy.typing as npt import pandas as pd @@ -134,6 +134,12 @@ def get_git_commit_sha() -> str: """ Returns the git commit SHA for the current repo """ + # Try to get the git commit SHA from the COMMIT_SHA env variable + commit_sha_var = os.getenv("COMMIT_SHA") + if commit_sha_var is not None: + return commit_sha_var + import git # Scoped import - this requires the git executable to exist on the machine + repo = git.Repo(search_parent_directories=True) hexsha: str = repo.head.object.hexsha return hexsha @@ -143,6 +149,8 @@ def is_git_repo_dirty() -> bool: """ Returns True if the git repo is dirty, i.e. there are uncommitted changes """ + import git # Scoped import - this requires the git executable to exist on the machine + repo = git.Repo(search_parent_directories=True) is_dirty: bool = repo.is_dirty() return is_dirty From 9bcf7ba90361af7cfa4a32456cf55c6a30471af1 Mon Sep 17 00:00:00 2001 From: Emanuele Bezzi Date: Thu, 16 Mar 2023 16:46:46 -0400 Subject: [PATCH 2/3] remove apt stuff --- Dockerfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3f44f3ae7..f813e2057 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,12 +5,7 @@ ENV DEBIAN_FRONTEND=noninteractive ARG COMMIT_SHA ENV COMMIT_SHA=${COMMIT_SHA} - -# RUN apt-get update && \ -# apt-get install -y python3 libhdf5-dev python3-h5py gettext moreutils build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests python3-aiohttp llvm jq && \ -# rm -rf /var/lib/apt/lists/* - -RUN apt update && apt -y install python3.10-venv python3-pip awscli gh jq +RUN apt update && apt -y install python3.10-venv python3-pip awscli ADD tools/cell_census_builder/ /tools/cell_census_builder ADD tools/scripts/requirements.txt . From be91bbadb8fbb513fc9f493a5b39feda59d2a90d Mon Sep 17 00:00:00 2001 From: Emanuele Bezzi Date: Fri, 17 Mar 2023 13:42:17 -0400 Subject: [PATCH 3/3] move stuff + readme.md --- entrypoint.py | 31 ------------- entrypoint.sh | 3 -- .../census-builder-workflow/Dockerfile | 4 +- tools/census-builder-workflow/README.md | 45 +++++++++++++++++++ .../census-builder-workflow/build-census.yaml | 0 tools/census-builder-workflow/entrypoint.py | 3 ++ 6 files changed, 50 insertions(+), 36 deletions(-) delete mode 100755 entrypoint.py delete mode 100755 entrypoint.sh rename Dockerfile => tools/census-builder-workflow/Dockerfile (75%) create mode 100644 tools/census-builder-workflow/README.md rename build-census.yaml => tools/census-builder-workflow/build-census.yaml (100%) create mode 100755 tools/census-builder-workflow/entrypoint.py diff --git a/entrypoint.py b/entrypoint.py deleted file mode 100755 index d16cb303c..000000000 --- a/entrypoint.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/python3 - -import yaml -import subprocess - -def add_args(opts, args): - for opt_key, opt_val in opts.items(): - if opt_key == "uri": - args.append(opt_val) - elif opt_key == "commands": - continue - elif isinstance(opt_val, bool): - args.append(f"--{opt_key}") - else: - args.append(f"--{opt_key}") - args.append(opt_val) - - -with open("build-census.yaml") as y: - args = ["python3", "-m", "tools.cell_census_builder"] - config = yaml.safe_load(y) - builder = config["census-builder"] - uri = builder["uri"] - add_args(builder, args) - commands = builder["commands"] - for cmd, opts in commands.items(): - subcommand_args = args.copy() - subcommand_args.append(cmd) - add_args(opts, subcommand_args) - print(subcommand_args) - subprocess.call(subcommand_args) \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh deleted file mode 100755 index 9e3646273..000000000 --- a/entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -python3 -m tools.cell_census_builder /data/cell-census-small/ -v build --manifest /data/manifest-small.csv --test-disable-dirty-git-check \ No newline at end of file diff --git a/Dockerfile b/tools/census-builder-workflow/Dockerfile similarity index 75% rename from Dockerfile rename to tools/census-builder-workflow/Dockerfile index f813e2057..899445ff3 100644 --- a/Dockerfile +++ b/tools/census-builder-workflow/Dockerfile @@ -7,8 +7,8 @@ ENV COMMIT_SHA=${COMMIT_SHA} RUN apt update && apt -y install python3.10-venv python3-pip awscli -ADD tools/cell_census_builder/ /tools/cell_census_builder -ADD tools/scripts/requirements.txt . +ADD cell_census_builder/ /tools/cell_census_builder +ADD scripts/requirements.txt . ADD entrypoint.py . ADD build-census.yaml . diff --git a/tools/census-builder-workflow/README.md b/tools/census-builder-workflow/README.md new file mode 100644 index 000000000..37512c232 --- /dev/null +++ b/tools/census-builder-workflow/README.md @@ -0,0 +1,45 @@ +# Cell Census Builder Workflow + +This subproject can be used to run a cell-census build using a Docker container and a custom workflow file. + +## Instructions + +### Build + +To build the docker container, `cd` into the parent folder (`tools/`) and run: + +```docker build --build-arg=COMMIT_SHA=$(git rev-parse --short HEAD) . -t census-builder``` + +This will build a Docker container named `census-builder`. + +### Prepare + +Before running the workflow, make sure that a `data` directory exists on the machine. This can contain any inputs for the builder (e.g. a manifest file and local `h5ad`s), and will also be used to output the built cell census. This folder will also need to contain a `build-census.yaml` file as defined in the next step. + + +### Create workflow file + +In the `data` folder, create a `build-census.yaml` file that contain a workflow that will be executed by the builder. This should also contain all the parameters for the workflow. + +Here is an example workflow that runs the builder using a manifest file: + +``` +census-builder: + uri: + /data/cell-census-small/ + verbose: + true + commands: + build: + manifest: + /data/manifest-small.csv + test-disable-dirty-git-check: + true +``` + + +### Run + +Run the builder workflow with: + +```docker run --mount type=bind,source="path/to/data",target=/data census-builder``` \ No newline at end of file diff --git a/build-census.yaml b/tools/census-builder-workflow/build-census.yaml similarity index 100% rename from build-census.yaml rename to tools/census-builder-workflow/build-census.yaml diff --git a/tools/census-builder-workflow/entrypoint.py b/tools/census-builder-workflow/entrypoint.py new file mode 100755 index 000000000..4e5e2cc8a --- /dev/null +++ b/tools/census-builder-workflow/entrypoint.py @@ -0,0 +1,3 @@ +#!/bin/python3 + +print("Calling the builder...")