Dockerfile for census builder

Dockerfile

@@ -0,0 +1,22 @@
+FROM ubuntu:22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+ARG COMMIT_SHA
+ENV COMMIT_SHA=${COMMIT_SHA}
+
+
+# RUN apt-get update && \
+#     apt-get install -y python3 libhdf5-dev python3-h5py gettext moreutils build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests python3-aiohttp llvm jq && \
+#     rm -rf /var/lib/apt/lists/*
+
+RUN apt update && apt -y install python3.10-venv python3-pip awscli gh jq
+
+ADD tools/cell_census_builder/ /tools/cell_census_builder
+ADD tools/scripts/requirements.txt .
+ADD entrypoint.py .
+ADD build-census.yaml .
+
+RUN python3 -m pip install -r requirements.txt
+
+ENTRYPOINT ["./entrypoint.py"]

build-census.yaml

@@ -0,0 +1,11 @@
+census-builder:
+  uri:
+    /data/cell-census-small/
+  verbose:
+    true
+  commands:
+    build:
+      manifest:
+        /data/manifest-small.csv
+      test-disable-dirty-git-check:
+        true

entrypoint.py

@@ -0,0 +1,31 @@
+#!/bin/python3
+
+import yaml
+import subprocess
+
+def add_args(opts, args):
+    for opt_key, opt_val in opts.items():
+        if opt_key == "uri":
+            args.append(opt_val)
+        elif opt_key == "commands":
+            continue
+        elif isinstance(opt_val, bool):
+            args.append(f"--{opt_key}")
+        else:
+            args.append(f"--{opt_key}")
+            args.append(opt_val)
+
+
+with open("build-census.yaml") as y:
+    args = ["python3", "-m", "tools.cell_census_builder"]
+    config = yaml.safe_load(y)
+    builder = config["census-builder"]
+    uri = builder["uri"]
+    add_args(builder, args)
+    commands = builder["commands"]
+    for cmd, opts in commands.items():
+        subcommand_args = args.copy()
+        subcommand_args.append(cmd)
+        add_args(opts, subcommand_args)
+        print(subcommand_args)
+        subprocess.call(subcommand_args)

entrypoint.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+python3 -m tools.cell_census_builder /data/cell-census-small/ -v build --manifest /data/manifest-small.csv --test-disable-dirty-git-check

tools/cell_census_builder/util.py

@@ -1,8 +1,8 @@
+import os
 import time
 import urllib.parse
 from typing import Any, Iterator, Optional, Union
 
-import git
 import numpy as np
 import numpy.typing as npt
 import pandas as pd
@@ -134,6 +134,12 @@ def get_git_commit_sha() -> str:
     """
     Returns the git commit SHA for the current repo
     """
+    # Try to get the git commit SHA from the COMMIT_SHA env variable
+    commit_sha_var = os.getenv("COMMIT_SHA")
+    if commit_sha_var is not None:
+        return commit_sha_var
+    import git  # Scoped import - this requires the git executable to exist on the machine
+
     repo = git.Repo(search_parent_directories=True)
     hexsha: str = repo.head.object.hexsha
     return hexsha
@@ -143,6 +149,8 @@ def is_git_repo_dirty() -> bool:
     """
     Returns True if the git repo is dirty, i.e. there are uncommitted changes
     """
+    import git  # Scoped import - this requires the git executable to exist on the machine
+
     repo = git.Repo(search_parent_directories=True)
     is_dirty: bool = repo.is_dirty()
     return is_dirty