Remove Dockerfile and use agibuild.*.yaml instead

Makefile

@@ -10,7 +10,6 @@ SHELL := /bin/bash
 NOS_VERSION := $(shell python -c "exec(open('nos/version.py').read().strip()); print(__version__)")
 NOS_VERSION_TAG = ${NOS_VERSION}
 DOCKER_IMAGE_NAME := autonomi/nos
-DOCKER_TARGET := server
 DOCKER_ARGS :=
 DOCKER_CMD :=
 

docker-compose.test.yml

@@ -20,24 +20,16 @@ x-nos-gpu:
 services:
   test-cpu:
     <<: *nos-common-test
-    image: autonomi/nos:latest-cpu-test
     build:
       context: .
-      dockerfile: docker/Dockerfile
-      target: test
-      args:
-        - TARGET=cpu
-        - BASE_IMAGE=python:3.8.10-slim
+      dockerfile: docker/Dockerfile.cpu
+      target: test-cpu
     command: ${DOCKER_CMD:-make test-cpu}
 
   test-gpu:
     <<: [*nos-common-test, *nos-common-gpu-test]
-    image: autonomi/nos:latest-gpu-test
     build:
       context: .
-      dockerfile: docker/Dockerfile
-      target: test
-      args:
-        - TARGET=gpu
-        - BASE_IMAGE=nvidia/cuda:11.8.0-base-ubuntu22.04
+      dockerfile: docker/Dockerfile.gpu
+      target: test-gpu
     command: ${DOCKER_CMD:-make test-gpu}

docker/Dockerfile.cpu

@@ -55,13 +55,11 @@ RUN pip install --upgrade pip
 # Install pip requirements, with cache mounting ${PIP_CACHE_DIR} for faster builds
 # Note: Cache mounts allow us to re-use the cache for pip packages
 # instead of having to re-download them every time we build.
-COPY requirements/requirements.torch.cpu.txt /tmp/reqs/requirements/requirements.torch.cpu.txt
 COPY requirements/requirements.txt /tmp/reqs/requirements/requirements.txt
 COPY requirements/requirements.http.txt /tmp/reqs/requirements/requirements.http.txt
 COPY requirements/requirements.server.txt /tmp/reqs/requirements/requirements.server.txt
 RUN --mount=type=cache,target=${PIP_CACHE_DIR} \
     pip install --upgrade pip \
-    && pip install -r /tmp/reqs/requirements/requirements.torch.cpu.txt \
     && pip install -r /tmp/reqs/requirements/requirements.txt \
     && pip install -r /tmp/reqs/requirements/requirements.http.txt \
     && pip install -r /tmp/reqs/requirements/requirements.server.txt \
@@ -81,6 +79,9 @@ ADD ./scripts/entrypoint.sh /app/entrypoint.sh
 
 # Run commands
 RUN echo "running commands"
+RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \
+    --mount=type=cache,target=${PIP_CACHE_DIR} \
+    pip install torch>=2.0.0+cpu torchaudio torchvision -i https://download.pytorch.org/whl/cpu
 RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \
     --mount=type=cache,target=${PIP_CACHE_DIR} \
     mamba install -y -c conda-forge x264=='1!161.3030' ffmpeg=4.3.2

docker/Dockerfile.gpu

@@ -99,6 +99,15 @@ RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \
     --mount=type=cache,target=${PIP_CACHE_DIR} \
     python -c "import torch as t; print(f'torch={t.__version__}, cuda={t.cuda.is_available()}, cudnn={t.backends.cudnn.is_available()}')"
 RUN echo "run commands complete"
+# Cleanup apt, mamba/conda and pip packages
+RUN apt-get -y autoclean \
+    && apt-get -y autoremove \
+    && rm -rf /var/lib/apt/lists/* \
+    && ${AGIPACK_PATH}/conda/bin/mamba clean -ya \
+    && rm -rf ${PIP_CACHE_DIR} \
+    && rm -rf ${CONDA_PKGS_DIRS} \
+    && rm -rf /tmp/reqs \
+    && echo "pip cleanup complete"
 
 # Setup environment variables
 ENV LOGURU_LEVEL=WARNING
@@ -134,6 +143,15 @@ RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \
     --mount=type=cache,target=${PIP_CACHE_DIR} \
     pip install --no-deps '.' && rm -rf build
 RUN echo "run commands complete"
+# Cleanup apt, mamba/conda and pip packages
+RUN apt-get -y autoclean \
+    && apt-get -y autoremove \
+    && rm -rf /var/lib/apt/lists/* \
+    && ${AGIPACK_PATH}/conda/bin/mamba clean -ya \
+    && rm -rf ${PIP_CACHE_DIR} \
+    && rm -rf ${CONDA_PKGS_DIRS} \
+    && rm -rf /tmp/reqs \
+    && echo "pip cleanup complete"
 CMD ["/app/entrypoint.sh"]
 # >>>>>>>>>>>>>>>>>>>>>>>>>>>
 # Auto-generated by agi-pack (version=0.1.19).
@@ -164,4 +182,13 @@ WORKDIR /app/$AGIPACK_PYENV
 ADD ./makefiles ./makefiles
 ADD ./Makefile ./Makefile
 ADD ./tests ./tests
+# Cleanup apt, mamba/conda and pip packages
+RUN apt-get -y autoclean \
+    && apt-get -y autoremove \
+    && rm -rf /var/lib/apt/lists/* \
+    && ${AGIPACK_PATH}/conda/bin/mamba clean -ya \
+    && rm -rf ${PIP_CACHE_DIR} \
+    && rm -rf ${CONDA_PKGS_DIRS} \
+    && rm -rf /tmp/reqs \
+    && echo "pip cleanup complete"
 CMD ["/app/entrypoint.sh"]

docker/Dockerfile.multiplatform.cpu

@@ -55,13 +55,11 @@ RUN pip install --upgrade pip
 # Install pip requirements, with cache mounting ${PIP_CACHE_DIR} for faster builds
 # Note: Cache mounts allow us to re-use the cache for pip packages
 # instead of having to re-download them every time we build.
-COPY requirements/requirements.torch.cpu.txt /tmp/reqs/requirements/requirements.torch.cpu.txt
 COPY requirements/requirements.txt /tmp/reqs/requirements/requirements.txt
 COPY requirements/requirements.http.txt /tmp/reqs/requirements/requirements.http.txt
 COPY requirements/requirements.server.txt /tmp/reqs/requirements/requirements.server.txt
 RUN --mount=type=cache,target=${PIP_CACHE_DIR} \
     pip install --upgrade pip \
-    && pip install -r /tmp/reqs/requirements/requirements.torch.cpu.txt \
     && pip install -r /tmp/reqs/requirements/requirements.txt \
     && pip install -r /tmp/reqs/requirements/requirements.http.txt \
     && pip install -r /tmp/reqs/requirements/requirements.server.txt \
@@ -81,6 +79,9 @@ ADD ./scripts/entrypoint.sh /app/entrypoint.sh
 
 # Run commands
 RUN echo "running commands"
+RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \
+    --mount=type=cache,target=${PIP_CACHE_DIR} \
+    pip install torch>=2.0.0+cpu torchaudio torchvision -i https://download.pytorch.org/whl/cpu
 RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \
     --mount=type=cache,target=${PIP_CACHE_DIR} \
     mamba install -y -c conda-forge x264=='1!161.3030' ffmpeg=4.3.2

docker/agibuild.cpu.yaml

@@ -11,13 +11,13 @@ images:
       - build-essential
       - git
     requirements:
-      - requirements/requirements.torch.cpu.txt
       - requirements/requirements.txt
       - requirements/requirements.http.txt
       - requirements/requirements.server.txt
     add:
       - ./scripts/entrypoint.sh:/app/entrypoint.sh
     run:
+      - pip install torch>=2.0.0+cpu torchaudio torchvision -i https://download.pytorch.org/whl/cpu
       - mamba install -y -c conda-forge x264=='1!161.3030' ffmpeg=4.3.2
       - python -c "import torch as t; print(f'torch={t.__version__}, cuda={t.cuda.is_available()}, cudnn={t.backends.cudnn.is_available()}')"
     env:

makefiles/Makefile.agibuild.mk

@@ -1,18 +1,20 @@
 # Copyright 2023 Autonomi AI, Inc. All rights reserved.
+AGIPACK_ARGS :=
 
 agi-build-cpu:  # equivalent to agi-build-py38-cpu (target=base-cpu, cpu, test-cpu)
-	agi-pack build \
+	agi-pack build ${AGIPACK_ARGS} \
 		-c docker/agibuild.cpu.yaml \
 		-o docker/Dockerfile.cpu \
 		-p 3.8.15 \
 		-b debian:buster-slim \
 		-t '${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-{target}'
+
 	docker tag \
 		${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-cpu \
 		${DOCKER_IMAGE_NAME}:latest-cpu
 
 agi-build-gpu:  # equivalent to agi-build-py38-cu118 (target=base-gpu, gpu, test-gpu)
-	agi-pack build \
+	agi-pack build ${AGIPACK_ARGS} \
 		-c docker/agibuild.gpu.yaml \
 		-o docker/Dockerfile.gpu \
 		-p 3.8.15 \
@@ -23,39 +25,39 @@ agi-build-gpu:  # equivalent to agi-build-py38-cu118 (target=base-gpu, gpu, test
 		${DOCKER_IMAGE_NAME}:latest-gpu
 
 agi-build-py38-cpu:
-	agi-pack build \
+	agi-pack build ${AGIPACK_ARGS} \
 		-c docker/agibuild.cpu.yaml \
 		-o docker/Dockerfile.py38-cpu \
 		-p 3.8.15 \
 		-b debian:buster-slim \
 		-t '${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-{target}-py38'
 
 agi-build-py38-cu118:
-	agi-pack build \
+	agi-pack build ${AGIPACK_ARGS} \
 		-c docker/agibuild.gpu.yaml \
 		-o docker/Dockerfile.py38-cu118 \
 		-p 3.8.15 \
 		-b nvidia/cuda:11.8.0-base-ubuntu22.04 \
 		-t '${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-{target}-py38-cu118'
 
 agi-build-py39-cu118:
-	agi-pack build \
+	agi-pack build ${AGIPACK_ARGS} \
 		-c docker/agibuild.gpu.yaml \
 		-o docker/Dockerfile.py39-cu118 \
 		-p 3.9.13 \
 		-b nvidia/cuda:11.8.0-base-ubuntu22.04 \
 		-t '${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-{target}-py39-cu118'
 
 agi-build-py310-cu118:
-	agi-pack build \
+	agi-pack build ${AGIPACK_ARGS} \
 		-c docker/agibuild.gpu.yaml \
 		-o docker/Dockerfile.py310-cu118 \
 		-p 3.10.11 \
 		-b nvidia/cuda:11.8.0-base-ubuntu22.04 \
 		-t '${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-{target}-py310-cu118'
 
 agi-build-py311-cu118:
-	agi-pack build \
+	agi-pack build ${AGIPACK_ARGS} \
 		-c docker/agibuild.gpu.yaml \
 		-o docker/Dockerfile.py310-cu118 \
 		-p 3.11.4 \

makefiles/Makefile.base.mk

@@ -19,22 +19,6 @@ help-base:
 	@echo "  docker-push-all     Push CPU and GPU docker images"
 	@echo ""
 
-.docker-build:
-	@echo "🛠️ Building docker image"
-	@echo "BASE_IMAGE: ${BASE_IMAGE}"
-	@echo "TARGET: ${TARGET}"
-	@echo "DOCKER_TARGET: ${DOCKER_TARGET}"
-	@echo "IMAGE: ${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-${TARGET}"
-	@echo ""
-	docker build -f docker/Dockerfile \
-		--target ${DOCKER_TARGET} \
-		-t ${DOCKER_IMAGE_NAME}:latest-${TARGET} \
-		-t ${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-${TARGET} \
-		--build-arg TARGET=${TARGET} \
-		--build-arg BASE_IMAGE=${BASE_IMAGE} \
-		--build-arg CUDA_VERSION=${CUDA_VERSION} \
-		.
-
 .docker-build-and-push-multiplatform:
 	@echo "🛠️ Building docker image"
 	@echo "BASE_IMAGE: ${BASE_IMAGE}"
@@ -47,7 +31,6 @@ help-base:
 		--target ${DOCKER_TARGET} \
 		-t ${DOCKER_IMAGE_NAME}:latest-${TARGET} \
 		-t ${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-${TARGET} \
-		--push \
 		.
 
 .docker-run:
@@ -60,18 +43,24 @@ help-base:
 	docker push ${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-${TARGET}
 
 docker-build-cpu: agi-build-cpu
+docker-build-cpu-prod:
+	make agi-build-cpu AGIPACK_ARGS=--prod
 
 docker-build-gpu: agi-build-gpu
+docker-build-gpu-prod:
+	make agi-build-gpu AGIPACK_ARGS=--prod
 
 docker-build-and-push-multiplatform-cpu:
-	agi-pack generate \
+	agi-pack generate ${AGIPACK_ARGS} \
 		-c docker/agibuild.cpu.yaml \
 		-o docker/Dockerfile.multiplatform.cpu \
 		-p 3.8.15 \
 		-b debian:buster-slim \
 		-t '${DOCKER_IMAGE_NAME}:${NOS_VERSION_TAG}-{target}'
 	make .docker-build-and-push-multiplatform \
 		TARGET=cpu DOCKER_TARGET=cpu
+docker-build-and-push-multiplatform-cpu-prod:
+	make .docker-build-and-push-multiplatform-cpu AGIPACK_ARGS=--prod
 
 docker-build-all: \
 	docker-build-cpu docker-build-gpu