Make "up" go Brrrrrrrr

data seed command synchronizes storage directory Remove storage volume (reuse olympia volume) Remove data_olympia (use anonymous volume) Remove sources from development DOCKER_TARGET Invert docker-compose.ci to docker-compose.dev (update defaults in setup script to support prod mode in ci)
mozilla · Nov 12, 2024 · da0acca · da0acca
1 parent b3d0a22
commit da0acca
Show file tree

Hide file tree

Showing 16 changed files with 111 additions and 84 deletions.
diff --git a/.github/actions/build-docker/action.yml b/.github/actions/build-docker/action.yml
@@ -10,9 +10,6 @@ inputs:
   version:
     required: true
     description: The image version to tag with
-  target:
-    required: true
-    description: The stage to target in the build
   push:
     required: false
     description: Push the image?
@@ -57,7 +54,7 @@ runs:
     - name: Create .env and version.json files
       shell: bash
       run: |
-        echo "DOCKER_TARGET=${{ inputs.target }}" >> $GITHUB_ENV
+        echo "DOCKER_TARGET=production" >> $GITHUB_ENV
         echo "DOCKER_VERSION=${{ steps.meta.outputs.version }}" >> $GITHUB_ENV
         echo "DOCKER_COMMIT=${{ steps.context.outputs.git_sha }}" >> $GITHUB_ENV
         echo "DOCKER_BUILD=${{ steps.context.outputs.git_build_url }}" >> $GITHUB_ENV

diff --git a/.github/actions/run-docker/action.yml b/.github/actions/run-docker/action.yml
@@ -15,7 +15,7 @@ inputs:
   compose_file:
     description: 'The docker-compose file to use'
     required: false
-    default: 'docker-compose.yml:docker-compose.ci.yml'
+    default: 'docker-compose.yml'
   logs:
     description: 'Show logs'
     required: false

diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml
@@ -41,36 +41,30 @@ jobs:
           -
             name: Needs Locale Compilation
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: |
               make compile_locales
               make test_needs_locales_compilation
           -
             name: Static Assets
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_static_assets
           -
             name: Internal Routes
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_internal_routes_allowed
           -
             name: Elastic Search
             services: ''
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make test_es_tests
           -
             name: Codestyle
             services: web
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make lint-codestyle
           -
             name: Manage Check
             services: web nginx
-            compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make check
-            data_backup_skip: true
+            data_backup_skip: false
     steps:
       - uses: actions/checkout@v4
       - name: Test (${{ matrix.name }})
@@ -79,6 +73,5 @@ jobs:
           version: ${{ inputs.version }}
           digest: ${{ inputs.digest }}
           services: ${{ matrix.services }}
-          compose_file: ${{ matrix.compose_file }}
           run: ${{ matrix.run }}
           data_backup_skip: ${{ matrix.data_backup_skip || 'true' }}
diff --git a/.github/workflows/_test_main.yml b/.github/workflows/_test_main.yml
@@ -88,7 +88,7 @@ jobs:
           services: ''
           digest: ${{ inputs.digest }}
           version: ${{ inputs.version }}
-          compose_file: docker-compose.yml
+          compose_file: docker-compose.yml:docker-compose.dev.yml
           run: |
             split="--splits ${{ needs.test_config.outputs.splits }}"
             group="--group ${{ matrix.group }}"

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -72,7 +72,6 @@ jobs:
           registry: ${{ steps.docker_hub.outputs.registry }}
           image: ${{ steps.docker_hub.outputs.image }}
           version: ci-${{ needs.context.outputs.docker_version }}
-          target: development
           push: true
 
   test_make_docker_configuration:
@@ -179,7 +178,7 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          compose_file: docker-compose.yml
+          compose_file: docker-compose.yml:docker-compose.dev.yml
           run: |
             make docs
 
@@ -230,7 +229,7 @@ jobs:
         with:
           digest: ${{ needs.build.outputs.digest }}
           version: ${{ needs.build.outputs.version }}
-          compose_file: docker-compose.yml
+          compose_file: docker-compose.yml:docker-compose.dev.yml
           run: make extract_locales
 
       - name: Push Locales
@@ -297,7 +296,6 @@ jobs:
           registry: ${{ steps.docker_hub.outputs.registry }}
           image: ${{ steps.docker_hub.outputs.image }}
           version: ${{ needs.context.outputs.docker_version }}
-          target: production
           push: true
 
   push_gar:
@@ -329,5 +327,4 @@ jobs:
           registry: ${{ steps.docker_gar.outputs.registry }}
           image: ${{ steps.docker_gar.outputs.image }}
           version: ${{ needs.context.outputs.docker_version }}
-          target: production
           push: true
diff --git a/.gitignore b/.gitignore
@@ -53,6 +53,6 @@ tmp/*
 # End of .gitignore. Please keep this in sync with the top section of .dockerignore
 
 # do not ignore the following files
-!docker-compose.development.yml
+!docker-compose.dev.yml
 !docker-compose.private.yml
 !private/README.md
diff --git a/Dockerfile b/Dockerfile
@@ -100,6 +100,10 @@ EOF
 ARG DOCKER_TARGET
 ENV DOCKER_TARGET=${DOCKER_TARGET}
 
+# Add our custom mime types (required for for ts/json/md files)
+COPY docker/etc/mime.types /etc/mime.types
+
+
 # Define production dependencies as a single layer
 # let's the rest of the stages inherit prod dependencies
 # and makes copying the /deps dir to the final layer easy.
@@ -119,7 +123,10 @@ ${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r require
 npm ci ${NPM_ARGS} --include=prod
 EOF
 
-FROM base AS pip_development
+# In the local development image, we stop with dependencies
+# Everything else is about compiling production assets which is not needed
+# in the typical development workflow.
+FROM base AS development
 
 RUN \
     # Files required to install pip dependencies
@@ -131,11 +138,7 @@ RUN \
     # Mounts for caching dependencies
     --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
     --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
-<<EOF
-${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
-${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
-npm install ${NPM_ARGS} --no-save
-EOF
+    make update_deps
 
 FROM base AS locales
 ARG LOCALE_DIR=${HOME}/locale
@@ -172,35 +175,23 @@ echo "from olympia.lib.settings_base import *" > settings_local.py
 DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets
 EOF
 
-FROM base AS sources
+FROM base AS production
 
 ARG DOCKER_BUILD DOCKER_COMMIT DOCKER_VERSION
 
 ENV DOCKER_BUILD=${DOCKER_BUILD}
 ENV DOCKER_COMMIT=${DOCKER_COMMIT}
 ENV DOCKER_VERSION=${DOCKER_VERSION}
 
-# Add our custom mime types (required for for ts/json/md files)
-COPY docker/etc/mime.types /etc/mime.types
 # Copy the rest of the source files from the host
 COPY --chown=olympia:olympia . ${HOME}
+# Copy compiled locales from builder
+COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy assets from assets
 COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
 COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-build
-
-# Set shell back to sh until we can prove we can use bash at runtime
-SHELL ["/bin/sh", "-c"]
-
-FROM sources AS development
-
-# Copy dependencies from `pip_development`
-COPY --from=pip_development --chown=olympia:olympia /deps /deps
-
-FROM sources AS production
-
-# Copy compiled locales from builder
-COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy dependencies from `pip_production`
 COPY --from=pip_production --chown=olympia:olympia /deps /deps
 
-
+# Set shell back to sh until we can prove we can use bash at runtime
+SHELL ["/bin/sh", "-c"]
diff --git a/Makefile-docker b/Makefile-docker
@@ -78,6 +78,11 @@ update_assets:
 	# Collect static files: This MUST be run last or files will be missing
 	$(PYTHON_COMMAND) manage.py collectstatic --noinput
 
+.PHONY: update_deps
+update_deps:
+	$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/prod.txt
+	$(PIP_COMMAND) install --progress-bar=off --no-deps --exists-action=w -r requirements/dev.txt
+	npm install $(NPM_ARGS) --no-save
 
 # TOOD: remove this after we migrate addons-frontned to not depend on it.
 .PHONY: setup-ui-tests

diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -0,0 +1,20 @@
+services:
+  worker:
+    environment:
+      - HOST_UID
+    volumes:
+      - .:/data/olympia
+      # Don't mount generated files. They only exist in the container
+      # and would otherwiser be deleted by mounting the cwd volume above
+      - /data/olympia/static-build
+      - /data/olympia/site-static
+      - ./package.json:/deps/package.json
+      - ./package-lock.json:/deps/package-lock.json
+
+  web:
+    extends:
+      service: worker
+
+  nginx:
+    volumes:
+      - .:/srv
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -20,7 +20,7 @@ x-env-mapping: &env
     - HISTIGNORE=ls:exit:"cd .."
     - HISTCONTROL=erasedups
     - CIRCLECI
-    - HOST_UID
+    - HOST_UID=9500
     - DEBUG
     - DATA_BACKUP_SKIP
 
@@ -51,14 +51,6 @@ services:
       "--",
       "celery -A olympia.amo.celery:app worker -E -c 2 --loglevel=INFO",
     ]
-    volumes:
-      - .:/data/olympia
-      # Don't mount generated files. They only exist in the container
-      # and would otherwiser be deleted by mounting the cwd volume above
-      - /data/olympia/static-build
-      - /data/olympia/site-static
-      - ./package.json:/deps/package.json
-      - ./package-lock.json:/deps/package-lock.json
     extra_hosts:
      - "olympia.test:127.0.0.1"
     restart: on-failure:5
@@ -91,14 +83,16 @@ services:
       interval: 90s
       start_interval: 1s
       start_period: 120s
+    volumes:
+      - storage:/data/olympia/storage
     command:
       - uwsgi --ini /data/olympia/docker/uwsgi.ini
 
   nginx:
     image: nginx
     volumes:
       - ./docker/nginx/addons.conf:/etc/nginx/conf.d/addons.conf
-      - .:/srv
+      - storage:/srv/storage
     ports:
       - "80:80"
     networks:
@@ -188,3 +182,5 @@ volumes:
     # External volumes must be manually created/destroyed
     name: addons-server_data_mysqld
     external: true
+  storage:
+
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
@@ -23,6 +23,17 @@ if [[ -n "${HOST_UID:-}" ]]; then
   echo "${OLYMPIA_USER} UID: ${OLYMPIA_UID} -> ${HOST_UID}"
 fi
 
+# TODO: this doesn't work but we have to check if the image we built is production
+# but we are for some reason trying to run it using development dependencies.
+# this is pretty typical for CI... we could consider just building ci images though as a simpler solution. It's just nice to test the image you already pushed cause muh CI/CD...
+if [[ "${DOCKER_VERSION:-}" != 'local' && "${DOCKER_TARGET:-}" == 'production' ]]; then
+  echo "Running remote image. Re-installing with development dependencies..."
+  rm -rf /deps
+  mkdir -p /deps
+  chown -R ${get_olympia_uid}:${get_olympia_gid} /deps
+  su -s /bin/bash $OLYMPIA_USER -c "make update_deps"
+fi
+
 cat <<EOF | su -s /bin/bash $OLYMPIA_USER
   echo "Running command as ${OLYMPIA_USER} $(get_olympia_uid):$(get_olympia_gid)"
   set -xue

diff --git a/docs/topics/development/building_and_running_services.md b/docs/topics/development/building_and_running_services.md
@@ -18,7 +18,7 @@ The Dockerfile for the **addons-server** project uses a multi-stage build to opt
 
 4. **Environment Variables for OLYMPIA_USER**:
    - **Development Setup**: The `HOST_UID` environment variable is set to the host user ID, ensuring that the container runs with the correct permissions.
-   - **CI Setup**: In CI environments, such as defined in `docker-compose.ci.yml`, the user ID is reset to the default 9500, and the Olympia mount is removed. This makes the container a closed system, mimicking production behavior closely.
+   - **CI Setup**: In CI environments, the user ID is reset to the default 9500, and the Olympia mount is removed. This makes the container a closed system, mimicking production behavior closely.
 
 ### Best Practices for the Dockerfile
 
@@ -146,7 +146,7 @@ We use docker compose under the hood to orchestrate container both locally and i
 The `docker-compose.yml` file defines the services, volumes, and networks required for the project.
 
 Our docker compose project is split into a root [docker-compose.yml](../../../docker-compose.yml) file and additional files for specific environments,
-such as [docker-compose.ci.yml](../../../docker-compose.ci.yml) for CI environments.
+such as [docker-compose.dev.yml](../../../docker-compose.dev.yml) for local environments.
 
 ### Healthchecks
 
@@ -155,15 +155,14 @@ The health checks ensure the django wsgi server and celery worker node are runni
 
 ### Environment specific compose files
 
-- **Local Development**: The `docker-compose.yml` file is used for local development. It defines services like `web`, `db`, `redis`, and `elasticsearch`.
-- **CI Environment**: The `docker-compose.ci.yml` file is used for CI environments. It overrides the HOST_UID as well as removing volumes to make the container more production like.
-- **Private**: This file includes the customs service that is not open source and should therefore not be included by default.
+- **Base**: The `docker-compose.yml` file is used for local development. It defines services like `web`, `db`, `redis`, and `elasticsearch`.
+- **Dev Environment**: The `docker-compose.dev.yml` file is used for local development. It overrides the HOST_UID as well as removing volumes to make the container more production like.
 - **Override**: This file allows modifying the default configuration without changing the main `docker-compose.yml` file. This file is larglely obsolete and should not be used.
 
 To mount with a specific  set of docker compose files you can add the COMPOSE_FILE argument to make up. This will persist your setting to .env.
 
 ```sh
-make up COMPOSE_FILE=docker-compose.yml:docker-compose.ci.yml
+make up COMPOSE_FILE=docker-compose.yml:docker-compose.dev.yml
 ```
 
 Files should be separated with a colon.