feat: optimize docker-compose up for faster boot time (apache#27953)

qleroy · Apr 28, 2024 · a9ca88d · a9ca88d
1 parent 591b192
commit a9ca88d
Show file tree

Hide file tree

Showing 10 changed files with 105 additions and 37 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -21,7 +21,7 @@ jobs:
     steps:
       - id: set_matrix
         run: |
-          MATRIX_CONFIG=$(if [ "${{ github.event_name }}" == "pull_request" ]; then echo '["ci"]'; else echo '["dev", "lean", "py310", "websocket", "dockerize"]'; fi)
+          MATRIX_CONFIG=$(if [ "${{ github.event_name }}" == "pull_request" ]; then echo '["dev"]'; else echo '["dev", "lean", "py310", "websocket", "dockerize"]'; fi)
           echo "matrix_config=${MATRIX_CONFIG}" >> $GITHUB_OUTPUT
           echo $GITHUB_OUTPUT
 

diff --git a/.gitignore b/.gitignore
@@ -110,6 +110,7 @@ release.json
 messages.mo
 
 docker/requirements-local.txt
+docker/.env-local
 
 cache/
 docker/*local*
diff --git a/Dockerfile b/Dockerfile
@@ -119,9 +119,15 @@ RUN apt-get update -qq \
         libasound2 \
         libxtst6 \
         wget \
-        pkg-config \
-    # Install GeckoDriver WebDriver
-    && wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \
+        git \
+        pkg-config
+
+RUN pip install playwright
+RUN playwright install-deps
+RUN playwright install chromium
+
+# Install GeckoDriver WebDriver
+RUN wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \
     # Install Firefox
     && wget -q https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \
     && ln -s /opt/firefox/firefox /usr/local/bin/firefox \

diff --git a/docker-compose-image-tag.yml b/docker-compose-image-tag.yml
@@ -33,7 +33,11 @@ services:
       - redis:/data
 
   db:
-    env_file: docker/.env
+    env_file:
+      # defaults
+      - docker/.env
+      # gitignored overrides
+      - docker/.env-local
     image: postgres:15
     container_name: superset_db
     restart: unless-stopped
@@ -42,7 +46,11 @@ services:
       - ./docker/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
 
   superset:
-    env_file: docker/.env
+    env_file:
+      # defaults
+      - docker/.env
+      # gitignored overrides
+      - docker/.env-local
     image: *superset-image
     container_name: superset_app
     command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
@@ -57,7 +65,11 @@ services:
     image: *superset-image
     container_name: superset_init
     command: ["/app/docker/docker-init.sh"]
-    env_file: docker/.env
+    env_file:
+      # defaults
+      - docker/.env
+      # gitignored overrides
+      - docker/.env-local
     depends_on: *superset-depends-on
     user: "root"
     volumes: *superset-volumes
@@ -68,7 +80,11 @@ services:
     image: *superset-image
     container_name: superset_worker
     command: ["/app/docker/docker-bootstrap.sh", "worker"]
-    env_file: docker/.env
+    env_file:
+      # defaults
+      - docker/.env
+      # gitignored overrides
+      - docker/.env-local
     restart: unless-stopped
     depends_on: *superset-depends-on
     user: "root"
@@ -84,7 +100,11 @@ services:
     image: *superset-image
     container_name: superset_worker_beat
     command: ["/app/docker/docker-bootstrap.sh", "beat"]
-    env_file: docker/.env
+    env_file:
+      # defaults
+      - docker/.env
+      # gitignored overrides
+      - docker/.env-local
     restart: unless-stopped
     depends_on: *superset-depends-on
     user: "root"

diff --git a/docker-compose-non-dev.yml b/docker-compose-non-dev.yml
@@ -38,7 +38,11 @@ services:
       - redis:/data
 
   db:
-    env_file: docker/.env
+    env_file:
+      # defaults
+      - docker/.env
+      # gitignored overrides
+      - docker/.env-local
     image: postgres:15
     container_name: superset_db
     restart: unless-stopped
@@ -47,7 +51,11 @@ services:
       - ./docker/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
 
   superset:
-    env_file: docker/.env
+    env_file:
+      # defaults
+      - docker/.env
+      # gitignored overrides
+      - docker/.env-local
     build:
       <<: *common-build
     container_name: superset_app
@@ -64,7 +72,11 @@ services:
     build:
       <<: *common-build
     command: ["/app/docker/docker-init.sh"]
-    env_file: docker/.env
+    env_file:
+      # defaults
+      - docker/.env
+      # gitignored overrides
+      - docker/.env-local
     depends_on: *superset-depends-on
     user: "root"
     volumes: *superset-volumes
@@ -76,7 +88,11 @@ services:
       <<: *common-build
     container_name: superset_worker
     command: ["/app/docker/docker-bootstrap.sh", "worker"]
-    env_file: docker/.env
+    env_file:
+      # defaults
+      - docker/.env
+      # gitignored overrides
+      - docker/.env-local
     restart: unless-stopped
     depends_on: *superset-depends-on
     user: "root"
@@ -93,7 +109,11 @@ services:
       <<: *common-build
     container_name: superset_worker_beat
     command: ["/app/docker/docker-bootstrap.sh", "beat"]
-    env_file: docker/.env
+    env_file:
+      # defaults
+      - docker/.env
+      # gitignored overrides
+      - docker/.env-local
     restart: unless-stopped
     depends_on: *superset-depends-on
     user: "root"

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -56,7 +56,11 @@ services:
       - redis:/data
 
   db:
-    env_file: docker/.env
+    env_file:
+      # defaults
+      - docker/.env
+      # gitignored overrides
+      - docker/.env-local
     image: postgres:15
     container_name: superset_db
     restart: unless-stopped
@@ -67,7 +71,11 @@ services:
       - ./docker/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d
 
   superset:
-    env_file: docker/.env
+    env_file:
+      # defaults
+      - docker/.env
+      # gitignored overrides
+      - docker/.env-local
     build:
       <<: *common-build
     container_name: superset_app
@@ -121,7 +129,11 @@ services:
       <<: *common-build
     container_name: superset_init
     command: [ "/app/docker/docker-init.sh" ]
-    env_file: docker/.env
+    env_file:
+      # defaults
+      - docker/.env
+      # gitignored overrides
+      - docker/.env-local
     depends_on: *superset-depends-on
     user: *superset-user
     volumes: *superset-volumes
@@ -135,12 +147,14 @@ services:
     environment:
       # set this to false if you have perf issues running the npm i; npm run dev in-docker
       # if you do so, you have to run this manually on the host, which should perform better!
-      BUILD_SUPERSET_FRONTEND_IN_DOCKER: ${BUILD_SUPERSET_FRONTEND_IN_DOCKER:-true}
       SCARF_ANALYTICS: "${SCARF_ANALYTICS}"
-      PUPPETEER_SKIP_CHROMIUM_DOWNLOAD: ${BUILD_SUPERSET_FRONTEND_IN_DOCKER:-false}
     container_name: superset_node
     command: [ "/app/docker/docker-frontend.sh" ]
-    env_file: docker/.env
+    env_file:
+      # defaults
+      - docker/.env
+      # gitignored overrides
+      - docker/.env-local
     depends_on: *superset-depends-on
     volumes: *superset-volumes
 
@@ -149,7 +163,11 @@ services:
       <<: *common-build
     container_name: superset_worker
     command: [ "/app/docker/docker-bootstrap.sh", "worker" ]
-    env_file: docker/.env
+    env_file:
+      # defaults
+      - docker/.env
+      # gitignored overrides
+      - docker/.env-local
     restart: unless-stopped
     depends_on: *superset-depends-on
     user: *superset-user
@@ -171,7 +189,11 @@ services:
       <<: *common-build
     container_name: superset_worker_beat
     command: [ "/app/docker/docker-bootstrap.sh", "beat" ]
-    env_file: docker/.env
+    env_file:
+      # defaults
+      - docker/.env
+      # gitignored overrides
+      - docker/.env-local
     restart: unless-stopped
     depends_on: *superset-depends-on
     user: *superset-user
@@ -184,7 +206,11 @@ services:
       <<: *common-build
     container_name: superset_tests_worker
     command: [ "/app/docker/docker-bootstrap.sh", "worker" ]
-    env_file: docker/.env
+    env_file:
+      # defaults
+      - docker/.env
+      # gitignored overrides
+      - docker/.env-local
     environment:
       DATABASE_HOST: localhost
       DATABASE_DB: test

diff --git a/docker/.env b/docker/.env
@@ -53,3 +53,7 @@ SUPERSET_PORT=8088
 MAPBOX_API_KEY=''
 
 SUPERSET_SECRET_KEY=TEST_NON_DEV_SECRET
+
+ENABLE_PLAYWRIGHT=false
+PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
+BUILD_SUPERSET_FRONTEND_IN_DOCKER=true
diff --git a/docker/docker-bootstrap.sh b/docker/docker-bootstrap.sh
@@ -39,14 +39,6 @@ else
   echo "Skipping local overrides"
 fi
 
-#
-# playwright is an optional package - run only if it is installed
-#
-if command -v playwright > /dev/null 2>&1; then
-  playwright install-deps
-  playwright install chromium
-fi
-
 case "${1}" in
   worker)
     echo "Starting Celery worker..."

diff --git a/docker/docker-frontend.sh b/docker/docker-frontend.sh
@@ -18,8 +18,8 @@
 set -e
 
 # Packages needed for puppeteer:
-apt update
 if [ "$PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" = "false" ]; then
+    apt update
     apt install -y chromium
 fi
 

diff --git a/docs/docs/installation/installing-superset-using-docker-compose.mdx b/docs/docs/installation/installing-superset-using-docker-compose.mdx
@@ -112,7 +112,6 @@ with docker compose will store that data in a PostgreSQL database contained in a
 [volume](https://docs.docker.com/storage/volumes/), which is not backed up.
 
 Again **DO NOT USE THIS FOR PRODUCTION**
-
 :::
 
 You should see a wall of logging output from the containers being launched on your machine. Once
@@ -127,10 +126,10 @@ can skip to the next section.
 You can install additional python packages and apply config overrides by following the steps
 mentioned in [docker/README.md](https://github.com/apache/superset/tree/master/docker#configuration)
 
-You can configure the Docker Compose environment variables for dev and non-dev mode with
-`docker/.env`. This environment file sets the environment
-for most containers in the Docker Compose setup, and some variables affect multiple containers and
-others only single ones.
+Note that `docker/.env` sets the default environment variables for all the docker images
+used by `docker-compose`, and that `docker/.env-local` can be used to override those defaults.
+Also note that `docker/.env-local` is referenced in our `.gitignore`,
+preventing developers from risking committing potentially sensitive configuration to the repository.
 
 One important variable is `SUPERSET_LOAD_EXAMPLES` which determines whether the `superset_init`
 container will populate example data and visualizations into the metadata database. These examples