Enable Production Mode via DEBUG or docker-compose.ci.yml (#22613)

* TMP: clean bake metadata file

* SPLIT: Log UWSGI in docker compose console

Remove PID

* SPLIT: Raise error in update_assets if file not found

* Enable Production mode via DEBUG and or docker-compose.ci.yml

* Option 1: Load node_modules from statifiles directory

* Option 2: Load node_modules via django-node-asset

* separate ./static from locally built static files

* Add docs for static file serving

* TMP: fixes from code review

* Update docs/topics/development/static-files.md

Co-authored-by: Mathieu Pillard <[email protected]>

* TMP: update docs

---------

Co-authored-by: Mathieu Pillard <[email protected]>

.github/workflows/_test.yml

@@ -47,12 +47,7 @@ jobs:
           -
             name: Static Assets
             services: ''
-            # TODO: we should remove this once we
-            # a) update the asset tests to look in the static-assets folder
-            # b) copy the static file into the container also.
-            run: |
-              make update_assets
-              make test_static_assets
+            run: make test_static_assets
           -
             name: Internal Routes
             services: ''

.gitignore

@@ -30,7 +30,6 @@ docs/_gh-pages
 docs/api/_build
 docs/_build
 local_settings.py
-logs/*
 MANIFEST
 node_modules
 pip-log.txt
@@ -40,7 +39,7 @@ shellng_local.py
 site-static/*
 src/olympia/discovery/strings.jinja2
 static/css/node_lib/*
-static/js/i18n/*.js
+static-build/*
 static/js/node_lib/*
 storage/files/*
 storage/git-storage/*

Dockerfile

@@ -169,6 +169,7 @@ COPY --chown=olympia:olympia . ${HOME}
 COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy assets from assets
 COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
+COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-build
 
 # Set shell back to sh until we can prove we can use bash at runtime
 SHELL ["/bin/sh", "-c"]

Makefile-docker

@@ -26,32 +26,6 @@ ifneq ($(NPM_DEBUG),)
 endif
 
 NODE_MODULES := $(NPM_CONFIG_PREFIX)node_modules/
-STATIC_CSS := static/css/node_lib/
-STATIC_JS := static/js/node_lib/
-STATIC_JQUERY_UI := static/js/node_lib/ui/
-
-NODE_LIBS_CSS := \
-@claviska/jquery-minicolors/jquery.minicolors.css \
-@claviska/jquery-minicolors/jquery.minicolors.png \
-
-# NODE_LIBS_JS and NODE_LIBS_JQUERY_UI are referenced in settings.MINIFY_BUNDLES - keep both lists in sync
-NODE_LIBS_JS := \
-less/dist/less.js \
-jquery/dist/jquery.js \
-jquery.browser/dist/jquery.browser.js \
-jquery.cookie/jquery.cookie.js \
-@claviska/jquery-minicolors/jquery.minicolors.js \
-jszip/dist/jszip.js \
-timeago/jquery.timeago.js \
-underscore/underscore.js \
-netmask/lib/netmask.js \
-
-NODE_LIBS_JQUERY_UI := \
-jquery-ui/ui/data.js \
-jquery-ui/ui/scroll-parent.js \
-jquery-ui/ui/widget.js \
-jquery-ui/ui/widgets/mouse.js \
-jquery-ui/ui/widgets/sortable.js
 
 REQUIRED_FILES := \
 	Makefile \
@@ -154,10 +128,6 @@ update_db: ## run the database migrations
 .PHONY: update_assets
 update_assets:
 	# Copy files required in compress_assets to the static folder
-	mkdir -p $(STATIC_CSS) $(STATIC_JS) $(STATIC_JQUERY_UI)
-	for dest in $(NODE_LIBS_CSS) ; do cp $(NODE_MODULES)$$dest $(STATIC_CSS) ; done
-	for dest in $(NODE_LIBS_JS) ; do cp $(NODE_MODULES)$$dest $(STATIC_JS) ; done
-	for dest in $(NODE_LIBS_JQUERY_UI) ; do cp $(NODE_MODULES)$$dest $(STATIC_JQUERY_UI) ; done
 	# If changing this here, make sure to adapt tests in amo/test_commands.py
 	$(PYTHON_COMMAND) manage.py compress_assets
 	$(PYTHON_COMMAND) manage.py generate_jsi18n_files
@@ -227,10 +197,6 @@ dbshell: ## connect to a database shell
 .PHONY: initialize
 initialize: update_deps initialize_db update_assets populate_data reindex_data ## init the dependencies, the database, and assets
 
-.PHONY: reload
-reload: ## force django code reload
-	uwsgi --reload ${HOME}/docker/artifacts/addons-server-uwsgi-master.pid
-
 reload-uwsgi: reload
 
 PYTEST_SRC := src/olympia/

Makefile-os

@@ -8,6 +8,7 @@ DOCKER_BUILDER ?= default
 DOCKER_PROGRESS ?= auto
 DOCKER_METADATA_FILE ?= buildx-bake-metadata.json
 DOCKER_PUSH ?=
+export DEBUG ?= True
 export DOCKER_COMMIT ?=
 export DOCKER_BUILD ?=
 export DOCKER_VERSION ?=
@@ -40,6 +41,8 @@ CLEAN_PATHS := \
 	src/olympia.egg-info \
 	supervisord.pid \
 	version.json \
+	logs \
+	buildx-bake-metadata.json \
 
 .PHONY: help_redirect
 help_redirect:
@@ -142,8 +145,8 @@ docker_clean_volumes: ## Remove dangling volumes
 	docker volume prune --force
 
 .PHONY: docker_clean_images
-docker_clean_images: ## Remove dangling images
-	docker image prune --filter "dangling=true" --force
+docker_clean_images: ## Remove dangling images, preserving layer cache
+	docker image prune --filter "dangling=true" --filter "label!=buildx.cache" --force
 
 .PHONY: docker_clean_build_cache
 docker_clean_build_cache: ## Remove buildx build cache

docker-compose.ci.yml

@@ -1,12 +1,21 @@
 services:
-  web: &web
-      environment:
-        - HOST_UID=9500
-      volumes:
+  worker:
+    environment:
+      - HOST_UID=9500
+      - DEBUG=
+    volumes:
       - /data/olympia
 
-  worker:
-    <<: *web
+  web:
+    extends:
+      service: worker
+    volumes:
+      - data_site_static:/data/olympia/site-static
+
+  nginx:
+    volumes:
+      - data_site_static:/srv/site-static
 
 volumes:
   data_olympia:
+  data_site_static:

docker-compose.yml

@@ -21,6 +21,7 @@ x-env-mapping: &env
     - HISTCONTROL=erasedups
     - CIRCLECI
     - HOST_UID
+    - DEBUG
 
 x-olympia: &olympia
   <<: *env
@@ -61,6 +62,10 @@ services:
     ]
     volumes:
       - data_olympia:/data/olympia
+      # Don't mount generated files. They only exist in the container
+      # and would otherwiser be deleted by mounbting data_olympia
+      - /data/olympia/static-build
+      - /data/olympia/site-static
       - storage:/data/olympia/storage
       - data_deps:/deps
       - ./package.json:/deps/package.json
@@ -87,8 +92,7 @@ services:
     image: nginx
     volumes:
       - ./docker/nginx/addons.conf:/etc/nginx/conf.d/addons.conf
-      - ./static:/srv/static
-      - ./site-static:/srv/site-static
+      - ./static:/srv/site-static
       - storage:/srv/user-media
     ports:
       - "80:80"

docker/nginx/addons.conf

@@ -11,29 +11,18 @@ server {
     }
 
     location /static/ {
-        alias /srv/static/;
-    }
+         alias /srv/site-static/;
 
-    location /site-static/ {
-        alias /srv/site-static/;
+         # Fallback to the uwsgi server if the file is not found in the static files directory.
+         # This will happen for vendor files from pytnon or npm dependencies that won't be available
+         # in the static files directory.
+         error_page 404 = @olympia;
     }
 
     location /user-media/ {
         alias /srv/user-media/;
     }
 
-    location /static/debug_toolbar/ {
-        alias /srv/site-static/debug_toolbar/;
-    }
-
-    location /static/admin/ {
-        alias /srv/site-static/admin/;
-    }
-
-    location /static/drf-yasg/ {
-        alias /srv/site-static/drf-yasg/;
-    }
-
     location ~ ^/api/ {
         try_files $uri @olympia;
     }

docker/uwsgi.ini

@@ -16,24 +16,13 @@ gid             = $(id -g olympia)
 memory-report   = true
 enable-threads  = true
 
-safe-pidfile = %(base)/docker/artifacts/addons-server-uwsgi-master.pid
-
 py-autoreload=1
 
 max-requests = 5000
 
 # Load apps in workers and not only in master
 lazy-apps = true
 
-# Open log file after we dropped privileges so that the file is being owned
-# by uid:gid and has proper permissions to be readable outside of docker
-logto2 = %(base)/logs/uwsgi-olympia.log
-
-# Limit log file size to 10MB
-log-maxsize = 1048576
-
-# And set the name for the previous log
-log-backupname = %(base)/logs/uwsgi-olympia.log.1
 
 # Set default settings as originally done by manage.py
 env = DJANGO_SETTINGS_MODULE=settings

docs/topics/development/index.md

@@ -22,6 +22,7 @@ branching
 vpn
 acl
 logging
+static-files
 search
 docs
 waffle

docs/topics/development/static-files.md

@@ -0,0 +1,81 @@
+# Static Files in addons-server
+
+This document explains how static files are served in the addons-server project during local development.
+
+## Overview
+
+addons-server uses a combination of nginx and Django's built-in static file serving capabilities to efficiently serve static files.
+These files come from multiple sources:
+
+1. The `./static` folder in the project
+2. Python dependencies
+3. npm dependencies
+
+## Static File Servers
+
+We use a combination of servers to serve static files:
+
+1. Nginx
+2. Django's built-in development server
+
+In development, the nginx server will attempt to serve static files from the `./static` directory mounted into the nginx cointainer.
+If the file cannot be found there the request is forwarded to django.
+Nginx serves our own static files quickly and any vendor files can be fetched from django directly during development.
+
+In production mode, we mount a data volume both to `web` anb `nginx` containers.
+The `web` container exposes the `site-static` directory to nginx that includes the collected static files.
+
+> In actual production environments, we upload the static files to a cloud bucket and serve them directly from the static path.
+
+## Static File Sources
+
+### Project Static Files
+
+Static files specific to the addons-server project are stored in the `./static` directory. These include CSS, JavaScript, images, and other assets used by the application.
+
+In reality there are 3 static directories in our docker compose container:
+
+- `/data/olympia/static`: Contains static files that are mounted directly from the host.
+- `/data/olympia/static-build`: Contains static files that are built by `compress_assets`.
+- `/data/olympia/site-static`: Contains static files that are collected by the `collectstatic` command.
+
+The only of these directories that is exposed to your host is the `./static` directory.
+
+### Compressing Static Files
+
+We currently use a `ducktape` script to compress our static files.
+Ideally we would migrate to a modern tool to replace manual scripting, but for now this works.
+
+Assets are compressed automatically during the docker build, but if you need to manually update files while developing,
+the easiest way is to run `make update_assets` which will compress and concatenate static assets as well as  collect all static files
+to the `site-static` directory.
+
+### Python Dependencies
+
+Some Python packages include their own static files. These assets are collected by the `collectstatic` command and included in the final static files directory.
+During development they are served by the django development server.
+
+### npm Dependencies
+
+We have a (complex) set of npm static assets that are built by the `compress_assets` management command.
+During development, these assets are served directly from the node_modules directory using a custom static finder.
+
+## DEBUG Property and Static File Serving
+
+The behavior of static file serving can be controlled using the `DEBUG` environment variable or via setting it directly in
+the `local_settings.py` file. Be careful directly setting this value, if DEBUG is set to false, and you don't have sufficient
+routing setup to serve files fron nginx only, it can cause failure to serve some static files.
+
+It is best to use  the compose file to control DEBUG.a
+
+This is set in the environment, and in CI environments, it's controlled by the `docker-compose.ci.yml` file.
+
+The `DEBUG` property is what is used by django to determine if it should serve static files or not. In development,
+you can manually override this in the make up command, but in general, you should rely on the `docker-compose.ci.yml` file
+to set the correct value as this will also set appropriate file mounts.
+
+```bash
+make up COMPOSE_FILE=docker-compose.yml:docker-compose.ci.yml
+```
+
+This will run addons-server in production mode, serving files from the `site-static` directory.

requirements/prod.txt

@@ -1228,3 +1228,6 @@ watchdog[watchmedo]==3.0.0 \
     --hash=sha256:c9d8c8ec7efb887333cf71e328e39cffbf771d8f8f95d308ea4125bf5f90ba64 \
     --hash=sha256:d00e6be486affb5781468457b21a6cbe848c33ef43f9ea4a73b4882e5f188a44 \
     --hash=sha256:d429c2430c93b7903914e4db9a966c7f2b068dd2ebdd2fa9b9ce094c7d459f33
+django-node-assets==0.9.14 \
+    --hash=sha256:80cbe3d10521808309712b2aa5ef6d69799bbcafef844cf7f223d3c93f405768 \
+    --hash=sha256:d5b5c472136084d533268f52ab77897327863a102e25c81f484aae85eb806987

settings.py

@@ -16,13 +16,8 @@
 
 INTERNAL_ROUTES_ALLOWED = True
 
-DEBUG = True
-
 # These apps are great during development.
-INSTALLED_APPS += (
-    'olympia.landfill',
-    'debug_toolbar',
-)
+INSTALLED_APPS += ('olympia.landfill',)
 
 # Override logging config to enable DEBUG logs for (almost) everything.
 LOGGING['root']['level'] = logging.DEBUG
@@ -48,7 +43,9 @@ def insert_debug_toolbar_middleware(middlewares):
     return tuple(ret_middleware)
 
 
-MIDDLEWARE = insert_debug_toolbar_middleware(MIDDLEWARE)
+if DEBUG:
+    INSTALLED_APPS += ('debug_toolbar',)
+    MIDDLEWARE = insert_debug_toolbar_middleware(MIDDLEWARE)
 
 DEBUG_TOOLBAR_CONFIG = {
     # Enable django-debug-toolbar locally, if DEBUG is True.