From 2a64618e9d1eebaed15639fb4f824f2844c92778 Mon Sep 17 00:00:00 2001 From: Kevin Meinhardt Date: Thu, 5 Sep 2024 16:20:03 +0200 Subject: [PATCH] Enable Production Mode via DEBUG or docker-compose.ci.yml (#22613) * TMP: clean bake metadata file * SPLIT: Log UWSGI in docker compose console Remove PID * SPLIT: Raise error in update_assets if file not found * Enable Production mode via DEBUG and or docker-compose.ci.yml * Option 1: Load node_modules from statifiles directory * Option 2: Load node_modules via django-node-asset * separate ./static from locally built static files * Add docs for static file serving * TMP: fixes from code review * Update docs/topics/development/static-files.md Co-authored-by: Mathieu Pillard * TMP: update docs --------- Co-authored-by: Mathieu Pillard --- .github/workflows/_test.yml | 7 +- .gitignore | 3 +- Dockerfile | 1 + Makefile-docker | 34 -------- Makefile-os | 7 +- docker-compose.ci.yml | 21 +++-- docker-compose.yml | 8 +- docker/nginx/addons.conf | 21 ++--- docker/uwsgi.ini | 11 --- docs/topics/development/index.md | 1 + docs/topics/development/static-files.md | 81 +++++++++++++++++++ logs/.gitkeep | 0 requirements/prod.txt | 3 + settings.py | 11 +-- .../management/commands/compress_assets.py | 39 ++++++--- .../commands/generate_jsi18n_files.py | 10 ++- src/olympia/amo/tests/test_commands.py | 4 +- src/olympia/core/apps.py | 41 ++++++++++ src/olympia/lib/settings_base.py | 50 ++++++++---- src/olympia/urls.py | 19 ++++- static/js/i18n/.keep | 0 tests/js/zamboni/stats.spec.js | 4 +- 22 files changed, 254 insertions(+), 122 deletions(-) create mode 100644 docs/topics/development/static-files.md delete mode 100644 logs/.gitkeep delete mode 100644 static/js/i18n/.keep diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index bfc8d3744b5..733b266cc96 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -47,12 +47,7 @@ jobs: - name: Static Assets services: '' - # TODO: we should remove this once we - # a) update the asset tests to look in the static-assets folder - # b) copy the static file into the container also. - run: | - make update_assets - make test_static_assets + run: make test_static_assets - name: Internal Routes services: '' diff --git a/.gitignore b/.gitignore index a74358efe57..ac6f4417733 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,6 @@ docs/_gh-pages docs/api/_build docs/_build local_settings.py -logs/* MANIFEST node_modules pip-log.txt @@ -40,7 +39,7 @@ shellng_local.py site-static/* src/olympia/discovery/strings.jinja2 static/css/node_lib/* -static/js/i18n/*.js +static-build/* static/js/node_lib/* storage/files/* storage/git-storage/* diff --git a/Dockerfile b/Dockerfile index be0e9779b99..2956d0036d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -169,6 +169,7 @@ COPY --chown=olympia:olympia . ${HOME} COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale # Copy assets from assets COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static +COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-build # Set shell back to sh until we can prove we can use bash at runtime SHELL ["/bin/sh", "-c"] diff --git a/Makefile-docker b/Makefile-docker index a68b39c5be1..0ec9a40332d 100644 --- a/Makefile-docker +++ b/Makefile-docker @@ -26,32 +26,6 @@ ifneq ($(NPM_DEBUG),) endif NODE_MODULES := $(NPM_CONFIG_PREFIX)node_modules/ -STATIC_CSS := static/css/node_lib/ -STATIC_JS := static/js/node_lib/ -STATIC_JQUERY_UI := static/js/node_lib/ui/ - -NODE_LIBS_CSS := \ -@claviska/jquery-minicolors/jquery.minicolors.css \ -@claviska/jquery-minicolors/jquery.minicolors.png \ - -# NODE_LIBS_JS and NODE_LIBS_JQUERY_UI are referenced in settings.MINIFY_BUNDLES - keep both lists in sync -NODE_LIBS_JS := \ -less/dist/less.js \ -jquery/dist/jquery.js \ -jquery.browser/dist/jquery.browser.js \ -jquery.cookie/jquery.cookie.js \ -@claviska/jquery-minicolors/jquery.minicolors.js \ -jszip/dist/jszip.js \ -timeago/jquery.timeago.js \ -underscore/underscore.js \ -netmask/lib/netmask.js \ - -NODE_LIBS_JQUERY_UI := \ -jquery-ui/ui/data.js \ -jquery-ui/ui/scroll-parent.js \ -jquery-ui/ui/widget.js \ -jquery-ui/ui/widgets/mouse.js \ -jquery-ui/ui/widgets/sortable.js REQUIRED_FILES := \ Makefile \ @@ -154,10 +128,6 @@ update_db: ## run the database migrations .PHONY: update_assets update_assets: # Copy files required in compress_assets to the static folder - mkdir -p $(STATIC_CSS) $(STATIC_JS) $(STATIC_JQUERY_UI) - for dest in $(NODE_LIBS_CSS) ; do cp $(NODE_MODULES)$$dest $(STATIC_CSS) ; done - for dest in $(NODE_LIBS_JS) ; do cp $(NODE_MODULES)$$dest $(STATIC_JS) ; done - for dest in $(NODE_LIBS_JQUERY_UI) ; do cp $(NODE_MODULES)$$dest $(STATIC_JQUERY_UI) ; done # If changing this here, make sure to adapt tests in amo/test_commands.py $(PYTHON_COMMAND) manage.py compress_assets $(PYTHON_COMMAND) manage.py generate_jsi18n_files @@ -227,10 +197,6 @@ dbshell: ## connect to a database shell .PHONY: initialize initialize: update_deps initialize_db update_assets populate_data reindex_data ## init the dependencies, the database, and assets -.PHONY: reload -reload: ## force django code reload - uwsgi --reload ${HOME}/docker/artifacts/addons-server-uwsgi-master.pid - reload-uwsgi: reload PYTEST_SRC := src/olympia/ diff --git a/Makefile-os b/Makefile-os index 0847f31ae1f..8b56c05ca60 100644 --- a/Makefile-os +++ b/Makefile-os @@ -8,6 +8,7 @@ DOCKER_BUILDER ?= default DOCKER_PROGRESS ?= auto DOCKER_METADATA_FILE ?= buildx-bake-metadata.json DOCKER_PUSH ?= +export DEBUG ?= True export DOCKER_COMMIT ?= export DOCKER_BUILD ?= export DOCKER_VERSION ?= @@ -40,6 +41,8 @@ CLEAN_PATHS := \ src/olympia.egg-info \ supervisord.pid \ version.json \ + logs \ + buildx-bake-metadata.json \ .PHONY: help_redirect help_redirect: @@ -142,8 +145,8 @@ docker_clean_volumes: ## Remove dangling volumes docker volume prune --force .PHONY: docker_clean_images -docker_clean_images: ## Remove dangling images - docker image prune --filter "dangling=true" --force +docker_clean_images: ## Remove dangling images, preserving layer cache + docker image prune --filter "dangling=true" --filter "label!=buildx.cache" --force .PHONY: docker_clean_build_cache docker_clean_build_cache: ## Remove buildx build cache diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index c6894984114..d45eebd5db3 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -1,12 +1,21 @@ services: - web: &web - environment: - - HOST_UID=9500 - volumes: + worker: + environment: + - HOST_UID=9500 + - DEBUG= + volumes: - /data/olympia - worker: - <<: *web + web: + extends: + service: worker + volumes: + - data_site_static:/data/olympia/site-static + + nginx: + volumes: + - data_site_static:/srv/site-static volumes: data_olympia: + data_site_static: diff --git a/docker-compose.yml b/docker-compose.yml index 20b65355db2..8dad9192588 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,7 @@ x-env-mapping: &env - HISTCONTROL=erasedups - CIRCLECI - HOST_UID + - DEBUG x-olympia: &olympia <<: *env @@ -61,6 +62,10 @@ services: ] volumes: - data_olympia:/data/olympia + # Don't mount generated files. They only exist in the container + # and would otherwiser be deleted by mounbting data_olympia + - /data/olympia/static-build + - /data/olympia/site-static - storage:/data/olympia/storage - data_deps:/deps - ./package.json:/deps/package.json @@ -87,8 +92,7 @@ services: image: nginx volumes: - ./docker/nginx/addons.conf:/etc/nginx/conf.d/addons.conf - - ./static:/srv/static - - ./site-static:/srv/site-static + - ./static:/srv/site-static - storage:/srv/user-media ports: - "80:80" diff --git a/docker/nginx/addons.conf b/docker/nginx/addons.conf index 58c0cda6c28..e53f6ac26a5 100644 --- a/docker/nginx/addons.conf +++ b/docker/nginx/addons.conf @@ -11,29 +11,18 @@ server { } location /static/ { - alias /srv/static/; - } + alias /srv/site-static/; - location /site-static/ { - alias /srv/site-static/; + # Fallback to the uwsgi server if the file is not found in the static files directory. + # This will happen for vendor files from pytnon or npm dependencies that won't be available + # in the static files directory. + error_page 404 = @olympia; } location /user-media/ { alias /srv/user-media/; } - location /static/debug_toolbar/ { - alias /srv/site-static/debug_toolbar/; - } - - location /static/admin/ { - alias /srv/site-static/admin/; - } - - location /static/drf-yasg/ { - alias /srv/site-static/drf-yasg/; - } - location ~ ^/api/ { try_files $uri @olympia; } diff --git a/docker/uwsgi.ini b/docker/uwsgi.ini index 2a02a13c5f4..4b52478f609 100644 --- a/docker/uwsgi.ini +++ b/docker/uwsgi.ini @@ -16,8 +16,6 @@ gid = $(id -g olympia) memory-report = true enable-threads = true -safe-pidfile = %(base)/docker/artifacts/addons-server-uwsgi-master.pid - py-autoreload=1 max-requests = 5000 @@ -25,15 +23,6 @@ max-requests = 5000 # Load apps in workers and not only in master lazy-apps = true -# Open log file after we dropped privileges so that the file is being owned -# by uid:gid and has proper permissions to be readable outside of docker -logto2 = %(base)/logs/uwsgi-olympia.log - -# Limit log file size to 10MB -log-maxsize = 1048576 - -# And set the name for the previous log -log-backupname = %(base)/logs/uwsgi-olympia.log.1 # Set default settings as originally done by manage.py env = DJANGO_SETTINGS_MODULE=settings diff --git a/docs/topics/development/index.md b/docs/topics/development/index.md index d10425fd0a1..44053026265 100644 --- a/docs/topics/development/index.md +++ b/docs/topics/development/index.md @@ -22,6 +22,7 @@ branching vpn acl logging +static-files search docs waffle diff --git a/docs/topics/development/static-files.md b/docs/topics/development/static-files.md new file mode 100644 index 00000000000..768d237f77d --- /dev/null +++ b/docs/topics/development/static-files.md @@ -0,0 +1,81 @@ +# Static Files in addons-server + +This document explains how static files are served in the addons-server project during local development. + +## Overview + +addons-server uses a combination of nginx and Django's built-in static file serving capabilities to efficiently serve static files. +These files come from multiple sources: + +1. The `./static` folder in the project +2. Python dependencies +3. npm dependencies + +## Static File Servers + +We use a combination of servers to serve static files: + +1. Nginx +2. Django's built-in development server + +In development, the nginx server will attempt to serve static files from the `./static` directory mounted into the nginx cointainer. +If the file cannot be found there the request is forwarded to django. +Nginx serves our own static files quickly and any vendor files can be fetched from django directly during development. + +In production mode, we mount a data volume both to `web` anb `nginx` containers. +The `web` container exposes the `site-static` directory to nginx that includes the collected static files. + +> In actual production environments, we upload the static files to a cloud bucket and serve them directly from the static path. + +## Static File Sources + +### Project Static Files + +Static files specific to the addons-server project are stored in the `./static` directory. These include CSS, JavaScript, images, and other assets used by the application. + +In reality there are 3 static directories in our docker compose container: + +- `/data/olympia/static`: Contains static files that are mounted directly from the host. +- `/data/olympia/static-build`: Contains static files that are built by `compress_assets`. +- `/data/olympia/site-static`: Contains static files that are collected by the `collectstatic` command. + +The only of these directories that is exposed to your host is the `./static` directory. + +### Compressing Static Files + +We currently use a `ducktape` script to compress our static files. +Ideally we would migrate to a modern tool to replace manual scripting, but for now this works. + +Assets are compressed automatically during the docker build, but if you need to manually update files while developing, +the easiest way is to run `make update_assets` which will compress and concatenate static assets as well as collect all static files +to the `site-static` directory. + +### Python Dependencies + +Some Python packages include their own static files. These assets are collected by the `collectstatic` command and included in the final static files directory. +During development they are served by the django development server. + +### npm Dependencies + +We have a (complex) set of npm static assets that are built by the `compress_assets` management command. +During development, these assets are served directly from the node_modules directory using a custom static finder. + +## DEBUG Property and Static File Serving + +The behavior of static file serving can be controlled using the `DEBUG` environment variable or via setting it directly in +the `local_settings.py` file. Be careful directly setting this value, if DEBUG is set to false, and you don't have sufficient +routing setup to serve files fron nginx only, it can cause failure to serve some static files. + +It is best to use the compose file to control DEBUG.a + +This is set in the environment, and in CI environments, it's controlled by the `docker-compose.ci.yml` file. + +The `DEBUG` property is what is used by django to determine if it should serve static files or not. In development, +you can manually override this in the make up command, but in general, you should rely on the `docker-compose.ci.yml` file +to set the correct value as this will also set appropriate file mounts. + +```bash +make up COMPOSE_FILE=docker-compose.yml:docker-compose.ci.yml +``` + +This will run addons-server in production mode, serving files from the `site-static` directory. diff --git a/logs/.gitkeep b/logs/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/requirements/prod.txt b/requirements/prod.txt index 7400a9f1cfa..c08bc635d62 100644 --- a/requirements/prod.txt +++ b/requirements/prod.txt @@ -1228,3 +1228,6 @@ watchdog[watchmedo]==3.0.0 \ --hash=sha256:c9d8c8ec7efb887333cf71e328e39cffbf771d8f8f95d308ea4125bf5f90ba64 \ --hash=sha256:d00e6be486affb5781468457b21a6cbe848c33ef43f9ea4a73b4882e5f188a44 \ --hash=sha256:d429c2430c93b7903914e4db9a966c7f2b068dd2ebdd2fa9b9ce094c7d459f33 +django-node-assets==0.9.14 \ + --hash=sha256:80cbe3d10521808309712b2aa5ef6d69799bbcafef844cf7f223d3c93f405768 \ + --hash=sha256:d5b5c472136084d533268f52ab77897327863a102e25c81f484aae85eb806987 diff --git a/settings.py b/settings.py index 6cf19584291..dbb5be584c9 100644 --- a/settings.py +++ b/settings.py @@ -16,13 +16,8 @@ INTERNAL_ROUTES_ALLOWED = True -DEBUG = True - # These apps are great during development. -INSTALLED_APPS += ( - 'olympia.landfill', - 'debug_toolbar', -) +INSTALLED_APPS += ('olympia.landfill',) # Override logging config to enable DEBUG logs for (almost) everything. LOGGING['root']['level'] = logging.DEBUG @@ -48,7 +43,9 @@ def insert_debug_toolbar_middleware(middlewares): return tuple(ret_middleware) -MIDDLEWARE = insert_debug_toolbar_middleware(MIDDLEWARE) +if DEBUG: + INSTALLED_APPS += ('debug_toolbar',) + MIDDLEWARE = insert_debug_toolbar_middleware(MIDDLEWARE) DEBUG_TOOLBAR_CONFIG = { # Enable django-debug-toolbar locally, if DEBUG is True. diff --git a/src/olympia/amo/management/commands/compress_assets.py b/src/olympia/amo/management/commands/compress_assets.py index f3c62bec372..180c32175d3 100644 --- a/src/olympia/amo/management/commands/compress_assets.py +++ b/src/olympia/amo/management/commands/compress_assets.py @@ -44,9 +44,20 @@ def add_arguments(self, parser): action='store_true', help='Ignores modified/created dates and forces compression.', ) + parser.add_argument( + '--dry-run', + action='store_true', + help=( + 'Do not actually compress assets, ' + 'just print the compressed file paths.' + ), + ) def handle(self, **options): self.force_compress = options.get('force', False) + self.dry_run = options.get('dry_run', False) + + concatted_file_names = set() # This will loop through every bundle, and do the following: # - Concat all files into one @@ -56,8 +67,7 @@ def handle(self, **options): for name, files in bundle.items(): # Set the paths to the files. concatted_file = os.path.join( - settings.ROOT, - 'static', + settings.STATIC_BUILD_PATH, ftype, '%s-all.%s' % ( @@ -66,8 +76,7 @@ def handle(self, **options): ), ) compressed_file = os.path.join( - settings.ROOT, - 'static', + settings.STATIC_BUILD_PATH, ftype, '%s-min.%s' % ( @@ -76,6 +85,11 @@ def handle(self, **options): ), ) + concatted_file_names.add(concatted_file) + + if self.dry_run: + continue + ensure_path_exists(concatted_file) ensure_path_exists(compressed_file) @@ -83,9 +97,7 @@ def handle(self, **options): contents = [] for filename in files: processed = self._preprocess_file(filename) - # If the file can't be processed, we skip it. - if processed is not None: - files_all.append(processed) + files_all.append(processed) with open(processed) as f: contents.append(f.read()) @@ -105,13 +117,16 @@ def handle(self, **options): if is_changed or not os.path.isfile(compressed_file): self._minify(ftype, concatted_file, compressed_file) else: - print( + self.stdout.write( 'File unchanged, skipping minification of %s' % (concatted_file) ) self.minify_skipped += 1 - if self.minify_skipped: - print( + if self.dry_run: + for file in sorted(concatted_file_names): + self.stdout.write(file) + elif self.minify_skipped: + self.stdout.write( 'Unchanged files skipped for minification: %s' % (self.minify_skipped) ) @@ -119,6 +134,10 @@ def _preprocess_file(self, filename): """Preprocess files and return new filenames.""" css_bin = filename.endswith('.less') and settings.LESS_BIN source = find_static_path(filename) + + if source is None: + raise CommandError('File not found: %s' % filename) + target = source if css_bin: target = '%s.css' % source diff --git a/src/olympia/amo/management/commands/generate_jsi18n_files.py b/src/olympia/amo/management/commands/generate_jsi18n_files.py index 619bc9fe5d1..c97639d6c88 100644 --- a/src/olympia/amo/management/commands/generate_jsi18n_files.py +++ b/src/olympia/amo/management/commands/generate_jsi18n_files.py @@ -15,10 +15,14 @@ class Command(BaseCommand): def handle(self, *args, **options): fake_request = HttpRequest() fake_request.method = 'GET' + + root = os.path.join(settings.STATIC_BUILD_PATH, 'js', 'i18n') + + if not os.path.exists(root): + os.makedirs(root) + for lang in settings.AMO_LANGUAGES: - filename = os.path.join( - settings.STATICFILES_DIRS[0], 'js', 'i18n', '%s.js' % lang - ) + filename = os.path.join(root, '%s.js' % lang) with translation.override(lang): response = JavaScriptCatalog.as_view()(fake_request) with open(filename, 'w') as f: diff --git a/src/olympia/amo/tests/test_commands.py b/src/olympia/amo/tests/test_commands.py index 933e52f94c7..27a620b4131 100644 --- a/src/olympia/amo/tests/test_commands.py +++ b/src/olympia/amo/tests/test_commands.py @@ -150,7 +150,7 @@ def test_compress_assets_correctly_compresses_js(settings, tmpdir): @pytest.mark.needs_locales_compilation def test_generate_jsi18n_files(): - dirname = os.path.join(settings.STATICFILES_DIRS[0], 'js', 'i18n') + dirname = os.path.join(settings.STATIC_BUILD_PATH, 'js', 'i18n') assert os.path.exists(dirname) filename = os.path.join(dirname, 'fr.js') call_command('generate_jsi18n_files') @@ -160,7 +160,7 @@ def test_generate_jsi18n_files(): # Spot-check: Look for a string we know should be in the french file # (Translation for "Error"). - filename = os.path.join(settings.STATICFILES_DIRS[0], 'js', 'i18n', 'fr.js') + filename = os.path.join(dirname, 'fr.js') with open(filename) as f: content = f.read() assert 'Erreur' in content diff --git a/src/olympia/core/apps.py b/src/olympia/core/apps.py index 5cf99f57635..6bc556df83e 100644 --- a/src/olympia/core/apps.py +++ b/src/olympia/core/apps.py @@ -1,10 +1,14 @@ import logging +import os import subprocess import warnings +from io import StringIO from django.apps import AppConfig from django.conf import settings from django.core.checks import Error, Tags, register +from django.core.management import call_command +from django.core.management.base import CommandError from django.utils.translation import gettext_lazy as _ from olympia.core.utils import get_version_json @@ -54,6 +58,43 @@ def version_check(app_configs, **kwargs): return [] +@register(CustomTags.custom_setup) +def static_check(app_configs, **kwargs): + errors = [] + output = StringIO() + + try: + call_command('compress_assets', dry_run=True, stdout=output) + file_paths = output.getvalue().strip().split('\n') + + if not file_paths: + errors.append( + Error( + 'No compressed asset files were found.', + id='setup.E003', + ) + ) + else: + for file_path in file_paths: + if not os.path.exists(file_path): + error = f'Compressed asset file does not exist: {file_path}' + errors.append( + Error( + error, + id='setup.E003', + ) + ) + except CommandError as e: + errors.append( + Error( + f'Error running compress_assets command: {str(e)}', + id='setup.E004', + ) + ) + + return errors + + class CoreConfig(AppConfig): name = 'olympia.core' verbose_name = _('Core') diff --git a/src/olympia/lib/settings_base.py b/src/olympia/lib/settings_base.py index edf0ad7818c..caf3fe03deb 100644 --- a/src/olympia/lib/settings_base.py +++ b/src/olympia/lib/settings_base.py @@ -57,7 +57,7 @@ def path(*folders): return os.path.join(ROOT, *folders) -DEBUG = False +DEBUG = env('DEBUG', default=False) DEBUG_TOOLBAR_CONFIG = { # Deactivate django debug toolbar by default. @@ -534,6 +534,7 @@ def get_db_config(environ_var, atomic_requests=True): 'rangefilter', 'django_recaptcha', 'drf_yasg', + 'django_node_assets', # Django contrib apps 'django.contrib.admin', 'django.contrib.auth', @@ -608,7 +609,7 @@ def get_db_config(environ_var, atomic_requests=True): 'css/devhub/buttons.less', 'css/devhub/in-app-config.less', 'css/devhub/static-theme.less', - 'css/node_lib/jquery.minicolors.css', + '@claviska/jquery-minicolors/jquery.minicolors.css', 'css/impala/devhub-api.less', 'css/devhub/dashboard.less', ), @@ -624,13 +625,12 @@ def get_db_config(environ_var, atomic_requests=True): }, 'js': { # JS files common to the entire site, apart from dev-landing. - # js/node_lib/* files are copied in Makefile-docker - keep both lists in sync 'common': ( - 'js/node_lib/underscore.js', + 'underscore/underscore.js', 'js/zamboni/init.js', 'js/zamboni/capabilities.js', 'js/lib/format.js', - 'js/node_lib/jquery.cookie.js', + 'jquery.cookie/jquery.cookie.js', 'js/zamboni/storage.js', 'js/common/keys.js', 'js/zamboni/helpers.js', @@ -644,8 +644,8 @@ def get_db_config(environ_var, atomic_requests=True): ), # Things to be loaded at the top of the page 'preload': ( - 'js/node_lib/jquery.js', - 'js/node_lib/jquery.browser.js', + 'jquery/dist/jquery.js', + 'jquery.browser/dist/jquery.browser.js', 'js/zamboni/analytics.js', ), 'zamboni/devhub': ( @@ -656,16 +656,16 @@ def get_db_config(environ_var, atomic_requests=True): 'js/common/upload-image.js', 'js/zamboni/devhub.js', 'js/zamboni/validator.js', - 'js/node_lib/jquery.timeago.js', + 'timeago/jquery.timeago.js', 'js/zamboni/static_theme.js', - 'js/node_lib/jquery.minicolors.js', - 'js/node_lib/jszip.js', + '@claviska/jquery-minicolors/jquery.minicolors.js', + 'jszip/dist/jszip.js', # jQuery UI for sortable - 'js/node_lib/ui/data.js', - 'js/node_lib/ui/scroll-parent.js', - 'js/node_lib/ui/widget.js', - 'js/node_lib/ui/mouse.js', - 'js/node_lib/ui/sortable.js', + 'jquery-ui/ui/data.js', + 'jquery-ui/ui/scroll-parent.js', + 'jquery-ui/ui/widget.js', + 'jquery-ui/ui/widgets/mouse.js', + 'jquery-ui/ui/widgets/sortable.js', ), 'devhub/new-landing/js': ( # Note that new-landing (devhub/index.html) doesn't include @@ -697,7 +697,7 @@ def get_db_config(environ_var, atomic_requests=True): # This is included when DEBUG is True. Bundle in . 'debug': ( 'js/debug/less_setup.js', - 'js/node_lib/less.js', + 'less/dist/less.js', 'js/debug/less_live.js', ), }, @@ -1289,7 +1289,23 @@ def read_only_mode(env): STATIC_ROOT = path('site-static') STATIC_URL = '/static/' -STATICFILES_DIRS = (path('static'),) +STATICFILES_FINDERS = ( + 'django.contrib.staticfiles.finders.FileSystemFinder', + 'django.contrib.staticfiles.finders.AppDirectoriesFinder', + 'django_node_assets.finders.NodeModulesFinder', +) + +NODE_MODULES_ROOT = os.path.join('/', 'deps', 'node_modules') +NODE_PACKAGE_JSON = os.path.join('/', 'deps', 'package.json') +NODE_PACKAGE_MANAGER_INSTALL_OPTIONS = ['--dry-run'] + +STATIC_BUILD_PATH = os.path.join('/', 'data', 'olympia', 'static-build') + +STATICFILES_DIRS = ( + path('static'), + STATIC_BUILD_PATH, +) + STATICFILES_STORAGE = 'olympia.lib.storage.ManifestStaticFilesStorageNotMaps' # Path related settings. In dev/stage/prod `NETAPP_STORAGE_ROOT` environment diff --git a/src/olympia/urls.py b/src/olympia/urls.py index 880a0ecfc89..2af1bd5ee62 100644 --- a/src/olympia/urls.py +++ b/src/olympia/urls.py @@ -107,8 +107,11 @@ ), ] -if settings.DEBUG and 'debug_toolbar' in settings.INSTALLED_APPS: - import debug_toolbar +if settings.DEBUG: + from django.contrib.staticfiles.views import serve as static_serve + + def serve_static_files(request, path, **kwargs): + return static_serve(request, path, insecure=True, **kwargs) # Remove leading and trailing slashes so the regex matches. media_url = settings.MEDIA_URL.lstrip('/').rstrip('/') @@ -120,6 +123,18 @@ serve_static, {'document_root': settings.MEDIA_ROOT}, ), + # fallback for static files that are not available directly over nginx. + # Mostly vendor files from python or npm dependencies that are not available + # in the static files directory. + re_path(r'^static/(?P.*)$', serve_static_files), + ] + ) + +if settings.DEBUG and 'debug_toolbar' in settings.INSTALLED_APPS: + import debug_toolbar + + urlpatterns.extend( + [ re_path(r'__debug__/', include(debug_toolbar.urls)), ] ) diff --git a/static/js/i18n/.keep b/static/js/i18n/.keep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/js/zamboni/stats.spec.js b/tests/js/zamboni/stats.spec.js index 3ffb277d5a7..381fb943f48 100644 --- a/tests/js/zamboni/stats.spec.js +++ b/tests/js/zamboni/stats.spec.js @@ -36,7 +36,7 @@ describe(__filename, () => { beforeEach(() => { stats_stats = - require('../../../static/js/zamboni/stats-all.js').stats_stats; + require('../../../static-build/js/zamboni/stats-all.js').stats_stats; }); describe('export links', () => { @@ -129,7 +129,7 @@ describe(__filename, () => { let stats_overview_make_handler; beforeEach(() => { - const stats_all = require('../../../static/js/zamboni/stats-all.js'); + const stats_all = require('../../../static-build/js/zamboni/stats-all.js'); stats_overview_make_handler = stats_all.stats_overview_make_handler; });