fix: expose off-web html and fix nginx static resources serving (#9865)

First thing is that this PR fix a long standing bug that was missed: we were using multiple locations for same files, one hiding the other. Changed the way we handle expiry so that it really works as intended, using nginx best practices.

Second openfoodfacts-web as an html/ folder that is currently not exposed, while it should… this is done through try_file directive in nginx configuration.

.env

@@ -19,7 +19,7 @@ TAG=latest
 
 # static lang dependent files for the website
 # we have a minimal set of files in server repo, in prod we use main repo
-WEB_LANG_PATH=./lang-default
+WEB_RESOURCES_PATH=./web-default
 
 # env vars
 PRODUCERS_PLATFORM=0

.github/workflows/container-deploy.yml

@@ -133,7 +133,7 @@ jobs:
 
           # Set App variables
           echo "TAG=sha-${{ github.sha }}" >> .env
-          echo "WEB_LANG_PATH=../off-web-net/lang" >> .env
+          echo "WEB_RESOURCES_PATH=../off-web-net/" >> .env
           echo "PRODUCERS_PLATFORM=0" >> .env
           echo "PRODUCT_OPENER_PORT=80" >> .env
           echo "PRODUCT_OPENER_DOMAIN=${{ env.PRODUCT_OPENER_DOMAIN }}" >> .env

Makefile

@@ -49,9 +49,10 @@ HOSTS=127.0.0.1 world.productopener.localhost fr.productopener.localhost static.
 # commands aliases
 DOCKER_COMPOSE=docker compose --env-file=${ENV_FILE} ${LOAD_EXTRA_ENV_FILE}
 # we run tests in a specific project name to be separated from dev instances
+# keep web-default for web contents
 # we also publish mongodb on a separate port to avoid conflicts
 # we also enable the possibility to fake services in po_test_runner
-DOCKER_COMPOSE_TEST=ROBOTOFF_URL="http://backend:8881/" GOOGLE_CLOUD_VISION_API_URL="http://backend:8881/" COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME}_test PO_COMMON_PREFIX=test_ MONGO_EXPOSE_PORT=27027 docker compose --env-file=${ENV_FILE}
+DOCKER_COMPOSE_TEST=WEB_RESOURCES_PATH=./web-default ROBOTOFF_URL="http://backend:8881/" GOOGLE_CLOUD_VISION_API_URL="http://backend:8881/" COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME}_test PO_COMMON_PREFIX=test_ MONGO_EXPOSE_PORT=27027 docker compose --env-file=${ENV_FILE}
 # Enable Redis only for integration tests
 DOCKER_COMPOSE_INT_TEST=REDIS_URL="redis:6379" ${DOCKER_COMPOSE_TEST}
 

conf/nginx.conf

@@ -11,7 +11,7 @@
 # Please see /usr/share/doc/nginx-doc/examples/ for more detailed examples.
 ##
 
-# Default server configuration
+# Default server configuration for DOCKER
 #
 
 # we need to have main domain for CORS (see nginx-cors includes)
@@ -20,13 +20,16 @@ map $host $main_domain {
     ~*.*\.(?<host_main_domain>[^.]+\.[^.]+) $host_main_domain;
 }
 
+include /etc/nginx/snippets/expires-no-json-xml.conf;
 
 server {
 	listen 80;
 	listen [::]:80;
 
 	# Product Opener needs a root domain + a wildcard for all subdomains
-	server_name 	server_name ${PRODUCT_OPENER_DOMAIN} *.${PRODUCT_OPENER_DOMAIN};
+	server_name ${PRODUCT_OPENER_DOMAIN} *.${PRODUCT_OPENER_DOMAIN};
+
+	access_log /var/log/${productopener_access_file_prefix}access.log;
 
 	# static file we serve are in html/
 	root /opt/product-opener/html/;
@@ -48,11 +51,9 @@ server {
 	# Add index.php to the list if you are using PHP
 	index index.html index.htm index.nginx-debian.html;
 
-	# handling expirations
-	include /etc/nginx/snippets/expires-no-json-xml.conf;
-
 	location ~ ^/images/products/ {
 		include /etc/nginx/snippets/off.cors-headers.include;
+		include /etc/nginx/snippets/expiry-headers.include;
 		add_header Link "<https://creativecommons.org/licenses/by-sa/3.0/>; rel='license'; title='CC-BY-SA 3.0'";
 		# optimize gzip compressed content (like OCR .json stored next to .jpg files)
 		gzip_static always;
@@ -63,6 +64,7 @@ server {
 
 	location ~ ^/files/(.*) {
 		include snippets/off.cors-headers.include;
+		include /etc/nginx/snippets/expiry-headers.include;
 		# fist try in files_resources
 		try_files files_resources/$1 $uri $uri/ =404;
 		gzip_static always;
@@ -71,9 +73,10 @@ server {
 
 	location ~ ^/(.well-known|images|fonts|css|js|rss|resources|foundation|bower_components)/ {
 		include /etc/nginx/snippets/off.cors-headers.include;
-		# First attempt to serve request as file, then
-		# as directory, then fall back to displaying a 404.
-		try_files $uri $uri/ =404;
+		include /etc/nginx/snippets/expiry-headers.include;
+		# First attempt to serve request as file, off_web_html acting as an override,
+		# then as directory, then fall back to displaying a 404.
+		try_files /off_web_html$uri $uri $uri/ =404;
 		gzip_static always;
 		gunzip on;
 	}
@@ -82,13 +85,13 @@ server {
 	# and much less when caching headers are sent
 	location = /.well-known/assetlinks.json {
 		include /etc/nginx/snippets/off.cors-headers.include;
-		expires 1d;
+		include /etc/nginx/snippets/expiry-headers.include;
 		try_files $uri $uri/ =404;
 	}
 
 	location ~ /(favicon\.ico)$ {
 		include /etc/nginx/snippets/off.cors-headers.include;
-		expires 1d;
+		include /etc/nginx/snippets/expiry-headers.include;
 		try_files $uri $uri/ =404;
 	}
 
@@ -100,8 +103,9 @@ server {
 
 	# this is the internal Docker DNS, cache only for 30s
 	resolver 127.0.0.11 valid=30s;
-	location ~ /donate\/.*$ {
+	location ~ ^/donate\/.*$ {
 		include /etc/nginx/snippets/off.cors-headers.include;
+		include /etc/nginx/snippets/expiry-headers.include;
 		try_files $uri  =404;
 	}
 	location / {

conf/nginx/sites-available/foodbattle

@@ -21,6 +21,7 @@ server {
 	return 301 http://foodbattle.net$request_uri;
 }
 
+include /etc/nginx/expires-no-json-xml.conf;
 
 server {
 	#listen 80 default_server;
@@ -104,5 +105,4 @@ server {
 	#       deny all;
 	#}
 
-	include /etc/nginx/expires-no-json-xml.conf;
 }

conf/nginx/sites-available/howmuchsugar

@@ -11,6 +11,8 @@ map $howmuchsugar_name $howmuchsugar_lang {
 	combiendesucres fr;
 }
 
+include /etc/nginx/snippets/expires-no-json-xml.conf;
+
 log_format proxied_requests2
   '$http_x_forwarded_for - $remote_user [$time_local] '
   '"$request" $status $body_bytes_sent '
@@ -22,8 +24,6 @@ server {
 
 	server_name howmuchsugar.in combiendesucres.fr;
 
-	include /etc/nginx/snippets/expires-no-json-xml.conf;
-
 	access_log /var/log/nginx/nginx.howmuchsugar.access.log proxied_requests2;
 	error_log /var/log/nginx/nginx.howmuchsugar.error.log;
 
@@ -36,16 +36,19 @@ server {
 	index index.html index.htm index.nginx-debian.html;
 
 	location ~* \.(eot|ttf|woff|woff2)$ {
+		include /etc/nginx/snippets/expiry-headers.conf;
 		add_header Access-Control-Allow-Origin *;
 	}
 
 	location ~ ^/(favicon.ico) {
+		include /etc/nginx/snippets/expiry-headers.conf;
 		# First attempt to serve request as file, then
 		# as directory, then fall back to displaying a 404.
 		try_files $howmuchsugar_name-favicon.ico =404;
 	}
 
 	location ~ ^/(.well-known|images|js|rss|data|files|resources|foundation)/ {
+		include /etc/nginx/snippets/expiry-headers.conf;
 		# First attempt to serve request as file, then
 		# as directory, then fall back to displaying a 404.
 		try_files $uri $uri/ =404;
@@ -56,11 +59,13 @@ server {
 	}
 
 	location ~ ^/$ {
+		include /etc/nginx/snippets/expires-no-json-xml.conf;
 		try_files $uri /$howmuchsugar_name.html;
 	}
 
 	# redirect to .html files
 	location / {
+		include /etc/nginx/snippets/expires-no-json-xml.conf;
 		try_files $uri $uri.html =404;
 	}
 

conf/nginx/sites-available/madenearme

@@ -6,6 +6,9 @@ map $host $madenearme_name {
             madenear.me.uk madenearme-uk;
 
 }
+
+include /etc/nginx/snippets/expires-no-json-xml.conf;
+
 server {
 	listen 80;
 	listen [::]:80;
@@ -18,32 +21,34 @@ server {
 	gzip on;
 	gzip_min_length 1000;
 
-	include /etc/nginx/snippets/expires-no-json-xml.conf;
-
 	root /srv/off/html;
 
 	# Add index.php to the list if you are using PHP
 	index index.html index.htm index.nginx-debian.html;
 
 	location ~* \.(eot|ttf|woff|woff2)$ {
 		add_header Access-Control-Allow-Origin *;
+		include /etc/nginx/snippets/expires-no-json-xml.conf;
 	}
 
 	location ~ ^/images/products/ {
 		add_header Link "<http://creativecommons.org/licenses/by-sa/3.0/>; rel='license'; title='CC-BY-SA 3.0'";
+		include /etc/nginx/snippets/expires-no-json-xml.conf;
 	}
 
 	location ~ ^/(favicon.ico) {
 		# First attempt to serve request as file, then
 		# as directory, then fall back to displaying a 404.
 		try_files $uri /images/misc/$madenearme_name.16x16.png;
+		include /etc/nginx/snippets/expires-no-json-xml.conf;
 	}
 
 
 	location ~ ^/(.well-known|images|js|rss|data|files|resources|foundation)/ {
 		# First attempt to serve request as file, then
 		# as directory, then fall back to displaying a 404.
 		try_files $uri $uri/ =404;
+		include /etc/nginx/snippets/expires-no-json-xml.conf;
 	}
 
 	location = /robots.txt {
@@ -52,6 +57,7 @@ server {
 
 	location / {
 		try_files $uri /data/$madenearme_name.html;
+		include /etc/nginx/snippets/expires-no-json-xml.conf;
 	}
 
 }

conf/nginx/sites-available/obf

@@ -11,6 +11,9 @@
 # Please see /usr/share/doc/nginx-doc/examples/ for more detailed examples.
 ##
 
+
+include /etc/nginx/snippets/expires-no-json-xml.conf;
+
 # Default server configuration
 #
 server {
@@ -66,17 +69,19 @@ server {
 
 	location /data/ {
 		include snippets/off.cors-headers.include;
+		include snippets/expiry-headers.include;
 		include snippets/ssl-headers.conf;
 		# First attempt to serve request as file, then
 		# as directory, then fall back to displaying a 404.
 		try_files $uri $uri/ =404;
-}
+	}
 
 	# Add index.php to the list if you are using PHP
 	index index.html index.htm index.nginx-debian.html;
 
 	location ~* \.(eot|ttf|woff|woff2)$ {
 		include snippets/off.cors-headers.include;
+		include snippets/expiry-headers.include;
 	}
 
 	location ~ ^/images/products/ {
@@ -96,6 +101,7 @@ server {
 
 	location ~ ^/(.well-known|images|fonts|css|js|rss|files|resources|foundation|bower_components)/ {
 		include snippets/off.cors-headers.include;
+		include snippets/expiry-headers.include;
 		# First attempt to serve request as file, then
 		# as directory, then fall back to displaying a 404.
 		try_files $uri $uri/ =404;
@@ -105,6 +111,7 @@ server {
         # and much less when caching headers are sent
         location = /.well-known/assetlinks.json {
                 include snippets/off.cors-headers.include;
+			include snippets/expiry-headers.include;
                 expires 1d;
                 try_files $uri $uri/ =404;
         }
@@ -130,5 +137,4 @@ server {
 	#       deny all;
 	#}
 
-	include /etc/nginx/expires-no-json-xml.conf;
 }

conf/nginx/sites-available/off

@@ -13,6 +13,10 @@ server {
 	}
 }
 
+# variables definitions for expiry headers
+include /etc/nginx/snippets/expires-no-json-xml.conf;
+
+
 server {
 
 	listen 80;
@@ -29,7 +33,7 @@ server {
 	client_header_timeout 120s;
 
 	# logs location
-	access_log /var/log/nginx/off-access.log proxied_requests buffer=256K flush=1s;
+	access_log /var/log/nginx/${productopener_access_file_prefix}off-access.log proxied_requests buffer=256K flush=1s;
 	error_log /var/log/nginx/off-error.log;
 
 	# some redirection for specific subdomains
@@ -43,15 +47,15 @@ server {
 
 	location /data/ {
         include snippets/off.cors-headers.include;
+		include snippets/expiry-headers.include;
 		# First attempt to serve request as file, then
 		# as directory, then fall back to displaying a 404.
 		try_files $uri $uri/ =404;
 	}
 
-	include /etc/nginx/snippets/expires-no-json-xml.conf; # 2018-06-18 Moved to the end because of CORS issue with top_translators.csv
-
 	location ~ ^/images/products/ {
 		include snippets/off.cors-headers.include;
+		include snippets/expiry-headers.include;
 		add_header Link "<http://creativecommons.org/licenses/by-sa/3.0/>; rel='license'; title='CC-BY-SA 3.0'";
 		# optimize gzip compressed content (like OCR .json stored next to .jpg files)
 		gzip_static always;
@@ -85,6 +89,7 @@ server {
 	# Static files are served directly by NGINX
 	location ~ ^/(.well-known|files)/ {
 		include snippets/off.cors-headers.include;
+		include snippets/expiry-headers.include;
 		# First attempt to serve request from resource, then as file,
 		# then as directory, then fall back to displaying a 404.
 		try_files resources/$uri $uri $uri/ =404;
@@ -93,9 +98,10 @@ server {
 	}
 	location ~ ^/(images|fonts|css|js|rss|foundation|bower_components)/ {
 		include snippets/off.cors-headers.include;
-		# First attempt to serve as file,
+		include snippets/expiry-headers.include;
+		# First attempt to serve request as file, off_web_html acting as an override,
 		# then as directory, then fall back to displaying a 404.
-		try_files $uri $uri/ =404;
+		try_files /off_web_html$uri $uri $uri/ =404;
 		gzip_static always;
 		gunzip on;
 	}
@@ -108,9 +114,10 @@ server {
 	# GoogleAssociationService made 2500 requests/min to assetlinks.json
 	# and much less when caching headers are sent
 	location = /.well-known/assetlinks.json {
-			include snippets/off.cors-headers.include;
-			expires 1d;
-			try_files $uri $uri/ =404;
+		include snippets/off.cors-headers.include;
+		include snippets/expiry-headers.include;
+		expires 1d;
+		try_files $uri =404;
 	}
 
 	include snippets/off.locations-redirects.include;