Sync storage with make up flow

.dockerignore

@@ -48,12 +48,7 @@ src/olympia/discovery/strings.jinja2
 static-build/*
 static/css/node_lib/*
 static/js/node_lib/*
-storage/files/*
-storage/git-storage/*
-storage/guarded-addons/*
-storage/mlbf/*
-storage/shared_storage/*
-storage/sitemaps/*
+storage
 tmp/*
 
 # Additionally ignore these files from the docker build that are not in .gitignore
@@ -62,7 +57,6 @@ tmp/*
 .github
 docs
 private
-storage
 docker-bake.hcl
 docker-compose*.yml
 Dockerfile*

.github/actions/run-docker/action.yml

@@ -19,6 +19,10 @@ inputs:
   logs:
     description: 'Show logs'
     required: false
+  data_backup_skip:
+    description: 'Skip data backup'
+    required: false
+    default: 'true'
 
 runs:
   using: 'composite'
@@ -35,7 +39,7 @@ runs:
         DOCKER_DIGEST: ${{ inputs.digest }}
         COMPOSE_FILE: ${{ inputs.compose_file }}
         HOST_UID: ${{ steps.id.outputs.id }}
-        DATA_BACKUP_SKIP: true
+        DATA_BACKUP_SKIP: ${{ inputs.data_backup_skip }}
       run: |
         # Start the specified services
         make up

.github/workflows/_test.yml

@@ -70,6 +70,7 @@ jobs:
             services: web nginx
             compose_file: docker-compose.yml:docker-compose.ci.yml
             run: make check
+            data_backup_skip: true
     steps:
       - uses: actions/checkout@v4
       - name: Test (${{ matrix.name }})
@@ -80,3 +81,4 @@ jobs:
           services: ${{ matrix.services }}
           compose_file: ${{ matrix.compose_file }}
           run: ${{ matrix.run }}
+          data_backup_skip: ${{ matrix.data_backup_skip || 'true' }}

.gitignore

@@ -47,12 +47,7 @@ src/olympia/discovery/strings.jinja2
 static-build/*
 static/css/node_lib/*
 static/js/node_lib/*
-storage/files/*
-storage/git-storage/*
-storage/guarded-addons/*
-storage/mlbf/*
-storage/shared_storage/*
-storage/sitemaps/*
+storage
 tmp/*
 
 # End of .gitignore. Please keep this in sync with the top section of .dockerignore

Dockerfile

@@ -60,6 +60,10 @@ ln -s /usr/bin/uwsgi /usr/sbin/uwsgi
 # link to the package*.json at ${HOME} so npm can install in /deps
 ln -s ${HOME}/package.json /deps/package.json
 ln -s ${HOME}/package-lock.json /deps/package-lock.json
+
+# Create the storage directory and the test file to verify nginx routing
+mkdir -p ${HOME}/storage
+chown -R olympia:olympia ${HOME}/storage
 EOF
 
 USER olympia:olympia

Makefile-docker

@@ -53,6 +53,8 @@ check_django: ## check if the django app is configured properly
 
 .PHONY: check_nginx
 check_nginx:  ## check if the nginx config for local development is configured properly
+	mkdir -p /data/olympia/storage/shared_storage/uploads
+	echo "OK" > /data/olympia/storage/shared_storage/uploads/.check
 	@if [ "$$(curl -sf http://nginx/user-media/.check)" != "OK" ]; then echo "Requesting http://nginx/user-media/.check failed"; exit 1; fi
 	@echo "Nginx user-media configuration looks correct."
 

docker-compose.ci.yml

@@ -8,4 +8,12 @@ services:
   web:
     extends:
       service: worker
+    volumes:
+      - storage:/data/olympia/storage
+
+  nginx:
+    volumes:
+      - storage:/srv/storage
 
+volumes:
+  storage:

docker/nginx/addons.conf

@@ -7,7 +7,7 @@ server {
 
     location /data/olympia/storage/ {
         internal;
-        alias /srv/user-media/;
+        alias /srv/storage/;
     }
 
     location /static/ {

src/olympia/amo/management/commands/data_seed.py

@@ -1,3 +1,6 @@
+import os
+import shutil
+
 from django.conf import settings
 from django.core.management import call_command
 
@@ -10,6 +13,39 @@ class Command(BaseDataCommand):
         'generated add-ons, and data from AMO production.'
     )
 
+    def _clean_storage(self, root: str, dir_dict: dict[str, str | dict]) -> None:
+        for key, value in dir_dict.items():
+            curr_path = os.path.join(root, key)
+            if isinstance(value, dict):
+                self._clean_storage(curr_path, value)
+            else:
+                shutil.rmtree(curr_path, ignore_errors=True)
+                os.makedirs(curr_path, exist_ok=True)
+
+    def clean_storage(self):
+        self.logger.info('Cleaning storage...')
+        self._clean_storage(
+            settings.STORAGE_ROOT,
+            {
+                'files': '',
+                'shared_storage': {
+                    'tmp': {
+                        'addons': '',
+                        'data': '',
+                        'file_viewer': '',
+                        'guarded-addons': '',
+                        'icon': '',
+                        'log': '',
+                        'persona_header': '',
+                        'preview': '',
+                        'test': '',
+                        'uploads': '',
+                    },
+                    'uploads': '',
+                },
+            },
+        )
+
     def handle(self, *args, **options):
         num_addons = 10
         num_themes = 5
@@ -18,6 +54,7 @@ def handle(self, *args, **options):
 
         self.logger.info('Resetting database...')
         call_command('flush', '--noinput')
+        self.clean_storage()
         # reindex --wipe will force the ES mapping to be re-installed.
         call_command('reindex', '--wipe', '--force', '--noinput')
         call_command('migrate', '--noinput')
@@ -43,4 +80,4 @@ def handle(self, *args, **options):
         call_command('generate_default_addons_for_frontend')
 
         call_command('data_dump', '--name', self.data_backup_init)
-        call_command('reindex', '--wipe', '--force', '--noinput')
+        call_command('data_load', '--name', self.data_backup_init)

src/olympia/amo/tests/test_commands.py

@@ -771,6 +771,6 @@ def test_default(self):
                 self.mock_commands.generate_themes(5),
                 self.mock_commands.generate_default_addons_for_frontend,
                 self.mock_commands.data_dump(self.base_data_command.data_backup_init),
-                self.mock_commands.reindex,
+                self.mock_commands.data_load(self.base_data_command.data_backup_init),
             ],
         )