Remove initialize_docker: Integrate data initialization to make up

.github/actions/run-docker/action.yml

@@ -12,17 +12,14 @@ inputs:
   run:
     description: 'Run command in container'
     required: true
-  services:
-    description: 'List of services to start'
-    required: false
-    default: 'web'
   compose_file:
     description: 'The docker-compose file to use'
     required: false
     default: 'docker-compose.yml:docker-compose.ci.yml'
   logs:
     description: 'Show logs'
     required: false
+
 runs:
   using: 'composite'
   steps:
@@ -37,8 +34,8 @@ runs:
         DOCKER_VERSION: ${{ inputs.version }}
         DOCKER_DIGEST: ${{ inputs.digest }}
         COMPOSE_FILE: ${{ inputs.compose_file }}
-        DOCKER_SERVICES: ${{ inputs.services }}
         HOST_UID: ${{ steps.id.outputs.id }}
+        DATA_BACKUP_SKIP: true
       run: |
         # Start the specified services
         make up

Makefile-docker

@@ -57,22 +57,6 @@ data_dump:
 data_load:
 	./manage.py data_load $(ARGS)
 
-.PHONY: initialize_db
-initialize_db: ## create a new database
-	rm -rf ./user-media/* ./tmp/*
-	./manage.py create_db --force
-	./manage.py migrate --noinput
-	# Seed the database with initial data
-	./manage.py data_seed
-
-.PHONY: reindex_data
-reindex_data: ## reindex the data in elasticsearch
-	$(PYTHON_COMMAND) manage.py reindex --force --noinput
-
-.PHONY: update_db
-update_db: ## run the database migrations
-	$(PYTHON_COMMAND) manage.py migrate --noinput
-
 .PHONY: update_assets
 update_assets:
 	# Copy files required in compress_assets to the static folder
@@ -82,45 +66,6 @@ update_assets:
 	# Collect static files: This MUST be run last or files will be missing
 	$(PYTHON_COMMAND) manage.py collectstatic --noinput
 
-.PHONY: update
-update: update_db update_assets ## update the dependencies, the database, and assets
-
-.PHONY: reindex
-reindex: ## reindex everything in elasticsearch, for AMO
-	$(PYTHON_COMMAND) manage.py reindex $(ARGS)
-
-.PHONY: setup-ui-tests
-setup-ui-tests:
-	rm -rf ./user-media/* ./tmp/*
-	# Reset the database and fake database migrations
-	$(PYTHON_COMMAND) manage.py create_db --force
-	$(PYTHON_COMMAND) manage.py migrate --noinput
-
-	# Reindex
-	$(PYTHON_COMMAND) manage.py reindex --force --noinput --wipe
-
-	# Let's load some initial data and import mozilla-product versions
-	$(PYTHON_COMMAND) manage.py loaddata initial.json
-	$(PYTHON_COMMAND) manage.py loaddata zadmin/users
-	$(PYTHON_COMMAND) manage.py loaddata src/olympia/access/fixtures/initial.json
-	$(PYTHON_COMMAND) manage.py import_prod_versions
-
-	# Create a proper superuser that can be used to access the API
-	$(PYTHON_COMMAND) manage.py waffle_switch super-create-accounts on --create
-	$(PYTHON_COMMAND) manage.py waffle_switch activate-autograph-signing on --create
-	$(PYTHON_COMMAND) manage.py generate_addons --app firefox $(NUM_ADDONS)
-	$(PYTHON_COMMAND) manage.py generate_addons --app android $(NUM_ADDONS)
-	$(PYTHON_COMMAND) manage.py generate_themes $(NUM_THEMES)
-	$(PYTHON_COMMAND) manage.py generate_default_addons_for_frontend
-
-	# Now that addons have been generated, reindex.
-	$(PYTHON_COMMAND) manage.py reindex --force --noinput
-
-.PHONY: perf-tests
-perf-tests: setup-ui-tests
-	$(PIP_COMMAND) install --progress-bar=off --no-deps -r requirements/perftests.txt
-	locust --no-web -c 1 -f tests/performance/locustfile.py --host "http://olympia.test"
-
 .PHONY: lint
 lint: ## lint the code
 	ruff check .
@@ -143,7 +88,10 @@ dbshell: ## connect to a database shell
 	$(PYTHON_COMMAND) ./manage.py dbshell
 
 .PHONY: initialize
-initialize: initialize_db update_assets reindex_data ## init the dependencies, the database, and assets
+initialize: ## ensure database exists
+	@echo "Initializing data..."
+	@echo "args: $(ARGS)"
+	$(PYTHON_COMMAND) ./manage.py initialize $(ARGS)
 
 PYTEST_SRC := src/olympia/
 

Makefile-os

@@ -11,11 +11,22 @@ export DEBUG ?= True
 export DOCKER_COMMIT ?=
 export DOCKER_BUILD ?=
 export DOCKER_VERSION ?=
+export DATA_BACKUP_SKIP ?=
 override DOCKER_MYSQLD_VOLUME = addons-server_data_mysqld
 
-override BACKUPS_DIR = $(shell pwd)/backups
-override EXPORT_DIR = $(BACKUPS_DIR)/$(shell date +%Y%m%d%H%M%S)
-RESTORE_DIR ?= $(BACKUPS_DIR)/$(shell ls -1 backups | sort -r | head -n 1)
+INITIALIZE_ARGS ?=
+INIT_CLEAN ?=
+INIT_LOAD ?=
+
+ifneq ($(INIT_CLEAN),)
+	INITIALIZE_ARGS += --clean
+endif
+
+ifneq ($(INIT_LOAD),)
+	INITIALIZE_ARGS += --load $(INIT_LOAD)
+endif
+
+
 
 DOCKER_BAKE_ARGS := \
 	--file docker-bake.hcl \
@@ -139,18 +150,18 @@ clean_docker: docker_compose_down docker_mysqld_volume_remove docker_clean_image
 
 .PHONY: docker_compose_up
 docker_compose_up: docker_mysqld_volume_create ## Start the docker containers
-	docker compose up $(DOCKER_SERVICES) $(DOCKER_COMPOSE_ARGS) $(ARGS)
+	docker compose up $(DOCKER_COMPOSE_ARGS) $(ARGS)
 
 .PHONY: up
 up: setup docker_pull_or_build docker_compose_up docker_clean_images docker_clean_volumes ## Create and start docker compose
+	# Explicitly run initialize via the web container as make can get confused
+	# both routing the command to the web container and
+	# routing the command to the proper target.
+	docker compose exec --user olympia web make -f Makefile-docker initialize ARGS=$(shell echo "'$(INITIALIZE_ARGS)'")
 
 .PHONY: down
 down: docker_compose_down docker_clean_images docker_clean_volumes ## Stop the docker containers and clean up non-peristent dangling resources
 
-.PHONY: initialize_docker
-initialize_docker: up
-	docker compose exec --user olympia web make initialize
-
 %: ## This directs any other recipe (command) to the web container's make.
 	docker compose exec --user olympia web make $(MAKECMDGOALS) ARGS="$(shell echo $(ARGS))"
 

docker-compose.yml

@@ -22,6 +22,7 @@ x-env-mapping: &env
     - CIRCLECI
     - HOST_UID
     - DEBUG
+    - DATA_BACKUP_SKIP
 
 x-olympia: &olympia
   <<: *env
@@ -70,6 +71,13 @@ services:
       start_interval: 1s
       # The start period is 60s
       start_period: 120s
+    depends_on:
+      - mysqld
+      - elasticsearch
+      - redis
+      - memcached
+      - rabbitmq
+      - autograph
 
   web:
     <<: *worker
@@ -112,6 +120,9 @@ services:
     - "3306:3306"
     volumes:
       - data_mysqld:/var/lib/mysql
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "--silent"]
+      start_interval: 1s
 
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:7.17.3

docs/topics/development/data_management.md

@@ -10,30 +10,76 @@ The project uses persistent data volumes to store MySQL data. This ensures that
 
 The use of an external mount allows for manual management of the data lifecycle. This ensures that data is preserved even if you run `make down`. By defining the MySQL data volume as external, it decouples the data lifecycle from the container lifecycle, allowing you to manually manage the data.
 
-## Data Population
+## Data Initialization
 
-The `make initialize_docker` command handles initial data population, including creating the database, running migrations, and seeding the database.
+When you run `make up` make will run the `initialize` command for you. This command will check if the database exists, and if the elasticsearch index exists.
 
-If you already have running containers, you can just run `make initialize` to reset the database, populate data, and reindex.
+If they don't exist it will create them. This command can be run manually as well.
 
-- **Database Initialization**:
+```sh
+make initialize
+```
+
+This command is responsible for ensuring your local mysql database is migrated, seeded, loaded with data and indexed.
+There are a number of different ways to execute this command. In most cases, the default behavior is what you want.
+But there are a few additional edge cases that it supports.
+
+### Clean the database
 
   ```sh
-  make initialize_docker
+  make initialize INIT_CLEAN=true
   ```
 
-- **Command Breakdown**:
-  - **`make up`**: Starts the Docker containers.
-  - **`make initialize`**: Runs database migrations and seeds the database with initial data.
+  This will force the database to be recreated, and re-initialized.
+
+### Load a data backup
+
+  ```sh
+  make initialize [INIT_LOAD=<backup_name>]
+  ```
+
+  This command will load a data backup from a specified path. The optional `INIT_LOAD` argument allows you to
+  specify the path to the data backup file. If not specified, the initialize command will determine if
+  data should be loaded based on the current state of the databse, and will load the `_init` data backup.
+
+### Skip seeding
+
+```sh
+make initialize INIT_SKIP_SEED=true
+```
+
+This will skip the seeding of the database. This can be useful in CI or if you specifically
+want to avoid touching the previous data or creating a new _init backup.
+
+### Skip index recreation
+
+```sh
+make initialize INIT_SKIP_INDEX=true
+```
+
+This will skip the recreation of the elasticsearch index. This can be useful in CI or if you specifically
+want to avoid touching the previous elasticsearch index.
+
+> NOTE: if your database is modified significantly and you don't re-index elasticsearch you could end up with
+> a broken addons-frontend.
+
+## Data seeding
+
+`addons-server` uses a a data seeding mechanism to populate the database with the initial data. This data is used to
+bootstrap the database with addons and other data to enable development.
+
+The data seed is treted just like a data backup with a special name `_init`. To recreate the dataseed run:
+
+```sh
+make seed_data
+```
 
-The `make initialize` command, executed as part of `make initialize_docker`, performs the following steps:
+This will flush the current database, remove the _init backup directory if it exists, run the seed commands,
+and finally dump the data back into the _init backup directory.
 
-1. **Create Database**: Sets up the initial database schema.
-2. **Run Migrations**: Applies any pending database migrations.
-3. **Seed Database**: Inserts initial data into the database.
-4. **Reindex**: Rebuilds the search index in Elasticsearch.
+The _init backup is used to populate the database with initial data during the initialization process.
 
-## Exporting and Loading Data Snapshots
+## Data backups
 
 You can export and load data snapshots to manage data states across different environments or for backup purposes.
 The Makefile provides commands to facilitate this.

docs/topics/development/makefile_commands.md

@@ -102,15 +102,7 @@ A common benefit of using Makefiles in this manner is the ability to coordinate
      make data_restore
      ```
 
-2. **`initialize_docker`**:
-   - **Purpose**: Sets up the initial Docker environment, including database initialization and data population.
-   - **Usage**:
-
-     ```sh
-     make initialize_docker
-     ```
-
-3. **`build_docker_image`**:
+2. **`build_docker_image`**:
    - **Purpose**: Builds the Docker image using BuildKit and Bake.
    - **Usage**: