Remove initialize_docker: Integrate data initialization to make up

Add Args for init

TMP: clean up

TMP: update command decsription

.github/actions/run-docker/action.yml

@@ -20,6 +20,18 @@ inputs:
     description: 'The docker-compose file to use'
     required: false
     default: 'docker-compose.yml:docker-compose.ci.yml'
+  init_force_db:
+    description: 'Force the initialization of the database'
+    required: false
+    default: ''
+  init_skip_seed:
+    description: 'Skip the seeding of the database'
+    required: false
+    default: 'true'
+  init_skip_index:
+    description: 'Skip the indexing of the database'
+    required: false
+    default: 'true'
 runs:
   using: 'composite'
   steps:
@@ -36,6 +48,10 @@ runs:
         COMPOSE_FILE: ${{ inputs.compose_file }}
         DOCKER_SERVICES: ${{ inputs.services }}
         HOST_UID: ${{ steps.id.outputs.id }}
+        INIT_FORCE_DB: ${{ inputs.init_force_db }}
+        INIT_SKIP_SEED: ${{ inputs.init_skip_seed }}
+        INIT_SKIP_INDEX: ${{ inputs.init_skip_index }}
+
       run: |
         # Start the specified services
         make up

Makefile-docker

@@ -52,41 +52,6 @@ check_django: ## check if the django app is configured properly
 .PHONY: check
 check: check_files check_olympia_user check_debian_packages check_pip_packages check_django
 
-.PHONY: initialize_db
-initialize_db: ## create a new database
-	rm -rf ./user-media/* ./tmp/*
-	$(PYTHON_COMMAND) manage.py create_db --force
-	$(PYTHON_COMMAND) manage.py migrate --noinput
-	$(PYTHON_COMMAND) manage.py loaddata initial.json
-	$(PYTHON_COMMAND) manage.py import_prod_versions
-	# The superuser needs to have a mozilla.com address for admin tools access
-	$(PYTHON_COMMAND) manage.py createsuperuser \
-		--no-input \
-		--username "local_admin" \
-		--email "[email protected]"
-	$(PYTHON_COMMAND) manage.py loaddata zadmin/users
-
-.PHONY: reindex_data
-reindex_data: ## reindex the data in elasticsearch
-	$(PYTHON_COMMAND) manage.py reindex --force --noinput
-
-.PHONY: populate_data
-populate_data: ## populate a new database
-	# reindex --wipe will force the ES mapping to be re-installed. Useful to
-	# make sure the mapping is correct before adding a bunch of add-ons.
-	$(PYTHON_COMMAND) manage.py reindex --wipe --force --noinput
-	$(PYTHON_COMMAND) manage.py generate_addons --app firefox $(NUM_ADDONS)
-	$(PYTHON_COMMAND) manage.py generate_addons --app android $(NUM_ADDONS)
-	$(PYTHON_COMMAND) manage.py generate_themes $(NUM_THEMES)
-	# These add-ons are specifically useful for the addons-frontend
-	# homepage. You may have to re-run this, in case the data there
-	# changes.
-	$(PYTHON_COMMAND) manage.py generate_default_addons_for_frontend
-
-.PHONY: update_db
-update_db: ## run the database migrations
-	$(PYTHON_COMMAND) manage.py migrate --noinput
-
 .PHONY: update_assets
 update_assets:
 	# Copy files required in compress_assets to the static folder
@@ -96,13 +61,6 @@ update_assets:
 	# Collect static files: This MUST be run last or files will be missing
 	$(PYTHON_COMMAND) manage.py collectstatic --noinput
 
-.PHONY: update
-update: update_db update_assets ## update the dependencies, the database, and assets
-
-.PHONY: reindex
-reindex: ## reindex everything in elasticsearch, for AMO
-	$(PYTHON_COMMAND) manage.py reindex $(ARGS)
-
 .PHONY: setup-ui-tests
 setup-ui-tests:
 	rm -rf ./user-media/* ./tmp/*
@@ -156,8 +114,11 @@ djshell: ## connect to django shell
 dbshell: ## connect to a database shell
 	$(PYTHON_COMMAND) ./manage.py dbshell
 
-.PHONY: initialize
-initialize: initialize_db update_assets populate_data reindex_data ## init the dependencies, the database, and assets
+.PHONY: initialize_data
+initialize_data: ## ensure database exists
+	@echo "Initializing data..."
+	@echo "args: $(ARGS)"
+	$(PYTHON_COMMAND) ./manage.py initialize_data $(ARGS)
 
 reload-uwsgi: reload
 

Makefile-os

@@ -12,11 +12,30 @@ export DOCKER_COMMIT ?=
 export DOCKER_BUILD ?=
 export DOCKER_VERSION ?=
 override DOCKER_MYSQLD_VOLUME = addons-server_data_mysqld
+override DOCKER_COMPOSE_EXEC = docker compose exec
+override DOCKER_COMPOSE_EXEC_WEB = $(DOCKER_COMPOSE_EXEC) --user olympia web
 
 override BACKUPS_DIR = $(shell pwd)/backups
 override EXPORT_DIR = $(BACKUPS_DIR)/$(shell date +%Y%m%d%H%M%S)
 RESTORE_DIR ?= $(BACKUPS_DIR)/$(shell ls -1 backups | sort -r | head -n 1)
 
+INITIALIZE_ARGS ?=
+INIT_FORCE_DB ?=
+INIT_SKIP_SEED ?=
+INIT_SKIP_INDEX ?=
+
+ifneq ($(INIT_FORCE_DB),)
+	INITIALIZE_ARGS += --force-db
+endif
+
+ifneq ($(INIT_SKIP_SEED),)
+	INITIALIZE_ARGS += --skip-seed
+endif
+
+ifneq ($(INIT_SKIP_INDEX),)
+	INITIALIZE_ARGS += --skip-index
+endif
+
 DOCKER_BAKE_ARGS := \
 	--file docker-bake.hcl \
 	--file .env \
@@ -69,29 +88,29 @@ update_docker: data_export up data_restore ## update all the docker images
 
 .PHONY: shell
 shell: ## connect to a running addons-server docker shell
-	docker compose exec --user olympia web bash
+	$(DOCKER_COMPOSE_EXEC_WEB) bash
 
 .PHONY: rootshell
 rootshell: ## connect to a running addons-server docker shell with root user
-	docker compose exec --user root web bash
+	$(DOCKER_COMPOSE_EXEC) --user root web bash
 
 .PHONY: data_export
 data_export:
 	@ mkdir -p $(EXPORT_DIR)
 
 	# Extracting mysql database
-	docker compose exec mysqld /usr/bin/mysqldump olympia > $(EXPORT_DIR)/data_mysqld.sql
+	$(DOCKER_COMPOSE_EXEC) mysqld /usr/bin/mysqldump olympia > $(EXPORT_DIR)/data_mysqld.sql
 
 .PHONY: data_restore
 data_restore:
 	@[ -d $(RESTORE_DIR) ] || (echo "Directory $(RESTORE_DIR) does not exist" && exit 1)
 
 	# Wait for MySQL server to be ready
-	docker compose exec mysqld bash \
+	$(DOCKER_COMPOSE_EXEC) mysqld bash \
 	-c 'while ! mysqladmin ping --silent; do echo "waiting"; sleep 1; done'
 
 	# Restoring mysql database
-	docker compose exec -T mysqld /usr/bin/mysql olympia < $(RESTORE_DIR)/data_mysqld.sql
+	$(DOCKER_COMPOSE_EXEC) -T mysqld /usr/bin/mysql olympia < $(RESTORE_DIR)/data_mysqld.sql
 
 	$(MAKE) reindex_data
 
@@ -161,16 +180,16 @@ docker_compose_up: docker_mysqld_volume_create ## Start the docker containers
 
 .PHONY: up
 up: setup docker_pull_or_build docker_compose_up docker_clean_images docker_clean_volumes ## Create and start docker compose
+	# Explicitly run initialize via the web container as make can get confused
+	# both routing the command to the web container and
+	# routing the command to the proper target.
+	$(DOCKER_COMPOSE_EXEC_WEB) make -f Makefile-docker initialize_data ARGS=$(shell echo "'$(INITIALIZE_ARGS)'")
 
 .PHONY: down
 down: docker_compose_down docker_clean_images docker_clean_volumes ## Stop the docker containers and clean up non-peristent dangling resources
 
-.PHONY: initialize_docker
-initialize_docker: up
-	docker compose exec --user olympia web make initialize
-
 %: ## This directs any other recipe (command) to the web container's make.
-	docker compose exec --user olympia web make $(MAKECMDGOALS) ARGS=$(ARGS)
+	$(DOCKER_COMPOSE_EXEC_WEB) make $(MAKECMDGOALS) ARGS=$(ARGS)
 
 # You probably want to put new commands in Makefile-docker, unless they operate
 # on multiple containers or are host-os specific.

docker-compose.yml

@@ -68,6 +68,13 @@ services:
       test: ["CMD-SHELL", "DJANGO_SETTINGS_MODULE=olympia celery -A olympia.amo.celery status"]
       interval: 1s
       retries: 100
+    depends_on:
+      - mysqld
+      - elasticsearch
+      - redis
+      - memcached
+      - rabbitmq
+      - autograph
 
   web:
     <<: *worker

docs/topics/development/data_management.md

@@ -12,26 +12,18 @@ The use of an external mount allows for manual management of the data lifecycle.
 
 ## Data Population
 
-The `make initialize_docker` command handles initial data population, including creating the database, running migrations, and seeding the database.
+When you run `make up` make will run the `initialize_data` command for you. This command will check if the database exists, and if the elasticsearch index exists.
 
-If you already have running containers, you can just run `make initialize` to reset the database, populate data, and reindex.
+If they don't exist it will create them. This command can be run manually as well.
 
 - **Database Initialization**:
 
   ```sh
-  make initialize_docker
+  make initialize_data
   ```
 
-- **Command Breakdown**:
-  - **`make up`**: Starts the Docker containers.
-  - **`make initialize`**: Runs database migrations and seeds the database with initial data.
-
-The `make initialize` command, executed as part of `make initialize_docker`, performs the following steps:
-
-1. **Create Database**: Sets up the initial database schema.
-2. **Run Migrations**: Applies any pending database migrations.
-3. **Seed Database**: Inserts initial data into the database.
-4. **Reindex**: Rebuilds the search index in Elasticsearch.
+This will create the database, run migrations, seed the database and create the index in elasticsearch.
+If any of these steps have already been run, they will be skipped.
 
 ## Exporting and Loading Data Snapshots
 
@@ -62,3 +54,11 @@ Refer to the Makefile for detailed instructions on these commands.
 This comprehensive setup ensures that the development environment is fully prepared with the necessary data.
 
 By following these practices, developers can manage data effectively in the **addons-server** project. The use of persistent volumes, external mounts, data snapshots, and automated data population ensures a robust and flexible data management strategy. For more detailed instructions, refer to the project's Makefile and Docker Compose configuration in the repository.
+
+- **Hard Reset Database**:
+
+In order to manually re-initialize the databse you can run the command with the `--foce` argument. This will delete the existing data. This will force recreate the database, seed it, and reindex.
+
+```bash
+make initialize_data INIT_FORCE_DB=true
+```

docs/topics/development/makefile_commands.md

@@ -102,15 +102,7 @@ A common benefit of using Makefiles in this manner is the ability to coordinate
      make data_restore
      ```
 
-2. **`initialize_docker`**:
-   - **Purpose**: Sets up the initial Docker environment, including database initialization and data population.
-   - **Usage**:
-
-     ```sh
-     make initialize_docker
-     ```
-
-3. **`build_docker_image`**:
+2. **`build_docker_image`**:
    - **Purpose**: Builds the Docker image using BuildKit and Bake.
    - **Usage**:
 

docs/topics/development/setup_and_configuration.md

@@ -15,49 +15,31 @@ Follow these steps to get started:
   cd addons-server
   ```
 
-(running-for-the-first-time)=
-## Running for the first time
-
-When running the project for the first time, execute:
-
-```sh
-make initialize_docker
-```
-
-This command will run:
-
-- `make up` to start the Docker containers.
-- `make initialize` to set up the initial Docker environment, including database initialization and data population.
-Detailed steps for `make initialize` will be covered in Section 6 on Data Management.
-
-If you run `make up` without running `make initialize` the docker compose services will be running, but you will not have a database
-and the app might crash or otherwise be unusable.
-
-Similarly, you can run `make initialize` even after you have an up and running environment, but this will totally reset your database
-as if you were running the application fresh.
-
-## Updating your environment
+## Running the docker compose project
 
 > TLDR; Just run `make up`.
 
 The _make up_ command ensures all necessary files are created on the host and starts the Docker Compose project,
-including volumes, containers, and networks. It is meant to be run frequently whenever you want to bring your environment "up".
+including volumes, containers, networks, databases and indexes.
+It is meant to be run frequently whenever you want to bring your environment "up".
 
 Here's a high-level overview of what _make up_ does:
 
 ```make
 .PHONY: up
-up: setup docker_pull_or_build docker_compose_up docker_clean_images docker_clean_volumes ## Create and start docker compose
+up: setup docker_pull_or_build docker_compose_up docker_clean_images docker_clean_volumes data
 ```
 
 - **setup**: Creates configuration files such as `.env` and `version.json`.
 - **docker_pull_or_build**: Pulls or builds the Docker image based on the image version.
 - **docker_compose_up**: Starts the Docker containers defined in [docker-compose.yml][docker-compose].
 - **docker_clean_images** and **docker_clean_volumes**: Cleans up unused Docker images and volumes.
+- **data**: Ensures the database, seed, and index are created.
 
-What happens if you run `make up` when your environment is already running?
-This will result in all services and volumes being recreated as if starting them for the first time,
-and will clear any local state from the containers. The `make up` command is {ref}`idempotent <idempotence>` so you can run it over and over.
+What happens if you run `make up` when your environment is already running?.
+Well that depends on what is changed since the last time you ran it.
+Because `make up` is {ref}`idempotent <idempotence>` it will only run the commands that are necessary to bring your environment up to date.
+If nothing has changed, nothing will happen because your environment is already in the desired state.
 
 ## Shutting down your environment
 
@@ -70,6 +52,19 @@ Running `make down` will free up resources on your machine and can help if your
 
 A common solution to many problems is to run `make down && make up`.
 
+> NOTE: When you run make down, it will clear all volumes except the data_mysqld volume.
+> This is where your database and other persisted data is stored.
+> If you want to start fresh, you can delete the data_mysqld volume.
+
+```sh
+make down
+make docker_mysqld_volume_remove # Remove the mysql database volume
+make up
+```
+
+If you want to completely nuke your environment and start over as if you had just cloned the repo,
+you can run `make clean_docker`. This will `make down` and remove all docker resources taking space on the host machine.
+
 ### Accessing the Development App
 
 - Add the following entry to your `/etc/hosts` file to access **addons-server** via a local domain:
@@ -219,11 +214,11 @@ Another way to find out what's wrong is to run `docker compose logs`.
 
 ### Getting "Programming error [table] doesn't exist"?
 
-Make sure you've run the `make initialize_docker` step as {ref}`detailed <running-for-the-first-time>` in the initial setup instructions.
+Make sure you've run `make up`.
 
 ### ConnectionError during initialize (elasticsearch container fails to start)
 
-When running `make initialize_docker` without a working elasticsearch container, you'll get a ConnectionError. Check the logs with `docker compose logs`. If elasticsearch is complaining about `vm.max_map_count`, run this command on your computer or your docker-machine VM:
+When running `make up` without a working elasticsearch container, you'll get a ConnectionError. Check the logs with `docker compose logs`. If elasticsearch is complaining about `vm.max_map_count`, run this command on your computer or your docker-machine VM:
 
 ```sh
     sudo sysctl -w vm.max_map_count=262144
@@ -233,7 +228,7 @@ This allows processes to allocate more [memory map areas](https://stackoverflow.
 
 ### Connection to elasticsearch timed out (elasticsearch container exits with code 137)
 
-`docker compose up -d` brings up all containers, but running `make initialize_docker` causes the elasticsearch container to go down. Running `docker compose ps` shows _Exited (137)_ against it.
+`docker compose up -d` brings up all containers, but running `make up` causes the elasticsearch container to go down. Running `docker compose ps` shows _Exited (137)_ against it.
 
 Update default settings in Docker Desktop - we suggest increasing RAM limit to at least 4 GB in the Resources/Advanced section and click on "Apply and Restart".