diff --git a/CHANGELOG.md b/CHANGELOG.md index a9f7af30b..37aefaada 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,8 @@ The types of changes are: * Parallelize CI safe checks to reduce run time [#717](https://github.com/ethyca/fidesops/pull/717) * Add dependabot to keep dependencies up to date [#718](https://github.com/ethyca/fidesops/pull/718) +* Make running a worker node optional [#770](https://github.com/ethyca/fidesops/pull/770) + ### Changed * Base64 encode passwords on frontend [#749](https://github.com/ethyca/fidesops/pull/749) diff --git a/Makefile b/Makefile index 5b2fed274..b87535201 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,9 @@ reset-db: server: compose-build @docker-compose up +server-with-worker: compose-build + @docker-compose -f docker-compose.yml -f docker-compose.worker.yml up + server-no-db: compose-build @docker-compose -f docker-compose.no-db.yml up diff --git a/docker-compose.worker.yml b/docker-compose.worker.yml new file mode 100644 index 000000000..c507e1499 --- /dev/null +++ b/docker-compose.worker.yml @@ -0,0 +1,22 @@ +services: + webserver: + depends_on: + - worker + environment: + - FIDESOPS__EXECUTION__WORKER_ENABLED=True + + worker: + build: + context: . + dockerfile: Dockerfile.worker + command: fidesops worker + depends_on: + redis: + condition: service_started + restart: always + volumes: + - type: bind + source: ./ + target: /fidesops + read_only: False + - /fidesops/src/fidesops.egg-info diff --git a/docker-compose.yml b/docker-compose.yml index 1f40bab40..a00967dcb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,6 @@ services: context: . dockerfile: Dockerfile.app depends_on: - - worker - db - redis expose: @@ -29,6 +28,7 @@ services: - FIDESOPS__LOG_PII=${FIDESOPS__LOG_PII} - FIDESOPS__HOT_RELOAD=${FIDESOPS__HOT_RELOAD} - FIDESOPS__ROOT_USER__ANALYTICS_ID=${FIDESOPS__ROOT_USER__ANALYTICS_ID} + - FIDESOPS__EXECUTION__WORKER_ENABLED=False db: image: postgres:12 @@ -59,22 +59,6 @@ services: ports: - "0.0.0.0:6379:6379" - worker: - build: - context: . - dockerfile: Dockerfile.worker - command: fidesops worker - depends_on: - redis: - condition: service_started - restart: always - volumes: - - type: bind - source: ./ - target: /fidesops - read_only: False - - /fidesops/src/fidesops.egg-info - docs: build: context: docs/fidesops/ diff --git a/docs/fidesops/docs/guides/configuration_reference.md b/docs/fidesops/docs/guides/configuration_reference.md index 1b28cce0c..a41ad68da 100644 --- a/docs/fidesops/docs/guides/configuration_reference.md +++ b/docs/fidesops/docs/guides/configuration_reference.md @@ -58,6 +58,9 @@ The `fidesops.toml` file should specify the following variables: |`TASK_RETRY_BACKOFF` | `FIDESOPS__EXECUTION__TASK_RETRY_BACKOFF` | int | 2 | 1 | The backoff factor for retries, to space out repeated retries. |`REQUIRE_MANUAL_REQUEST_APPROVAL` | `FIDESOPS__EXECUTION__REQUIRE_MANUAL_REQUEST_APPROVAL` | bool | False | False | Whether privacy requests require explicit approval to execute |`MASKING_STRICT` | `FIDESOPS__EXECUTION__MASKING_STRICT` | bool | True | True | If MASKING_STRICT is True, we only use "update" requests to mask data. (For third-party integrations, you should define an `update` endpoint to use.) If MASKING_STRICT is False, you are allowing fidesops to use any defined DELETE or GDPR DELETE endpoints to remove PII. In this case, you should define `delete` or `data_protection_request` endpoints for your third-party integrations. Note that setting MASKING_STRICT to False means that data may be deleted beyond the specific data categories that you've configured in your Policy. +|`CELERY_BROKER_URL` | `FIDESOPS__EXECUTION__CELERY_BROKER_URL` | str | redis://:testpassword@redis:6379/1 | N/A | The datastore to maintain ordered queues of tasks. +|`CELERY_RESULT_BACKEND` | `FIDESOPS__EXECUTION__RESULT_BACKEND` | str | redis://:testpassword@redis:6379/1 | N/A | The datastore to put results from asynchronously processed tasks. +|`WORKER_ENABLED` | `FIDESOPS__EXECUTION__WORKER_ENABLED` | bool | True | True | Whether Fidesops is running with a dedicated worker to process privacy requests asynchronously. |---|---|---|---|---|---| |`ANALYTICS_OPT_OUT` | `FIDESOPS__USER__ANALYTICS_OPT_OUT` | bool | False | False | Opt out of sending anonymous usage data to Ethyca to improve the product experience | Admin UI Variables|---|---|---|---|---| @@ -98,6 +101,10 @@ TASK_RETRY_DELAY=20 TASK_RETRY_BACKOFF=2 REQUIRE_MANUAL_REQUEST_APPROVAL=true MASKING_STRICT=true +CELERY_BROKER_URL="redis://:testpassword@redis:6379/1" +CELERY_RESULT_BACKEND="redis://:testpassword@redis:6379/1" +WORKER_ENABLED=true + [root_user] ANALYTICS_OPT_OUT=false diff --git a/fidesops.toml b/fidesops.toml index d9c17c75c..1ce87750c 100644 --- a/fidesops.toml +++ b/fidesops.toml @@ -34,6 +34,7 @@ REQUIRE_MANUAL_REQUEST_APPROVAL = false TASK_RETRY_COUNT = 0 TASK_RETRY_DELAY = 1 TASK_RETRY_BACKOFF = 1 +WORKER_ENABLED = false [root_user] ANALYTICS_OPT_OUT = false diff --git a/src/fidesops/core/config.py b/src/fidesops/core/config.py index 369f5ad30..eab986ea8 100644 --- a/src/fidesops/core/config.py +++ b/src/fidesops/core/config.py @@ -41,6 +41,7 @@ class ExecutionSettings(FidesSettings): MASKING_STRICT: bool = True CELERY_BROKER_URL: str = "redis://:testpassword@redis:6379/1" CELERY_RESULT_BACKEND: str = "redis://:testpassword@redis:6379/1" + WORKER_ENABLED: bool = True class Config: env_prefix = "FIDESOPS__EXECUTION__" diff --git a/src/fidesops/main.py b/src/fidesops/main.py index 0caa1af2f..271e17676 100644 --- a/src/fidesops/main.py +++ b/src/fidesops/main.py @@ -1,4 +1,5 @@ import logging +import subprocess from datetime import datetime, timezone from pathlib import Path from typing import Callable, Optional @@ -179,6 +180,10 @@ def start_webserver() -> None: ) ) + if not config.execution.WORKER_ENABLED: + logger.info("Starting worker...") + subprocess.Popen(["fidesops", "worker"]) + logger.info("Starting web server...") uvicorn.run( "fidesops.main:app",