From 740f12dc3aee0157401bdadc8c4766ff1b121062 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Tue, 19 Apr 2022 10:43:13 +0200 Subject: [PATCH 1/3] Add some example configurations for worker --- .../workers/background_worker.yaml | 8 ++++ .../workers/event_persister.yaml | 23 ++++++++++ .../workers/generic_worker.yaml | 3 +- docs/workers.md | 45 +++++++++---------- 4 files changed, 55 insertions(+), 24 deletions(-) create mode 100644 docs/systemd-with-workers/workers/background_worker.yaml create mode 100644 docs/systemd-with-workers/workers/event_persister.yaml diff --git a/docs/systemd-with-workers/workers/background_worker.yaml b/docs/systemd-with-workers/workers/background_worker.yaml new file mode 100644 index 000000000000..9fbfbda7db94 --- /dev/null +++ b/docs/systemd-with-workers/workers/background_worker.yaml @@ -0,0 +1,8 @@ +worker_app: synapse.app.generic_worker +worker_name: background_worker + +# The replication listener on the main synapse process. +worker_replication_host: 127.0.0.1 +worker_replication_http_port: 9093 + +worker_log_config: /etc/matrix-synapse/background-worker-log.yaml diff --git a/docs/systemd-with-workers/workers/event_persister.yaml b/docs/systemd-with-workers/workers/event_persister.yaml new file mode 100644 index 000000000000..ef26c7bce2cf --- /dev/null +++ b/docs/systemd-with-workers/workers/event_persister.yaml @@ -0,0 +1,23 @@ +worker_app: synapse.app.generic_worker +worker_name: event_persister1 + +# The replication listener on the main synapse process. +worker_replication_host: 127.0.0.1 +worker_replication_http_port: 9093 + +worker_listeners: + - type: http + port: 8034 + resources: + - names: [replication] + + # Enable listener if this stream writer handles endpoints like `typing` or `to_device` + # stream. Uses a different port than `replication` listener to protect + # `replication` listener for exposing. + # + #- type: http + # port: 8035 + # resources: + # - names: [client] + +worker_log_config: /etc/matrix-synapse/event-persister-log.yaml diff --git a/docs/systemd-with-workers/workers/generic_worker.yaml b/docs/systemd-with-workers/workers/generic_worker.yaml index 8561e2cda59c..a82f9c161f81 100644 --- a/docs/systemd-with-workers/workers/generic_worker.yaml +++ b/docs/systemd-with-workers/workers/generic_worker.yaml @@ -1,12 +1,13 @@ worker_app: synapse.app.generic_worker worker_name: generic_worker1 +# The replication listener on the main synapse process. worker_replication_host: 127.0.0.1 worker_replication_http_port: 9093 worker_listeners: - type: http - port: 8011 + port: 8083 resources: - names: [client, federation] diff --git a/docs/workers.md b/docs/workers.md index 858411b15ec3..afdcd785e408 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -138,20 +138,7 @@ as the `listeners` option in the shared config. For example: ```yaml -worker_app: synapse.app.generic_worker -worker_name: worker1 - -# The replication listener on the main synapse process. -worker_replication_host: 127.0.0.1 -worker_replication_http_port: 9093 - -worker_listeners: - - type: http - port: 8083 - resources: - - names: [client, federation] - -worker_log_config: /home/matrix/synapse/config/worker1_log_config.yaml +{{#include systemd-with-workers/workers/generic_worker.yaml}} ``` ...is a full configuration for a generic worker instance, which will expose a @@ -363,6 +350,12 @@ stream_writers: events: event_persister1 ``` +An example for a stream writer instance: + +```yaml +{{#include systemd-with-workers/workers/event_persister.yaml}} +``` + Some of the streams have associated endpoints which, for maximum efficiency, should be routed to the workers handling that stream. See below for the currently supported streams and the endpoints associated with them: @@ -436,6 +429,12 @@ run_background_tasks_on: background_worker You might also wish to investigate the `update_user_directory` and `media_instance_running_background_jobs` settings. +An example for a dedicated background worker instance: + +```yaml +{{#include systemd-with-workers/workers/background_worker.yaml}} +``` + ### `synapse.app.pusher` Handles sending push notifications to sygnal and email. Doesn't handle any @@ -615,14 +614,14 @@ The following shows an example setup using Redis and a reverse proxy: | Main | | Generic | | Generic | | Event | | Process | | Worker 1 | | Worker 2 | | Persister | +--------------+ +--------------+ +--------------+ +--------------+ - ^ ^ | ^ | | ^ | ^ ^ - | | | | | | | | | | - | | | | | HTTP | | | | | - | +----------+<--|---|---------+ | | | | - | | +-------------|-->+----------+ | - | | | | - | | | | - v v v v -==================================================================== + ^ ^ | ^ | | ^ | | ^ ^ + | | | | | | | | | | | + | | | | | HTTP | | | | | | + | +----------+<--|---|---------+<--|---|---------+ | | + | | +-------------|-->+-------------+ | + | | | | + | | | | + v v v v +====================================================================== Redis pub/sub channel ``` From 63e9ed00866f551cb795a89e8c8fb5e67ef9ef86 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Tue, 19 Apr 2022 10:53:21 +0200 Subject: [PATCH 2/3] newsfile --- changelog.d/12492.doc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/12492.doc diff --git a/changelog.d/12492.doc b/changelog.d/12492.doc new file mode 100644 index 000000000000..1246fc2c4726 --- /dev/null +++ b/changelog.d/12492.doc @@ -0,0 +1 @@ +Add some example configurations for worker and update architectural diagram. From 92aaaac5faab3a229a217b8885fce5b938ff7be0 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Mon, 25 Apr 2022 13:05:04 +0200 Subject: [PATCH 3/3] Apply suggestions from code review Co-authored-by: Sean Quah <8349537+squahtx@users.noreply.github.com> --- changelog.d/12492.doc | 2 +- docs/systemd-with-workers/workers/event_persister.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/changelog.d/12492.doc b/changelog.d/12492.doc index 1246fc2c4726..4a3e2f4f0698 100644 --- a/changelog.d/12492.doc +++ b/changelog.d/12492.doc @@ -1 +1 @@ -Add some example configurations for worker and update architectural diagram. +Add some example configurations for workers and update architectural diagram. diff --git a/docs/systemd-with-workers/workers/event_persister.yaml b/docs/systemd-with-workers/workers/event_persister.yaml index ef26c7bce2cf..9bc6997bad99 100644 --- a/docs/systemd-with-workers/workers/event_persister.yaml +++ b/docs/systemd-with-workers/workers/event_persister.yaml @@ -11,9 +11,9 @@ worker_listeners: resources: - names: [replication] - # Enable listener if this stream writer handles endpoints like `typing` or `to_device` - # stream. Uses a different port than `replication` listener to protect - # `replication` listener for exposing. + # Enable listener if this stream writer handles endpoints for the `typing` or + # `to_device` streams. Uses a different port to the `replication` listener to + # avoid exposing the `replication` listener publicly. # #- type: http # port: 8035