[WPB-10308] Use RabbitMQ queues for notifications

Makefile

@@ -51,7 +51,12 @@ install: init
 
 .PHONY: rabbit-clean
 rabbit-clean:
-	rabbitmqadmin -f pretty_json list queues vhost name messages | jq -r '.[] | "rabbitmqadmin delete queue name=\(.name) --vhost=\(.vhost)"' | bash
+	rabbitmqadmin -f pretty_json list queues vhost name \
+		| jq -r '.[] | "rabbitmqadmin delete queue name=\(.name) --vhost=\(.vhost)"' \
+		| bash
+	rabbitmqadmin -f pretty_json list exchanges name vhost \
+		| jq -r '.[] |select(.name | startswith("amq") | not) | select (.name != "") | "rabbitmqadmin delete exchange name=\(.name) --vhost=\(.vhost)"' \
+		| bash
 
 # Clean
 .PHONY: full-clean

cassandra-schema.cql

@@ -1729,6 +1729,26 @@ CREATE TABLE gundeck_test.meta (
     AND read_repair_chance = 0.0
     AND speculative_retry = '99PERCENTILE';
 
+CREATE TABLE gundeck_test.missed_notifications (
+    user_id uuid,
+    client_id text,
+    PRIMARY KEY (user_id, client_id)
+) WITH CLUSTERING ORDER BY (client_id ASC)
+    AND bloom_filter_fp_chance = 0.01
+    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
+    AND comment = ''
+    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
+    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
+    AND crc_check_chance = 1.0
+    AND dclocal_read_repair_chance = 0.1
+    AND default_time_to_live = 0
+    AND gc_grace_seconds = 864000
+    AND max_index_interval = 2048
+    AND memtable_flush_period_in_ms = 0
+    AND min_index_interval = 128
+    AND read_repair_chance = 0.0
+    AND speculative_retry = '99PERCENTILE';
+
 CREATE TABLE gundeck_test.push (
     ptoken text,
     app text,

changelog.d/0-release-notes/WBP-10308

@@ -0,0 +1 @@
+Notifications are now also sent via RabbitMQ. Therefore, if federation is enabled, RabbitMQ is a required configuration in Brig.

changelog.d/1-api-changes/WPB-10308

@@ -0,0 +1,3 @@
+New endpoint `GET /events` for consuming events is added.
+
+When a client misses notifications because it was offline for too long, it needs to know this information so it can do a full synchronisation. This appears as the first notification in `GET /events` endpoint whenever the system detects this happening. The next acknowledgement of the message makes this notification not appear anymore until the next notification is missed.

changelog.d/5-internal/WPB-10308

@@ -0,0 +1,2 @@
+- RabbitMQ queues have been introduced for client notifications
+- New internal endpoint `POST /i/users/:uid/clients/:cid/consumable-notifications` is added

charts/background-worker/templates/configmap.yaml

@@ -21,6 +21,12 @@ data:
       host: federator
       port: 8080
 
+    cassandra:
+      endpoint:
+        host: {{ .cassandra.host }}
+        port: 9042
+      keyspace: gundeck
+
     {{- with .rabbitmq }}
     rabbitmq:
       host: {{ .host }}

charts/background-worker/values.yaml

@@ -29,6 +29,8 @@ config:
     # tlsCaSecretRef:
     #   name: <secret-name>
     #   key: <ca-attribute>
+  cassandra:
+    host: aws-cassandra
 
   backendNotificationPusher:
     pushBackoffMinWait: 10000 # in microseconds, so 10ms

charts/cannon/templates/configmap.yaml

@@ -1,25 +1,46 @@
 apiVersion: v1
 data:
+  {{- with .Values }}
   cannon.yaml: |
-    logFormat: {{ .Values.config.logFormat }}
-    logLevel: {{ .Values.config.logLevel }}
-    logNetStrings: {{ .Values.config.logNetStrings }}
+    logFormat: {{ .config.logFormat }}
+    logLevel: {{ .config.logLevel }}
+    logNetStrings: {{ .config.logNetStrings }}
 
     cannon:
       host: 0.0.0.0
-      port: {{ .Values.service.externalPort }}
+      port: {{ .service.externalPort }}
       externalHostFile: /etc/wire/cannon/externalHost/host.txt
 
     gundeck:
       host: gundeck
       port: 8080
 
+    cassandra:
+      endpoint:
+        host: {{ .config.cassandra.host }}
+        port: 9042
+      keyspace: gundeck
+
+    {{- with .config.rabbitmq }}
+    rabbitmq:
+      host: {{ .host }}
+      port: {{ .port }}
+      vHost: {{ .vHost }}
+      enableTls: {{ .enableTls }}
+      insecureSkipVerifyTls: {{ .insecureSkipVerifyTls }}
+      {{- if .tlsCaSecretRef }}
+      caCert: /etc/wire/gundeck/rabbitmq-ca/{{ .tlsCaSecretRef.key }}
+      {{- end }}
+    {{- end }}
+
     drainOpts:
-      gracePeriodSeconds: {{ .Values.config.drainOpts.gracePeriodSeconds }}
-      millisecondsBetweenBatches: {{ .Values.config.drainOpts.millisecondsBetweenBatches }}
-      minBatchSize: {{ .Values.config.drainOpts.minBatchSize }}
+      gracePeriodSeconds: {{ .config.drainOpts.gracePeriodSeconds }}
+      millisecondsBetweenBatches: {{ .config.drainOpts.millisecondsBetweenBatches }}
+      minBatchSize: {{ .config.drainOpts.minBatchSize }}
+
+    disabledAPIVersions: {{ toJson .config.disabledAPIVersions }}
+  {{- end }}
 
-    disabledAPIVersions: {{ toJson .Values.config.disabledAPIVersions }}
 
 kind: ConfigMap
 metadata:

charts/cannon/values.yaml

@@ -11,6 +11,35 @@ config:
   logLevel: Info
   logFormat: StructuredJSON
   logNetStrings: false
+  rabbitmq:
+    host: rabbitmq
+    port: 5672
+    vHost: /
+    enableTls: false
+    insecureSkipVerifyTls: false
+  cassandra:
+    host: aws-cassandra
+    # To enable TLS provide a CA:
+    # tlsCa: <CA in PEM format (can be self-signed)>
+    #
+    # Or refer to an existing secret (containing the CA):
+    # tlsCaSecretRef:
+    #   name: <secret-name>
+    #   key: <ca-attribute>
+
+  redis:
+    host: redis-ephemeral-master
+    port: 6379
+    connectionMode: "master" # master | cluster
+    enableTls: false
+    insecureSkipVerifyTls: false
+    # To configure custom TLS CA, please provide one of these:
+    # tlsCa: <CA in PEM format (can be self-signed)>
+    #
+    # Or refer to an existing secret (containing the CA):
+    # tlsCaSecretRef:
+    #   name: <secret-name>
+    #   key: <ca-attribute>
 
   # See also the section 'Controlling the speed of websocket draining during
   # cannon pod replacement' in docs/how-to/install/configuration-options.rst

charts/gundeck/templates/configmap.yaml

@@ -29,6 +29,18 @@ data:
       tlsCa: /etc/wire/gundeck/cassandra/{{- (include "tlsSecretRef" . | fromYaml).key }}
       {{- end }}
 
+    {{- with .rabbitmq }}
+    rabbitmq:
+      host: {{ .host }}
+      port: {{ .port }}
+      vHost: {{ .vHost }}
+      enableTls: {{ .enableTls }}
+      insecureSkipVerifyTls: {{ .insecureSkipVerifyTls }}
+      {{- if .tlsCaSecretRef }}
+      caCert: /etc/wire/gundeck/rabbitmq-ca/{{ .tlsCaSecretRef.key }}
+      {{- end }}
+    {{- end }}
+
     redis:
       host: {{ .redis.host }}
       port: {{ .redis.port }}

charts/gundeck/templates/deployment.yaml

@@ -39,6 +39,11 @@ spec:
         - name: "gundeck-config"
           configMap:
             name: "gundeck"
+        {{- if .Values.config.rabbitmq.tlsCaSecretRef }}
+        - name: "rabbitmq-ca"
+          secret:
+            secretName: {{ .Values.config.rabbitmq.tlsCaSecretRef.name }}
+        {{- end }}
         {{- if eq (include "useCassandraTLS" .Values.config) "true" }}
         - name: "gundeck-cassandra"
           secret:
@@ -77,7 +82,21 @@ spec:
           - name: "additional-redis-ca"
             mountPath: "/etc/wire/gundeck/additional-redis-ca/"
           {{- end }}
+          {{- if .Values.config.rabbitmq.tlsCaSecretRef }}
+          - name: "rabbitmq-ca"
+            mountPath: "/etc/wire/gundeck/rabbitmq-ca/"
+          {{- end }}
           env:
+          - name: RABBITMQ_USERNAME
+            valueFrom:
+              secretKeyRef:
+                name: gundeck
+                key: rabbitmqUsername
+          - name: RABBITMQ_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: gundeck
+                key: rabbitmqPassword
           {{- if hasKey .Values.secrets "awsKeyId" }}
           - name: AWS_ACCESS_KEY_ID
             valueFrom:

charts/gundeck/templates/secret.yaml

@@ -11,6 +11,8 @@ metadata:
 type: Opaque
 data:
   {{- with .Values.secrets }}
+  rabbitmqUsername: {{ .rabbitmq.username | b64enc | quote }}
+  rabbitmqPassword: {{ .rabbitmq.password | b64enc | quote }}
   {{- if hasKey . "awsKeyId" }}
   awsKeyId: {{ .awsKeyId | b64enc | quote }}
   {{- end }}

charts/gundeck/templates/tests/gundeck-integration.yaml

@@ -23,6 +23,11 @@ spec:
       secret:
         secretName: {{ include "redisTlsSecretName" .Values.config }}
     {{- end }}
+    {{- if .Values.config.rabbitmq.tlsCaSecretRef }}
+    - name: "rabbitmq-ca"
+      secret:
+        secretName: {{ .Values.config.rabbitmq.tlsCaSecretRef.name }}
+    {{- end }}
   containers:
   - name: integration
     # TODO: When deployed to staging (or real AWS env), _all_ tests should be run
@@ -72,6 +77,10 @@ spec:
     - name: "redis-ca"
       mountPath: "/etc/wire/gundeck/redis-ca/"
     {{- end }}
+    {{- if .Values.config.rabbitmq.tlsCaSecretRef }}
+    - name: "rabbitmq-ca"
+      mountPath: "/etc/wire/gundeck/rabbitmq-ca/"
+    {{- end }}
     env:
     # these dummy values are necessary for Amazonka's "Discover"
     - name: AWS_ACCESS_KEY_ID
@@ -82,6 +91,11 @@ spec:
       value: "eu-west-1"
     - name: TEST_XML
       value: /tmp/result.xml
+    # RabbitMQ needs dummy credentials for the tests to run
+    - name: RABBITMQ_USERNAME
+      value: "guest"
+    - name: RABBITMQ_PASSWORD
+      value: "guest"
     {{- if hasKey .Values.secrets "redisUsername" }}
     - name: REDIS_USERNAME
       valueFrom:

charts/gundeck/values.yaml

@@ -18,6 +18,13 @@ config:
   logLevel: Info
   logFormat: StructuredJSON
   logNetStrings: false
+  rabbitmq:
+    host: rabbitmq
+    port: 5672
+    adminPort: 15672
+    vHost: /
+    enableTls: false
+    insecureSkipVerifyTls: false
   cassandra:
     host: aws-cassandra
     # To enable TLS provide a CA:

charts/integration/templates/integration-integration.yaml

@@ -261,6 +261,9 @@ spec:
     - name: rabbitmq-ca
       mountPath: /etc/wire/background-worker/rabbitmq-ca
 
+    - name: rabbitmq-ca
+      mountPath: /etc/wire/gundeck/rabbitmq-ca
+
     {{- if eq (include "useCassandraTLS" .Values.config) "true" }}
     - name: "integration-cassandra"
       mountPath: "/certs"

deploy/dockerephemeral/docker-compose.yaml

@@ -271,6 +271,7 @@ services:
     ports:
       - '127.0.0.1:5671:5671'
       - '127.0.0.1:15671:15671'
+      - '127.0.0.1:15672:15672'
     volumes:
       - ./rabbitmq-config/rabbitmq.conf:/etc/rabbitmq/conf.d/20-wire.conf
       - ./rabbitmq-config/certificates:/etc/rabbitmq/certificates

hack/helm_vars/wire-server/values.yaml.gotmpl

@@ -80,7 +80,7 @@ brig:
       enableTls: true
       insecureSkipVerifyTls: false
       tlsCaSecretRef:
-        name: rabbitmq-certificate
+        name: "rabbitmq-certificate"
         key: "ca.crt"
     authSettings:
       userTokenTimeout: 120
@@ -205,7 +205,23 @@ cannon:
       memory: 512Mi
   drainTimeout: 0
   config:
+    cassandra:
+      host: {{ .Values.cassandraHost }}
+      replicaCount: 1
     disabledAPIVersions: []
+    rabbitmq:
+      port: 5671
+      adminPort: 15671
+      enableTls: true
+      insecureSkipVerifyTls: false
+      tlsCaSecretRef:
+        name: "rabbitmq-certificate"
+        key: "ca.crt"
+  secrets:
+    rabbitmq:
+      username: {{ .Values.rabbitmqUsername }}
+      password: {{ .Values.rabbitmqPassword }}
+
 cargohold:
   replicaCount: 1
   imagePullPolicy: {{ .Values.imagePullPolicy }}
@@ -252,7 +268,7 @@ galley:
       enableTls: true
       insecureSkipVerifyTls: false
       tlsCaSecretRef:
-        name: rabbitmq-certificate
+        name: "rabbitmq-certificate"
         key: "ca.crt"
     enableFederation: true # keep in sync with brig.config.enableFederation, cargohold.config.enableFederation and tags.federator!
     settings:
@@ -373,6 +389,14 @@ gundeck:
           name: "cassandra-jks-keystore"
           key: "ca.crt"
     {{- end }}
+    rabbitmq:
+      port: 5671
+      adminPort: 15671
+      enableTls: true
+      insecureSkipVerifyTls: false
+      tlsCaSecretRef:
+        name: "rabbitmq-certificate"
+        key: "ca.crt"
     redis:
       host: redis-ephemeral-master
       connectionMode: master
@@ -395,6 +419,9 @@ gundeck:
     awsKeyId: dummykey
     awsSecretKey: dummysecret
     redisPassword: very-secure-redis-master-password
+    rabbitmq:
+      username: {{ .Values.rabbitmqUsername }}
+      password: {{ .Values.rabbitmqPassword }}
   tests:
     {{- if .Values.uploadXml }}
     config:
@@ -518,13 +545,21 @@ background-worker:
       pushBackoffMinWait: 1000 # 1ms
       pushBackoffMaxWait: 500000 # 0.5s
       remotesRefreshInterval: 1000000 # 1s
+    cassandra:
+      host: {{ .Values.cassandraHost }}
+      replicaCount: 1
+      {{- if .Values.useK8ssandraSSL.enabled }}
+      tlsCaSecretRef:
+          name: "cassandra-jks-keystore"
+          key: "ca.crt"
+      {{- end }}
     rabbitmq:
       port: 5671
       adminPort: 15671
       enableTls: true
       insecureSkipVerifyTls: false
       tlsCaSecretRef:
-        name: rabbitmq-certificate
+        name: "rabbitmq-certificate"
         key: "ca.crt"
   secrets:
     rabbitmq:

hack/helmfile.yaml

@@ -1,5 +1,5 @@
 ---
-# This helfile is used for the setup of two ephemeral backends on kubernetes
+# This helmfile is used for the setup of two ephemeral backends on kubernetes
 # during integration testing (including federation integration tests spanning
 # over 2 backends)
 # This helmfile is used via the './hack/bin/integration-setup-federation.sh' via

integration/integration.cabal

@@ -121,6 +121,7 @@ library
     Test.Demo
     Test.EJPD
     Test.Errors
+    Test.Events
     Test.ExternalPartner
     Test.FeatureFlags
     Test.FeatureFlags.AppLock