Add dead letter exchanges and queues to rabbitmq

There is a need from the govuk-chat app to be able to control the
amount of delay that is used when a message from the
govuk_chat_pulished_documents is retried. At the moment, a message
will be retried immediately. The desired outcome is that we could
have a message be sent away for a fixed amount of time before it
is back in the queue.
The way we do this is we set a dead letter exchange for the
govuk_chat_published_documents called govuk_chat_retry_dlx
This dlx then send all incoming messages to a queue called
govuk_chat_retry. this queue has a message-ttl value set and a
dead letter exchange set to an exchange called govuk_chat_dlx.
This last exchange route all incoming messages back to
govuk_chat_published_documents.

The result of all this is that when a message is rejected in
govuk_chat_published_documents, it ends up in the govuk_chat_retry
queue, it will then wait here for the message-ttl value before
expiring. After expiration it is sent back to
govuk_chat_published_documents.

terraform/projects/app-publishing-amazonmq/publishing-rabbitmq-schema.json.tpl

@@ -120,6 +120,31 @@
       },
       "priority": 0
     },
+    {
+      "vhost": "publishing",
+      "name": "govuk_chat_retry_dlx",
+      "pattern": "govuk_chat_published_documents",
+      "apply-to": "queues",
+      "definition": {
+        "dead-letter-exchange": "govuk_chat_retry_dlx",
+        "ha-mode": "all",
+        "ha-sync-mode": "automatic"
+      },
+      "priority": 0
+    },
+    {
+      "vhost": "publishing",
+      "name": "govuk_chat_retry",
+      "pattern": "govuk_chat_retry",
+      "apply-to": "queues",
+      "definition": {
+        "dead-letter-exchange": "govuk_chat_dlx",
+        "message-ttl":"${govuk_chat_retry_message-ttl}",
+        "ha-mode": "all",
+        "ha-sync-mode": "automatic"
+      },
+      "priority": 0
+    },
     {
       "vhost": "publishing",
       "name": "ha-all",
@@ -210,6 +235,13 @@
       "auto_delete": false,
       "arguments": {}
     },
+    {
+      "name": "govuk_chat_retry",
+      "vhost": "publishing",
+      "durable": true,
+      "auto_delete": false,
+      "arguments": {}
+    },
     {
       "name": "email_unpublishing",
       "vhost": "publishing",
@@ -228,6 +260,24 @@
       "internal": false,
       "arguments": {}
     },
+    {
+      "name": "govuk_chat_retry_dlx",
+      "vhost": "publishing",
+      "type": "topic",
+      "durable": true,
+      "auto_delete": false,
+      "internal": false,
+      "arguments": {}
+    },
+    {
+      "name": "govuk_chat_dlx",
+      "vhost": "publishing",
+      "type": "topic",
+      "durable": true,
+      "auto_delete": false,
+      "internal": false,
+      "arguments": {}
+    },
     {
       "name": "content_data_api_dlx",
       "vhost": "publishing",
@@ -414,6 +464,22 @@
       "destination_type": "queue",
       "routing_key": "*.unpublish.#",
       "arguments": {}
+    },
+    {
+      "source": "govuk_chat_retry_dlx",
+      "vhost": "publishing",
+      "destination": "govuk_chat_retry",
+      "destination_type": "queue",
+      "routing_key": "#",
+      "arguments": {}
+    },
+    {
+      "source": "govuk_chat_dlx",
+      "vhost": "publishing",
+      "destination": "govuk_chat_published_documents",
+      "destination_type": "queue",
+      "routing_key": "#",
+      "arguments": {}
     }
   ]
 }

terraform/projects/app-publishing-amazonmq/variables.tf

@@ -45,6 +45,12 @@ variable "publishing_amazonmq_broker_name" {
   default     = "PublishingMQ"
 }
 
+variable "govuk_chat_retry_message-ttl" {
+  type        = string
+  description = "Time in miliseconds before messages in the govuk_chat_retry queue expires and are sent back to the govuk_chat_published_ducoments queue through the dead letter mechanism"
+  default     = "300000"
+}
+
 variable "elb_internal_certname" {
   type        = string
   description = "The ACM cert domain name to find the ARN of, so that it can be applied to the Network Load Balancer"