feat(rds): add alerting on CA certificates expiration

This commit adds a new alert that will trigger if it detects any instance
with a CA certificate with an expiration date scheduled within the next
15 days.

charts/prometheus-rds-alerts/prometheus_tests/RDSCACertificateCloseToExpiration.yml

@@ -0,0 +1,43 @@
+rule_files:
+    - rules.yml
+
+evaluation_interval: 1m
+
+tests:
+
+    - name: RDSCACertificateCloseToExpiration
+      interval: 1m
+      input_series:
+          - series: 'rds_certificate_expiry_timestamp_seconds{aws_account_id="111111111111",aws_region="eu-west-3",dbidentifier="db1"}'
+            values: '1728000' # 20 days
+          - series: 'rds_certificate_expiry_timestamp_seconds{aws_account_id="111111111111",aws_region="eu-west-3",dbidentifier="db2"}'
+            values: '2629800' # 1 month
+          - series: 'rds_certificate_expiry_timestamp_seconds{aws_account_id="111111111111",aws_region="eu-west-1",dbidentifier="db1"}'
+            values: '1728000' # 20 days
+          - series: 'rds_certificate_expiry_timestamp_seconds{aws_account_id="222222222222",aws_region="eu-west-3",dbidentifier="db1"}'
+            values: '2629800' # 1 month
+      alert_rule_test:
+          - alertname: RDSCACertificateCloseToExpiration
+            eval_time: 4d
+            exp_alerts: []
+          - alertname: RDSCACertificateCloseToExpiration
+            eval_time: 6d
+            exp_alerts:
+                - exp_labels:
+                      aws_account_id: 111111111111
+                      aws_region: eu-west-3
+                      dbidentifier: db1
+                      severity: warning
+                  exp_annotations:
+                      description: "1 instance(s) of the AWS account ID=111111111111 in region=eu-west-3 use(s) a certificate with an expiration date inferior to 15 days"
+                      summary: "RDS instance(s) use(s) a certificate with an expiration date inferior to 15 days"
+                      runbook_url: "https://qonto.github.io/database-monitoring-framework/0.0.0/runbooks/rds/RDSCACertificateCloseToExpiration"
+                - exp_labels:
+                      aws_account_id: 111111111111
+                      aws_region: eu-west-1
+                      dbidentifier: db1
+                      severity: warning
+                  exp_annotations:
+                      description: "1 instance(s) of the AWS account ID=111111111111 in region=eu-west-1 use(s) a certificate with an expiration date inferior to 15 days"
+                      summary: "RDS instance(s) use(s) a certificate with an expiration date inferior to 15 days"
+                      runbook_url: "https://qonto.github.io/database-monitoring-framework/0.0.0/runbooks/rds/RDSCACertificateCloseToExpiration"

charts/prometheus-rds-alerts/values.yaml

@@ -184,3 +184,13 @@ rules:
       description: "{{ $labels.dbidentifier }} has forced maintenance"
     pintComments:
       - disable promql/series
+
+  RDSCACertificateCloseToExpiration:
+    expr: |
+      # 1296000 seconds = 15 days
+      count by (aws_account_id, aws_region) (rds_certificate_expiry_timestamp_seconds - time() <= 1296000) > 0
+    labels:
+      severity: critical
+    annotations:
+      summary: "RDS instance(s) use(s) a certificate with an expiration date inferior to 15 days"
+      description: "{{ $value }} instance(s) of the AWS account ID={{ $labels.aws_account_id}} in region={{ $labels.aws_region }} use(s) a certificate with an expiration date inferior to 15 days"

content/runbooks/rds/RDSCACertificateCloseToExpiration.md

@@ -0,0 +1,59 @@
+---
+title: CA Certificate Close to Expiration
+---
+
+# RDSCACertificateCloseToExpiration
+
+## Meaning
+
+Alert is triggered when an RDS instance is detected using a CA certificate which is going to expire in less than 15 days.
+
+## Impact
+
+If the certificate is not renewed before expiration, all attempts to initiate an SSL/TLS connection to the RDS instance will fail.
+
+## Diagnosis
+
+- Identify the instance(s) concerned by either:
+  - opening the `RDS instances` dashboard
+  - or using the following AWS CLI command
+
+```bash
+aws rds describe-db-instances | jq '
+  [
+    .DBInstances[] |
+    {
+      db_instance_identifier: .DBInstanceIdentifier,
+      ca_certificate_identifier: .CACertificateIdentifier,
+      ca_certificate_valid_until: .CertificateDetails.ValidTill
+    } |
+    (now + 1296000) as $date |
+    select (
+      (.ca_certificate_valid_until | split("+")[0] + "Z" | fromdate) < $date
+    )
+  ]'
+```
+
+Note: `1296000` seconds = 15 days
+
+## Mitigation
+
+- Renew your certificate for the instances retrieved above by running:
+
+```bash
+aws rds modify-db-instance \
+    --db-instance-identifier <your_db_instance> \
+    --ca-certificate-identifier <your_new_certificate>
+```
+
+Use the `--apply-immediately` flag if you wish to change the certificate immediately, otherwise it will apply during your next scheduled maintenance window.
+
+Moving away from `rds-ca-2019`, we recommend using the `rds-ca-rsa2048-g1` certificate authority which:
+
+- has the same properties as `rds-ca-2019` (2048 private key, SHA256 signing alg.) so no risk of incompatibility
+- is valid until 2061
+
+## Additional resources
+
+- [Using SSL with RDS](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.SSL.html)
+- [SSL Certificate Rotation](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.SSL-certificate-rotation.html)