[filebeat] Add SSL and AUTH username support for Redis input

[gpop63]

Overview

This PR adds SSL and username AUTH support for redis input.

Checklist

• My code follows the style guidelines of this project
• I have commented my code, particularly in hard-to-understand areas
• I have made corresponding changes to the documentation
• I have made corresponding change to the default configuration files
• I have added tests that prove my fix is effective or that my feature works
• I have added an entry in CHANGELOG.next.asciidoc or CHANGELOG-developer.next.asciidoc.

Disruptive User Impact

Author's Checklist

• [ ]

How to test this PR locally

Related issues

• Closes: filebeat/input/redis: Add AUTH (username) and SSL/TLS support #39412

Use cases

Screenshots

Logs

[mergify]

This pull request is now in conflicts. Could you fix it? 🙏
To fixup this pull request, you can check out it locally. See documentation: https://help.github.com/articles/checking-out-pull-requests-locally/

git fetch upstream
git checkout -b add_redis-tls_filebeat upstream/add_redis-tls_filebeat
git merge upstream/main
git push upstream add_redis-tls_filebeat

[mergify]

This pull request does not have a backport label.
If this is a bug or security fix, could you label this PR @gpop63? 🙏.
For such, you'll need to label your PR with:

• The upcoming major version of the Elastic Stack
• The upcoming minor version of the Elastic Stack (if you're not pushing a breaking change)

To fixup this pull request, you need to add the backport labels for the needed
branches, such as:

• backport-v8./d.0 is the label to automatically backport to the 8./d branch. /d is the digit

[gpop63]

How to test

Generate certificates sh certs.sh.

openssl.cnf

[req]
distinguished_name = req_distinguished_name
req_extensions = v3_req

[req_distinguished_name]
commonName = Common Name
commonName_default = localhost

[v3_req]
subjectAltName = @alt_names

[alt_names]
IP.1 = 0.0.0.0
DNS.1 = localhost

certs.sh

#!/bin/bash

# Set the certificates directory
CERTS_DIR="./certs"

# Create certs directory
mkdir -p "$CERTS_DIR"

# Set OpenSSL subject information
OPENSSL_SUBJ="/C=US/ST=California/L=Santa Clara"
OPENSSL_CA="${OPENSSL_SUBJ}/CN=fake-CA"
OPENSSL_SERVER="${OPENSSL_SUBJ}/CN=fake-server"

# Generate root CA
openssl genrsa 2048 > "$CERTS_DIR/root-ca-key.pem"

openssl req -new -x509 -nodes -days 3600 \
    -subj "${OPENSSL_CA}" \
    -key "$CERTS_DIR/root-ca-key.pem" -out "$CERTS_DIR/root-ca.pem"

# Generate server certificate
openssl req -newkey rsa:2048 -days 3600 -nodes \
    -subj "${OPENSSL_SERVER}" \
    -keyout "$CERTS_DIR/server-key.pem" -out "$CERTS_DIR/server-req.pem" \
    -config openssl.cnf

openssl rsa -in "$CERTS_DIR/server-key.pem" -out "$CERTS_DIR/server-key.pem"

openssl x509 -req -in "$CERTS_DIR/server-req.pem" -days 3600 \
    -CA "$CERTS_DIR/root-ca.pem" -CAkey "$CERTS_DIR/root-ca-key.pem" \
    -set_serial 01 -out "$CERTS_DIR/server-cert.pem" \
    -extensions v3_req -extfile openssl.cnf

openssl verify -CAfile "$CERTS_DIR/root-ca.pem" "$CERTS_DIR/server-cert.pem"

echo "Certificate generation complete. Certificates are stored in $CERTS_DIR"

docker-compose.redis.yml

services:
  redis:
    image: redis:latest
    command: redis-server /usr/local/etc/redis/redis.conf
    ports:
      - "6379:6379"
    volumes:
      - ./redis.conf:/usr/local/etc/redis/redis.conf
      - ./certs:/certs
    environment:
      - REDIS_PASSWORD=password

redis.conf

port 0
tls-port 6379
tls-cert-file /certs/server-cert.pem
tls-key-file /certs/server-key.pem
tls-ca-cert-file /certs/root-ca.pem
requirepass password
user default on >password ~* &* +@all
slowlog-log-slower-than 100
slowlog-max-len 128

redis.yml

- module: redis
  slowlog:
    enabled: true
    var.hosts: ["localhost:6379"]
    var.password: password

slowlog.yml

type: redis
hosts:
{{ range $i, $host := .hosts }}
 - {{$host}}
{{ end }}
password: {{ .password }}
ssl.enabled: true
ssl.certificate_authorities: ["/path/to/certs/root-ca.pem"]
ssl.certificate: "/path/to/certs/server-cert.pem"
ssl.key: "/path/to/certs/server-key.pem"
processors:
  - add_fields:
      target: ''
      fields:
        ecs.version: 1.12.0

redis-cli -h localhost -p 6379 --tls --cert /certs/server-cert.pem --key /certs/server-key.pem --cacert /certs/root-ca.pem

Run a command inside redis container to generate some logs: EVAL "local i = 0 for j=1,500000 do i = i + j end return i" 0

[shmsr]

The changes look really good. I'll take a good look again in the morning.

Appreciate making the script and using openssl directly. But if someone wants to follow easier steps (for testing) then certificate generation is included in Redis' repo. See: #35240 (comment) (see: "how to test this PR locally")

[elasticmachine]

Pinging @elastic/elastic-agent-data-plane (Team:Elastic-Agent-Data-Plane)

[shmsr]

Please update the doc: https://github.com/elastic/beats/blob/21457f1485f83a02c78e8002fea17b385d20b139/filebeat/docs/inputs/input-redis.asciidoc

[shmsr]

Please update the doc: https://github.com/elastic/beats/blob/21457f1485f83a02c78e8002fea17b385d20b139/filebeat/docs/inputs/input-redis.asciidoc

Add username doc as well. It is important.

[belimawr]

The implementation looks good, but I'm missing some tests. An integration tests would probably be better.

[shmsr]

Tests look good but here are some nitpicks:

diff --git a/filebeat/input/redis/redis_integration_test.go b/filebeat/input/redis/redis_integration_test.go
index 1d46ed0e5c..f8968f0ea5 100644
--- a/filebeat/input/redis/redis_integration_test.go
+++ b/filebeat/input/redis/redis_integration_test.go
@@ -20,6 +20,7 @@
 package redis
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"sync"
@@ -97,10 +98,12 @@ func TestInput(t *testing.T) {
 
 	// Route input events through our captor instead of sending through ES.
 	eventsCh := make(chan beat.Event)
-	defer close(eventsCh)
-
 	captor := newEventCaptor(eventsCh)
-	defer captor.Close()
+
+	t.Cleanup(func() {
+		close(eventsCh)
+		captor.Close()
+	})
 
 	connector := channel.ConnectorFunc(func(_ *conf.C, _ beat.ClientConfig) (channel.Outleter, error) {
 		return channel.SubOutlet(captor), nil
@@ -117,27 +120,32 @@ func TestInput(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, input)
 
+	t.Cleanup(func() {
+		input.Stop()
+	})
+
 	// Run the input.
 	input.Run()
 
-	// Create Redis Client
+	// Create Redis client.
 	redisClient := createRedisClient(t)
 
-	// Verify that event has been received
-	verifiedCh := make(chan struct{})
-	defer close(verifiedCh)
-
-	emitInputData(t, verifiedCh, redisClient)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
 
-	event := <-eventsCh
-	verifiedCh <- struct{}{}
+	emitInputData(t, ctx, redisClient)
 
-	val, err := event.GetValue("message")
-	require.NoError(t, err)
-	require.Equal(t, message, val)
+	select {
+	case event := <-eventsCh:
+		val, err := event.GetValue("message")
+		require.NoError(t, err)
+		require.Equal(t, message, val)
+	case <-time.After(30 * time.Second):
+		t.Fatal("Timeout waiting for event")
+	}
 }
 
-func emitInputData(t *testing.T, verifiedCh <-chan struct{}, pool *rd.Pool) {
+func emitInputData(t *testing.T, ctx context.Context, pool *rd.Pool) {
 	script := "local i = 0 for j=1,500000 do i = i + j end return i"
 
 	go func() {
@@ -145,11 +153,14 @@ func emitInputData(t *testing.T, verifiedCh <-chan struct{}, pool *rd.Pool) {
 		defer ticker.Stop()
 
 		conn := pool.Get()
-		defer conn.Close()
+		defer func() {
+			err := conn.Close()
+			require.NoError(t, err)
+		}()
 
 		for {
 			select {
-			case <-verifiedCh:
+			case <-ctx.Done():
 				return
 			case <-ticker.C:
 				_, err := conn.Do("EVAL", script, 0)
@@ -177,7 +188,9 @@ func createRedisClient(t *testing.T) *rd.Pool {
 	}
 
 	return &rd.Pool{
-		MaxIdle:     10,
+		MaxActive:   10,
+		MaxIdle:     5,
+		Wait:        true,
 		IdleTimeout: idleTimeout,
 		Dial: func() (rd.Conn, error) {
 			dialOptions := []rd.DialOption{