Feature/32089 generic sql enchancement

CHANGELOG.next.asciidoc

@@ -113,6 +113,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 - Enhance Oracle Module: Connection string for Oracle does not handle special characters properly {issue}24609[24609] {pull}31368[#31368]
 - Enhance Oracle Module: New sysmetric metricset {issue}30946[30946] {pull}31462[#31462]
 - Upgrade Mongodb library in Beats to v5 {pull}31185[31185]
+- Upgrade Generic SQL to support new connection string and special character support {issue}32089[32089]
 
 *Packetbeat*
 

x-pack/metricbeat/module/sql/docker-compose.yml

@@ -10,3 +10,8 @@ services:
     extends:
       file: ../../../../metricbeat/module/postgresql/docker-compose.yml
       service: postgresql
+
+  oracle:
+    extends:
+      file: ../../../../x-pack/metricbeat/module/oracle/docker-compose.yml
+      service: oracle

x-pack/metricbeat/module/sql/query/dsn.go

@@ -5,17 +5,54 @@
 package query
 
 import (
+	"fmt"
 	"net/url"
 
 	"github.com/go-sql-driver/mysql"
+	"github.com/godror/godror"
+	"github.com/godror/godror/dsn"
 
 	"github.com/elastic/beats/v7/metricbeat/mb"
 )
 
+// ConnectionDetails contains all possible data that can be used to create a connection with
+// an Oracle db
+type ConnectionDetails struct {
+	Username string `config:"username"`
+	Password string `config:"password"`
+	Driver   string `config:"driver"`
+}
+
 // ParseDSN tries to parse the host
 func ParseDSN(mod mb.Module, host string) (mb.HostData, error) {
 	// TODO: Add support for `username` and `password` as module options
+	config := ConnectionDetails{}
+	if err := mod.UnpackConfig(&config); err != nil {
+		return mb.HostData{}, fmt.Errorf("error parsing config file: %w", err)
+	}
 
+	if config.Driver == "oracle" {
+		params, err := godror.ParseDSN(host)
+		if err != nil {
+			return mb.HostData{}, fmt.Errorf("error trying to parse connection string in field 'hosts': %w", err)
+		}
+
+		if params.Username == "" {
+			params.Username = config.Username
+		}
+
+		if params.Password.Secret() == "" {
+			params.StandaloneConnection = true
+			params.Password = dsn.NewPassword(config.Password)
+		}
+		return mb.HostData{
+			URI:          params.StringWithPassword(),
+			SanitizedURI: params.ConnectString,
+			Host:         params.String(),
+			User:         params.Username,
+			Password:     params.Password.Secret(),
+		}, nil
+	}
 	sanitized := sanitize(host)
 
 	return mb.HostData{

x-pack/metricbeat/module/sql/query/query_integration_test.go

@@ -10,8 +10,11 @@ package query
 import (
 	"fmt"
 	"net"
+	"os"
 	"testing"
+	"time"
 
+	"github.com/godror/godror"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -48,7 +51,6 @@ func TestMySQL(t *testing.T) {
 		Host:      mysql.GetMySQLEnvDSN(service.Host()),
 		Assertion: assertFieldNotContains("service.address", ":test@"),
 	}
-
 	t.Run("fetch", func(t *testing.T) {
 		testFetch(t, cfg)
 	})
@@ -186,6 +188,27 @@ func TestPostgreSQL(t *testing.T) {
 	})
 }
 
+func TestOracle(t *testing.T) {
+	service := compose.EnsureUp(t, "oracle")
+	host, port, _ := net.SplitHostPort(service.Host())
+	cfg := testFetchConfig{
+		config: config{
+			Driver:         "oracle",
+			Query:          `SELECT name, physical_reads, db_block_gets, consistent_gets, 1 - (physical_reads / (db_block_gets + consistent_gets)) "Hit_Ratio" FROM V$BUFFER_POOL_STATISTICS`,
+			ResponseFormat: tableResponseFormat,
+
+			RawData: rawData{
+				Enabled: true,
+			},
+		},
+		Host:      GetOracleConnectionDetails(host, port),
+		Assertion: assertFieldContainsFloat64("Hit_Ratio", 0.0),
+	}
+	t.Run("fetch", func(t *testing.T) {
+		testFetch(t, cfg)
+	})
+}
+
 func testFetch(t *testing.T, cfg testFetchConfig) {
 	m := mbtest.NewFetcher(t, getConfig(cfg))
 	events, errs := m.FetchEvents()
@@ -230,3 +253,56 @@ func assertFieldNotContains(field, s string) func(t *testing.T, event beat.Event
 		require.NotContains(t, value.(string), s)
 	}
 }
+
+func assertFieldContainsFloat64(field string, limit float64) func(t *testing.T, event beat.Event) {
+	return func(t *testing.T, event beat.Event) {
+		value, err := event.GetValue("sql.metrics.hit_ratio")
+		assert.NoError(t, err)
+		require.GreaterOrEqual(t, value.(float64), limit)
+	}
+}
+
+func GetOracleConnectionDetails(host string, port string) string {
+
+	params, _ := godror.ParseDSN(GetOracleConnectString(host, port))
+	return params.StringWithPassword()
+}
+
+// GetOracleEnvServiceName returns the service name to use with Oracle testing server or the value of the environment variable ORACLE_SERVICE_NAME if not empty
+func GetOracleEnvServiceName() string {
+	serviceName := os.Getenv("ORACLE_SERVICE_NAME")
+
+	if len(serviceName) == 0 {
+		serviceName = "ORCLCDB.localdomain"
+	}
+	return serviceName
+}
+
+// GetOracleEnvUsername returns the username to use with Oracle testing server or the value of the environment variable ORACLE_USERNAME if not empty
+func GetOracleEnvUsername() string {
+	username := os.Getenv("ORACLE_USERNAME")
+
+	if len(username) == 0 {
+		username = "sys"
+	}
+	return username
+}
+
+// GetOracleEnvUsername returns the port of the Oracle server or the value of the environment variable ORACLE_PASSWORD if not empty
+func GetOracleEnvPassword() string {
+	password := os.Getenv("ORACLE_PASSWORD")
+
+	if len(password) == 0 {
+		password = "Oradoc_db1" // #nosec
+	}
+	return password
+}
+
+func GetOracleConnectString(host string, port string) string {
+	time.Sleep(300 * time.Second)
+	connectString := os.Getenv("ORACLE_CONNECT_STRING")
+	if len(connectString) == 0 {
+		connectString = fmt.Sprintf("%s/%s@%s:%s/%s as sysdba", GetOracleEnvUsername(), GetOracleEnvPassword(), host, port, GetOracleEnvServiceName())
+	}
+	return connectString
+}

x-pack/metricbeat/module/sql/query/test_sql_oracle.py

@@ -0,0 +1,50 @@
+import os
+import sys
+import unittest
+import time
+from xpack_metricbeat import XPackTest, metricbeat
+
+
+class Test(XPackTest):
+
+    COMPOSE_SERVICES = ['oracle']
+
+    @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
+    def test_query(self):
+        """
+        sql oracle custom query test
+        """
+        self.render_config_template(modules=[{
+            "name": "sql",
+            "metricsets": ["query"],
+            "hosts": self.get_hosts(),
+            "period": "5s",
+            "additional_content": """
+  driver: oracle
+  sql_query: 'SELECT name, physical_reads, db_block_gets, consistent_gets, 1 - (physical_reads / (db_block_gets + consistent_gets)) FROM V$BUFFER_POOL_STATISTICS'
+  sql_response_format: table"""
+        }])
+        proc = self.start_beat(home=self.beat_path)
+        self.wait_until(lambda: self.output_lines() > 0)
+        self.wait_until(lambda: self.check_for_events(), max_timeout=300)
+        proc.check_kill_and_wait()
+        self.assert_no_logged_warnings()
+
+        output = self.read_output_json()
+        self.assertGreater(len(output), 0)
+
+        event_valid_counter = False
+        for evt in output:
+            if evt.get("sql") and evt["sql"].get("query"):
+                event_valid_counter = True
+
+    def check_for_events(self):
+        output = self.read_output_json()
+        for evt in output:
+            if evt.get("sql") and evt["sql"].get("query"):
+                return True
+
+        return False
+
+    def get_hosts(self):
+        return ['user="sys" password="Oradoc_db1" connectString="{}/ORCLPDB1.localdomain" sysdba=true'.format(self.compose_host())]