Feature/32089 generic sql enchancement

CHANGELOG.next.asciidoc

@@ -92,6 +92,7 @@ https://github.com/elastic/beats/compare/v8.2.0\...main[Check the HEAD diff]
 - Fix logstash cgroup mappings {pull}33131[33131]
 - Remove unused `elasticsearch.node_stats.indices.bulk.avg_time.bytes` mapping {pull}33263[33263]
 - Add tags to events based on parsed identifier. {pull}33472[33472]
+- Upgrade Generic SQL module to support new connection string and special character support for Oracle Database. {issue}32089[32089]
 
 *Packetbeat*
 

metricbeat/docs/modules/sql.asciidoc

@@ -52,6 +52,7 @@ each row.
 
 `sql_query`:: The single query you want to run. (`Backward Compatibility`). Also provide corresponding  `sql_response_format`: either `variables` or `table`
 
+
 [float]
 === Example: capture Innodb-related metrics
 
@@ -228,7 +229,7 @@ for each row. For example, this event is created for the first row:
 ----
 
 [float]
-=== Example: get the buffer catch hit ratio in Oracle
+==== Example: get the buffer catch hit ratio in Oracle
 
 This `sql.yml` configuration shows how to get the buffer cache hit ratio:
 
@@ -298,6 +299,8 @@ The example generates this event:
     "ephemeral_id": "49e00060-0fa4-4b34-80f1-446881f7a788"
   }
 }
+
+
 ----
 
 [float]
@@ -551,6 +554,52 @@ This creates a combined event as below, where `blks_hit`, `blks_read`, `checkpoi
 }
 ----
 
+== Host Setup (Oracle)
+
+==== Oracle Database Connection Pre-requisites
+
+To get connected with the Oracle Database ORACLE_SID, ORACLE_BASE, ORACLE_HOME environment variables should be set.
+
+For example: Let’s consider Oracle Database 21c installation using RPM manually by following https://docs.oracle.com/en/database/oracle/oracle-database/21/ladbi/running-rpm-packages-to-install-oracle-database.html[this] link, environment variables should be set as follows:
+
+    `ORACLE_BASE=/opt/oracle/oradata`
+    `ORACLE_HOME=/opt/oracle/product/21c/dbhome_1`
+Also, add `ORACLE_HOME/bin` to the `PATH` environment variable. 
+
+===== Oracle Instant Client Installation
+
+Oracle Instant Client enables development and deployment of applications that connect to Oracle Database. The Instant Client libraries provide the necessary network connectivity and advanced data features to make full use of Oracle Database. If you have OCI Oracle server which comes with these libraries pre-installed, you don't need a separate client installation.
+
+The OCI library install few Client Shared Libraries that must be referenced on the machine where Metricbeat is installed. Please follow https://docs.oracle.com/en/database/oracle/oracle-database/21/lacli/install-instant-client-using-zip.html#GUID-D3DCB4FB-D3CA-4C25-BE48-3A1FB5A22E84[this] link for OCI Instant Client set up. The OCI Instant Client is available with the Oracle Universal Installer, RPM file or ZIP file. Download links can be found https://www.oracle.com/database/technologies/instant-client/downloads.html[here].
+
+===== Enable Oracle Listener
+
+The Oracle listener is a service that runs on the database host and receives requests from Oracle clients. Make sure that https://docs.oracle.com/cd/B19306_01/network.102/b14213/lsnrctl.htm[listener] should be running. 
+To check if the listener is running or not, run: 
+
+`lsnrctl STATUS`
+
+If the listener is not running, use the command to start:
+
+`lsnrctl START`
+
+Then, Metricbeat can be launched.
+
+==== Host Configuration for Oracle
+
+The following two types of host configurations are supported:
+
+1. Old style host configuration for backwards compatibility:
+    a. `hosts: ["user/[email protected]:1521/ORCLPDB1.localdomain"]`
+    b. `hosts: ["user/[email protected]:1521/ORCLPDB1.localdomain as sysdba"]`
+
+2. DSN host configuration:
+    a. `hosts: ['user="user" password="pass" connectString="0.0.0.0:1521/ORCLPDB1.localdomain"']`
+    b. `hosts: ['user="user" password="password" connectString="host:port/service_name" sysdba=true']`
+
+Note: If the password contains the backslash (`\`) character, it must be escaped with a backslash. For example, if the password is `my\_password`, it should be written as `my\\_password`.
+
+
 :edit_url:
 
 [float]

x-pack/metricbeat/module/sql/_meta/docs.asciidoc

@@ -38,6 +38,7 @@ each row.
 
 `sql_query`:: The single query you want to run. (`Backward Compatibility`). Also provide corresponding  `sql_response_format`: either `variables` or `table`
 
+
 [float]
 === Example: capture Innodb-related metrics
 
@@ -214,7 +215,7 @@ for each row. For example, this event is created for the first row:
 ----
 
 [float]
-=== Example: get the buffer catch hit ratio in Oracle
+==== Example: get the buffer catch hit ratio in Oracle
 
 This `sql.yml` configuration shows how to get the buffer cache hit ratio:
 
@@ -284,6 +285,8 @@ The example generates this event:
     "ephemeral_id": "49e00060-0fa4-4b34-80f1-446881f7a788"
   }
 }
+
+
 ----
 
 [float]
@@ -535,4 +538,49 @@ This creates a combined event as below, where `blks_hit`, `blks_read`, `checkpoi
     "dataset": "sql.query"
   }
 }
-----
+----
+
+== Host Setup (Oracle)
+
+==== Oracle Database Connection Pre-requisites
+
+To get connected with the Oracle Database ORACLE_SID, ORACLE_BASE, ORACLE_HOME environment variables should be set.
+
+For example: Let’s consider Oracle Database 21c installation using RPM manually by following https://docs.oracle.com/en/database/oracle/oracle-database/21/ladbi/running-rpm-packages-to-install-oracle-database.html[this] link, environment variables should be set as follows:
+
+    `ORACLE_BASE=/opt/oracle/oradata`
+    `ORACLE_HOME=/opt/oracle/product/21c/dbhome_1`
+Also, add `ORACLE_HOME/bin` to the `PATH` environment variable. 
+
+===== Oracle Instant Client Installation
+
+Oracle Instant Client enables development and deployment of applications that connect to Oracle Database. The Instant Client libraries provide the necessary network connectivity and advanced data features to make full use of Oracle Database. If you have OCI Oracle server which comes with these libraries pre-installed, you don't need a separate client installation.
+
+The OCI library install few Client Shared Libraries that must be referenced on the machine where Metricbeat is installed. Please follow https://docs.oracle.com/en/database/oracle/oracle-database/21/lacli/install-instant-client-using-zip.html#GUID-D3DCB4FB-D3CA-4C25-BE48-3A1FB5A22E84[this] link for OCI Instant Client set up. The OCI Instant Client is available with the Oracle Universal Installer, RPM file or ZIP file. Download links can be found https://www.oracle.com/database/technologies/instant-client/downloads.html[here].
+
+===== Enable Oracle Listener
+
+The Oracle listener is a service that runs on the database host and receives requests from Oracle clients. Make sure that https://docs.oracle.com/cd/B19306_01/network.102/b14213/lsnrctl.htm[listener] should be running. 
+To check if the listener is running or not, run: 
+
+`lsnrctl STATUS`
+
+If the listener is not running, use the command to start:
+
+`lsnrctl START`
+
+Then, Metricbeat can be launched.
+
+==== Host Configuration for Oracle
+
+The following two types of host configurations are supported:
+
+1. Old style host configuration for backwards compatibility:
+    a. `hosts: ["user/[email protected]:1521/ORCLPDB1.localdomain"]`
+    b. `hosts: ["user/[email protected]:1521/ORCLPDB1.localdomain as sysdba"]`
+
+2. DSN host configuration:
+    a. `hosts: ['user="user" password="pass" connectString="0.0.0.0:1521/ORCLPDB1.localdomain"']`
+    b. `hosts: ['user="user" password="password" connectString="host:port/service_name" sysdba=true']`
+
+Note: If the password contains the backslash (`\`) character, it must be escaped with a backslash. For example, if the password is `my\_password`, it should be written as `my\\_password`.

x-pack/metricbeat/module/sql/docker-compose.yml

@@ -10,3 +10,8 @@ services:
     extends:
       file: ../../../../metricbeat/module/postgresql/docker-compose.yml
       service: postgresql
+
+  oracle:
+    extends:
+      file: ../../../../x-pack/metricbeat/module/oracle/docker-compose.yml
+      service: oracle

x-pack/metricbeat/module/sql/query/dsn.go

@@ -5,19 +5,52 @@
 package query
 
 import (
+	"fmt"
 	"net/url"
 
 	"github.com/go-sql-driver/mysql"
+	"github.com/godror/godror"
+	"github.com/godror/godror/dsn"
 
 	"github.com/elastic/beats/v7/metricbeat/mb"
 )
 
+// ConnectionDetails contains all possible data that can be used to create a connection with
+// an Oracle db
+type ConnectionDetails struct {
+	Username string `config:"username"`
+	Password string `config:"password"`
+	Driver   string `config:"driver"`
+}
+
 // ParseDSN tries to parse the host
 func ParseDSN(mod mb.Module, host string) (mb.HostData, error) {
 	// TODO: Add support for `username` and `password` as module options
-
+	config := ConnectionDetails{}
+	if err := mod.UnpackConfig(&config); err != nil {
+		return mb.HostData{}, fmt.Errorf("error parsing config file: %w", err)
+	}
+	if config.Driver == "oracle" {
+		params, err := godror.ParseDSN(host)
+		if err != nil {
+			return mb.HostData{}, fmt.Errorf("error trying to parse connection string in field 'hosts': %w", err)
+		}
+		if params.Username == "" {
+			params.Username = config.Username
+		}
+		if params.Password.Secret() == "" {
+			params.StandaloneConnection = true
+			params.Password = dsn.NewPassword(config.Password)
+		}
+		return mb.HostData{
+			URI:          params.StringWithPassword(),
+			SanitizedURI: params.ConnectString,
+			Host:         params.String(),
+			User:         params.Username,
+			Password:     params.Password.Secret(),
+		}, nil
+	}
 	sanitized := sanitize(host)
-
 	return mb.HostData{
 		URI:          host,
 		SanitizedURI: sanitized,

x-pack/metricbeat/module/sql/query/query_integration_test.go

@@ -10,8 +10,11 @@ package query
 import (
 	"fmt"
 	"net"
+	"os"
 	"testing"
+	"time"
 
+	"github.com/godror/godror"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -40,15 +43,13 @@ func TestMySQL(t *testing.T) {
 			Driver:         "mysql",
 			Query:          "select table_schema, table_name, engine, table_rows from information_schema.tables where table_rows > 0;",
 			ResponseFormat: tableResponseFormat,
-
 			RawData: rawData{
 				Enabled: true,
 			},
 		},
 		Host:      mysql.GetMySQLEnvDSN(service.Host()),
 		Assertion: assertFieldNotContains("service.address", ":test@"),
 	}
-
 	t.Run("fetch", func(t *testing.T) {
 		testFetch(t, cfg)
 	})
@@ -226,6 +227,26 @@ func TestPostgreSQL(t *testing.T) {
 	})
 }
 
+func TestOracle(t *testing.T) {
+	service := compose.EnsureUp(t, "oracle")
+	host, port, _ := net.SplitHostPort(service.Host())
+	cfg := testFetchConfig{
+		config: config{
+			Driver:         "oracle",
+			Query:          `SELECT name, physical_reads, db_block_gets, consistent_gets, 1 - (physical_reads / (db_block_gets + consistent_gets)) "Hit_Ratio" FROM V$BUFFER_POOL_STATISTICS`,
+			ResponseFormat: tableResponseFormat,
+			RawData: rawData{
+				Enabled: true,
+			},
+		},
+		Host:      GetOracleConnectionDetails(t, host, port),
+		Assertion: assertFieldContainsFloat64("Hit_Ratio", 0.0),
+	}
+	t.Run("fetch", func(t *testing.T) {
+		testFetch(t, cfg)
+	})
+}
+
 func testFetch(t *testing.T, cfg testFetchConfig) {
 	m := mbtest.NewFetcher(t, getConfig(cfg))
 	events, errs := m.FetchEvents()
@@ -256,11 +277,9 @@ func getConfig(cfg testFetchConfig) map[string]interface{} {
 		"raw_data.enabled": cfg.config.RawData.Enabled,
 		"merge_results":    cfg.config.MergeResults,
 	}
-
 	if cfg.config.ResponseFormat != "" {
 		values["sql_response_format"] = cfg.config.ResponseFormat
 	}
-
 	return values
 }
 
@@ -272,3 +291,53 @@ func assertFieldNotContains(field, s string) func(t *testing.T, event beat.Event
 		require.NotContains(t, value.(string), s)
 	}
 }
+
+func assertFieldContainsFloat64(field string, limit float64) func(t *testing.T, event beat.Event) {
+	return func(t *testing.T, event beat.Event) {
+		value, err := event.GetValue("sql.metrics.hit_ratio")
+		assert.NoError(t, err)
+		require.GreaterOrEqual(t, value.(float64), limit)
+	}
+}
+
+func GetOracleConnectionDetails(t *testing.T, host string, port string) string {
+	params, err := godror.ParseDSN(GetOracleConnectString(host, port))
+	require.Empty(t, err)
+	return params.StringWithPassword()
+}
+
+// GetOracleEnvServiceName returns the service name to use with Oracle testing server or the value of the environment variable ORACLE_SERVICE_NAME if not empty
+func GetOracleEnvServiceName() string {
+	serviceName := os.Getenv("ORACLE_SERVICE_NAME")
+	if len(serviceName) == 0 {
+		serviceName = "ORCLCDB.localdomain"
+	}
+	return serviceName
+}
+
+// GetOracleEnvUsername returns the username to use with Oracle testing server or the value of the environment variable ORACLE_USERNAME if not empty
+func GetOracleEnvUsername() string {
+	username := os.Getenv("ORACLE_USERNAME")
+	if len(username) == 0 {
+		username = "sys"
+	}
+	return username
+}
+
+// GetOracleEnvUsername returns the port of the Oracle server or the value of the environment variable ORACLE_PASSWORD if not empty
+func GetOracleEnvPassword() string {
+	password := os.Getenv("ORACLE_PASSWORD")
+	if len(password) == 0 {
+		password = "Oradoc_db1" // #nosec
+	}
+	return password
+}
+
+func GetOracleConnectString(host string, port string) string {
+	time.Sleep(300 * time.Second)
+	connectString := os.Getenv("ORACLE_CONNECT_STRING")
+	if len(connectString) == 0 {
+		connectString = fmt.Sprintf("%s/%s@%s:%s/%s as sysdba", GetOracleEnvUsername(), GetOracleEnvPassword(), host, port, GetOracleEnvServiceName())
+	}
+	return connectString
+}