From 84f2d771358ce72d92635d1a8f085bc2cf4482ed Mon Sep 17 00:00:00 2001 From: machinespeak Date: Mon, 20 Nov 2023 15:05:56 -0500 Subject: [PATCH 01/14] remove smart agent postgresql configs and use native receiver for config --- ...scovery.yaml => postgresql.discovery.yaml} | 44 ++++++++----------- ...scovery.yaml => postgresql.discovery.yaml} | 44 ++++++++----------- ...ml.tmpl => postgresql.discovery.yaml.tmpl} | 36 ++++++--------- .../discovery/bundle/bundle_gen.go | 4 +- .../discovery/bundle/bundle_other_test.go | 2 +- 5 files changed, 53 insertions(+), 77 deletions(-) rename cmd/otelcol/config/collector/config.d.linux/receivers/{smartagent-postgresql.discovery.yaml => postgresql.discovery.yaml} (68%) rename internal/confmapprovider/discovery/bundle/bundle.d/receivers/{smartagent-postgresql.discovery.yaml => postgresql.discovery.yaml} (66%) rename internal/confmapprovider/discovery/bundle/bundle.d/receivers/{smartagent-postgresql.discovery.yaml.tmpl => postgresql.discovery.yaml.tmpl} (76%) diff --git a/cmd/otelcol/config/collector/config.d.linux/receivers/smartagent-postgresql.discovery.yaml b/cmd/otelcol/config/collector/config.d.linux/receivers/postgresql.discovery.yaml similarity index 68% rename from cmd/otelcol/config/collector/config.d.linux/receivers/smartagent-postgresql.discovery.yaml rename to cmd/otelcol/config/collector/config.d.linux/receivers/postgresql.discovery.yaml index 86f4cf33ee..a5bea517e0 100644 --- a/cmd/otelcol/config/collector/config.d.linux/receivers/smartagent-postgresql.discovery.yaml +++ b/cmd/otelcol/config/collector/config.d.linux/receivers/postgresql.discovery.yaml @@ -6,39 +6,22 @@ # Please note that any changes made to this file will need to be reconciled during # # upgrades of the Collector. # ##################################################################################### -# smartagent/postgresql: +# postgresql: # enabled: true # rule: # docker_observer: type == "container" and any([name, image, command], {# matches "(?i)postgres"}) and not (command matches "splunk.discovery") # host_observer: type == "hostport" and command matches "(?i)postgres" and not (command matches "splunk.discovery") # k8s_observer: type == "port" and pod.name matches "(?i)postgres" # config: -# default: -# type: postgresql -# connectionString: 'sslmode=disable user={{.username}} password={{.password}}' -# params: -# username: splunk.discovery.default -# password: splunk.discovery.default -# masterDBName: splunk.discovery.default +# default: {} # status: # metrics: # successful: -# - strict: postgres_query_count +# - strict: postgresql.commits # first_only: true # log_record: # severity_text: info # body: PostgreSQL receiver is working! -# partial: -# - strict: postgres_rows_inserted -# first_only: true -# log_record: -# severity_text: info -# body: >- -# Make sure that -# `CREATE EXTENSION IF NOT EXISTS pg_stat_statements;` -# has been run for each database you would like to monitor. -# For example: -# `psql --dbname "" -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;"` # statements: # failed: # - regexp: 'connect: network is unreachable' @@ -61,10 +44,10 @@ # append_pattern: true # body: >- # Please ensure your user credentials are correctly specified with -# `--set splunk.discovery.receivers.smartagent/postgresql.config.params::username=""` and -# `--set splunk.discovery.receivers.smartagent/postgresql.config.params::password=""` or -# `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_params_x3a__x3a_username=""` and -# `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_params_x3a__x3a_password=""` environment variables. +# `--set splunk.discovery.receivers.postgresql.config.params::username=""` and +# `--set splunk.discovery.receivers.postgresql.config.params::password=""` or +# `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_params_x3a__x3a_username=""` and +# `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_params_x3a__x3a_password=""` environment variables. # - regexp: 'pq: database .* does not exist' # first_only: true # log_record: @@ -72,8 +55,17 @@ # append_pattern: true # body: >- # Make sure the target database is correctly specified using the -# `--set splunk.discovery.receivers.smartagent/postgresql.config.masterDBName=""` command or the -# `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_masterDBName=""` environment variable. +# `--set splunk.discovery.receivers.postgresql.config.databases="[]"` command or the +# `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_daatabases="[]"` environment variable. +# - regexp: 'pq: SSL is not enabled on the server' +# first_only: true +# log_record: +# severity_text: info +# append_pattern: true +# body: >- +# Make sure the target database has SSL enabled or set insecure using the +# `--set splunk.discovery.receivers.postgresql.config.tls::insecure=""` command or the +# `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_tls_x3a__x3a_insecure=""` environment variable. # - regexp: 'pq: pg_stat_statements must be loaded via shared_preload_libraries' # first_only: true # log_record: diff --git a/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml similarity index 66% rename from internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml rename to internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml index 6c911fe71a..d34dfde7d1 100644 --- a/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml +++ b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml @@ -2,39 +2,22 @@ # Do not edit manually! # # All changes must be made to associated .tmpl file before running 'make bundle.d'. # ##################################################################################### -smartagent/postgresql: +postgresql: enabled: true rule: docker_observer: type == "container" and any([name, image, command], {# matches "(?i)postgres"}) and not (command matches "splunk.discovery") host_observer: type == "hostport" and command matches "(?i)postgres" and not (command matches "splunk.discovery") k8s_observer: type == "port" and pod.name matches "(?i)postgres" config: - default: - type: postgresql - connectionString: 'sslmode=disable user={{.username}} password={{.password}}' - params: - username: splunk.discovery.default - password: splunk.discovery.default - masterDBName: splunk.discovery.default + default: {} status: metrics: successful: - - strict: postgres_query_count + - strict: postgresql.commits first_only: true log_record: severity_text: info body: PostgreSQL receiver is working! - partial: - - strict: postgres_rows_inserted - first_only: true - log_record: - severity_text: info - body: >- - Make sure that - `CREATE EXTENSION IF NOT EXISTS pg_stat_statements;` - has been run for each database you would like to monitor. - For example: - `psql --dbname "" -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;"` statements: failed: - regexp: 'connect: network is unreachable' @@ -57,10 +40,10 @@ smartagent/postgresql: append_pattern: true body: >- Please ensure your user credentials are correctly specified with - `--set splunk.discovery.receivers.smartagent/postgresql.config.params::username=""` and - `--set splunk.discovery.receivers.smartagent/postgresql.config.params::password=""` or - `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_params_x3a__x3a_username=""` and - `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_params_x3a__x3a_password=""` environment variables. + `--set splunk.discovery.receivers.postgresql.config.params::username=""` and + `--set splunk.discovery.receivers.postgresql.config.params::password=""` or + `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_params_x3a__x3a_username=""` and + `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_params_x3a__x3a_password=""` environment variables. - regexp: 'pq: database .* does not exist' first_only: true log_record: @@ -68,8 +51,17 @@ smartagent/postgresql: append_pattern: true body: >- Make sure the target database is correctly specified using the - `--set splunk.discovery.receivers.smartagent/postgresql.config.masterDBName=""` command or the - `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_masterDBName=""` environment variable. + `--set splunk.discovery.receivers.postgresql.config.databases="[]"` command or the + `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_daatabases="[]"` environment variable. + - regexp: 'pq: SSL is not enabled on the server' + first_only: true + log_record: + severity_text: info + append_pattern: true + body: >- + Make sure the target database has SSL enabled or set insecure using the + `--set splunk.discovery.receivers.postgresql.config.tls::insecure=""` command or the + `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_tls_x3a__x3a_insecure=""` environment variable. - regexp: 'pq: pg_stat_statements must be loaded via shared_preload_libraries' first_only: true log_record: diff --git a/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml.tmpl similarity index 76% rename from internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl rename to internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml.tmpl index 4a243f6449..5ec83bd4f0 100644 --- a/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl +++ b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml.tmpl @@ -1,36 +1,19 @@ -{{ receiver "smartagent/postgresql" }}: +{{ receiver "postgresql" }}: enabled: true rule: docker_observer: type == "container" and any([name, image, command], {# matches "(?i)postgres"}) and not (command matches "splunk.discovery") host_observer: type == "hostport" and command matches "(?i)postgres" and not (command matches "splunk.discovery") k8s_observer: type == "port" and pod.name matches "(?i)postgres" config: - default: - type: postgresql - connectionString: 'sslmode=disable user={{ "{{.username}}" }} password={{ "{{.password}}" }}' - params: - username: {{ defaultValue }} - password: {{ defaultValue }} - masterDBName: {{ defaultValue }} + default: {} status: metrics: successful: - - strict: postgres_query_count + - strict: postgresql.commits first_only: true log_record: severity_text: info body: PostgreSQL receiver is working! - partial: - - strict: postgres_rows_inserted - first_only: true - log_record: - severity_text: info - body: >- - Make sure that - `CREATE EXTENSION IF NOT EXISTS pg_stat_statements;` - has been run for each database you would like to monitor. - For example: - `psql --dbname "" -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;"` statements: failed: - regexp: 'connect: network is unreachable' @@ -64,8 +47,17 @@ append_pattern: true body: >- Make sure the target database is correctly specified using the - `--set {{ configProperty "masterDBName" "" }}` command or the - `{{ configPropertyEnvVar "masterDBName" "" }}` environment variable. + `--set {{ configProperty "databases" "[]" }}` command or the + `{{ configPropertyEnvVar "daatabases" "[]" }}` environment variable. + - regexp: 'pq: SSL is not enabled on the server' + first_only: true + log_record: + severity_text: info + append_pattern: true + body: >- + Make sure the target database has SSL enabled or set insecure using the + `--set {{ configProperty "tls::insecure" "" }}` command or the + `{{ configPropertyEnvVar "tls::insecure" "" }}` environment variable. - regexp: 'pq: pg_stat_statements must be loaded via shared_preload_libraries' first_only: true log_record: diff --git a/internal/confmapprovider/discovery/bundle/bundle_gen.go b/internal/confmapprovider/discovery/bundle/bundle_gen.go index d9fbbdbff9..2954cc7896 100644 --- a/internal/confmapprovider/discovery/bundle/bundle_gen.go +++ b/internal/confmapprovider/discovery/bundle/bundle_gen.go @@ -19,12 +19,12 @@ // TODO: add a pre-bundle.d step that generates these generate directives for easier maintenance/management across platforms //go:generate go install github.com/signalfx/splunk-otel-collector/internal/confmapprovider/discovery/bundle/cmd/discoverybundler -//go:generate discoverybundler -r -t bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl -//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl //go:generate discoverybundler -r -t bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml.tmpl //go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml.tmpl //go:generate discoverybundler -r -t bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml.tmpl //go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml.tmpl +//go:generate discoverybundler -r -t bundle.d/receivers/postgresql.discovery.yaml.tmpl +//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/postgresql.discovery.yaml.tmpl //go:generate discoverybundler -r -t bundle.d/receivers/oracledb.discovery.yaml.tmpl //go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/oracledb.discovery.yaml.tmpl //go:generate discoverybundler -r -t bundle.d/receivers/redis.discovery.yaml.tmpl diff --git a/internal/confmapprovider/discovery/bundle/bundle_other_test.go b/internal/confmapprovider/discovery/bundle/bundle_other_test.go index 64f14c0baf..ac5f933146 100644 --- a/internal/confmapprovider/discovery/bundle/bundle_other_test.go +++ b/internal/confmapprovider/discovery/bundle/bundle_other_test.go @@ -27,11 +27,11 @@ func TestBundleDir(t *testing.T) { receivers, err := fs.Glob(BundledFS, "bundle.d/receivers/*.discovery.yaml") require.NoError(t, err) require.Equal(t, []string{ + "bundle.d/receivers/postgresql.discovery.yaml", "bundle.d/receivers/oracledb.discovery.yaml", "bundle.d/receivers/redis.discovery.yaml", "bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml", "bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml", - "bundle.d/receivers/smartagent-postgresql.discovery.yaml", }, receivers) extensions, err := fs.Glob(BundledFS, "bundle.d/extensions/*.discovery.yaml") From 0fa2afb415c8ca959298a4483c79079fbb8fdd8a Mon Sep 17 00:00:00 2001 From: machinespeak Date: Mon, 20 Nov 2023 15:06:43 -0500 Subject: [PATCH 02/14] write bundled test for otel postgresql receiver --- tests/receivers/postgresql/bundled_test.go | 60 +++++++++ .../postgresql/testdata/otlp_exporter.yaml | 13 ++ .../testdata/resource_metrics/bundled.yaml | 121 ++++++++++++++++++ 3 files changed, 194 insertions(+) create mode 100644 tests/receivers/postgresql/bundled_test.go create mode 100644 tests/receivers/postgresql/testdata/otlp_exporter.yaml create mode 100644 tests/receivers/postgresql/testdata/resource_metrics/bundled.yaml diff --git a/tests/receivers/postgresql/bundled_test.go b/tests/receivers/postgresql/bundled_test.go new file mode 100644 index 0000000000..6f2028e23b --- /dev/null +++ b/tests/receivers/postgresql/bundled_test.go @@ -0,0 +1,60 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build integration + +package tests + +import ( + "fmt" + "runtime" + "testing" + + "github.com/signalfx/splunk-otel-collector/tests/testutils" +) + +func TestPostgresqlDockerObserver(t *testing.T) { + testutils.SkipIfNotContainerTest(t) + if runtime.GOOS == "darwin" { + t.Skip("unable to share sockets between mac and d4m vm: https://github.com/docker/for-mac/issues/483#issuecomment-758836836") + } + + testutils.AssertAllMetricsReceived(t, "bundled.yaml", "otlp_exporter.yaml", + postgresqldb, []testutils.CollectorBuilder{ + func(c testutils.Collector) testutils.Collector { + cc := c.(*testutils.CollectorContainer) + cc.Container = cc.Container.WithBinds("/var/run/docker.sock:/var/run/docker.sock:ro") + cc.Container = cc.Container.WillWaitForLogs("Discovering for next") + cc.Container = cc.Container.WithUser(fmt.Sprintf("999:%d", testutils.GetDockerGID(t))) + return cc + }, + func(collector testutils.Collector) testutils.Collector { + return collector.WithEnv(map[string]string{ + "SPLUNK_DISCOVERY_DURATION": "10s", + // confirm that debug logging doesn't affect runtime + "SPLUNK_DISCOVERY_LOG_LEVEL": "debug", + }).WithArgs( + "--discovery", + "--set", "splunk.discovery.receivers.postgresql.config.username=otelu", + "--set", "splunk.discovery.receivers.postgresql.config.password=otelp", + "--set", "splunk.discovery.receivers.postgresql.config.tls::insecure=true", + //Disabling postgresql.backends metric that doesn't dependably show up in metrics during testing + "--set", "splunk.discovery.receivers.postgresql.config.metrics::postgresql.backends::enabled=false", + "--set", `splunk.discovery.extensions.k8s_observer.enabled=false`, + "--set", `splunk.discovery.extensions.host_observer.enabled=false`, + ) + }, + }, + ) +} diff --git a/tests/receivers/postgresql/testdata/otlp_exporter.yaml b/tests/receivers/postgresql/testdata/otlp_exporter.yaml new file mode 100644 index 0000000000..fa00f3c646 --- /dev/null +++ b/tests/receivers/postgresql/testdata/otlp_exporter.yaml @@ -0,0 +1,13 @@ +exporters: + otlp: + endpoint: "${OTLP_ENDPOINT}" + tls: + insecure: true + +service: + telemetry: + logs: + level: debug + pipelines: + metrics: + exporters: [otlp] diff --git a/tests/receivers/postgresql/testdata/resource_metrics/bundled.yaml b/tests/receivers/postgresql/testdata/resource_metrics/bundled.yaml new file mode 100644 index 0000000000..c56cc1e1e0 --- /dev/null +++ b/tests/receivers/postgresql/testdata/resource_metrics/bundled.yaml @@ -0,0 +1,121 @@ +resource_metrics: + - attributes: + container.image.name: + container.name: + postgresql.database.name: + scope_metrics: + - instrumentation_scope: + attributes: {} + name: otelcol/postgresqlreceiver + version: + metrics: + - name: postgresql.commits + type: IntMonotonicCumulativeSum + - name: postgresql.db_size + type: IntNonmonotonicCumulativeSum + - name: postgresql.rollbacks + type: IntMonotonicCumulativeSum + - name: postgresql.table.count + type: IntNonmonotonicCumulativeSum + - attributes: + container.image.name: + container.name: + postgresql.database.name: + postgresql.index.name: + postgresql.table.name: + scope_metrics: + - instrumentation_scope: + attributes: {} + name: otelcol/postgresqlreceiver + version: + metrics: + - name: postgresql.index.scans + type: IntMonotonicCumulativeSum + - name: postgresql.index.size + type: IntGauge + - attributes: + container.image.name: + container.name: + postgresql.database.name: + postgresql.table.name: + scope_metrics: + - instrumentation_scope: + attributes: {} + name: otelcol/postgresqlreceiver + version: + metrics: + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: heap_read + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: heap_hit + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: idx_read + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: idx_hit + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: toast_hit + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: tidx_read + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: tidx_hit + - name: postgresql.operations + type: IntMonotonicCumulativeSum + attributes: + operation: del + - name: postgresql.operations + type: IntMonotonicCumulativeSum + attributes: + operation: upd + - name: postgresql.operations + type: IntMonotonicCumulativeSum + attributes: + operation: hot_upd + - name: postgresql.rows + type: IntNonmonotonicCumulativeSum + attributes: + state: dead + - name: postgresql.rows + type: IntNonmonotonicCumulativeSum + attributes: + state: live + - name: postgresql.table.size + type: IntNonmonotonicCumulativeSum + - name: postgresql.table.vacuum.count + type: IntMonotonicCumulativeSum + - attributes: + container.image.name: + container.name: + scope_metrics: + - instrumentation_scope: + attributes: {} + name: otelcol/postgresqlreceiver + version: + metrics: + - name: postgresql.bgwriter.buffers.allocated + type: IntMonotonicCumulativeSum + - name: postgresql.bgwriter.buffers.writes + type: IntMonotonicCumulativeSum + - name: postgresql.bgwriter.checkpoint.count + type: IntMonotonicCumulativeSum + - name: postgresql.bgwriter.duration + type: DoubleMonotonicCumulativeSum + - name: postgresql.bgwriter.maxwritten + type: IntMonotonicCumulativeSum + - name: postgresql.connection.max + type: IntGauge + - name: postgresql.database.count + type: IntNonmonotonicCumulativeSum \ No newline at end of file From edba6dedec209ba60f08a0324ca20333c6418005 Mon Sep 17 00:00:00 2001 From: machinespeak Date: Mon, 20 Nov 2023 15:53:59 -0500 Subject: [PATCH 03/14] ordering --- internal/confmapprovider/discovery/bundle/bundle_other_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/confmapprovider/discovery/bundle/bundle_other_test.go b/internal/confmapprovider/discovery/bundle/bundle_other_test.go index ac5f933146..b72b5856a6 100644 --- a/internal/confmapprovider/discovery/bundle/bundle_other_test.go +++ b/internal/confmapprovider/discovery/bundle/bundle_other_test.go @@ -27,8 +27,8 @@ func TestBundleDir(t *testing.T) { receivers, err := fs.Glob(BundledFS, "bundle.d/receivers/*.discovery.yaml") require.NoError(t, err) require.Equal(t, []string{ - "bundle.d/receivers/postgresql.discovery.yaml", "bundle.d/receivers/oracledb.discovery.yaml", + "bundle.d/receivers/postgresql.discovery.yaml", "bundle.d/receivers/redis.discovery.yaml", "bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml", "bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml", From 16db21e0d3a25be3843e19391a3d3031377f459b Mon Sep 17 00:00:00 2001 From: machinespeak Date: Tue, 21 Nov 2023 10:29:48 -0500 Subject: [PATCH 04/14] adjust windows bundles --- internal/confmapprovider/discovery/bundle/bundle_windows.go | 2 +- .../confmapprovider/discovery/bundle/bundle_windows_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/confmapprovider/discovery/bundle/bundle_windows.go b/internal/confmapprovider/discovery/bundle/bundle_windows.go index 175b19ee13..36bd7e952a 100644 --- a/internal/confmapprovider/discovery/bundle/bundle_windows.go +++ b/internal/confmapprovider/discovery/bundle/bundle_windows.go @@ -28,6 +28,6 @@ import ( // //go:embed bundle.d/extensions/*.discovery.yaml //go:embed bundle.d/receivers/oracledb.discovery.yaml +//go:embed bundle.d/receivers/postgresql.discovery.yaml //go:embed bundle.d/receivers/redis.discovery.yaml -//go:embed bundle.d/receivers/smartagent-postgresql.discovery.yaml var BundledFS embed.FS diff --git a/internal/confmapprovider/discovery/bundle/bundle_windows_test.go b/internal/confmapprovider/discovery/bundle/bundle_windows_test.go index 8eb5d07084..776b65f6f4 100644 --- a/internal/confmapprovider/discovery/bundle/bundle_windows_test.go +++ b/internal/confmapprovider/discovery/bundle/bundle_windows_test.go @@ -28,8 +28,8 @@ func TestBundleDir(t *testing.T) { require.NoError(t, err) require.Equal(t, []string{ "bundle.d/receivers/oracledb.discovery.yaml", + "bundle.d/receivers/postgresql.discovery.yaml", "bundle.d/receivers/redis.discovery.yaml", - "bundle.d/receivers/smartagent-postgresql.discovery.yaml", }, receivers) extensions, err := fs.Glob(BundledFS, "bundle.d/extensions/*.discovery.yaml") From a53318f6a105caff65a64519e5e64282e85e571a Mon Sep 17 00:00:00 2001 From: machinespeak Date: Tue, 21 Nov 2023 14:15:50 -0500 Subject: [PATCH 05/14] remove old bunded test and add k8s bits to otel native bundled test fmt fmt config tweak and bundle --- .../receivers/postgresql.discovery.yaml | 10 +- .../receivers/postgresql.discovery.yaml | 10 +- .../receivers/postgresql.discovery.yaml.tmpl | 10 +- tests/receivers/postgresql/bundled_test.go | 352 ++++++++++++++++ .../postgresql/testdata/client/Dockerfile | 6 + .../postgresql/testdata/client/requests.sh | 95 +++++ .../testdata/resource_metrics/all_k8s.yaml | 125 ++++++ .../postgresql/testdata/server_k8s/Dockerfile | 9 + .../testdata/server_k8s/initdb.d/db.sql | 30 ++ .../testdata/server_k8s/initdb.d/init.sh | 12 + .../smartagent/postgresql/bundled_test.go | 376 ------------------ 11 files changed, 644 insertions(+), 391 deletions(-) create mode 100644 tests/receivers/postgresql/testdata/client/Dockerfile create mode 100755 tests/receivers/postgresql/testdata/client/requests.sh create mode 100644 tests/receivers/postgresql/testdata/resource_metrics/all_k8s.yaml create mode 100644 tests/receivers/postgresql/testdata/server_k8s/Dockerfile create mode 100755 tests/receivers/postgresql/testdata/server_k8s/initdb.d/db.sql create mode 100755 tests/receivers/postgresql/testdata/server_k8s/initdb.d/init.sh delete mode 100644 tests/receivers/smartagent/postgresql/bundled_test.go diff --git a/cmd/otelcol/config/collector/config.d.linux/receivers/postgresql.discovery.yaml b/cmd/otelcol/config/collector/config.d.linux/receivers/postgresql.discovery.yaml index a5bea517e0..fd7d86b1b2 100644 --- a/cmd/otelcol/config/collector/config.d.linux/receivers/postgresql.discovery.yaml +++ b/cmd/otelcol/config/collector/config.d.linux/receivers/postgresql.discovery.yaml @@ -44,10 +44,10 @@ # append_pattern: true # body: >- # Please ensure your user credentials are correctly specified with -# `--set splunk.discovery.receivers.postgresql.config.params::username=""` and -# `--set splunk.discovery.receivers.postgresql.config.params::password=""` or -# `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_params_x3a__x3a_username=""` and -# `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_params_x3a__x3a_password=""` environment variables. +# `--set splunk.discovery.receivers.postgresql.config.username=""` and +# `--set splunk.discovery.receivers.postgresql.config.password=""` or +# `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_username=""` and +# `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_password=""` environment variables. # - regexp: 'pq: database .* does not exist' # first_only: true # log_record: @@ -56,7 +56,7 @@ # body: >- # Make sure the target database is correctly specified using the # `--set splunk.discovery.receivers.postgresql.config.databases="[]"` command or the -# `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_daatabases="[]"` environment variable. +# `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_databases="[]"` environment variable. # - regexp: 'pq: SSL is not enabled on the server' # first_only: true # log_record: diff --git a/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml index d34dfde7d1..b0b9e0fd7a 100644 --- a/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml +++ b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml @@ -40,10 +40,10 @@ postgresql: append_pattern: true body: >- Please ensure your user credentials are correctly specified with - `--set splunk.discovery.receivers.postgresql.config.params::username=""` and - `--set splunk.discovery.receivers.postgresql.config.params::password=""` or - `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_params_x3a__x3a_username=""` and - `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_params_x3a__x3a_password=""` environment variables. + `--set splunk.discovery.receivers.postgresql.config.username=""` and + `--set splunk.discovery.receivers.postgresql.config.password=""` or + `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_username=""` and + `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_password=""` environment variables. - regexp: 'pq: database .* does not exist' first_only: true log_record: @@ -52,7 +52,7 @@ postgresql: body: >- Make sure the target database is correctly specified using the `--set splunk.discovery.receivers.postgresql.config.databases="[]"` command or the - `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_daatabases="[]"` environment variable. + `SPLUNK_DISCOVERY_RECEIVERS_postgresql_CONFIG_databases="[]"` environment variable. - regexp: 'pq: SSL is not enabled on the server' first_only: true log_record: diff --git a/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml.tmpl b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml.tmpl index 5ec83bd4f0..3dc31c2374 100644 --- a/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml.tmpl +++ b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml.tmpl @@ -36,10 +36,10 @@ append_pattern: true body: >- Please ensure your user credentials are correctly specified with - `--set {{ configProperty "params" "username" "" }}` and - `--set {{ configProperty "params" "password" "" }}` or - `{{ configPropertyEnvVar "params" "username" "" }}` and - `{{ configPropertyEnvVar "params" "password" "" }}` environment variables. + `--set {{ configProperty "username" "" }}` and + `--set {{ configProperty "password" "" }}` or + `{{ configPropertyEnvVar "username" "" }}` and + `{{ configPropertyEnvVar "password" "" }}` environment variables. - regexp: 'pq: database .* does not exist' first_only: true log_record: @@ -48,7 +48,7 @@ body: >- Make sure the target database is correctly specified using the `--set {{ configProperty "databases" "[]" }}` command or the - `{{ configPropertyEnvVar "daatabases" "[]" }}` environment variable. + `{{ configPropertyEnvVar "databases" "[]" }}` environment variable. - regexp: 'pq: SSL is not enabled on the server' first_only: true log_record: diff --git a/tests/receivers/postgresql/bundled_test.go b/tests/receivers/postgresql/bundled_test.go index 6f2028e23b..0db56506ff 100644 --- a/tests/receivers/postgresql/bundled_test.go +++ b/tests/receivers/postgresql/bundled_test.go @@ -17,11 +17,25 @@ package tests import ( + "context" "fmt" + "os" + "path/filepath" "runtime" + "strings" "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "gopkg.in/yaml.v3" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/signalfx/splunk-otel-collector/tests/testutils" + "github.com/signalfx/splunk-otel-collector/tests/testutils/kubeutils" + "github.com/signalfx/splunk-otel-collector/tests/testutils/kubeutils/manifests" ) func TestPostgresqlDockerObserver(t *testing.T) { @@ -58,3 +72,341 @@ func TestPostgresqlDockerObserver(t *testing.T) { }, ) } + +func TestK8sObserver(t *testing.T) { + testutils.SkipIfNotContainerTest(t) + tc := testutils.NewTestcase(t, testutils.OTLPReceiverSinkAllInterfaces) + defer tc.PrintLogsOnFailure() + defer tc.ShutdownOTLPReceiverSink() + + cluster := testCluster{kubeutils.NewKindCluster(tc)} + defer cluster.Teardown() + cluster.Create() + cluster.LoadLocalCollectorImageIfNecessary() + + namespace := manifests.Namespace{Name: "test-namespace"} + serviceAccount := manifests.ServiceAccount{Name: "some.serviceacount", Namespace: namespace.Name} + clusterRole, clusterRoleBinding := clusterRoleAndBinding(namespace.Name, serviceAccount.Name) + sout, serr, err := cluster.Apply(manifests.RenderAll(t, namespace, serviceAccount, clusterRole, clusterRoleBinding)) + require.NoError(t, err, "stdout: %s, stderr: %s", sout, serr) + + postgresUID := cluster.createPostgres("target.postgres", namespace.Name, serviceAccount.Name) + + configMap := manifests.ConfigMap{ + Name: "collector.config", Namespace: namespace.Name, + Data: `config: | + exporters: + otlp: + endpoint: ${OTLP_ENDPOINT} + tls: + insecure: true + service: + pipelines: + metrics: + exporters: + - otlp +`} + + ds := cluster.daemonSet(namespace.Name, serviceAccount.Name, configMap.Name) + sout, serr, err = cluster.Apply(manifests.RenderAll(t, configMap, ds)) + require.NoError(t, err, "stdout: %s, stderr: %s", sout, serr) + + pods := cluster.WaitForPods(ds.Name, namespace.Name, 10*time.Minute) + require.Len(t, pods, 1) + collectorName := pods[0].Name + + expectedMetrics := tc.ResourceMetrics("all_k8s.yaml") + require.NoError(t, tc.OTLPReceiverSink.AssertAllMetricsReceived(t, *expectedMetrics, 30*time.Second)) + + stdOut, stdErr, err := cluster.Kubectl("logs", "-n", namespace.Name, collectorName) + require.NoError(t, err) + require.Contains( + t, stdOut.String(), + fmt.Sprintf(`Successfully discovered "postgresql" using "k8s_observer" endpoint "k8s_observer/%s/(5432)`, postgresUID), + stdErr.String(), + ) +} + +type testCluster struct{ *kubeutils.KindCluster } + +func (cluster testCluster) createPostgres(name, namespace, serviceAccount string) string { + dbsql, err := os.ReadFile(filepath.Join(".", "testdata", "server_k8s", "initdb.d", "db.sql")) + require.NoError(cluster.Testcase, err) + cmContent := map[string]any{"db.sql": string(dbsql)} + + initsh, err := os.ReadFile(filepath.Join(".", "testdata", "server_k8s", "initdb.d", "init.sh")) + require.NoError(cluster.Testcase, err) + cmContent["init.sh"] = string(initsh) + + requests, err := os.ReadFile(filepath.Join(".", "testdata", "client", "requests.sh")) + require.NoError(cluster.Testcase, err) + cmContent["requests.sh"] = string(requests) + + configMapContent, err := yaml.Marshal(cmContent) + require.NoError(cluster.Testcase, err) + + cm := manifests.ConfigMap{ + Namespace: namespace, + Name: "postgres", + Data: string(configMapContent), + } + sout, serr, err := cluster.Apply(cm.Render(cluster.Testcase)) + cluster.Testcase.Logger.Debug("applying ConfigMap", zap.String("stdout", sout.String()), zap.String("stderr", serr.String())) + require.NoError(cluster.Testcase, err) + + fileMode := int32(0777) + postgresID := int64(70) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + postgres, err := cluster.Clientset.CoreV1().Pods(namespace).Create( + ctx, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Image: "postgres:13-alpine", + Name: "postgres-server", + Ports: []corev1.ContainerPort{{ContainerPort: 5432}}, + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &postgresID, + RunAsGroup: &postgresID, + }, + Command: []string{ + "docker-entrypoint.sh", + "-c", "shared_preload_libraries=pg_stat_statements", + "-c", "wal_level=logical", + "-c", "max_replication_slots=2", + }, + Env: []corev1.EnvVar{ + { + Name: "POSTGRES_DB", + Value: "test_db", + }, + { + Name: "POSTGRES_USER", + Value: "postgres", + }, + { + Name: "POSTGRES_PASSWORD", + Value: "postgres", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "initdb", MountPath: "/docker-entrypoint-initdb.d", + }, + }, + }, + { + Image: "postgres:13-alpine", + Name: "postgres-client", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/opt/requests/requests.sh"}, + Env: []corev1.EnvVar{ + { + Name: "POSTGRES_SERVER", + Value: "localhost", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "requests", MountPath: "/opt/requests", + }, + }, + }, + }, + ServiceAccountName: serviceAccount, + HostNetwork: true, + Volumes: []corev1.Volume{ + { + Name: "initdb", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cm.Name, + }, + Items: []corev1.KeyToPath{ + { + Key: "init.sh", + Path: "init.sh", + Mode: &fileMode, + }, + { + Key: "db.sql", + Path: "db.sql", + }, + }, + }, + }, + }, + { + Name: "requests", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cm.Name, + }, + Items: []corev1.KeyToPath{ + { + Key: "requests.sh", + Path: "requests.sh", + Mode: &fileMode, + }, + }, + }, + }, + }, + }, + }, + }, metav1.CreateOptions{}, + ) + require.NoError(cluster.Testcase, err) + + cluster.WaitForPods(postgres.Name, namespace, 10*time.Minute) + return string(postgres.UID) +} + +func clusterRoleAndBinding(namespace, serviceAccount string) (manifests.ClusterRole, manifests.ClusterRoleBinding) { + cr := manifests.ClusterRole{ + Name: "cluster-role", + Namespace: namespace, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{ + "events", + "namespaces", + "namespaces/status", + "nodes", + "nodes/spec", + "nodes/stats", + "nodes/proxy", + "pods", + "pods/status", + "persistentvolumeclaims", + "persistentvolumes", + "replicationcontrollers", + "replicationcontrollers/status", + "resourcequotas", + "services", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{ + "daemonsets", + "deployments", + "replicasets", + "statefulsets", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"extensions"}, + Resources: []string{ + "daemonsets", + "deployments", + "replicasets", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"batch"}, + Resources: []string{ + "jobs", + "cronjobs", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"autoscaling"}, + Resources: []string{ + "horizontalpodautoscalers", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + NonResourceURLs: []string{"/metrics"}, + Verbs: []string{"get", "list", "watch"}, + }, + }, + } + + crb := manifests.ClusterRoleBinding{ + Namespace: namespace, + Name: "cluster-role-binding", + ClusterRoleName: cr.Name, + ServiceAccountName: serviceAccount, + } + return cr, crb +} + +func (cluster testCluster) daemonSet(namespace, serviceAccount, configMap string) manifests.DaemonSet { + splat := strings.Split(cluster.Testcase.OTLPEndpoint, ":") + port := splat[len(splat)-1] + var hostFromContainer string + if runtime.GOOS == "darwin" { + hostFromContainer = "host.docker.internal" + } else { + hostFromContainer = cluster.GetDefaultGatewayIP() + } + otlpEndpoint := fmt.Sprintf("%s:%s", hostFromContainer, port) + + return manifests.DaemonSet{ + Name: "an.agent.daemonset", + Namespace: namespace, + ServiceAccount: serviceAccount, + Labels: map[string]string{"label.key": "label.value"}, + Containers: []corev1.Container{ + { + Image: testutils.GetCollectorImageOrSkipTest(cluster.Testcase), + Command: []string{ + "/otelcol", "--config=/config/config.yaml", "--discovery", + "--set", "splunk.discovery.receivers.postgresql.config.username='${env:PG_USERNAME}'", + "--set", "splunk.discovery.receivers.postgresql.config.password='${env:PG_PASSWORD}'", + "--set", "splunk.discovery.receivers.postgresql.config.tls::insecure=true", + //Disabling postgresql.backends metric that doesn't dependably show up in metrics during testing + "--set", "splunk.discovery.receivers.postgresql.config.metrics::postgresql.backends::enabled=false", + }, + Env: []corev1.EnvVar{ + {Name: "PG_USERNAME", Value: "test_user"}, + {Name: "PG_PASSWORD", Value: "test_password"}, + {Name: "OTLP_ENDPOINT", Value: otlpEndpoint}, + // Helpful for debugging + // {Name: "SPLUNK_DISCOVERY_DURATION", Value: "20s"}, + // {Name: "SPLUNK_DISCOVERY_LOG_LEVEL", Value: "debug"}, + }, + Name: "otel-collector", + ImagePullPolicy: corev1.PullIfNotPresent, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "config-map-volume", MountPath: "/config", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "config-map-volume", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMap, + }, + Items: []corev1.KeyToPath{ + { + Key: "config", + Path: "config.yaml", + }, + }, + }, + }, + }, + }, + } +} diff --git a/tests/receivers/postgresql/testdata/client/Dockerfile b/tests/receivers/postgresql/testdata/client/Dockerfile new file mode 100644 index 0000000000..a22875b55e --- /dev/null +++ b/tests/receivers/postgresql/testdata/client/Dockerfile @@ -0,0 +1,6 @@ +ARG POSTGRES_VERSION=13-alpine +FROM postgres:${POSTGRES_VERSION} + +COPY requests.sh /usr/local/bin/requests.sh + +CMD ["requests.sh"] diff --git a/tests/receivers/postgresql/testdata/client/requests.sh b/tests/receivers/postgresql/testdata/client/requests.sh new file mode 100755 index 0000000000..ab1304dd4c --- /dev/null +++ b/tests/receivers/postgresql/testdata/client/requests.sh @@ -0,0 +1,95 @@ +#!/bin/bash +################################################################################################## +# This file will exercise a target POSTGRES_SERVER's test_db whose schema is defined in the +# corresponding server's initdb.d/db.sql. It will create the necessary replication slots and make +# a series of random insertions, updates, and deletions without end to help generate values for +# produced metrics. +# As implemented, test_db size will grow without bound if left is running, so this script isn't +# currently suitable for extended demonstrations or soak test purposes. +################################################################################################## + +echopsql() { + set -x + psql postgresql://test_user:test_password@${POSTGRES_SERVER:-localhost}:5432/test_db "$@" + local r=$? + { set +x; } 2>/dev/null + return $r +} + +# create replication slots for postgres_replication_state metrics +create_replication_slots() { +for query in \ + "SELECT * FROM pg_create_physical_replication_slot('some_physical_replication_slot');" \ + "SELECT * FROM pg_create_logical_replication_slot('some_logical_replication_slot', 'test_decoding');"; do + while true; do + if echopsql -c "${query}"; then + echo "${query} successful" + break + fi + sleep 1 + done +done +} + +random_str() { + tr -dc A-Za-z0-9 + k8s.namespace.name: + k8s.pod.name: + k8s.pod.uid: + scope_metrics: + - instrumentation_scope: + name: otelcol/postgresqlreceiver + version: + attributes: {} + metrics: + - name: postgresql.commits + type: IntMonotonicCumulativeSum + - name: postgresql.db_size + type: IntNonmonotonicCumulativeSum + - name: postgresql.rollbacks + type: IntMonotonicCumulativeSum + - name: postgresql.table.count + type: IntNonmonotonicCumulativeSum + - attributes: + postgresql.database.name: + postgresql.index.name: + postgresql.table.name: + k8s.namespace.name: + k8s.pod.name: + k8s.pod.uid: + scope_metrics: + - instrumentation_scope: + name: otelcol/postgresqlreceiver + version: + attributes: {} + metrics: + - name: postgresql.index.scans + type: IntMonotonicCumulativeSum + - name: postgresql.index.size + type: IntGauge + - attributes: + postgresql.database.name: + postgresql.table.name: + k8s.namespace.name: + k8s.pod.name: + k8s.pod.uid: + scope_metrics: + - instrumentation_scope: + name: otelcol/postgresqlreceiver + version: + attributes: {} + metrics: + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: heap_read + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: heap_hit + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: idx_read + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: idx_hit + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: toast_hit + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: tidx_read + - name: postgresql.blocks_read + type: IntMonotonicCumulativeSum + attributes: + source: tidx_hit + - name: postgresql.operations + type: IntMonotonicCumulativeSum + attributes: + operation: del + - name: postgresql.operations + type: IntMonotonicCumulativeSum + attributes: + operation: upd + - name: postgresql.operations + type: IntMonotonicCumulativeSum + attributes: + operation: hot_upd + - name: postgresql.rows + type: IntNonmonotonicCumulativeSum + attributes: + state: dead + - name: postgresql.rows + type: IntNonmonotonicCumulativeSum + attributes: + state: live + - name: postgresql.table.size + type: IntNonmonotonicCumulativeSum + - name: postgresql.table.vacuum.count + type: IntMonotonicCumulativeSum + - attributes: + k8s.namespace.name: + k8s.pod.name: + k8s.pod.uid: + scope_metrics: + - instrumentation_scope: + name: otelcol/postgresqlreceiver + version: + attributes: {} + metrics: + - name: postgresql.bgwriter.buffers.allocated + type: IntMonotonicCumulativeSum + - name: postgresql.bgwriter.buffers.writes + type: IntMonotonicCumulativeSum + - name: postgresql.bgwriter.checkpoint.count + type: IntMonotonicCumulativeSum + - name: postgresql.bgwriter.duration + type: DoubleMonotonicCumulativeSum + - name: postgresql.bgwriter.maxwritten + type: IntMonotonicCumulativeSum + - name: postgresql.connection.max + type: IntGauge + - name: postgresql.database.count + type: IntNonmonotonicCumulativeSum \ No newline at end of file diff --git a/tests/receivers/postgresql/testdata/server_k8s/Dockerfile b/tests/receivers/postgresql/testdata/server_k8s/Dockerfile new file mode 100644 index 0000000000..16223f7d2a --- /dev/null +++ b/tests/receivers/postgresql/testdata/server_k8s/Dockerfile @@ -0,0 +1,9 @@ +ARG POSTGRES_VERSION=13-alpine +FROM postgres:${POSTGRES_VERSION} + +COPY initdb.d /docker-entrypoint-initdb.d + +CMD ["postgres", \ +"-c", "shared_preload_libraries=pg_stat_statements", \ +"-c", "wal_level=logical", \ +"-c", "max_replication_slots=2"] diff --git a/tests/receivers/postgresql/testdata/server_k8s/initdb.d/db.sql b/tests/receivers/postgresql/testdata/server_k8s/initdb.d/db.sql new file mode 100755 index 0000000000..16be09fd82 --- /dev/null +++ b/tests/receivers/postgresql/testdata/server_k8s/initdb.d/db.sql @@ -0,0 +1,30 @@ +-- This sql script is intended to be run via initdb or with the postgres container's +-- docker-entrypoint-initdb.d autorun functionality. It will creeate the test schema +-- and test user to be exercised by the corresponding pg client's requests script to +-- assist in the postgresql monitor in generating metrics. + +drop schema if exists test_schema cascade; +create schema test_schema; +set schema 'test_schema'; + +drop table if exists table_one; +create table table_one ( + string_one varchar(64) primary key, + string_two varchar(64), + timestamp_one timestamp not null, + timestamp_two timestamp +); + +drop table if exists table_two; +create table table_two ( + int_one integer primary key, + int_two integer, + float_one decimal(11,4) not null, + float_two decimal(11,4) +); + +drop role if exists test_user; +create role test_user with login password 'test_password'; + +alter user test_user with superuser; +grant pg_read_all_settings to test_user; diff --git a/tests/receivers/postgresql/testdata/server_k8s/initdb.d/init.sh b/tests/receivers/postgresql/testdata/server_k8s/initdb.d/init.sh new file mode 100755 index 0000000000..0f52ccea2f --- /dev/null +++ b/tests/receivers/postgresql/testdata/server_k8s/initdb.d/init.sh @@ -0,0 +1,12 @@ +#!/bin/bash +################################################################################################## +# This file will enable the required pg_stat_statements extension for a database named test_db to +# allow the postgresql monitor queries to return valid results to generate metrics. Any desired +# functionality that should occur on server startup should be added here. It is intended to run at +# server startup or via the postgresql container's docker-entrypoint-initdb.d autorun +# functionality. It does not create the test_db database and requires that to have been created +# earlier (e.g. by the container's POSTGRES_DB env var). +################################################################################################## +set -euo pipefail + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "test_db" -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;" diff --git a/tests/receivers/smartagent/postgresql/bundled_test.go b/tests/receivers/smartagent/postgresql/bundled_test.go deleted file mode 100644 index d21ef22aff..0000000000 --- a/tests/receivers/smartagent/postgresql/bundled_test.go +++ /dev/null @@ -1,376 +0,0 @@ -// Copyright Splunk, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build integration - -package tests - -import ( - "context" - "fmt" - "os" - "path/filepath" - "runtime" - "strings" - "testing" - "time" - - "github.com/stretchr/testify/require" - "go.uber.org/zap" - "gopkg.in/yaml.v3" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/signalfx/splunk-otel-collector/tests/testutils" - "github.com/signalfx/splunk-otel-collector/tests/testutils/kubeutils" - "github.com/signalfx/splunk-otel-collector/tests/testutils/kubeutils/manifests" -) - -func TestK8sObserver(t *testing.T) { - testutils.SkipIfNotContainerTest(t) - tc := testutils.NewTestcase(t, testutils.OTLPReceiverSinkAllInterfaces) - defer tc.PrintLogsOnFailure() - defer tc.ShutdownOTLPReceiverSink() - - cluster := testCluster{kubeutils.NewKindCluster(tc)} - defer cluster.Teardown() - cluster.Create() - cluster.LoadLocalCollectorImageIfNecessary() - - namespace := manifests.Namespace{Name: "test-namespace"} - serviceAccount := manifests.ServiceAccount{Name: "some.serviceacount", Namespace: namespace.Name} - clusterRole, clusterRoleBinding := clusterRoleAndBinding(namespace.Name, serviceAccount.Name) - sout, serr, err := cluster.Apply(manifests.RenderAll(t, namespace, serviceAccount, clusterRole, clusterRoleBinding)) - require.NoError(t, err, "stdout: %s, stderr: %s", sout, serr) - - postgresUID := cluster.createPostgres("target.postgres", namespace.Name, serviceAccount.Name) - - configMap := manifests.ConfigMap{ - Name: "collector.config", Namespace: namespace.Name, - Data: `config: | - exporters: - otlp: - endpoint: ${OTLP_ENDPOINT} - tls: - insecure: true - service: - pipelines: - metrics: - exporters: - - otlp -`} - - ds := cluster.daemonSet(namespace.Name, serviceAccount.Name, configMap.Name) - sout, serr, err = cluster.Apply(manifests.RenderAll(t, configMap, ds)) - require.NoError(t, err, "stdout: %s, stderr: %s", sout, serr) - - pods := cluster.WaitForPods(ds.Name, namespace.Name, 5*time.Minute) - require.Len(t, pods, 1) - collectorName := pods[0].Name - - expectedMetrics := tc.ResourceMetrics("all.yaml") - require.NoError(t, tc.OTLPReceiverSink.AssertAllMetricsReceived(t, *expectedMetrics, 30*time.Second)) - - stdOut, stdErr, err := cluster.Kubectl("logs", "-n", namespace.Name, collectorName) - require.NoError(t, err) - require.Contains( - t, stdOut.String(), - fmt.Sprintf(`Successfully discovered "smartagent/postgresql" using "k8s_observer" endpoint "k8s_observer/%s/(5432)`, postgresUID), - stdErr.String(), - ) -} - -type testCluster struct{ *kubeutils.KindCluster } - -func (cluster testCluster) createPostgres(name, namespace, serviceAccount string) string { - dbsql, err := os.ReadFile(filepath.Join(".", "testdata", "server", "initdb.d", "db.sql")) - require.NoError(cluster.Testcase, err) - cmContent := map[string]any{"db.sql": string(dbsql)} - - initsh, err := os.ReadFile(filepath.Join(".", "testdata", "server", "initdb.d", "init.sh")) - require.NoError(cluster.Testcase, err) - cmContent["init.sh"] = string(initsh) - - requests, err := os.ReadFile(filepath.Join(".", "testdata", "client", "requests.sh")) - require.NoError(cluster.Testcase, err) - cmContent["requests.sh"] = string(requests) - - configMapContent, err := yaml.Marshal(cmContent) - require.NoError(cluster.Testcase, err) - - cm := manifests.ConfigMap{ - Namespace: namespace, - Name: "postgres", - Data: string(configMapContent), - } - sout, serr, err := cluster.Apply(cm.Render(cluster.Testcase)) - cluster.Testcase.Logger.Debug("applying ConfigMap", zap.String("stdout", sout.String()), zap.String("stderr", serr.String())) - require.NoError(cluster.Testcase, err) - - fileMode := int32(0777) - postgresID := int64(70) - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - postgres, err := cluster.Clientset.CoreV1().Pods(namespace).Create( - ctx, &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Image: "postgres:13-alpine", - Name: "postgres-server", - Ports: []corev1.ContainerPort{{ContainerPort: 5432}}, - ImagePullPolicy: corev1.PullIfNotPresent, - SecurityContext: &corev1.SecurityContext{ - RunAsUser: &postgresID, - RunAsGroup: &postgresID, - }, - Command: []string{ - "docker-entrypoint.sh", - "-c", "shared_preload_libraries=pg_stat_statements", - "-c", "wal_level=logical", - "-c", "max_replication_slots=2", - }, - Env: []corev1.EnvVar{ - { - Name: "POSTGRES_DB", - Value: "test_db", - }, - { - Name: "POSTGRES_USER", - Value: "postgres", - }, - { - Name: "POSTGRES_PASSWORD", - Value: "postgres", - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "initdb", MountPath: "/docker-entrypoint-initdb.d", - }, - }, - }, - { - Image: "postgres:13-alpine", - Name: "postgres-client", - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"/opt/requests/requests.sh"}, - Env: []corev1.EnvVar{ - { - Name: "POSTGRES_SERVER", - Value: "localhost", - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "requests", MountPath: "/opt/requests", - }, - }, - }, - }, - ServiceAccountName: serviceAccount, - HostNetwork: true, - Volumes: []corev1.Volume{ - { - Name: "initdb", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: cm.Name, - }, - Items: []corev1.KeyToPath{ - { - Key: "init.sh", - Path: "init.sh", - Mode: &fileMode, - }, - { - Key: "db.sql", - Path: "db.sql", - }, - }, - }, - }, - }, - { - Name: "requests", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: cm.Name, - }, - Items: []corev1.KeyToPath{ - { - Key: "requests.sh", - Path: "requests.sh", - Mode: &fileMode, - }, - }, - }, - }, - }, - }, - }, - }, metav1.CreateOptions{}, - ) - require.NoError(cluster.Testcase, err) - - cluster.WaitForPods(postgres.Name, namespace, 5*time.Minute) - return string(postgres.UID) -} - -func clusterRoleAndBinding(namespace, serviceAccount string) (manifests.ClusterRole, manifests.ClusterRoleBinding) { - cr := manifests.ClusterRole{ - Name: "cluster-role", - Namespace: namespace, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{ - "events", - "namespaces", - "namespaces/status", - "nodes", - "nodes/spec", - "nodes/stats", - "nodes/proxy", - "pods", - "pods/status", - "persistentvolumeclaims", - "persistentvolumes", - "replicationcontrollers", - "replicationcontrollers/status", - "resourcequotas", - "services", - }, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"apps"}, - Resources: []string{ - "daemonsets", - "deployments", - "replicasets", - "statefulsets", - }, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"extensions"}, - Resources: []string{ - "daemonsets", - "deployments", - "replicasets", - }, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"batch"}, - Resources: []string{ - "jobs", - "cronjobs", - }, - Verbs: []string{"get", "list", "watch"}, - }, - { - APIGroups: []string{"autoscaling"}, - Resources: []string{ - "horizontalpodautoscalers", - }, - Verbs: []string{"get", "list", "watch"}, - }, - { - NonResourceURLs: []string{"/metrics"}, - Verbs: []string{"get", "list", "watch"}, - }, - }, - } - - crb := manifests.ClusterRoleBinding{ - Namespace: namespace, - Name: "cluster-role-binding", - ClusterRoleName: cr.Name, - ServiceAccountName: serviceAccount, - } - return cr, crb -} - -func (cluster testCluster) daemonSet(namespace, serviceAccount, configMap string) manifests.DaemonSet { - splat := strings.Split(cluster.Testcase.OTLPEndpoint, ":") - port := splat[len(splat)-1] - var hostFromContainer string - if runtime.GOOS == "darwin" { - hostFromContainer = "host.docker.internal" - } else { - hostFromContainer = cluster.GetDefaultGatewayIP() - } - otlpEndpoint := fmt.Sprintf("%s:%s", hostFromContainer, port) - - return manifests.DaemonSet{ - Name: "an.agent.daemonset", - Namespace: namespace, - ServiceAccount: serviceAccount, - Labels: map[string]string{"label.key": "label.value"}, - Containers: []corev1.Container{ - { - Image: testutils.GetCollectorImageOrSkipTest(cluster.Testcase), - Command: []string{ - "/otelcol", "--config=/config/config.yaml", "--discovery", - "--set", "splunk.discovery.receivers.smartagent/postgresql.config.params::username='${env:PG_USERNAME}'", - "--set", "splunk.discovery.receivers.smartagent/postgresql.config.params::password='${env:PG_PASSWORD}'", - "--set", "splunk.discovery.receivers.smartagent/postgresql.config.masterDBName=test_db", - "--set", `splunk.discovery.receivers.smartagent/postgresql.config.extraMetrics=["*"]`, - }, - Env: []corev1.EnvVar{ - {Name: "PG_USERNAME", Value: "test_user"}, - {Name: "PG_PASSWORD", Value: "test_password"}, - {Name: "OTLP_ENDPOINT", Value: otlpEndpoint}, - // Helpful for debugging - // {Name: "SPLUNK_DISCOVERY_DURATION", Value: "20s"}, - // {Name: "SPLUNK_DISCOVERY_LOG_LEVEL", Value: "debug"}, - }, - Name: "otel-collector", - ImagePullPolicy: corev1.PullIfNotPresent, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "config-map-volume", MountPath: "/config", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "config-map-volume", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: configMap, - }, - Items: []corev1.KeyToPath{ - { - Key: "config", - Path: "config.yaml", - }, - }, - }, - }, - }, - }, - } -} From 38293b7ee4d9effc98d94d1d66b5e2ab7215f955 Mon Sep 17 00:00:00 2001 From: machinespeak Date: Mon, 27 Nov 2023 13:56:37 -0500 Subject: [PATCH 06/14] re-add smartagent-postgresql discovery with enabled set to false --- .../smartagent-postgresql.discovery.yaml | 89 +++++++++++++++++++ .../smartagent-postgresql.discovery.yaml | 85 ++++++++++++++++++ .../smartagent-postgresql.discovery.yaml.tmpl | 81 +++++++++++++++++ .../discovery/bundle/bundle_gen.go | 2 + .../discovery/bundle/bundle_other_test.go | 1 + .../discovery/bundle/bundle_windows.go | 1 + .../discovery/bundle/bundle_windows_test.go | 1 + 7 files changed, 260 insertions(+) create mode 100644 cmd/otelcol/config/collector/config.d.linux/receivers/smartagent-postgresql.discovery.yaml create mode 100644 internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml create mode 100644 internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl diff --git a/cmd/otelcol/config/collector/config.d.linux/receivers/smartagent-postgresql.discovery.yaml b/cmd/otelcol/config/collector/config.d.linux/receivers/smartagent-postgresql.discovery.yaml new file mode 100644 index 0000000000..40ee4a400c --- /dev/null +++ b/cmd/otelcol/config/collector/config.d.linux/receivers/smartagent-postgresql.discovery.yaml @@ -0,0 +1,89 @@ +##################################################################################### +# This file is generated by the Splunk Distribution of the OpenTelemetry Collector. # +# # +# It reflects the default configuration bundled in the Collector executable for use # +# in discovery mode (--discovery) and is provided for reference or customization. # +# Please note that any changes made to this file will need to be reconciled during # +# upgrades of the Collector. # +##################################################################################### +# smartagent/postgresql: +# enabled: false +# rule: +# docker_observer: type == "container" and any([name, image, command], {# matches "(?i)postgres"}) and not (command matches "splunk.discovery") +# host_observer: type == "hostport" and command matches "(?i)postgres" and not (command matches "splunk.discovery") +# k8s_observer: type == "port" and pod.name matches "(?i)postgres" +# config: +# default: +# type: postgresql +# connectionString: 'sslmode=disable user={{.username}} password={{.password}}' +# params: +# username: splunk.discovery.default +# password: splunk.discovery.default +# masterDBName: splunk.discovery.default +# status: +# metrics: +# successful: +# - strict: postgres_query_count +# first_only: true +# log_record: +# severity_text: info +# body: PostgreSQL receiver is working! +# partial: +# - strict: postgres_rows_inserted +# first_only: true +# log_record: +# severity_text: info +# body: >- +# Make sure that +# `CREATE EXTENSION IF NOT EXISTS pg_stat_statements;` +# has been run for each database you would like to monitor. +# For example: +# `psql --dbname "" -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;"` +# statements: +# failed: +# - regexp: 'connect: network is unreachable' +# first_only: true +# log_record: +# severity_text: info +# append_pattern: true +# body: The container cannot be reached by the Collector. Make sure they're in the same network. +# - regexp: 'connect: connection refused' +# first_only: true +# log_record: +# severity_text: info +# append_pattern: true +# body: The container is refusing PostgreSQL connections. +# partial: +# - regexp: 'pq: password authentication failed for user' +# first_only: true +# log_record: +# severity_text: info +# append_pattern: true +# body: >- +# Please ensure your user credentials are correctly specified with +# `--set splunk.discovery.receivers.smartagent/postgresql.config.params::username=""` and +# `--set splunk.discovery.receivers.smartagent/postgresql.config.params::password=""` or +# `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_params_x3a__x3a_username=""` and +# `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_params_x3a__x3a_password=""` environment variables. +# - regexp: 'pq: database .* does not exist' +# first_only: true +# log_record: +# severity_text: info +# append_pattern: true +# body: >- +# Make sure the target database is correctly specified using the +# `--set splunk.discovery.receivers.smartagent/postgresql.config.masterDBName=""` command or the +# `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_masterDBName=""` environment variable. +# - regexp: 'pq: pg_stat_statements must be loaded via shared_preload_libraries' +# first_only: true +# log_record: +# severity_text: info +# append_pattern: true +# body: >- +# Make sure your PostgreSQL database has +# `shared_preload_libraries = 'pg_stat_statements'` +# in the postgresql.conf file and that +# `CREATE EXTENSION IF NOT EXISTS pg_stat_statements;` +# has been run for each database you would like to monitor. +# For example: +# `psql --dbname "" -c "CREATE EXTENSION pg_stat_statements;"` \ No newline at end of file diff --git a/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml new file mode 100644 index 0000000000..117044e6ee --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml @@ -0,0 +1,85 @@ +##################################################################################### +# Do not edit manually! # +# All changes must be made to associated .tmpl file before running 'make bundle.d'. # +##################################################################################### +smartagent/postgresql: + enabled: false + rule: + docker_observer: type == "container" and any([name, image, command], {# matches "(?i)postgres"}) and not (command matches "splunk.discovery") + host_observer: type == "hostport" and command matches "(?i)postgres" and not (command matches "splunk.discovery") + k8s_observer: type == "port" and pod.name matches "(?i)postgres" + config: + default: + type: postgresql + connectionString: 'sslmode=disable user={{.username}} password={{.password}}' + params: + username: splunk.discovery.default + password: splunk.discovery.default + masterDBName: splunk.discovery.default + status: + metrics: + successful: + - strict: postgres_query_count + first_only: true + log_record: + severity_text: info + body: PostgreSQL receiver is working! + partial: + - strict: postgres_rows_inserted + first_only: true + log_record: + severity_text: info + body: >- + Make sure that + `CREATE EXTENSION IF NOT EXISTS pg_stat_statements;` + has been run for each database you would like to monitor. + For example: + `psql --dbname "" -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;"` + statements: + failed: + - regexp: 'connect: network is unreachable' + first_only: true + log_record: + severity_text: info + append_pattern: true + body: The container cannot be reached by the Collector. Make sure they're in the same network. + - regexp: 'connect: connection refused' + first_only: true + log_record: + severity_text: info + append_pattern: true + body: The container is refusing PostgreSQL connections. + partial: + - regexp: 'pq: password authentication failed for user' + first_only: true + log_record: + severity_text: info + append_pattern: true + body: >- + Please ensure your user credentials are correctly specified with + `--set splunk.discovery.receivers.smartagent/postgresql.config.params::username=""` and + `--set splunk.discovery.receivers.smartagent/postgresql.config.params::password=""` or + `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_params_x3a__x3a_username=""` and + `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_params_x3a__x3a_password=""` environment variables. + - regexp: 'pq: database .* does not exist' + first_only: true + log_record: + severity_text: info + append_pattern: true + body: >- + Make sure the target database is correctly specified using the + `--set splunk.discovery.receivers.smartagent/postgresql.config.masterDBName=""` command or the + `SPLUNK_DISCOVERY_RECEIVERS_smartagent_x2f_postgresql_CONFIG_masterDBName=""` environment variable. + - regexp: 'pq: pg_stat_statements must be loaded via shared_preload_libraries' + first_only: true + log_record: + severity_text: info + append_pattern: true + body: >- + Make sure your PostgreSQL database has + `shared_preload_libraries = 'pg_stat_statements'` + in the postgresql.conf file and that + `CREATE EXTENSION IF NOT EXISTS pg_stat_statements;` + has been run for each database you would like to monitor. + For example: + `psql --dbname "" -c "CREATE EXTENSION pg_stat_statements;"` \ No newline at end of file diff --git a/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl new file mode 100644 index 0000000000..6fab6001ca --- /dev/null +++ b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl @@ -0,0 +1,81 @@ +{{ receiver "smartagent/postgresql" }}: + enabled: false + rule: + docker_observer: type == "container" and any([name, image, command], {# matches "(?i)postgres"}) and not (command matches "splunk.discovery") + host_observer: type == "hostport" and command matches "(?i)postgres" and not (command matches "splunk.discovery") + k8s_observer: type == "port" and pod.name matches "(?i)postgres" + config: + default: + type: postgresql + connectionString: 'sslmode=disable user={{ "{{.username}}" }} password={{ "{{.password}}" }}' + params: + username: {{ defaultValue }} + password: {{ defaultValue }} + masterDBName: {{ defaultValue }} + status: + metrics: + successful: + - strict: postgres_query_count + first_only: true + log_record: + severity_text: info + body: PostgreSQL receiver is working! + partial: + - strict: postgres_rows_inserted + first_only: true + log_record: + severity_text: info + body: >- + Make sure that + `CREATE EXTENSION IF NOT EXISTS pg_stat_statements;` + has been run for each database you would like to monitor. + For example: + `psql --dbname "" -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;"` + statements: + failed: + - regexp: 'connect: network is unreachable' + first_only: true + log_record: + severity_text: info + append_pattern: true + body: The container cannot be reached by the Collector. Make sure they're in the same network. + - regexp: 'connect: connection refused' + first_only: true + log_record: + severity_text: info + append_pattern: true + body: The container is refusing PostgreSQL connections. + partial: + - regexp: 'pq: password authentication failed for user' + first_only: true + log_record: + severity_text: info + append_pattern: true + body: >- + Please ensure your user credentials are correctly specified with + `--set {{ configProperty "params" "username" "" }}` and + `--set {{ configProperty "params" "password" "" }}` or + `{{ configPropertyEnvVar "params" "username" "" }}` and + `{{ configPropertyEnvVar "params" "password" "" }}` environment variables. + - regexp: 'pq: database .* does not exist' + first_only: true + log_record: + severity_text: info + append_pattern: true + body: >- + Make sure the target database is correctly specified using the + `--set {{ configProperty "masterDBName" "" }}` command or the + `{{ configPropertyEnvVar "masterDBName" "" }}` environment variable. + - regexp: 'pq: pg_stat_statements must be loaded via shared_preload_libraries' + first_only: true + log_record: + severity_text: info + append_pattern: true + body: >- + Make sure your PostgreSQL database has + `shared_preload_libraries = 'pg_stat_statements'` + in the postgresql.conf file and that + `CREATE EXTENSION IF NOT EXISTS pg_stat_statements;` + has been run for each database you would like to monitor. + For example: + `psql --dbname "" -c "CREATE EXTENSION pg_stat_statements;"` \ No newline at end of file diff --git a/internal/confmapprovider/discovery/bundle/bundle_gen.go b/internal/confmapprovider/discovery/bundle/bundle_gen.go index 2954cc7896..6d1e18790d 100644 --- a/internal/confmapprovider/discovery/bundle/bundle_gen.go +++ b/internal/confmapprovider/discovery/bundle/bundle_gen.go @@ -19,6 +19,8 @@ // TODO: add a pre-bundle.d step that generates these generate directives for easier maintenance/management across platforms //go:generate go install github.com/signalfx/splunk-otel-collector/internal/confmapprovider/discovery/bundle/cmd/discoverybundler +//go:generate discoverybundler -r -t bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl +//go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-postgresql.discovery.yaml.tmpl //go:generate discoverybundler -r -t bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml.tmpl //go:generate discoverybundler -r -c -d ../../../../cmd/otelcol/config/collector/config.d.linux/receivers -t bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml.tmpl //go:generate discoverybundler -r -t bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml.tmpl diff --git a/internal/confmapprovider/discovery/bundle/bundle_other_test.go b/internal/confmapprovider/discovery/bundle/bundle_other_test.go index b72b5856a6..2230c9756c 100644 --- a/internal/confmapprovider/discovery/bundle/bundle_other_test.go +++ b/internal/confmapprovider/discovery/bundle/bundle_other_test.go @@ -32,6 +32,7 @@ func TestBundleDir(t *testing.T) { "bundle.d/receivers/redis.discovery.yaml", "bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml", "bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml", + "bundle.d/receivers/smartagent-postgresql.discovery.yaml", }, receivers) extensions, err := fs.Glob(BundledFS, "bundle.d/extensions/*.discovery.yaml") diff --git a/internal/confmapprovider/discovery/bundle/bundle_windows.go b/internal/confmapprovider/discovery/bundle/bundle_windows.go index 36bd7e952a..b2ef5d29b1 100644 --- a/internal/confmapprovider/discovery/bundle/bundle_windows.go +++ b/internal/confmapprovider/discovery/bundle/bundle_windows.go @@ -30,4 +30,5 @@ import ( //go:embed bundle.d/receivers/oracledb.discovery.yaml //go:embed bundle.d/receivers/postgresql.discovery.yaml //go:embed bundle.d/receivers/redis.discovery.yaml +//go:embed bundle.d/receivers/smartagent-postgresql.discovery.yaml var BundledFS embed.FS diff --git a/internal/confmapprovider/discovery/bundle/bundle_windows_test.go b/internal/confmapprovider/discovery/bundle/bundle_windows_test.go index 776b65f6f4..1f77367baa 100644 --- a/internal/confmapprovider/discovery/bundle/bundle_windows_test.go +++ b/internal/confmapprovider/discovery/bundle/bundle_windows_test.go @@ -30,6 +30,7 @@ func TestBundleDir(t *testing.T) { "bundle.d/receivers/oracledb.discovery.yaml", "bundle.d/receivers/postgresql.discovery.yaml", "bundle.d/receivers/redis.discovery.yaml", + "bundle.d/receivers/smartagent-postgresql.discovery.yaml", }, receivers) extensions, err := fs.Glob(BundledFS, "bundle.d/extensions/*.discovery.yaml") From 3167875c518335812814887b06febe8dcf3ac836 Mon Sep 17 00:00:00 2001 From: machinespeak Date: Fri, 1 Dec 2023 10:20:55 -0500 Subject: [PATCH 07/14] add back smartagent postgresql bundled_test --- .../smartagent/postgresql/bundled_test.go | 377 ++++++++++++++++++ 1 file changed, 377 insertions(+) create mode 100644 tests/receivers/smartagent/postgresql/bundled_test.go diff --git a/tests/receivers/smartagent/postgresql/bundled_test.go b/tests/receivers/smartagent/postgresql/bundled_test.go new file mode 100644 index 0000000000..75c974abc0 --- /dev/null +++ b/tests/receivers/smartagent/postgresql/bundled_test.go @@ -0,0 +1,377 @@ +// Copyright Splunk, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build integration + +package tests + +import ( + "context" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "gopkg.in/yaml.v3" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/signalfx/splunk-otel-collector/tests/testutils" + "github.com/signalfx/splunk-otel-collector/tests/testutils/kubeutils" + "github.com/signalfx/splunk-otel-collector/tests/testutils/kubeutils/manifests" +) + +func TestK8sObserver(t *testing.T) { + testutils.SkipIfNotContainerTest(t) + tc := testutils.NewTestcase(t, testutils.OTLPReceiverSinkAllInterfaces) + defer tc.PrintLogsOnFailure() + defer tc.ShutdownOTLPReceiverSink() + + cluster := testCluster{kubeutils.NewKindCluster(tc)} + defer cluster.Teardown() + cluster.Create() + cluster.LoadLocalCollectorImageIfNecessary() + + namespace := manifests.Namespace{Name: "test-namespace"} + serviceAccount := manifests.ServiceAccount{Name: "some.serviceacount", Namespace: namespace.Name} + clusterRole, clusterRoleBinding := clusterRoleAndBinding(namespace.Name, serviceAccount.Name) + sout, serr, err := cluster.Apply(manifests.RenderAll(t, namespace, serviceAccount, clusterRole, clusterRoleBinding)) + require.NoError(t, err, "stdout: %s, stderr: %s", sout, serr) + + postgresUID := cluster.createPostgres("target.postgres", namespace.Name, serviceAccount.Name) + + configMap := manifests.ConfigMap{ + Name: "collector.config", Namespace: namespace.Name, + Data: `config: | + exporters: + otlp: + endpoint: ${OTLP_ENDPOINT} + tls: + insecure: true + service: + pipelines: + metrics: + exporters: + - otlp +`} + + ds := cluster.daemonSet(namespace.Name, serviceAccount.Name, configMap.Name) + sout, serr, err = cluster.Apply(manifests.RenderAll(t, configMap, ds)) + require.NoError(t, err, "stdout: %s, stderr: %s", sout, serr) + + pods := cluster.WaitForPods(ds.Name, namespace.Name, 5*time.Minute) + require.Len(t, pods, 1) + collectorName := pods[0].Name + + expectedMetrics := tc.ResourceMetrics("all.yaml") + require.NoError(t, tc.OTLPReceiverSink.AssertAllMetricsReceived(t, *expectedMetrics, 30*time.Second)) + + stdOut, stdErr, err := cluster.Kubectl("logs", "-n", namespace.Name, collectorName) + require.NoError(t, err) + require.Contains( + t, stdOut.String(), + fmt.Sprintf(`Successfully discovered "smartagent/postgresql" using "k8s_observer" endpoint "k8s_observer/%s/(5432)`, postgresUID), + stdErr.String(), + ) +} + +type testCluster struct{ *kubeutils.KindCluster } + +func (cluster testCluster) createPostgres(name, namespace, serviceAccount string) string { + dbsql, err := os.ReadFile(filepath.Join(".", "testdata", "server", "initdb.d", "db.sql")) + require.NoError(cluster.Testcase, err) + cmContent := map[string]any{"db.sql": string(dbsql)} + + initsh, err := os.ReadFile(filepath.Join(".", "testdata", "server", "initdb.d", "init.sh")) + require.NoError(cluster.Testcase, err) + cmContent["init.sh"] = string(initsh) + + requests, err := os.ReadFile(filepath.Join(".", "testdata", "client", "requests.sh")) + require.NoError(cluster.Testcase, err) + cmContent["requests.sh"] = string(requests) + + configMapContent, err := yaml.Marshal(cmContent) + require.NoError(cluster.Testcase, err) + + cm := manifests.ConfigMap{ + Namespace: namespace, + Name: "postgres", + Data: string(configMapContent), + } + sout, serr, err := cluster.Apply(cm.Render(cluster.Testcase)) + cluster.Testcase.Logger.Debug("applying ConfigMap", zap.String("stdout", sout.String()), zap.String("stderr", serr.String())) + require.NoError(cluster.Testcase, err) + + fileMode := int32(0777) + postgresID := int64(70) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + postgres, err := cluster.Clientset.CoreV1().Pods(namespace).Create( + ctx, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Image: "postgres:13-alpine", + Name: "postgres-server", + Ports: []corev1.ContainerPort{{ContainerPort: 5432}}, + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &postgresID, + RunAsGroup: &postgresID, + }, + Command: []string{ + "docker-entrypoint.sh", + "-c", "shared_preload_libraries=pg_stat_statements", + "-c", "wal_level=logical", + "-c", "max_replication_slots=2", + }, + Env: []corev1.EnvVar{ + { + Name: "POSTGRES_DB", + Value: "test_db", + }, + { + Name: "POSTGRES_USER", + Value: "postgres", + }, + { + Name: "POSTGRES_PASSWORD", + Value: "postgres", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "initdb", MountPath: "/docker-entrypoint-initdb.d", + }, + }, + }, + { + Image: "postgres:13-alpine", + Name: "postgres-client", + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/opt/requests/requests.sh"}, + Env: []corev1.EnvVar{ + { + Name: "POSTGRES_SERVER", + Value: "localhost", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "requests", MountPath: "/opt/requests", + }, + }, + }, + }, + ServiceAccountName: serviceAccount, + HostNetwork: true, + Volumes: []corev1.Volume{ + { + Name: "initdb", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cm.Name, + }, + Items: []corev1.KeyToPath{ + { + Key: "init.sh", + Path: "init.sh", + Mode: &fileMode, + }, + { + Key: "db.sql", + Path: "db.sql", + }, + }, + }, + }, + }, + { + Name: "requests", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cm.Name, + }, + Items: []corev1.KeyToPath{ + { + Key: "requests.sh", + Path: "requests.sh", + Mode: &fileMode, + }, + }, + }, + }, + }, + }, + }, + }, metav1.CreateOptions{}, + ) + require.NoError(cluster.Testcase, err) + + cluster.WaitForPods(postgres.Name, namespace, 5*time.Minute) + return string(postgres.UID) +} + +func clusterRoleAndBinding(namespace, serviceAccount string) (manifests.ClusterRole, manifests.ClusterRoleBinding) { + cr := manifests.ClusterRole{ + Name: "cluster-role", + Namespace: namespace, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{ + "events", + "namespaces", + "namespaces/status", + "nodes", + "nodes/spec", + "nodes/stats", + "nodes/proxy", + "pods", + "pods/status", + "persistentvolumeclaims", + "persistentvolumes", + "replicationcontrollers", + "replicationcontrollers/status", + "resourcequotas", + "services", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"apps"}, + Resources: []string{ + "daemonsets", + "deployments", + "replicasets", + "statefulsets", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"extensions"}, + Resources: []string{ + "daemonsets", + "deployments", + "replicasets", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"batch"}, + Resources: []string{ + "jobs", + "cronjobs", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + APIGroups: []string{"autoscaling"}, + Resources: []string{ + "horizontalpodautoscalers", + }, + Verbs: []string{"get", "list", "watch"}, + }, + { + NonResourceURLs: []string{"/metrics"}, + Verbs: []string{"get", "list", "watch"}, + }, + }, + } + + crb := manifests.ClusterRoleBinding{ + Namespace: namespace, + Name: "cluster-role-binding", + ClusterRoleName: cr.Name, + ServiceAccountName: serviceAccount, + } + return cr, crb +} + +func (cluster testCluster) daemonSet(namespace, serviceAccount, configMap string) manifests.DaemonSet { + splat := strings.Split(cluster.Testcase.OTLPEndpoint, ":") + port := splat[len(splat)-1] + var hostFromContainer string + if runtime.GOOS == "darwin" { + hostFromContainer = "host.docker.internal" + } else { + hostFromContainer = cluster.GetDefaultGatewayIP() + } + otlpEndpoint := fmt.Sprintf("%s:%s", hostFromContainer, port) + + return manifests.DaemonSet{ + Name: "an.agent.daemonset", + Namespace: namespace, + ServiceAccount: serviceAccount, + Labels: map[string]string{"label.key": "label.value"}, + Containers: []corev1.Container{ + { + Image: testutils.GetCollectorImageOrSkipTest(cluster.Testcase), + Command: []string{ + "/otelcol", "--config=/config/config.yaml", "--discovery", + "--set", "splunk.discovery.receivers.smartagent/postgresql.config.params::username='${env:PG_USERNAME}'", + "--set", "splunk.discovery.receivers.smartagent/postgresql.config.params::password='${env:PG_PASSWORD}'", + "--set", "splunk.discovery.receivers.smartagent/postgresql.config.masterDBName=test_db", + "--set", `splunk.discovery.receivers.smartagent/postgresql.config.extraMetrics=["*"]`, + "--set", `splunk.discovery.receivers.smartagent/postgresql.enabled=true`, + }, + Env: []corev1.EnvVar{ + {Name: "PG_USERNAME", Value: "test_user"}, + {Name: "PG_PASSWORD", Value: "test_password"}, + {Name: "OTLP_ENDPOINT", Value: otlpEndpoint}, + // Helpful for debugging + // {Name: "SPLUNK_DISCOVERY_DURATION", Value: "20s"}, + // {Name: "SPLUNK_DISCOVERY_LOG_LEVEL", Value: "debug"}, + }, + Name: "otel-collector", + ImagePullPolicy: corev1.PullIfNotPresent, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "config-map-volume", MountPath: "/config", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "config-map-volume", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMap, + }, + Items: []corev1.KeyToPath{ + { + Key: "config", + Path: "config.yaml", + }, + }, + }, + }, + }, + }, + } +} \ No newline at end of file From 23370bc6fc20a2431109d9da73d1501154993387 Mon Sep 17 00:00:00 2001 From: machinespeak Date: Fri, 1 Dec 2023 13:49:22 -0500 Subject: [PATCH 08/14] gofmt --- tests/receivers/smartagent/postgresql/bundled_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/receivers/smartagent/postgresql/bundled_test.go b/tests/receivers/smartagent/postgresql/bundled_test.go index 75c974abc0..07d72c1d80 100644 --- a/tests/receivers/smartagent/postgresql/bundled_test.go +++ b/tests/receivers/smartagent/postgresql/bundled_test.go @@ -374,4 +374,4 @@ func (cluster testCluster) daemonSet(namespace, serviceAccount, configMap string }, }, } -} \ No newline at end of file +} From 4a15e9554e1b340040e628d48db6d8f2243a4b05 Mon Sep 17 00:00:00 2001 From: greatestusername-splunk <83726369+greatestusername-splunk@users.noreply.github.com> Date: Wed, 13 Dec 2023 13:57:10 -0500 Subject: [PATCH 09/14] Update internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml.tmpl Co-authored-by: Ryan Fitzpatrick <10867373+rmfitzpatrick@users.noreply.github.com> --- .../bundle/bundle.d/receivers/postgresql.discovery.yaml.tmpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml.tmpl b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml.tmpl index 3dc31c2374..d2d2a6f583 100644 --- a/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml.tmpl +++ b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml.tmpl @@ -5,7 +5,9 @@ host_observer: type == "hostport" and command matches "(?i)postgres" and not (command matches "splunk.discovery") k8s_observer: type == "port" and pod.name matches "(?i)postgres" config: - default: {} + default: + username: {{ defaultValue }} + password: {{ defaultValue }} status: metrics: successful: From 0e22df70be238bc1b85db1b59a5cd9ac463933e1 Mon Sep 17 00:00:00 2001 From: obutts Date: Wed, 13 Dec 2023 11:53:50 -0800 Subject: [PATCH 10/14] reference existing testdata files and remove copies. --- .../receivers/postgresql.discovery.yaml | 4 +- .../receivers/postgresql.discovery.yaml | 4 +- tests/receivers/postgresql/bundled_test.go | 6 +- .../postgresql/testdata/client/Dockerfile | 6 -- .../postgresql/testdata/client/requests.sh | 95 ------------------- .../postgresql/testdata/server_k8s/Dockerfile | 9 -- .../testdata/server_k8s/initdb.d/db.sql | 30 ------ .../testdata/server_k8s/initdb.d/init.sh | 12 --- 8 files changed, 9 insertions(+), 157 deletions(-) delete mode 100644 tests/receivers/postgresql/testdata/client/Dockerfile delete mode 100755 tests/receivers/postgresql/testdata/client/requests.sh delete mode 100644 tests/receivers/postgresql/testdata/server_k8s/Dockerfile delete mode 100755 tests/receivers/postgresql/testdata/server_k8s/initdb.d/db.sql delete mode 100755 tests/receivers/postgresql/testdata/server_k8s/initdb.d/init.sh diff --git a/cmd/otelcol/config/collector/config.d.linux/receivers/postgresql.discovery.yaml b/cmd/otelcol/config/collector/config.d.linux/receivers/postgresql.discovery.yaml index fd7d86b1b2..c3e6559338 100644 --- a/cmd/otelcol/config/collector/config.d.linux/receivers/postgresql.discovery.yaml +++ b/cmd/otelcol/config/collector/config.d.linux/receivers/postgresql.discovery.yaml @@ -13,7 +13,9 @@ # host_observer: type == "hostport" and command matches "(?i)postgres" and not (command matches "splunk.discovery") # k8s_observer: type == "port" and pod.name matches "(?i)postgres" # config: -# default: {} +# default: +# username: splunk.discovery.default +# password: splunk.discovery.default # status: # metrics: # successful: diff --git a/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml index b0b9e0fd7a..cc4446be29 100644 --- a/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml +++ b/internal/confmapprovider/discovery/bundle/bundle.d/receivers/postgresql.discovery.yaml @@ -9,7 +9,9 @@ postgresql: host_observer: type == "hostport" and command matches "(?i)postgres" and not (command matches "splunk.discovery") k8s_observer: type == "port" and pod.name matches "(?i)postgres" config: - default: {} + default: + username: splunk.discovery.default + password: splunk.discovery.default status: metrics: successful: diff --git a/tests/receivers/postgresql/bundled_test.go b/tests/receivers/postgresql/bundled_test.go index 0db56506ff..ae0a723295 100644 --- a/tests/receivers/postgresql/bundled_test.go +++ b/tests/receivers/postgresql/bundled_test.go @@ -130,15 +130,15 @@ func TestK8sObserver(t *testing.T) { type testCluster struct{ *kubeutils.KindCluster } func (cluster testCluster) createPostgres(name, namespace, serviceAccount string) string { - dbsql, err := os.ReadFile(filepath.Join(".", "testdata", "server_k8s", "initdb.d", "db.sql")) + dbsql, err := os.ReadFile(filepath.Join("..", "smartagent", "postgresql", "testdata", "server", "initdb.d", "db.sql")) require.NoError(cluster.Testcase, err) cmContent := map[string]any{"db.sql": string(dbsql)} - initsh, err := os.ReadFile(filepath.Join(".", "testdata", "server_k8s", "initdb.d", "init.sh")) + initsh, err := os.ReadFile(filepath.Join("..", "smartagent", "postgresql", "testdata", "server", "initdb.d", "init.sh")) require.NoError(cluster.Testcase, err) cmContent["init.sh"] = string(initsh) - requests, err := os.ReadFile(filepath.Join(".", "testdata", "client", "requests.sh")) + requests, err := os.ReadFile(filepath.Join("..", "smartagent", "postgresql", "testdata", "client", "requests.sh")) require.NoError(cluster.Testcase, err) cmContent["requests.sh"] = string(requests) diff --git a/tests/receivers/postgresql/testdata/client/Dockerfile b/tests/receivers/postgresql/testdata/client/Dockerfile deleted file mode 100644 index a22875b55e..0000000000 --- a/tests/receivers/postgresql/testdata/client/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -ARG POSTGRES_VERSION=13-alpine -FROM postgres:${POSTGRES_VERSION} - -COPY requests.sh /usr/local/bin/requests.sh - -CMD ["requests.sh"] diff --git a/tests/receivers/postgresql/testdata/client/requests.sh b/tests/receivers/postgresql/testdata/client/requests.sh deleted file mode 100755 index ab1304dd4c..0000000000 --- a/tests/receivers/postgresql/testdata/client/requests.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/bin/bash -################################################################################################## -# This file will exercise a target POSTGRES_SERVER's test_db whose schema is defined in the -# corresponding server's initdb.d/db.sql. It will create the necessary replication slots and make -# a series of random insertions, updates, and deletions without end to help generate values for -# produced metrics. -# As implemented, test_db size will grow without bound if left is running, so this script isn't -# currently suitable for extended demonstrations or soak test purposes. -################################################################################################## - -echopsql() { - set -x - psql postgresql://test_user:test_password@${POSTGRES_SERVER:-localhost}:5432/test_db "$@" - local r=$? - { set +x; } 2>/dev/null - return $r -} - -# create replication slots for postgres_replication_state metrics -create_replication_slots() { -for query in \ - "SELECT * FROM pg_create_physical_replication_slot('some_physical_replication_slot');" \ - "SELECT * FROM pg_create_logical_replication_slot('some_logical_replication_slot', 'test_decoding');"; do - while true; do - if echopsql -c "${query}"; then - echo "${query} successful" - break - fi - sleep 1 - done -done -} - -random_str() { - tr -dc A-Za-z0-9 Date: Thu, 14 Dec 2023 09:27:04 -0800 Subject: [PATCH 11/14] update properties.discovery.yaml.example --- .../collector/config.d.linux/properties.discovery.yaml.example | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/otelcol/config/collector/config.d.linux/properties.discovery.yaml.example b/cmd/otelcol/config/collector/config.d.linux/properties.discovery.yaml.example index 279c8a8595..c134c55e76 100644 --- a/cmd/otelcol/config/collector/config.d.linux/properties.discovery.yaml.example +++ b/cmd/otelcol/config/collector/config.d.linux/properties.discovery.yaml.example @@ -16,4 +16,6 @@ splunk.discovery: smartagent/collectd/nginx: enabled: true smartagent/postgresql: + enabled: false + postgresql: enabled: true From 299fe7864eccf1247021d16ad131cfaff4c71005 Mon Sep 17 00:00:00 2001 From: obutts Date: Thu, 14 Dec 2023 11:39:44 -0800 Subject: [PATCH 12/14] fix for bundledfs --- internal/confmapprovider/discovery/bundle/bundledfs_others.go | 1 + internal/confmapprovider/discovery/bundle/components.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/internal/confmapprovider/discovery/bundle/bundledfs_others.go b/internal/confmapprovider/discovery/bundle/bundledfs_others.go index cfbd2de7e5..b46c7ca5ce 100644 --- a/internal/confmapprovider/discovery/bundle/bundledfs_others.go +++ b/internal/confmapprovider/discovery/bundle/bundledfs_others.go @@ -26,6 +26,7 @@ import ( //go:embed bundle.d/extensions/host-observer.discovery.yaml //go:embed bundle.d/extensions/k8s-observer.discovery.yaml //go:embed bundle.d/receivers/oracledb.discovery.yaml +//go:embed bundle.d/receivers/postgresql.discovery.yaml //go:embed bundle.d/receivers/redis.discovery.yaml //go:embed bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml //go:embed bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml diff --git a/internal/confmapprovider/discovery/bundle/components.go b/internal/confmapprovider/discovery/bundle/components.go index c1a927d252..b936bb46b2 100644 --- a/internal/confmapprovider/discovery/bundle/components.go +++ b/internal/confmapprovider/discovery/bundle/components.go @@ -32,6 +32,7 @@ var ( // in Components.Linux. If desired in windows BundledFS, ensure they are included in Components.Windows. receivers = []string{ "oracledb", + "postgresql", "redis", "smartagent-collectd-mysql", "smartagent-collectd-nginx", @@ -60,6 +61,7 @@ var ( Windows: func() map[string]struct{} { windows := map[string]struct{}{ "oracledb": {}, + "postgresql": {}, "redis": {}, "smartagent-postgresql": {}, } From 33e0bd605daa68e8ee84768bc037103d67c56d3b Mon Sep 17 00:00:00 2001 From: obutts Date: Thu, 14 Dec 2023 13:56:40 -0800 Subject: [PATCH 13/14] fix alpha ordering --- .../config.d.linux/properties.discovery.yaml.example | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/otelcol/config/collector/config.d.linux/properties.discovery.yaml.example b/cmd/otelcol/config/collector/config.d.linux/properties.discovery.yaml.example index c134c55e76..996df50aa7 100644 --- a/cmd/otelcol/config/collector/config.d.linux/properties.discovery.yaml.example +++ b/cmd/otelcol/config/collector/config.d.linux/properties.discovery.yaml.example @@ -11,11 +11,11 @@ splunk.discovery: k8s_observer: enabled: true receivers: + postgresql: + enabled: true smartagent/collectd/mysql: enabled: true smartagent/collectd/nginx: enabled: true smartagent/postgresql: enabled: false - postgresql: - enabled: true From 0389e2d9dd9bceb8f7abb96f0649b86d1975f543 Mon Sep 17 00:00:00 2001 From: obutts Date: Mon, 18 Dec 2023 09:40:09 -0800 Subject: [PATCH 14/14] add breaking change to postgresql discovery in changelog and update discovery readme --- CHANGELOG.md | 4 ++++ internal/confmapprovider/discovery/README.md | 1 + 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3fd4c1f6f..9b6741bf71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ - (Splunk) Update splunk-otel-js to [v2.6.0](https://github.com/signalfx/splunk-otel-js/releases/tag/v2.6.0) ([#4064](https://github.com/signalfx/splunk-otel-collector/pull/4064)) +### 🛑 Breaking changes 🛑 + +- `postgresql` discovery now uses the OpenTelemetry Collector Contrib receiver by default instead of the smartagent receiver. + ## v0.91.0 This Splunk OpenTelemetry Collector release includes changes from the [opentelemetry-collector v0.91.0](https://github.com/open-telemetry/opentelemetry-collector/releases/tag/v0.91.0) and the [opentelemetry-collector-contrib v0.91.0](https://github.com/open-telemetry/opentelemetry-collector-contrib/releases/tag/v0.91.0) releases where appropriate. diff --git a/internal/confmapprovider/discovery/README.md b/internal/confmapprovider/discovery/README.md index 83aede17dd..e1d23a7ef3 100644 --- a/internal/confmapprovider/discovery/README.md +++ b/internal/confmapprovider/discovery/README.md @@ -137,6 +137,7 @@ The following components have bundled discovery configurations in the last Splun I. Receivers * `oracledb` ([Linux and Windows](./bundle/bundle.d/receivers/oracledb.discovery.yaml)) +* `postgresql` ([Linux and Windows](./bundle/bundle.d/receivers/postgresql.discovery.yaml)) * `redis` ([Linux and Windows](./bundle/bundle.d/receivers/redis.discovery.yaml)) * `smartagent` with `collectd/mysql` monitor type ([Linux](./bundle/bundle.d/receivers/smartagent-collectd-mysql.discovery.yaml)) * `smartagent` with `collectd/nginx` monitor type ([Linux](./bundle/bundle.d/receivers/smartagent-collectd-nginx.discovery.yaml))