From aac7c67039c57ce05b29fe122b6e60f131b2330d Mon Sep 17 00:00:00 2001 From: Curtis Robert Date: Mon, 5 Feb 2024 11:05:48 -0800 Subject: [PATCH 1/5] [exporter/loadbalancing] Fix memory leak --- .chloggen/goleak_loadbalancingexp.yaml | 27 +++++++++++++++++++ exporter/loadbalancingexporter/go.mod | 1 + exporter/loadbalancingexporter/go.sum | 1 + .../loadbalancingexporter/loadbalancer.go | 5 ++-- .../loadbalancer_test.go | 1 + .../loadbalancingexporter/log_exporter.go | 5 ++-- .../loadbalancingexporter/metrics_exporter.go | 3 ++- .../loadbalancingexporter/package_test.go | 17 ++++++++++++ .../resolver_dns_test.go | 1 + .../loadbalancingexporter/resolver_static.go | 13 ++++++++- .../loadbalancingexporter/trace_exporter.go | 5 ++-- 11 files changed, 71 insertions(+), 8 deletions(-) create mode 100755 .chloggen/goleak_loadbalancingexp.yaml create mode 100644 exporter/loadbalancingexporter/package_test.go diff --git a/.chloggen/goleak_loadbalancingexp.yaml b/.chloggen/goleak_loadbalancingexp.yaml new file mode 100755 index 000000000000..5eb02f90e98e --- /dev/null +++ b/.chloggen/goleak_loadbalancingexp.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: bug_fix + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: loadbalancingexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Fix memory leak on shutdown + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/loadbalancingexporter/go.mod b/exporter/loadbalancingexporter/go.mod index 4214331b7e58..74221da764d5 100644 --- a/exporter/loadbalancingexporter/go.mod +++ b/exporter/loadbalancingexporter/go.mod @@ -17,6 +17,7 @@ require ( go.opentelemetry.io/collector/semconv v0.93.1-0.20240202170612-7abb9622312d go.opentelemetry.io/otel/metric v1.22.0 go.opentelemetry.io/otel/trace v1.22.0 + go.uber.org/goleak v1.3.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.26.0 k8s.io/api v0.28.4 diff --git a/exporter/loadbalancingexporter/go.sum b/exporter/loadbalancingexporter/go.sum index 1ad37f262cce..7b6e9890093b 100644 --- a/exporter/loadbalancingexporter/go.sum +++ b/exporter/loadbalancingexporter/go.sum @@ -277,6 +277,7 @@ go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40 go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= diff --git a/exporter/loadbalancingexporter/loadbalancer.go b/exporter/loadbalancingexporter/loadbalancer.go index 862111d06c76..69007d5c2906 100644 --- a/exporter/loadbalancingexporter/loadbalancer.go +++ b/exporter/loadbalancingexporter/loadbalancer.go @@ -172,9 +172,10 @@ func endpointFound(endpoint string, endpoints []string) bool { return false } -func (lb *loadBalancerImp) Shutdown(context.Context) error { +func (lb *loadBalancerImp) Shutdown(ctx context.Context) error { + err := lb.res.shutdown(ctx) lb.stopped = true - return nil + return err } func (lb *loadBalancerImp) Endpoint(identifier []byte) string { diff --git a/exporter/loadbalancingexporter/loadbalancer_test.go b/exporter/loadbalancingexporter/loadbalancer_test.go index fbc884ff1863..d6720c01957f 100644 --- a/exporter/loadbalancingexporter/loadbalancer_test.go +++ b/exporter/loadbalancingexporter/loadbalancer_test.go @@ -134,6 +134,7 @@ func TestWithDNSResolverNoEndpoints(t *testing.T) { err = p.Start(context.Background(), componenttest.NewNopHost()) require.NoError(t, err) + defer func() { assert.NoError(t, p.Shutdown(context.Background())) }() // test e := p.Endpoint([]byte{128, 128, 0, 0}) diff --git a/exporter/loadbalancingexporter/log_exporter.go b/exporter/loadbalancingexporter/log_exporter.go index 6cfe82edca65..cd94e8d5042c 100644 --- a/exporter/loadbalancingexporter/log_exporter.go +++ b/exporter/loadbalancingexporter/log_exporter.go @@ -58,13 +58,14 @@ func (e *logExporterImp) Start(ctx context.Context, host component.Host) error { return e.loadBalancer.Start(ctx, host) } -func (e *logExporterImp) Shutdown(context.Context) error { +func (e *logExporterImp) Shutdown(ctx context.Context) error { if !e.started { return nil } + err := e.loadBalancer.Shutdown(ctx) e.started = false e.shutdownWg.Wait() - return nil + return err } func (e *logExporterImp) ConsumeLogs(ctx context.Context, ld plog.Logs) error { diff --git a/exporter/loadbalancingexporter/metrics_exporter.go b/exporter/loadbalancingexporter/metrics_exporter.go index 0b685801e346..fe1b02ac9a81 100644 --- a/exporter/loadbalancingexporter/metrics_exporter.go +++ b/exporter/loadbalancingexporter/metrics_exporter.go @@ -75,7 +75,8 @@ func (e *metricExporterImp) Start(ctx context.Context, host component.Host) erro return e.loadBalancer.Start(ctx, host) } -func (e *metricExporterImp) Shutdown(context.Context) error { +func (e *metricExporterImp) Shutdown(ctx context.Context) error { + e.loadBalancer.Shutdown(ctx) e.stopped = true e.shutdownWg.Wait() return nil diff --git a/exporter/loadbalancingexporter/package_test.go b/exporter/loadbalancingexporter/package_test.go new file mode 100644 index 000000000000..4e898c447576 --- /dev/null +++ b/exporter/loadbalancingexporter/package_test.go @@ -0,0 +1,17 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package loadbalancingexporter + +import ( + "testing" + + "go.uber.org/goleak" +) + +// The IgnoreTopFunction call prevents catching the leak generated by opencensus +// defaultWorker.Start which at this time is part of the package's init call. +// See https://github.com/census-instrumentation/opencensus-go/issues/1191 for more information. +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m, goleak.IgnoreTopFunction("go.opencensus.io/stats/view.(*worker).start")) +} diff --git a/exporter/loadbalancingexporter/resolver_dns_test.go b/exporter/loadbalancingexporter/resolver_dns_test.go index 6f4ca264e403..ed55f3ce19fc 100644 --- a/exporter/loadbalancingexporter/resolver_dns_test.go +++ b/exporter/loadbalancingexporter/resolver_dns_test.go @@ -107,6 +107,7 @@ func TestCantResolve(t *testing.T) { // verify assert.NoError(t, err) + assert.NoError(t, res.shutdown(context.Background())) } func TestOnChange(t *testing.T) { diff --git a/exporter/loadbalancingexporter/resolver_static.go b/exporter/loadbalancingexporter/resolver_static.go index 8527669db834..1c1633feeec3 100644 --- a/exporter/loadbalancingexporter/resolver_static.go +++ b/exporter/loadbalancingexporter/resolver_static.go @@ -25,6 +25,7 @@ type staticResolver struct { endpoints []string onChangeCallbacks []func([]string) once sync.Once // we trigger the onChange only once + mx sync.Mutex } func newStaticResolver(endpoints []string) (*staticResolver, error) { @@ -49,13 +50,23 @@ func (r *staticResolver) start(ctx context.Context) error { return err } -func (r *staticResolver) shutdown(_ context.Context) error { +func (r *staticResolver) shutdown(ctx context.Context) error { + r.endpoints = nil + + r.mx.Lock() + defer r.mx.Unlock() + for _, callback := range r.onChangeCallbacks { + callback(r.endpoints) + } + return nil } func (r *staticResolver) resolve(ctx context.Context) ([]string, error) { _ = stats.RecordWithTags(ctx, staticResolverMutators, mNumResolutions.M(1)) + r.mx.Lock() + defer r.mx.Unlock() r.once.Do(func() { _ = stats.RecordWithTags(ctx, staticResolverMutators, mNumBackends.M(int64(len(r.endpoints)))) diff --git a/exporter/loadbalancingexporter/trace_exporter.go b/exporter/loadbalancingexporter/trace_exporter.go index d3ce46d8e2f6..254a754048df 100644 --- a/exporter/loadbalancingexporter/trace_exporter.go +++ b/exporter/loadbalancingexporter/trace_exporter.go @@ -73,10 +73,11 @@ func (e *traceExporterImp) Start(ctx context.Context, host component.Host) error return e.loadBalancer.Start(ctx, host) } -func (e *traceExporterImp) Shutdown(context.Context) error { +func (e *traceExporterImp) Shutdown(ctx context.Context) error { + err := e.loadBalancer.Shutdown(ctx) e.stopped = true e.shutdownWg.Wait() - return nil + return err } func (e *traceExporterImp) ConsumeTraces(ctx context.Context, td ptrace.Traces) error { From ab21ead49d717c7f49bfc6a674d967186bf29205 Mon Sep 17 00:00:00 2001 From: Curtis Robert Date: Mon, 5 Feb 2024 13:29:08 -0800 Subject: [PATCH 2/5] Update chlog --- .chloggen/goleak_loadbalancingexp.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.chloggen/goleak_loadbalancingexp.yaml b/.chloggen/goleak_loadbalancingexp.yaml index 5eb02f90e98e..ccb58dd1c189 100755 --- a/.chloggen/goleak_loadbalancingexp.yaml +++ b/.chloggen/goleak_loadbalancingexp.yaml @@ -7,10 +7,10 @@ change_type: bug_fix component: loadbalancingexporter # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: Fix memory leak on shutdown +note: Fix memory leaks on shutdown # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. -issues: [] +issues: [31050] # (Optional) One or more lines of additional information to render under the primary note. # These lines will be padded with 2 spaces and then inserted directly into the document. From 7ad4e08edb65cb1eca4e4dfb94327ff80da24a24 Mon Sep 17 00:00:00 2001 From: Curtis Robert Date: Mon, 5 Feb 2024 13:53:26 -0800 Subject: [PATCH 3/5] lint fixes --- exporter/loadbalancingexporter/metrics_exporter.go | 4 ++-- exporter/loadbalancingexporter/resolver_static.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/exporter/loadbalancingexporter/metrics_exporter.go b/exporter/loadbalancingexporter/metrics_exporter.go index fe1b02ac9a81..1faca2d95b58 100644 --- a/exporter/loadbalancingexporter/metrics_exporter.go +++ b/exporter/loadbalancingexporter/metrics_exporter.go @@ -76,10 +76,10 @@ func (e *metricExporterImp) Start(ctx context.Context, host component.Host) erro } func (e *metricExporterImp) Shutdown(ctx context.Context) error { - e.loadBalancer.Shutdown(ctx) + err := e.loadBalancer.Shutdown(ctx) e.stopped = true e.shutdownWg.Wait() - return nil + return err } func (e *metricExporterImp) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) error { diff --git a/exporter/loadbalancingexporter/resolver_static.go b/exporter/loadbalancingexporter/resolver_static.go index 1c1633feeec3..258384cb1118 100644 --- a/exporter/loadbalancingexporter/resolver_static.go +++ b/exporter/loadbalancingexporter/resolver_static.go @@ -50,7 +50,7 @@ func (r *staticResolver) start(ctx context.Context) error { return err } -func (r *staticResolver) shutdown(ctx context.Context) error { +func (r *staticResolver) shutdown(context.Context) error { r.endpoints = nil r.mx.Lock() From 689a26cf983736efba9d38436d9dab8f308869e4 Mon Sep 17 00:00:00 2001 From: Curtis Robert Date: Wed, 28 Feb 2024 16:15:01 -0800 Subject: [PATCH 4/5] Remove unecessary mutex --- exporter/loadbalancingexporter/resolver_static.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/exporter/loadbalancingexporter/resolver_static.go b/exporter/loadbalancingexporter/resolver_static.go index 258384cb1118..86759f132bbe 100644 --- a/exporter/loadbalancingexporter/resolver_static.go +++ b/exporter/loadbalancingexporter/resolver_static.go @@ -25,7 +25,6 @@ type staticResolver struct { endpoints []string onChangeCallbacks []func([]string) once sync.Once // we trigger the onChange only once - mx sync.Mutex } func newStaticResolver(endpoints []string) (*staticResolver, error) { @@ -53,8 +52,6 @@ func (r *staticResolver) start(ctx context.Context) error { func (r *staticResolver) shutdown(context.Context) error { r.endpoints = nil - r.mx.Lock() - defer r.mx.Unlock() for _, callback := range r.onChangeCallbacks { callback(r.endpoints) } @@ -65,8 +62,6 @@ func (r *staticResolver) shutdown(context.Context) error { func (r *staticResolver) resolve(ctx context.Context) ([]string, error) { _ = stats.RecordWithTags(ctx, staticResolverMutators, mNumResolutions.M(1)) - r.mx.Lock() - defer r.mx.Unlock() r.once.Do(func() { _ = stats.RecordWithTags(ctx, staticResolverMutators, mNumBackends.M(int64(len(r.endpoints)))) From 1c33384be7498b4ca5543d1122b25e607313f5d5 Mon Sep 17 00:00:00 2001 From: Curtis Robert Date: Thu, 7 Mar 2024 11:31:45 -0800 Subject: [PATCH 5/5] make gotidy --- exporter/loadbalancingexporter/go.mod | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/exporter/loadbalancingexporter/go.mod b/exporter/loadbalancingexporter/go.mod index 5bdce7e07374..5a4430e94d8a 100644 --- a/exporter/loadbalancingexporter/go.mod +++ b/exporter/loadbalancingexporter/go.mod @@ -6,16 +6,6 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/pkg/batchpersignal v0.96.0 github.com/stretchr/testify v1.9.0 go.opencensus.io v0.24.0 - go.opentelemetry.io/collector/component v0.95.0 - go.opentelemetry.io/collector/confmap v0.95.0 - go.opentelemetry.io/collector/consumer v0.95.0 - go.opentelemetry.io/collector/exporter v0.95.0 - go.opentelemetry.io/collector/exporter/otlpexporter v0.95.0 - go.opentelemetry.io/collector/otelcol v0.95.0 - go.opentelemetry.io/collector/pdata v1.2.0 - go.opentelemetry.io/collector/semconv v0.95.0 - go.opentelemetry.io/otel/metric v1.23.1 - go.opentelemetry.io/otel/trace v1.23.1 go.opentelemetry.io/collector/component v0.96.1-0.20240306115632-b2693620eff6 go.opentelemetry.io/collector/confmap v0.96.1-0.20240306115632-b2693620eff6 go.opentelemetry.io/collector/consumer v0.96.1-0.20240306115632-b2693620eff6