From 2dad2b3736c2833ab09d659d86703d0aaf3eb966 Mon Sep 17 00:00:00 2001 From: Rueian Date: Fri, 15 May 2020 11:33:13 +0800 Subject: [PATCH] [Issue 244][zstdProvider] Fix zstd memory leak (#245) * Upgrade klauspost/compress to v1.10.5 * Fix zstdProvider.Decompress memory leak --- go.mod | 3 +-- go.sum | 7 +++---- pulsar/internal/compression/zstd.go | 21 ++++++--------------- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 34bae4e6ef..911facfa54 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,8 @@ require ( github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6 github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b github.com/golang/protobuf v1.3.1 - github.com/google/go-cmp v0.3.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/klauspost/compress v1.9.2 + github.com/klauspost/compress v1.10.5 github.com/pierrec/lz4 v2.0.5+incompatible github.com/pkg/errors v0.8.1 github.com/sirupsen/logrus v1.4.1 diff --git a/go.sum b/go.sum index f66879a83a..305907fbfd 100644 --- a/go.sum +++ b/go.sum @@ -14,16 +14,14 @@ github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4w github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jawher/mow.cli v1.0.4/go.mod h1:5hQj2V8g+qYmLUVWqu4Wuja1pI57M83EChYLVZ0sMKk= github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg= -github.com/klauspost/compress v1.9.2 h1:LfVyl+ZlLlLDeQ/d2AqfGIIH4qEDu0Ed2S5GyhCWIWY= -github.com/klauspost/compress v1.9.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.10.5 h1:7q6vHIqubShURwQz8cQK6yIe/xC3IF0Vm7TGfqjewrc= +github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= @@ -69,6 +67,7 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190808195139-e713427fea3f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= diff --git a/pulsar/internal/compression/zstd.go b/pulsar/internal/compression/zstd.go index 41fba0b4c8..13b681f234 100644 --- a/pulsar/internal/compression/zstd.go +++ b/pulsar/internal/compression/zstd.go @@ -18,19 +18,19 @@ package compression import ( - "bytes" - "github.com/klauspost/compress/zstd" "github.com/pkg/errors" ) type zstdProvider struct { encoder *zstd.Encoder + decoder *zstd.Decoder } func NewZStdProvider() Provider { p := &zstdProvider{} p.encoder, _ = zstd.NewWriter(nil) + p.decoder, _ = zstd.NewReader(nil) return p } @@ -42,19 +42,10 @@ func (p *zstdProvider) Compress(data []byte) []byte { return p.encoder.EncodeAll(data, []byte{}) } -func (p *zstdProvider) Decompress(compressedData []byte, originalSize int) ([]byte, error) { - d, err := zstd.NewReader(bytes.NewReader(compressedData)) - if err != nil { - return nil, err - } - - uncompressed := make([]byte, originalSize) - size, err := d.Read(uncompressed) - if err != nil { - return nil, err - } else if size != originalSize { +func (p *zstdProvider) Decompress(compressedData []byte, originalSize int) (dst []byte, err error) { + dst, err = p.decoder.DecodeAll(compressedData, nil) + if err == nil && len(dst) != originalSize { return nil, errors.New("Invalid uncompressed size") - } else { - return uncompressed, nil } + return }