From 1ad856e67e156061d4389a9eb07f0c2bf53e9417 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Mon, 8 Jun 2020 18:14:07 -0700 Subject: [PATCH] When CGO is enabled, use C version of ZStd (#270) * When CGO is enabled, use C version of ZStd * Fixed formatting --- go.mod | 3 +- go.sum | 3 ++ pulsar/internal/compression/zstd.go | 2 ++ pulsar/internal/compression/zstd_cgo.go | 46 +++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 pulsar/internal/compression/zstd_cgo.go diff --git a/go.mod b/go.mod index 911facfa54..706e3822b3 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b github.com/golang/protobuf v1.3.1 github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/klauspost/compress v1.10.5 + github.com/klauspost/compress v1.10.8 github.com/pierrec/lz4 v2.0.5+incompatible github.com/pkg/errors v0.8.1 github.com/sirupsen/logrus v1.4.1 @@ -16,4 +16,5 @@ require ( github.com/spf13/pflag v1.0.3 // indirect github.com/stretchr/testify v1.4.0 github.com/yahoo/athenz v1.8.55 + github.com/valyala/gozstd v1.7.0 ) diff --git a/go.sum b/go.sum index 305907fbfd..1008e02a67 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,7 @@ github.com/jawher/mow.cli v1.0.4/go.mod h1:5hQj2V8g+qYmLUVWqu4Wuja1pI57M83EChYLV github.com/jawher/mow.cli v1.1.0/go.mod h1:aNaQlc7ozF3vw6IJ2dHjp2ZFiA4ozMIYY6PyuRJwlUg= github.com/klauspost/compress v1.10.5 h1:7q6vHIqubShURwQz8cQK6yIe/xC3IF0Vm7TGfqjewrc= github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= @@ -47,6 +48,8 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/valyala/gozstd v1.7.0 h1:Ljh5c9zboqLhwTI33al32R72iCZfn0mCbVGcFWbGwRQ= +github.com/valyala/gozstd v1.7.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ= github.com/yahoo/athenz v1.8.55 h1:xGhxN3yLq334APyn0Zvcc+aqu78Q7BBhYJevM3EtTW0= github.com/yahoo/athenz v1.8.55/go.mod h1:G7LLFUH7Z/r4QAB7FfudfuA7Am/eCzO1GlzBhDL6Kv0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/pulsar/internal/compression/zstd.go b/pulsar/internal/compression/zstd.go index 13b681f234..ccf84ba10b 100644 --- a/pulsar/internal/compression/zstd.go +++ b/pulsar/internal/compression/zstd.go @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +// +build !cgo + package compression import ( diff --git a/pulsar/internal/compression/zstd_cgo.go b/pulsar/internal/compression/zstd_cgo.go new file mode 100644 index 0000000000..3f74b934ba --- /dev/null +++ b/pulsar/internal/compression/zstd_cgo.go @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build cgo + +// If CGO is enabled, use ZSTD library that links with official +// C based zstd which provides better performance compared with +// respect to the native Go implementation of ZStd. + +package compression + +import ( + zstd "github.com/valyala/gozstd" +) + +type zstdCGoProvider struct{} + +func NewZStdProvider() Provider { + return &zstdCGoProvider{} +} + +func (*zstdCGoProvider) CanCompress() bool { + return true +} + +func (*zstdCGoProvider) Compress(data []byte) []byte { + return zstd.Compress(nil, data) +} + +func (*zstdCGoProvider) Decompress(compressedData []byte, originalSize int) ([]byte, error) { + return zstd.Decompress(nil, compressedData) +}