Skip to content

Commit

Permalink
Upgrade to clickhouse-go/v2 (#5020)
Browse files Browse the repository at this point in the history 
We upgrade Flow Aggregator to use the ClickHouse go client v2,
for better performance and future-proofness.
We are using the standard database/sql interface, and not the new
"native" ClickHouse interface,
To open the connection, we replace the DSN with the OpenDB function
with options.

Fixes: #4901

Signed-off-by: heanlan <[email protected]>
  • Loading branch information
@heanlan
heanlan authored Jun 6, 2023
1 parent 96b5228 commit 3d1eb00
Show file tree
Hide file tree
Showing 9 changed files with 239 additions and 191 deletions.
2 changes: 1 addition & 1 deletion build/yamls/flow-visibility-e2e.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,7 @@ spec:
spec:
containers:
- name: clickhouse
image: projects.registry.vmware.com/antrea/theia-clickhouse-server:21.11
image: projects.registry.vmware.com/antrea/clickhouse-server:23.4
imagePullPolicy: IfNotPresent
volumeMounts:
- name: clickhouse-configmap-volume
Expand Down
11 changes: 7 additions & 4 deletions ci/kind/test-e2e-kind.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,9 +171,9 @@ COMMON_IMAGES_LIST=("registry.k8s.io/e2e-test-images/agnhost:2.29" \
"projects.registry.vmware.com/antrea/perftool")

FLOW_VISIBILITY_IMAGE_LIST=("projects.registry.vmware.com/antrea/ipfix-collector:v0.6.1" \
"projects.registry.vmware.com/antrea/theia-clickhouse-operator:0.18.2" \
"projects.registry.vmware.com/antrea/theia-metrics-exporter:0.18.2" \
"projects.registry.vmware.com/antrea/theia-clickhouse-server:21.11")
"projects.registry.vmware.com/antrea/clickhouse-operator:0.21.0" \
"projects.registry.vmware.com/antrea/metrics-exporter:0.21.0" \
"projects.registry.vmware.com/antrea/clickhouse-server:23.4")
if $coverage; then
manifest_args="$manifest_args --coverage"
COMMON_IMAGES_LIST+=("antrea/antrea-ubuntu-coverage:latest")
Expand Down Expand Up @@ -245,7 +245,10 @@ function run_test {
$FLOWAGGREGATOR_YML_CMD | docker exec -i kind-control-plane dd of=/root/flow-aggregator.yml
fi
cat $FLOW_VISIBILITY_YML | docker exec -i kind-control-plane dd of=/root/flow-visibility.yml
curl -o $CH_OPERATOR_YML https://raw.githubusercontent.com/antrea-io/theia/main/build/charts/theia/crds/clickhouse-operator-install-bundle.yaml
curl -o $CH_OPERATOR_YML https://raw.githubusercontent.com/Altinity/clickhouse-operator/release-0.21.0/deploy/operator/clickhouse-operator-install-bundle.yaml
sed -i -e "s|\"image\": \"clickhouse/clickhouse-server:22.3\"|\"image\": \"projects.registry.vmware.com/antrea/clickhouse-server:23.4\"|g" $CH_OPERATOR_YML
sed -i -e "s|image: altinity/clickhouse-operator:0.21.0|image: projects.registry.vmware.com/antrea/clickhouse-operator:0.21.0|g" $CH_OPERATOR_YML
sed -i -e "s|image: altinity/metrics-exporter:0.21.0|image: projects.registry.vmware.com/antrea/metrics-exporter:0.21.0|g" $CH_OPERATOR_YML
cat $CH_OPERATOR_YML | docker exec -i kind-control-plane dd of=/root/clickhouse-operator-install-bundle.yml
fi

Expand Down
34 changes: 21 additions & 13 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.19
require (
antrea.io/libOpenflow v0.10.2
antrea.io/ofnet v0.7.1
github.com/ClickHouse/clickhouse-go v1.5.4
github.com/ClickHouse/clickhouse-go/v2 v2.6.1
github.com/DATA-DOG/go-sqlmock v1.5.0
github.com/Mellanox/sriovnet v1.1.0
github.com/Microsoft/go-winio v0.6.1
Expand Down Expand Up @@ -87,8 +87,10 @@ require (
)

require (
github.com/ClickHouse/ch-go v0.51.2 // indirect
github.com/NYTimes/gziphandler v1.1.1 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.4 // indirect
Expand All @@ -108,12 +110,11 @@ require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/cenk/hub v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/cenkalti/backoff/v4 v4.2.0 // indirect
github.com/cenkalti/hub v1.0.1 // indirect
github.com/cenkalti/rpc2 v0.0.0-20180727162946-9642ea02d0aa // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect
github.com/containerd/cgroups v1.0.1 // indirect
github.com/containerd/cgroups v1.0.4 // indirect
github.com/contiv/libovsdb v0.0.0-20170227191248-d0061a53e358 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
Expand All @@ -124,6 +125,8 @@ require (
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/fatih/color v1.14.1 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/go-faster/city v1.0.1 // indirect
github.com/go-faster/errors v0.6.1 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
Expand All @@ -149,6 +152,7 @@ require (
github.com/josharian/intern v1.0.0 // indirect
github.com/josharian/native v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.15.14 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/mailru/easyjson v0.7.6 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
Expand All @@ -162,6 +166,8 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect
github.com/paulmach/orb v0.8.0 // indirect
github.com/pierrec/lz4/v4 v4.1.17 // indirect
github.com/pion/dtls/v2 v2.2.4 // indirect
github.com/pion/logging v0.2.2 // indirect
github.com/pion/transport/v2 v2.0.0 // indirect
Expand All @@ -172,6 +178,8 @@ require (
github.com/rivo/uniseg v0.2.0 // indirect
github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1 // indirect
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
github.com/segmentio/asm v1.2.0 // indirect
github.com/shopspring/decimal v1.3.1 // indirect
github.com/stoewer/go-strcase v1.2.0 // indirect
github.com/streamrail/concurrent-map v0.0.0-20160823150647-8bf1e9bacbf6 // indirect
github.com/ti-mo/netfilter v0.3.1 // indirect
Expand All @@ -183,16 +191,16 @@ require (
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.0 // indirect
go.opentelemetry.io/otel v1.10.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 // indirect
go.opentelemetry.io/otel/metric v0.31.0 // indirect
go.opentelemetry.io/otel/sdk v1.10.0 // indirect
go.opentelemetry.io/otel/trace v1.10.0 // indirect
go.opentelemetry.io/otel v1.11.2 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.2 // indirect
go.opentelemetry.io/otel/metric v0.34.0 // indirect
go.opentelemetry.io/otel/sdk v1.11.2 // indirect
go.opentelemetry.io/otel/trace v1.11.2 // indirect
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/oauth2 v0.8.0 // indirect
golang.org/x/term v0.8.0 // indirect
Expand Down
Loading

0 comments on commit 3d1eb00

Please sign in to comment.