diff --git a/.codecov.yml b/.codecov.yml index 58e1eb75695..f2e07efbbef 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -11,6 +11,7 @@ ignore: - "thrift-gen/*/*" - "**/thrift-0.9.2/*" - "**/main.go" + - "cmd/jaeger/integration" - "examples/hotrod" coverage: diff --git a/.github/workflows/ci-kafka.yml b/.github/workflows/ci-kafka.yml index 5481acaf69d..a0458cac51a 100644 --- a/.github/workflows/ci-kafka.yml +++ b/.github/workflows/ci-kafka.yml @@ -18,6 +18,10 @@ permissions: # added using https://github.com/step-security/secure-workflows jobs: kafka: runs-on: ubuntu-latest + strategy: + matrix: + name: [v1, v2] + steps: - name: Harden Runner uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 @@ -31,7 +35,15 @@ jobs: go-version: 1.21.x - name: Run kafka integration tests - run: bash scripts/kafka-integration-test.sh -k + run: | + case ${{ matrix.name }} in + v1) + bash scripts/kafka-integration-test.sh -k + ;; + v2) + bash scripts/otel-kafka-integration-test.sh 3.6 + ;; + esac - name: Output Kafka logs run: docker logs kafka diff --git a/.gitignore b/.gitignore index a66dcb760cd..af6f463bdda 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ cmd/collector/collector cmd/collector/collector-* cmd/ingester/ingester cmd/ingester/ingester-* +cmd/jaeger/integration/results cmd/remote-storage/remote-storage cmd/remote-storage/remote-storage-* cmd/es-index-cleaner/es-index-cleaner-* diff --git a/Makefile b/Makefile index 40d8905848e..0e80e1a0f9e 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ SHELL := /bin/bash JAEGER_IMPORT_PATH = github.com/jaegertracing/jaeger STORAGE_PKGS = ./plugin/storage/integration/... +OTEL_INTEGRATION_PATH = ./cmd/jaeger/integration/... # These DOCKER_xxx vars are used when building Docker images. DOCKER_NAMESPACE?=jaegertracing @@ -141,6 +142,13 @@ index-rollover-integration-test: docker-images-elastic go clean -testcache bash -c "set -e; set -o pipefail; $(GOTEST) -tags index_rollover -coverpkg=./... -coverprofile cover-index-rollover.out $(STORAGE_PKGS) $(COLORIZE)" +.PHONY: otel-integration-test +otel-integration-test: + # Expire tests results for storage integration tests since the environment might change + # even though the code remains the same. + go clean -testcache + bash -c "set -e; set -o pipefail; $(GOTEST) -coverpkg=./... -coverprofile cover.out $(OTEL_INTEGRATION_PATH) $(COLORIZE)" + .PHONY: cover cover: nocover bash -c "set -e; set -o pipefail; $(GOTEST) -tags=memory_storage_integration -timeout 5m -coverprofile cover.out ./... | tee test-results.json" diff --git a/cmd/jaeger/collector-with-kafka.yaml b/cmd/jaeger/collector-with-kafka.yaml new file mode 100644 index 00000000000..41dc206badd --- /dev/null +++ b/cmd/jaeger/collector-with-kafka.yaml @@ -0,0 +1,30 @@ +service: + pipelines: + traces: + receivers: [otlp, jaeger, zipkin] + processors: [batch] + exporters: [kafka] + +receivers: + otlp: + protocols: + grpc: + http: + + jaeger: + protocols: + grpc: + thrift_binary: + thrift_compact: + thrift_http: + + zipkin: + +processors: + batch: + +exporters: + kafka: + brokers: + - localhost:9092 + encoding: otlp_proto # available encodings are otlp_proto, jaeger_proto, jaeger_json, zipkin_proto, zipkin_json, zipkin_thrift diff --git a/cmd/jaeger/ingester-with-remote.yaml b/cmd/jaeger/ingester-with-remote.yaml new file mode 100644 index 00000000000..52ff7f68ef0 --- /dev/null +++ b/cmd/jaeger/ingester-with-remote.yaml @@ -0,0 +1,31 @@ +service: + extensions: [jaeger_storage] + pipelines: + traces: + receivers: [kafka] + processors: [batch] + exporters: [jaeger_storage_exporter] # same as in the default cmd/jaeger-v2/config.yaml + telemetry: + metrics: + address: 0.0.0.0:8889 # to avoid port conflict with collector-with-kafka.yaml + +extensions: + jaeger_storage: + grpc: + memstore: + server: localhost:17271 + connection-timeout: 5s + +receivers: + kafka: + brokers: + - localhost:9092 + encoding: otlp_proto # available encodings are otlp_proto, jaeger_proto, jaeger_json, zipkin_proto, zipkin_json, zipkin_thrift + initial_offset: earliest # consume messages from the beginning + +processors: + batch: + +exporters: + jaeger_storage_exporter: + trace_storage: memstore diff --git a/cmd/jaeger/ingester.yaml b/cmd/jaeger/ingester.yaml new file mode 100644 index 00000000000..cfc02b91936 --- /dev/null +++ b/cmd/jaeger/ingester.yaml @@ -0,0 +1,32 @@ +service: + extensions: [jaeger_storage] + pipelines: + traces: + receivers: [kafka] + processors: [batch] + exporters: [jaeger_storage_exporter] # same as in the default cmd/jaeger-v2/config.yaml + telemetry: + metrics: + address: 0.0.0.0:8889 # to avoid port conflict with collector-with-kafka.yaml + +extensions: + jaeger_storage: + memory: + memstore: + max_traces: 100000 + memstore_archive: + max_traces: 100000 + +receivers: + kafka: + brokers: + - localhost:9092 + encoding: otlp_proto # available encodings are otlp_proto, jaeger_proto, jaeger_json, zipkin_proto, zipkin_json, zipkin_thrift + initial_offset: earliest # consume messages from the beginning + +processors: + batch: + +exporters: + jaeger_storage_exporter: + trace_storage: memstore diff --git a/cmd/jaeger/integration/README.md b/cmd/jaeger/integration/README.md new file mode 100644 index 00000000000..b382c31c7cd --- /dev/null +++ b/cmd/jaeger/integration/README.md @@ -0,0 +1,16 @@ +# Integration + +Jaeger v2 integration tests are built on top of [OTEL Testbed module](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/testbed). OTEL Testbed provide comprehensive tools for conducting end-to-end tests for the OTEL Collector, such as reproducible short-term benchmarks, correctness tests, long-running stability tests and maximum load stress tests. To learn more about OTEL Testbed, please refer to the their [README](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/testbed/README.md) + +## kafka_test + +Kafka e2e test checks if the pipelines through `kafka` and finally at `remote-storage` have stored match exactly with the provided data using `GoldenDataProvider` (Provides data from the "Golden" dataset generated using pairwise combinatorial testing a.k.a PICT techniques for use in correctness tests) and validated using `CorrectnessTestValidator`. + +The pipelines are checked in 2 steps, which the first test case verifies if the spans sent to Kafka are correct, and the second one checks the spans stored in the remote storage. +![kafka diagram](kafka_diagram.jpeg) + +To conduct the tests, run the following command: + +``` +scripts/otel-kafka-integration-test.sh [kafka_version=latest] [remote_storage_version=latest] +``` diff --git a/cmd/jaeger/integration/datareceivers/.nocover b/cmd/jaeger/integration/datareceivers/.nocover new file mode 100644 index 00000000000..9d6cf4b7fb6 --- /dev/null +++ b/cmd/jaeger/integration/datareceivers/.nocover @@ -0,0 +1 @@ +FIXME diff --git a/cmd/jaeger/integration/datareceivers/jaegerstorage.go b/cmd/jaeger/integration/datareceivers/jaegerstorage.go new file mode 100644 index 00000000000..f171584fbef --- /dev/null +++ b/cmd/jaeger/integration/datareceivers/jaegerstorage.go @@ -0,0 +1,59 @@ +// Copyright (c) 2024 The Jaeger Authors. +// SPDX-License-Identifier: Apache-2.0 + +package datareceivers + +import ( + "context" + "fmt" + "time" + + "github.com/open-telemetry/opentelemetry-collector-contrib/testbed/testbed" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/receiver" + "go.opentelemetry.io/collector/receiver/receivertest" + + "github.com/jaegertracing/jaeger/cmd/jaeger/integration/receivers/storagereceiver" +) + +type jaegerStorageDataReceiver struct { + Port int + receiver receiver.Traces +} + +func NewJaegerStorageDataReceiver(port int) testbed.DataReceiver { + return &jaegerStorageDataReceiver{Port: port} +} + +func (dr *jaegerStorageDataReceiver) Start(tc consumer.Traces, _ consumer.Metrics, _ consumer.Logs) error { + factory := storagereceiver.NewFactory() + cfg := factory.CreateDefaultConfig().(*storagereceiver.Config) + cfg.GRPC.RemoteServerAddr = fmt.Sprintf("localhost:%d", dr.Port) + cfg.GRPC.RemoteConnectTimeout = time.Duration(5 * time.Second) + // TODO add support for other backends + + var err error + set := receivertest.NewNopCreateSettings() + dr.receiver, err = factory.CreateTracesReceiver(context.Background(), set, cfg, tc) + if err != nil { + return err + } + + return dr.receiver.Start(context.Background(), componenttest.NewNopHost()) +} + +func (dr *jaegerStorageDataReceiver) Stop() error { + return dr.receiver.Shutdown(context.Background()) +} + +func (dr *jaegerStorageDataReceiver) GenConfigYAMLStr() string { + return fmt.Sprintf(` + jaeger_storage_receiver: + grpc: + server: localhost:%d`, dr.Port) +} + +func (dr *jaegerStorageDataReceiver) ProtocolName() string { + return "jaeger_storage_receiver" +} diff --git a/cmd/jaeger/integration/datareceivers/kafka.go b/cmd/jaeger/integration/datareceivers/kafka.go new file mode 100644 index 00000000000..09b0b7c3e46 --- /dev/null +++ b/cmd/jaeger/integration/datareceivers/kafka.go @@ -0,0 +1,57 @@ +// Copyright (c) 2024 The Jaeger Authors. +// SPDX-License-Identifier: Apache-2.0 + +package datareceivers + +import ( + "context" + "fmt" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kafkareceiver" + "github.com/open-telemetry/opentelemetry-collector-contrib/testbed/testbed" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/receiver" + "go.opentelemetry.io/collector/receiver/receivertest" +) + +type kafkaDataReceiver struct { + testbed.DataReceiverBase + receiver receiver.Traces +} + +func NewKafkaDataReceiver(port int) testbed.DataReceiver { + return &kafkaDataReceiver{DataReceiverBase: testbed.DataReceiverBase{Port: port}} +} + +func (dr *kafkaDataReceiver) Start(tc consumer.Traces, _ consumer.Metrics, _ consumer.Logs) error { + factory := kafkareceiver.NewFactory() + cfg := factory.CreateDefaultConfig().(*kafkareceiver.Config) + cfg.Brokers = []string{fmt.Sprintf("localhost:%d", dr.Port)} + cfg.GroupID = "testbed_collector" + + var err error + set := receivertest.NewNopCreateSettings() + dr.receiver, err = factory.CreateTracesReceiver(context.Background(), set, cfg, tc) + if err != nil { + return err + } + + return dr.receiver.Start(context.Background(), componenttest.NewNopHost()) +} + +func (dr *kafkaDataReceiver) Stop() error { + return dr.receiver.Shutdown(context.Background()) +} + +func (dr *kafkaDataReceiver) GenConfigYAMLStr() string { + return fmt.Sprintf(` + kafka: + brokers: + - localhost:%d + encoding: otlp_proto`, dr.Port) +} + +func (dr *kafkaDataReceiver) ProtocolName() string { + return "kafka" +} diff --git a/cmd/jaeger/integration/fixtures/generated_pict_pairs_spans.txt b/cmd/jaeger/integration/fixtures/generated_pict_pairs_spans.txt new file mode 100644 index 00000000000..138f2974806 --- /dev/null +++ b/cmd/jaeger/integration/fixtures/generated_pict_pairs_spans.txt @@ -0,0 +1,307 @@ +Parent Tracestate Kind Attributes Events Links Status +Root Empty Producer FaaSPubSub Empty Empty Ok +Child One Internal Internal Empty Empty Ok +Child Four Producer Empty Empty Empty Ok +Child One Unspecified gRPCServer Eight Empty Unset +Root Four Server HTTPServer One Empty Unset +Child Four Unspecified Internal Two Empty Ok +Root Four Producer Empty Empty Empty Unset +Child Four Server FaaSHTTP Empty Empty Unset +Child Empty Server FaaSHTTP Two Empty Error +Child Four Unspecified MessagingConsumer Two Empty Ok +Child One Producer MessagingProducer One Empty Unset +Child One Internal Internal Eight Empty Error +Child One Server FaaSTimer Eight Empty Ok +Root Empty Server FaaSTimer Two Empty Ok +Child Four Unspecified FaaSDatasource One Empty Ok +Child One Unspecified HTTPServer One Empty Ok +Root One Server gRPCServer One Empty Ok +Child Empty Unspecified DatabaseNoSQL One Empty Ok +Child One Client DatabaseNoSQL Empty Empty Error +Child Four Client HTTPClient Eight Empty Ok +Root Four Producer Empty One Empty Error +Root One Server FaaSTimer Empty Empty Ok +Child Four Consumer FaaSDatasource Eight Empty Unset +Child One Server FaaSOther Two Empty Ok +Child Four Server FaaSHTTP Two Empty Error +Child Four Unspecified gRPCClient Two Empty Ok +Root Four Producer MessagingProducer Two Empty Ok +Root One Server MaxCount Eight Empty Ok +Child Four Server MaxCount Two Empty Unset +Child One Client gRPCClient Empty Empty Error +Root Empty Producer FaaSPubSub One Empty Unset +Child Four Server FaaSTimer Two Empty Ok +Child One Producer MessagingProducer Two Empty Unset +Child Four Unspecified Empty One Empty Ok +Root One Server FaaSHTTP Eight Empty Error +Child Four Client DatabaseNoSQL Two Empty Ok +Child Empty Unspecified FaaSPubSub Eight Empty Ok +Child Empty Server gRPCServer Eight Empty Error +Child Four Client DatabaseNoSQL Two Empty Error +Root One Producer MessagingProducer One Empty Unset +Root One Server FaaSTimer One Empty Ok +Child Four Internal Internal Eight Empty Error +Child One Client DatabaseNoSQL Two Empty Error +Child Four Unspecified gRPCClient One Empty Error +Root Four Server FaaSTimer Empty Empty Ok +Child Four Client HTTPClient Eight Empty Error +Child Empty Unspecified Empty Two Empty Ok +Child One Server HTTPServer Eight Empty Ok +Child Empty Unspecified gRPCClient Empty Empty Ok +Child One Unspecified Internal Two Empty Error +Child Four Unspecified MessagingConsumer Empty Empty Ok +Child Empty Unspecified FaaSDatasource Empty Empty Error +Root One Producer MessagingProducer One Empty Ok +Child Four Unspecified DatabaseNoSQL One Empty Error +Root One Server HTTPServer Two Empty Unset +Child One Producer Empty Eight Empty Unset +Child One Unspecified DatabaseNoSQL One Empty Unset +Child Four Client DatabaseNoSQL Eight Empty Unset +Child Empty Unspecified Internal One Empty Ok +Child One Unspecified MessagingProducer Eight Empty Unset +Child Four Server FaaSOther Empty Empty Ok +Child Empty Consumer MessagingConsumer Eight Empty Unset +Child One Server FaaSOther Empty Empty Ok +Child Empty Unspecified MaxCount Empty Empty Error +Child Four Producer FaaSPubSub Two Empty Unset +Child One Client Empty One Empty Ok +Child Empty Producer Empty One Empty Unset +Child Four Client DatabaseSQL Eight Empty Error +Child Four Server FaaSOther Eight Empty Unset +Root Empty Server FaaSHTTP Two Empty Ok +Child One Client DatabaseSQL Eight Empty Unset +Child One Unspecified FaaSHTTP Empty Empty Error +Child One Unspecified MessagingProducer Eight Empty Ok +Child Empty Consumer MessagingConsumer Two Empty Unset +Child Empty Unspecified DatabaseNoSQL Two Empty Error +Child Four Unspecified FaaSDatasource Empty Empty Error +Child One Unspecified MaxCount Empty Empty Ok +Child One Unspecified FaaSDatasource Eight Empty Unset +Child Four Consumer MessagingConsumer Empty Empty Error +Child Four Client Empty Eight Empty Ok +Child One Consumer MessagingConsumer One Empty Error +Child One Client gRPCClient Eight Empty Error +Child Four Unspecified FaaSTimer Two Empty Error +Child One Client gRPCClient Eight Empty Ok +Child Empty Server FaaSHTTP Empty Empty Error +Child Four Unspecified FaaSOther Empty Empty Unset +Child Empty Consumer MessagingConsumer One Empty Ok +Child Four Client DatabaseSQL Eight Empty Unset +Child Four Unspecified MessagingProducer Empty Empty Ok +Child One Client DatabaseSQL Empty Empty Ok +Root Four Server FaaSOther Empty Empty Ok +Child Four Unspecified gRPCClient One Empty Unset +Child Empty Unspecified HTTPServer Two Empty Ok +Root One Producer Empty Eight Empty Ok +Child One Producer FaaSPubSub One Empty Error +Root Four Server FaaSHTTP Two Empty Ok +Child One Unspecified DatabaseSQL Two Empty Ok +Root Empty Server FaaSTimer Empty Empty Error +Child One Server MaxCount One Empty Unset +Child One Producer Empty Two Empty Unset +Child Four Server HTTPServer Eight Empty Unset +Root Empty Server HTTPServer One Empty Ok +Child Four Server FaaSOther One Empty Unset +Child One Producer Empty One Empty Unset +Child One Producer MessagingProducer One Empty Error +Child One Unspecified gRPCClient Two Empty Ok +Root Empty Server HTTPServer Eight Empty Error +Child Empty Unspecified FaaSHTTP One Empty Ok +Root One Server FaaSHTTP One Empty Unset +Child Four Unspecified MessagingConsumer Eight Empty Error +Child Empty Unspecified FaaSPubSub Two Empty Unset +Child Empty Client DatabaseSQL Two Empty Ok +Child Four Unspecified Empty Empty Empty Error +Child Empty Producer FaaSPubSub Eight Empty Ok +Child Four Producer Empty Two Empty Unset +Child Four Unspecified MessagingProducer Eight Empty Ok +Child Empty Unspecified MessagingProducer One Empty Error +Child Empty Server MaxCount One Empty Unset +Root Four Server FaaSHTTP One Empty Ok +Child Empty Server HTTPServer Two Empty Unset +Child One Unspecified Empty One Empty Ok +Child Empty Server gRPCServer Empty Empty Error +Child Four Unspecified gRPCServer One Empty Ok +Child Four Consumer MessagingConsumer Eight Empty Ok +Child Empty Unspecified DatabaseSQL Empty Empty Unset +Child Four Producer MessagingProducer Empty Empty Unset +Child One Client Empty Two Empty Error +Child Four Unspecified gRPCServer Empty Empty Unset +Child Empty Producer FaaSPubSub Two Empty Error +Child One Unspecified FaaSTimer Two Empty Unset +Child Empty Internal Internal One Empty Ok +Child One Client DatabaseSQL Empty Empty Error +Child Four Producer Empty Empty Empty Error +Child Four Unspecified FaaSTimer Empty Empty Unset +Child Four Client gRPCClient Two Empty Unset +Child Empty Unspecified Internal One Empty Error +Root Empty Server FaaSHTTP Empty Empty Error +Child Empty Server FaaSTimer Two Empty Unset +Child Four Consumer MessagingConsumer Empty Empty Unset +Child Empty Unspecified Internal Empty Empty Ok +Child Empty Producer MessagingProducer Two Empty Error +Child Four Client Empty Eight Empty Error +Child One Client DatabaseNoSQL One Empty Unset +Child Empty Client Empty Eight Empty Ok +Child One Server FaaSTimer One Empty Error +Child One Producer FaaSPubSub Eight Empty Ok +Child One Unspecified FaaSPubSub One Empty Ok +Child Empty Producer Empty Eight Empty Ok +Child Four Unspecified FaaSHTTP Empty Empty Unset +Child One Unspecified FaaSHTTP Empty Empty Ok +Child Empty Producer MessagingProducer One Empty Error +Child One Consumer FaaSDatasource One Empty Ok +Child One Client HTTPClient Eight Empty Error +Child Four Unspecified MaxCount Two Empty Ok +Root Empty Server HTTPServer Empty Empty Ok +Child Four Server HTTPServer Empty Empty Error +Child Four Internal Internal One Empty Error +Child Empty Server FaaSTimer Two Empty Error +Root Four Producer FaaSPubSub Two Empty Ok +Child Four Unspecified FaaSDatasource Eight Empty Error +Child One Unspecified DatabaseNoSQL Two Empty Error +Root One Server FaaSHTTP One Empty Error +Child Empty Server gRPCServer Two Empty Unset +Root Empty Producer Empty Empty Empty Error +Child Four Unspecified FaaSPubSub One Empty Error +Child One Unspecified FaaSHTTP Eight Empty Error +Child One Server MaxCount Two Empty Unset +Root Four Server FaaSOther Eight Empty Error +Child Empty Client DatabaseNoSQL Empty Empty Error +Child One Client Empty Empty Empty Error +Root Empty Server FaaSTimer One Empty Ok +Child One Consumer FaaSDatasource Empty Empty Ok +Child Empty Unspecified MessagingProducer Empty Empty Error +Child One Unspecified HTTPServer Eight Empty Error +Child Empty Server gRPCServer One Empty Ok +Child One Server MaxCount Empty Empty Error +Child Four Server FaaSOther Two Empty Error +Child Four Server HTTPServer One Empty Unset +Root One Server FaaSOther One Empty Error +Child Empty Unspecified FaaSOther One Empty Ok +Root Empty Server MaxCount Empty Empty Unset +Child Four Unspecified DatabaseSQL One Empty Unset +Child Four Unspecified Internal Two Empty Error +Child Four Unspecified DatabaseNoSQL Two Empty Unset +Child One Producer FaaSPubSub Eight Empty Error +Child Four Internal Internal Empty Empty Unset +Child Four Consumer FaaSDatasource Empty Empty Unset +Child One Client DatabaseNoSQL Two Empty Ok +Child Empty Producer FaaSPubSub One Empty Unset +Root One Server FaaSHTTP Two Empty Ok +Root Four Server gRPCServer Two Empty Ok +Child Four Unspecified DatabaseNoSQL Two Empty Error +Child Four Unspecified FaaSTimer Two Empty Unset +Child One Unspecified FaaSHTTP Eight Empty Ok +Child Empty Client DatabaseNoSQL Eight Empty Ok +Child Empty Client DatabaseSQL One Empty Error +Child Four Unspecified FaaSTimer One Empty Ok +Child Four Producer FaaSPubSub One Empty Error +Child One Server FaaSTimer Eight Empty Unset +Child Four Unspecified MessagingConsumer One Empty Unset +Root Four Server MaxCount Empty Empty Ok +Root Empty Server MaxCount Eight Empty Error +Child One Consumer FaaSDatasource Eight Empty Ok +Child Four Unspecified FaaSHTTP One Empty Error +Child Empty Server MaxCount One Empty Error +Child One Unspecified Internal One Empty Error +Child One Unspecified DatabaseSQL Eight Empty Unset +Root One Server FaaSTimer One Empty Error +Root Empty Server HTTPServer Eight Empty Unset +Child Four Server MaxCount One Empty Unset +Child One Unspecified FaaSTimer Eight Empty Error +Root Four Server FaaSTimer One Empty Unset +Root One Producer MessagingProducer Two Empty Error +Child Empty Unspecified gRPCServer Two Empty Ok +Child Four Server FaaSHTTP Two Empty Ok +Child Four Unspecified FaaSPubSub One Empty Unset +Child One Unspecified HTTPServer One Empty Unset +Child Empty Unspecified FaaSTimer One Empty Ok +Child One Unspecified gRPCServer Empty Empty Unset +Child Four Unspecified FaaSDatasource Two Empty Error +Child One Unspecified FaaSPubSub Empty Empty Unset +Root Four Server MaxCount Two Empty Error +Root Empty Producer MessagingProducer Empty Empty Ok +Child Four Unspecified FaaSOther Two Empty Unset +Child Four Server HTTPServer One Empty Error +Root Four Server HTTPServer Empty Empty Ok +Root One Server gRPCServer Two Empty Error +Root Four Server HTTPServer One Empty Error +Child Empty Client Empty Two Empty Error +Child Empty Unspecified MaxCount Empty Empty Unset +Child Empty Unspecified HTTPClient Eight Empty Error +Child Empty Producer Empty Eight Empty Unset +Child One Consumer FaaSDatasource One Empty Unset +Child Four Producer FaaSPubSub Empty Empty Error +Child One Consumer MessagingConsumer Two Empty Ok +Child Four Unspecified gRPCServer Two Empty Ok +Child One Unspecified FaaSHTTP Eight Empty Unset +Child Empty Client DatabaseNoSQL Empty Empty Ok +Child One Server FaaSHTTP One Empty Error +Child Four Unspecified MaxCount Eight Empty Ok +Child Four Unspecified MaxCount Empty Empty Ok +Child Empty Consumer FaaSDatasource Two Empty Error +Root One Server MaxCount Empty Empty Ok +Child Four Consumer FaaSDatasource One Empty Unset +Child Four Unspecified FaaSDatasource Empty Empty Ok +Child Four Unspecified gRPCClient Eight Empty Unset +Child One Unspecified DatabaseSQL One Empty Unset +Child One Internal Internal Two Empty Ok +Child Empty Client DatabaseNoSQL Two Empty Unset +Child Empty Unspecified gRPCServer Eight Empty Ok +Root One Server gRPCServer Empty Empty Ok +Child One Unspecified Empty Two Empty Error +Root Four Server MaxCount Two Empty Unset +Child Empty Unspecified HTTPClient Eight Empty Ok +Child One Server FaaSHTTP Empty Empty Ok +Child Four Unspecified HTTPServer One Empty Error +Child Empty Server FaaSOther One Empty Error +Child One Unspecified HTTPServer Empty Empty Unset +Child Four Unspecified HTTPClient Two Empty Unset +Child Empty Unspecified gRPCServer One Empty Unset +Child Empty Unspecified DatabaseNoSQL Eight Empty Unset +Child Four Unspecified HTTPServer Eight Empty Error +Root Empty Producer Empty Two Empty Ok +Root One Server gRPCServer Empty Empty Unset +Root One Producer FaaSPubSub Empty Empty Ok +Child Four Producer FaaSPubSub Empty Empty Ok +Child Four Unspecified Empty Eight Empty Ok +Child Empty Unspecified FaaSTimer One Empty Error +Child Four Unspecified FaaSTimer Two Empty Ok +Root One Server HTTPServer Eight Empty Error +Root Empty Server FaaSTimer One Empty Unset +Root Empty Server MaxCount Two Empty Unset +Child One Unspecified Empty Eight Empty Unset +Child One Unspecified HTTPServer Two Empty Ok +Child Empty Producer FaaSPubSub Empty Empty Ok +Root Four Server FaaSHTTP Eight Empty Error +Child Empty Unspecified Internal Eight Empty Ok +Child Four Unspecified HTTPServer Empty Empty Ok +Child One Client HTTPClient Two Empty Unset +Child One Unspecified FaaSTimer One Empty Unset +Child Empty Client Empty One Empty Unset +Child Four Unspecified MessagingConsumer Empty Empty Error +Child Empty Unspecified gRPCClient Eight Empty Error +Root Four Server FaaSOther Two Empty Error +Child Four Unspecified FaaSHTTP Eight Empty Ok +Child One Client Empty Eight Empty Ok +Child Empty Unspecified MaxCount Eight Empty Ok +Child Empty Unspecified FaaSOther One Empty Unset +Root Four Server FaaSTimer Eight Empty Unset +Child One Client DatabaseSQL One Empty Ok +Child Four Unspecified DatabaseNoSQL Empty Empty Unset +Child Empty Internal Internal Empty Empty Unset +Root Empty Server gRPCServer One Empty Ok +Child Four Server HTTPServer Two Empty Unset +Child One Unspecified MaxCount One Empty Unset +Child Four Client DatabaseSQL One Empty Error +Child Four Unspecified FaaSTimer One Empty Unset +Child One Unspecified gRPCClient One Empty Ok +Child Four Unspecified FaaSPubSub Eight Empty Ok +Root One Server FaaSOther Eight Empty Error +Child Empty Unspecified DatabaseSQL One Empty Error +Child Four Unspecified HTTPServer Eight Empty Unset +Root Four Server FaaSTimer Two Empty Error +Child One Unspecified MessagingConsumer Eight Empty Ok +Child Empty Unspecified HTTPServer Eight Empty Error +Root One Server gRPCServer Eight Empty Unset diff --git a/cmd/jaeger/integration/fixtures/generated_pict_pairs_traces.txt b/cmd/jaeger/integration/fixtures/generated_pict_pairs_traces.txt new file mode 100644 index 00000000000..ccbc54ba6c1 --- /dev/null +++ b/cmd/jaeger/integration/fixtures/generated_pict_pairs_traces.txt @@ -0,0 +1,33 @@ +Resource InstrumentationLibrary Spans +Empty None Several +K8sCloud None One +VMCloud None One +VMOnPrem None All +VMCloud None Several +Exec None One +VMCloud Two One +VMOnPrem One Several +K8sCloud One All +VMCloud Two None +Faas None Several +K8sCloud Two All +Empty Two None +Empty One None +VMCloud One All +VMOnPrem None One +K8sOnPrem Two Several +Empty Two Several +Exec Two Several +K8sOnPrem None One +K8sCloud One None +Empty None All +Empty None One +VMCloud None None +K8sCloud Two One +Exec None None +K8sOnPrem One Several +VMOnPrem Two All +VMOnPrem Two Several +K8sOnPrem One One +Faas One Several +Faas None One diff --git a/cmd/jaeger/integration/fixtures/pict_input_spans.txt b/cmd/jaeger/integration/fixtures/pict_input_spans.txt new file mode 100644 index 00000000000..10cea9916ff --- /dev/null +++ b/cmd/jaeger/integration/fixtures/pict_input_spans.txt @@ -0,0 +1,14 @@ +Parent: Root, Child +Tracestate: Empty, One, Four +Kind: Unspecified, Internal, Server, Client, Producer, Consumer +Attributes: Empty, DatabaseSQL, DatabaseNoSQL, FaaSDatasource, FaaSHTTP, FaaSPubSub, FaaSTimer, FaaSOther, HTTPClient, HTTPServer, MessagingProducer, MessagingConsumer, gRPCClient, gRPCServer, Internal, MaxCount +Events: Empty, One, Two, Eight +Links: Empty +Status: Unset, Ok, Error + +IF [Parent] = "Root" THEN [Kind] in {"Server", "Producer"}; +IF [Kind] = "Internal" THEN [Attributes] in {"Nil", "Internal"}; +IF [Kind] = "Server" THEN [Attributes] in {"Nil", "FaaSHTTP", "FaaSTimer", "FaaSOther", "HTTPServer", "gRPCServer", "MaxCount"}; +IF [Kind] = "Client" THEN [Attributes] in {"Empty", "DatabaseSQL", "DatabaseNoSQL", "HTTPClient", "gRPCClient"}; +IF [Kind] = "Producer" THEN [Attributes] in {"Empty", "MessagingProducer", "FaaSPubSub"}; +IF [Kind] = "Consumer" THEN [Attributes] in {"Nil", "MessagingConsumer", "FaaSDatasource"}; diff --git a/cmd/jaeger/integration/fixtures/pict_input_traces.txt b/cmd/jaeger/integration/fixtures/pict_input_traces.txt new file mode 100644 index 00000000000..9eb0b9a29c1 --- /dev/null +++ b/cmd/jaeger/integration/fixtures/pict_input_traces.txt @@ -0,0 +1,3 @@ +Resource: Empty, VMOnPrem, VMCloud, K8sOnPrem, K8sCloud, Faas, Exec +InstrumentationLibrary: None, One, Two +Spans: None, One, Several, All diff --git a/cmd/jaeger/integration/kafka_diagram.jpeg b/cmd/jaeger/integration/kafka_diagram.jpeg new file mode 100644 index 00000000000..9dfa2ca64f3 Binary files /dev/null and b/cmd/jaeger/integration/kafka_diagram.jpeg differ diff --git a/cmd/jaeger/integration/kafka_test.go b/cmd/jaeger/integration/kafka_test.go new file mode 100644 index 00000000000..ebe1c5b494d --- /dev/null +++ b/cmd/jaeger/integration/kafka_test.go @@ -0,0 +1,100 @@ +// Copyright (c) 2024 The Jaeger Authors. +// SPDX-License-Identifier: Apache-2.0 + +package integration + +import ( + "os" + "testing" + "time" + + "github.com/open-telemetry/opentelemetry-collector-contrib/testbed/testbed" + "github.com/stretchr/testify/require" + + "github.com/jaegertracing/jaeger/cmd/jaeger/integration/datareceivers" + "github.com/jaegertracing/jaeger/cmd/jaeger/internal" +) + +var correctnessResults testbed.TestResultsSummary = &testbed.CorrectnessResults{} + +type KafkaIntegration struct { + receiver testbed.DataReceiver + configFile string +} + +func TestKafkaStorage(t *testing.T) { + if os.Getenv("STORAGE") != "otel_kafka" { + t.Skip("Integration test against Jaeger v2 Kafka; set STORAGE env var to otel_kafka to run this") + } + + dataProvider := testbed.NewGoldenDataProvider( + "fixtures/generated_pict_pairs_traces.txt", + "fixtures/generated_pict_pairs_spans.txt", + "", + ) + sender := testbed.NewOTLPTraceDataSender(testbed.DefaultHost, 4317) + // LoadGenerator will be shared across testbed testcases + // since we will validate the same origin data provided and the received traces + loadGenerator, err := testbed.NewLoadGenerator(dataProvider, sender) + require.NoError(t, err, "Cannot create generator") + + tests := []KafkaIntegration{ + { + receiver: datareceivers.NewKafkaDataReceiver(9092), + configFile: "../collector-with-kafka.yaml", + }, + { + receiver: datareceivers.NewJaegerStorageDataReceiver(17271), + configFile: "../ingester-with-remote.yaml", + }, + } + + for i, test := range tests { + factories, err := internal.Components() + require.NoError(t, err, "default components resulted in: %v", err) + + runner := testbed.NewInProcessCollector(factories) + validator := testbed.NewCorrectTestValidator(sender.ProtocolName(), test.receiver.ProtocolName(), dataProvider) + + config, err := os.ReadFile(test.configFile) + if err != nil { + t.Fatal(err) + } + configCleanup, err := runner.PrepareConfig(string(config)) + require.NoError(t, err, "collector configuration resulted in: %v", err) + defer configCleanup() + + tc := testbed.NewLoadGeneratorTestCase( + t, + loadGenerator, + test.receiver, + runner, + validator, + correctnessResults, + ) + defer tc.Stop() + + tc.EnableRecording() + tc.StartBackend() + tc.StartAgent() + + load := i == 0 + if load { + tc.StartLoad(testbed.LoadOptions{ + DataItemsPerSecond: 16, + ItemsPerBatch: 1, + }) + } + tc.Sleep(5 * time.Second) + if load { + tc.StopLoad() + } + + tc.WaitForN(func() bool { return tc.LoadGenerator.DataItemsSent() == tc.MockBackend.DataItemsReceived() }, + 10*time.Second, "all data items received") + + tc.StopBackend() + + tc.ValidateData() + } +} diff --git a/cmd/jaeger/integration/receivers/storagereceiver/.nocover b/cmd/jaeger/integration/receivers/storagereceiver/.nocover new file mode 100644 index 00000000000..4c28ea5d152 --- /dev/null +++ b/cmd/jaeger/integration/receivers/storagereceiver/.nocover @@ -0,0 +1 @@ +FIXME \ No newline at end of file diff --git a/cmd/jaeger/integration/receivers/storagereceiver/config.go b/cmd/jaeger/integration/receivers/storagereceiver/config.go new file mode 100644 index 00000000000..2e76d122c18 --- /dev/null +++ b/cmd/jaeger/integration/receivers/storagereceiver/config.go @@ -0,0 +1,19 @@ +// Copyright (c) 2024 The Jaeger Authors. +// SPDX-License-Identifier: Apache-2.0 + +package storagereceiver + +import ( + "github.com/asaskevich/govalidator" + + grpcCfg "github.com/jaegertracing/jaeger/plugin/storage/grpc/config" +) + +type Config struct { + GRPC grpcCfg.Configuration `mapstructure:"grpc"` +} + +func (cfg *Config) Validate() error { + _, err := govalidator.ValidateStruct(cfg) + return err +} diff --git a/cmd/jaeger/integration/receivers/storagereceiver/factory.go b/cmd/jaeger/integration/receivers/storagereceiver/factory.go new file mode 100644 index 00000000000..c992f8aa11a --- /dev/null +++ b/cmd/jaeger/integration/receivers/storagereceiver/factory.go @@ -0,0 +1,33 @@ +// Copyright (c) 2024 The Jaeger Authors. +// SPDX-License-Identifier: Apache-2.0 + +package storagereceiver + +import ( + "context" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/receiver" +) + +// componentType is the name of this extension in configuration. +const componentType = component.Type("jaeger_storage_receiver") + +func NewFactory() receiver.Factory { + return receiver.NewFactory( + componentType, + createDefaultConfig, + receiver.WithTraces(createTraces, component.StabilityLevelDevelopment), + ) +} + +func createDefaultConfig() component.Config { + return &Config{} +} + +func createTraces(ctx context.Context, set receiver.CreateSettings, config component.Config, nextConsumer consumer.Traces) (receiver.Traces, error) { + cfg := config.(*Config) + + return newReceiver(cfg, set.TelemetrySettings, nextConsumer) +} diff --git a/cmd/jaeger/integration/receivers/storagereceiver/receiver.go b/cmd/jaeger/integration/receivers/storagereceiver/receiver.go new file mode 100644 index 00000000000..903193ee01c --- /dev/null +++ b/cmd/jaeger/integration/receivers/storagereceiver/receiver.go @@ -0,0 +1,142 @@ +// Copyright (c) 2024 The Jaeger Authors. +// SPDX-License-Identifier: Apache-2.0 + +package storagereceiver + +import ( + "context" + "fmt" + + jaeger2otlp "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.uber.org/zap" + + "github.com/jaegertracing/jaeger/model" + "github.com/jaegertracing/jaeger/pkg/metrics" + "github.com/jaegertracing/jaeger/plugin/storage/grpc" + "github.com/jaegertracing/jaeger/storage/spanstore" +) + +type storageReceiver struct { + cancelConsumeLoop context.CancelFunc + config *Config + logger *zap.Logger + consumedTraces map[model.TraceID]*consumedTrace + nextConsumer consumer.Traces + spanReader spanstore.Reader +} + +type consumedTrace struct { + spanIDs map[model.SpanID]struct{} +} + +func newReceiver(config *Config, otel component.TelemetrySettings, nextConsumer consumer.Traces) (*storageReceiver, error) { + f, err := grpc.NewFactoryWithConfig( + config.GRPC, + metrics.NullFactory, + otel.Logger, + ) + if err != nil { + return nil, fmt.Errorf("failed to init storage factory: %w", err) + } + // TODO add support for other backends + + spanReader, err := f.CreateSpanReader() + if err != nil { + return nil, fmt.Errorf("failed to create span reader: %w", err) + } + + return &storageReceiver{ + config: config, + logger: otel.Logger, + consumedTraces: make(map[model.TraceID]*consumedTrace), + nextConsumer: nextConsumer, + spanReader: spanReader, + }, nil +} + +func (r *storageReceiver) Start(_ context.Context, host component.Host) error { + ctx, cancel := context.WithCancel(context.Background()) + r.cancelConsumeLoop = cancel + + go func() { + if err := r.consumeLoop(ctx); err != nil { + host.ReportFatalError(err) + } + }() + + return nil +} + +func (r *storageReceiver) consumeLoop(ctx context.Context) error { + for { + services, err := r.spanReader.GetServices(ctx) + if err != nil { + r.logger.Error("Failed to get services from consumer", zap.Error(err)) + return err + } + + for _, svc := range services { + if err := r.consumeTraces(ctx, svc); err != nil { + r.logger.Error("Failed to consume traces from consumer", zap.Error(err)) + } + if ctx.Err() != nil { + r.logger.Error("Consumer stopped", zap.Error(ctx.Err())) + return ctx.Err() + } + } + } +} + +func (r *storageReceiver) consumeTraces(ctx context.Context, serviceName string) error { + traces, err := r.spanReader.FindTraces(ctx, &spanstore.TraceQueryParameters{ + ServiceName: serviceName, + }) + if err != nil { + return err + } + + cnt := 0 + for _, trace := range traces { + cnt += len(trace.Spans) + traceID := trace.Spans[0].TraceID + if _, ok := r.consumedTraces[traceID]; !ok { + r.consumedTraces[traceID] = &consumedTrace{ + spanIDs: make(map[model.SpanID]struct{}), + } + } + if len(trace.Spans) > len(r.consumedTraces[traceID].spanIDs) { + r.consumeSpans(ctx, r.consumedTraces[traceID], trace.Spans) + } + } + + return nil +} + +func (r *storageReceiver) consumeSpans(ctx context.Context, tc *consumedTrace, spans []*model.Span) error { + // Spans are consumed one at a time because we don't know whether all spans + // in a trace have been completely exported + for _, span := range spans { + if _, ok := tc.spanIDs[span.SpanID]; !ok { + tc.spanIDs[span.SpanID] = struct{}{} + td, err := jaeger2otlp.ProtoToTraces([]*model.Batch{ + { + Spans: []*model.Span{span}, + Process: span.Process, + }, + }) + if err != nil { + return err + } + r.nextConsumer.ConsumeTraces(ctx, td) + } + } + + return nil +} + +func (r *storageReceiver) Shutdown(_ context.Context) error { + r.cancelConsumeLoop() + return nil +} diff --git a/cmd/jaeger/internal/command.go b/cmd/jaeger/internal/command.go index 072a05e8cc1..b53e22e4307 100644 --- a/cmd/jaeger/internal/command.go +++ b/cmd/jaeger/internal/command.go @@ -29,7 +29,7 @@ func Command() *cobra.Command { settings := otelcol.CollectorSettings{ BuildInfo: info, - Factories: components, + Factories: Components, } cmd := otelcol.NewCommand(settings) diff --git a/cmd/jaeger/internal/components.go b/cmd/jaeger/internal/components.go index 2b758a55990..d98582023ba 100644 --- a/cmd/jaeger/internal/components.go +++ b/cmd/jaeger/internal/components.go @@ -116,6 +116,6 @@ func (b builders) build() (otelcol.Factories, error) { return factories, nil } -func components() (otelcol.Factories, error) { +func Components() (otelcol.Factories, error) { return defaultBuilders().build() } diff --git a/cmd/jaeger/internal/components_test.go b/cmd/jaeger/internal/components_test.go index fed360d0bdd..fd333453691 100644 --- a/cmd/jaeger/internal/components_test.go +++ b/cmd/jaeger/internal/components_test.go @@ -29,7 +29,7 @@ import ( ) func TestComponents(t *testing.T) { - factories, err := components() + factories, err := Components() require.NoError(t, err) diff --git a/cmd/jaeger/internal/extension/jaegerstorage/config.go b/cmd/jaeger/internal/extension/jaegerstorage/config.go index db32f6c79cd..f0d71053dce 100644 --- a/cmd/jaeger/internal/extension/jaegerstorage/config.go +++ b/cmd/jaeger/internal/extension/jaegerstorage/config.go @@ -9,12 +9,14 @@ import ( memoryCfg "github.com/jaegertracing/jaeger/pkg/memory/config" badgerCfg "github.com/jaegertracing/jaeger/plugin/storage/badger" + grpcCfg "github.com/jaegertracing/jaeger/plugin/storage/grpc/config" ) // Config has the configuration for jaeger-query, type Config struct { Memory map[string]memoryCfg.Configuration `mapstructure:"memory"` Badger map[string]badgerCfg.NamespaceConfig `mapstructure:"badger"` + GRPC map[string]grpcCfg.Configuration `mapstructure:"grpc"` // TODO add other storage types here // TODO how will this work with 3rd party storage implementations? // Option: instead of looking for specific name, check interface. diff --git a/cmd/jaeger/internal/extension/jaegerstorage/extension.go b/cmd/jaeger/internal/extension/jaegerstorage/extension.go index ae743313046..0dd15b647c3 100644 --- a/cmd/jaeger/internal/extension/jaegerstorage/extension.go +++ b/cmd/jaeger/internal/extension/jaegerstorage/extension.go @@ -15,6 +15,7 @@ import ( "github.com/jaegertracing/jaeger/pkg/metrics" "github.com/jaegertracing/jaeger/plugin/storage/badger" + "github.com/jaegertracing/jaeger/plugin/storage/grpc" "github.com/jaegertracing/jaeger/plugin/storage/memory" "github.com/jaegertracing/jaeger/storage" ) @@ -88,6 +89,20 @@ func (s *storageExt) Start(ctx context.Context, host component.Host) error { s.factories[name] = factory } + for name, g := range s.config.GRPC { + if _, ok := s.factories[name]; ok { + return fmt.Errorf("duplicate grpc storage name %s", name) + } + var err error + s.factories[name], err = grpc.NewFactoryWithConfig( + g, + metrics.NullFactory, + s.logger.With(zap.String("storage_name", name)), + ) + if err != nil { + return fmt.Errorf("failed to initialize grpc storage: %w", err) + } + } // TODO add support for other backends return nil } diff --git a/cmd/jaeger/internal/extension/jaegerstorage/extension_test.go b/cmd/jaeger/internal/extension/jaegerstorage/extension_test.go index 2ec6c05306e..87f21aa3d95 100644 --- a/cmd/jaeger/internal/extension/jaegerstorage/extension_test.go +++ b/cmd/jaeger/internal/extension/jaegerstorage/extension_test.go @@ -19,14 +19,17 @@ import ( memoryCfg "github.com/jaegertracing/jaeger/pkg/memory/config" "github.com/jaegertracing/jaeger/pkg/metrics" badgerCfg "github.com/jaegertracing/jaeger/plugin/storage/badger" + grpcCfg "github.com/jaegertracing/jaeger/plugin/storage/grpc/config" + "github.com/jaegertracing/jaeger/plugin/storage/memory" "github.com/jaegertracing/jaeger/storage" "github.com/jaegertracing/jaeger/storage/dependencystore" "github.com/jaegertracing/jaeger/storage/spanstore" ) const ( - memstoreName = "memstore" - badgerName = "badgerstore" + memstoreName = "memstore" + badgerName = "badgerstore" + grpcstoreName = "grpcstore" ) type storageHost struct { @@ -82,15 +85,45 @@ func TestStorageExtensionConfigError(t *testing.T) { require.EqualError(t, err, fmt.Sprintf("%s: no storage type present in config", ID)) } -func TestStorageExtensionStartTwiceError(t *testing.T) { - ctx := context.Background() +func TestStorageExtensionDuplicateNameError(t *testing.T) { + tests := []struct { + name string + config *Config + }{ + { + name: "memory", + config: &Config{ + Memory: map[string]memoryCfg.Configuration{ + memstoreName: {}, + }, + }, + }, + { + name: "grpc", + config: &Config{ + GRPC: map[string]grpcCfg.Configuration{ + memstoreName: {}, + }, + }, + }, + } - storageExtension := makeStorageExtension(t, memstoreName) + ctx := context.Background() + ext := storageExt{ + factories: map[string]storage.Factory{ + memstoreName: memory.NewFactoryWithConfig(memoryCfg.Configuration{}, metrics.NullFactory, zap.NewNop()), + }, + } - host := componenttest.NewNopHost() - err := storageExtension.Start(ctx, host) - require.Error(t, err) - require.EqualError(t, err, fmt.Sprintf("duplicate memory storage name %s", memstoreName)) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ext.config = test.config + err := ext.Start(ctx, componenttest.NewNopHost()) + require.Error(t, err) + require.EqualError(t, err, fmt.Sprintf("duplicate %s storage name %s", test.name, memstoreName)) + }) + } + t.Cleanup(func() { require.NoError(t, ext.Shutdown(ctx)) }) } func TestStorageFactoryBadHostError(t *testing.T) { @@ -133,6 +166,27 @@ func TestStorageExtension(t *testing.T) { require.NoError(t, err) } +// This test is only to cover the grpc.NewFactoryWithConfig +// since it can't be mocked yet require external grpc connection +func TestGRPCStorageExtensionError(t *testing.T) { + ctx := context.Background() + ext := storageExt{ + config: &Config{ + GRPC: map[string]grpcCfg.Configuration{ + grpcstoreName: {}, + }, + }, + logger: zap.NewNop(), + factories: map[string]storage.Factory{ + memstoreName: memory.NewFactoryWithConfig(memoryCfg.Configuration{}, metrics.NullFactory, zap.NewNop()), + }, + } + + err := ext.Start(ctx, componenttest.NewNopHost()) + require.Error(t, err) + require.Contains(t, err.Error(), "failed to initialize grpc storage: grpc-plugin builder failed to create a store: error connecting to remote storage") +} + func TestBadgerStorageExtension(t *testing.T) { ctx := context.Background() telemetrySettings := component.TelemetrySettings{ diff --git a/go.mod b/go.mod index 24345c341a0..099013602a2 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jaegerreceiver v0.93.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kafkareceiver v0.93.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/zipkinreceiver v0.93.0 + github.com/open-telemetry/opentelemetry-collector-contrib/testbed v0.93.0 github.com/prometheus/client_golang v1.18.0 github.com/prometheus/client_model v0.5.0 github.com/prometheus/common v0.46.0 @@ -89,6 +90,7 @@ require ( github.com/aws/aws-sdk-go v1.50.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect + github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect @@ -99,10 +101,11 @@ require ( github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.4.0 // indirect - github.com/fatih/color v1.14.1 // indirect + github.com/expr-lang/expr v1.15.8 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-kit/log v0.2.1 // indirect - github.com/go-logfmt/logfmt v0.5.1 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect @@ -112,7 +115,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v1.12.1 // indirect github.com/google/uuid v1.5.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -122,6 +125,7 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/yamux v0.1.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/influxdata/go-syslog/v3 v3.0.1-0.20230911200830-875f5bc594a4 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect @@ -129,18 +133,20 @@ require ( github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.4 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect github.com/knadh/koanf/providers/confmap v0.1.0 // indirect github.com/knadh/koanf/v2 v2.0.1 // indirect github.com/kr/text v0.2.0 // indirect + github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165 // indirect github.com/lightstep/go-expohisto v1.0.0 // indirect - github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect @@ -150,12 +156,22 @@ require ( github.com/mostynb/go-grpc-compression v1.2.2 // indirect github.com/oklog/run v1.1.0 // indirect github.com/onsi/ginkgo v1.16.5 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/exporter/opencensusexporter v0.93.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/exporter/syslogexporter v0.93.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/exporter/zipkinexporter v0.93.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.93.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.93.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/kafka v0.93.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.93.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/batchpersignal v0.93.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.93.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.93.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.93.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure v0.93.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus v0.93.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/zipkin v0.93.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/opencensusreceiver v0.93.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/syslogreceiver v0.93.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/openzipkin/zipkin-go v0.4.2 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect @@ -163,7 +179,7 @@ require ( github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/prometheus/statsd_exporter v0.22.7 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect @@ -184,6 +200,7 @@ require ( github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/uber/jaeger-lib v2.4.1+incompatible // indirect + github.com/valyala/fastjson v1.6.4 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect @@ -216,8 +233,8 @@ require ( golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect golang.org/x/text v0.14.0 // indirect gonum.org/v1/gonum v0.14.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 06c57030087..b5fff92dec7 100644 --- a/go.sum +++ b/go.sum @@ -79,6 +79,8 @@ github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZ github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -130,9 +132,11 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/expr-lang/expr v1.15.8 h1:FL8+d3rSSP4tmK9o+vKfSMqqpGL8n15pEPiHcnBpxoI= +github.com/expr-lang/expr v1.15.8/go.mod h1:uCkhfG+x7fcZ5A5sXHKuQ07jGZRl6J0FCAaf2k4PtVQ= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= -github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= @@ -159,8 +163,9 @@ github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBj github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= +github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -261,8 +266,8 @@ github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+ github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF/w5E9CNxSwbpD6No= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -292,6 +297,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/influxdata/go-syslog/v3 v3.0.1-0.20230911200830-875f5bc594a4 h1:2r2WiFeAwiJ/uyx1qIKnV1L4C9w/2V8ehlbJY4gjFaM= +github.com/influxdata/go-syslog/v3 v3.0.1-0.20230911200830-875f5bc594a4/go.mod h1:1yEQhaLb/cETXCqQmdh7lDjupNAReO7c83AHyK2dJ48= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -349,10 +356,13 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165 h1:bCiVCRCs1Heq84lurVinUPy19keqGEe4jh5vtK37jcg= +github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg= github.com/lightstep/go-expohisto v1.0.0 h1:UPtTS1rGdtehbbAF7o/dhkWLTDI73UifG8LbfQI7cA4= github.com/lightstep/go-expohisto v1.0.0/go.mod h1:xDXD0++Mu2FOaItXtdDfksfgxfV0z1TMPa+e/EUd0cs= -github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c h1:VtwQ41oftZwlMnOEbMWQtSEUgU64U4s+GHk7hZK+jtY= +github.com/lufia/plan9stats v0.0.0-20220913051719-115f729f3c8c/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= @@ -365,8 +375,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= @@ -409,28 +419,50 @@ github.com/open-telemetry/opentelemetry-collector-contrib/connector/spanmetricsc github.com/open-telemetry/opentelemetry-collector-contrib/connector/spanmetricsconnector v0.93.0/go.mod h1:hIINFdrXGiZyykfuqisb+co0UaWbMDmUr5wU+t5+eVw= github.com/open-telemetry/opentelemetry-collector-contrib/exporter/kafkaexporter v0.93.0 h1:7ZHKvHwppr1T6Onv+V/4CorPycsGQfsXJjRCX4PP0iE= github.com/open-telemetry/opentelemetry-collector-contrib/exporter/kafkaexporter v0.93.0/go.mod h1:rLxL7P9JEbwIW6BpDrygRuVy3uK3jZVuCForANZ8L9U= +github.com/open-telemetry/opentelemetry-collector-contrib/exporter/opencensusexporter v0.93.0 h1:sgWoDGxhdmU+v5RYRwtIzQAMNNoL5sZ4c+BG7D1s7Qk= +github.com/open-telemetry/opentelemetry-collector-contrib/exporter/opencensusexporter v0.93.0/go.mod h1:MqHN6NdWXycHIU3LC2gPfQJOGh2YL4Sbc3/+dZcb1kk= +github.com/open-telemetry/opentelemetry-collector-contrib/exporter/syslogexporter v0.93.0 h1:SkgK8YtgEabeWCan9qRVXY6UylbKHJe0YSQXLe0d31A= +github.com/open-telemetry/opentelemetry-collector-contrib/exporter/syslogexporter v0.93.0/go.mod h1:RwI8hARtD4QBP8bR11QDKr4uuoGKyj3JMtVxdUxnP68= +github.com/open-telemetry/opentelemetry-collector-contrib/exporter/zipkinexporter v0.93.0 h1:8INixKIwvufGpeyLJy1tmQR9SIoGw0EI0ypwTj5iwvs= +github.com/open-telemetry/opentelemetry-collector-contrib/exporter/zipkinexporter v0.93.0/go.mod h1:wCCQKDKqw2LlshQWW616MiTx/oKjS7+SgGYr3opd3qw= +github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage v0.93.0 h1:Ffq1pCM1BNzLSOZETYRkiJV7TlOBF10s0PsNBHnrDn0= github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.93.0 h1:l2tQFbMQWLEYPiXFWPEhhCRwLDtJuC8o+wll2jNeKXc= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.93.0/go.mod h1:paPB+PvAyGZuxLSUBawTPpu0Cbw6tUa/mleY/OUun38= github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.93.0 h1:Ei+8BMFhMb6jkZhZoz6w6LG0wb+APZ1xxTfYTtCYlr4= github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.93.0/go.mod h1:1xxL5U1cpBaAY9r14z06qX55YEVZSglq9xv1JKZdODw= github.com/open-telemetry/opentelemetry-collector-contrib/internal/kafka v0.93.0 h1:zn1G8V07/jNd+cKz9M35baxOZVlgFFp2pIw88Wa7rdo= github.com/open-telemetry/opentelemetry-collector-contrib/internal/kafka v0.93.0/go.mod h1:k4L6sS5aP0IEC3MBNSP1gH0er7BLHIauW/5R7P02rpk= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.93.0 h1:qsu9lDUWwwGA+73JpLVECROE8uQZQj99AN7m+F/iXiw= +github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.93.0/go.mod h1:wEv4N8NtUTUWTYd8z5K0VBI/DiBVk2bO33EqNxsylcE= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/batchpersignal v0.93.0 h1:C20edg7CvAVXPL2dJrQzguKV0zmzHXw21fUtNCSHM0g= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/batchpersignal v0.93.0/go.mod h1:ly4/uU4RYU5s9Rickntn4a0ElMU9MU+FFiCkRmr30Es= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.93.0 h1:sQ2+pQ1vkZil4CpsjdzSwetleNKMgLoO1uAXwwbbZSY= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.93.0/go.mod h1:hn5eC3vatwBMZXQLNc9lzZ8vxTXjVpJ3DjyUu9M2CiM= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.93.0 h1:R0GGB09OWAP1bSVps2brDOOln8pgSgzHU658bdzIb+g= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.93.0 h1:2AKT1xw2KNenv/nfRoWueYxQiLJe76Axhb5++4jdgiA= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.93.0/go.mod h1:VmP4/AOAPwtpAG4TYHtWha603iLbLRmO7k0d0WbRqrM= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.93.0 h1:4CQnh6M2vR1j4jcw7m0q4dsdQUQ9wfE0Gdsm8vFmu60= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.93.0/go.mod h1:c4O1ckz309eAPjY2B7ea6SM2WmERcptM8yGpBfoyPQw= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure v0.93.0 h1:75yql+EFg5ZFV0/0uSuLNrEjJeDBz5dgdYm4qq4ek5w= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure v0.93.0/go.mod h1:377/UNBryP/Uov8cISY/6wkvoz625kASt21u3IutnbI= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.93.0 h1:04cMlZkov/1UbBLJfJraAI7SZTu9BtWxXnrQn/Hq7c8= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger v0.93.0/go.mod h1:wrFGO273hn+u6k3EfWa5puMKqAWpX2kbX+4hszWB3sE= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus v0.93.0 h1:sLpsZw9z9cXG3iKcdyVIIy3V7uPt0FVaFMHg9lNJ4z4= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus v0.93.0/go.mod h1:WNykU+uOM/xPhBiCocHeeEAArZccXH8FM2E8WQdiQBI= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/zipkin v0.93.0 h1:mtYiK/ZfNfvL9l6/+eq+aknwsTpWKOeLUtjzJqFjozE= github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/zipkin v0.93.0/go.mod h1:6JIMEJ0SFk6WsSycrRjiumfOTr9BNicC194ZDvqO7j0= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jaegerreceiver v0.93.0 h1:D9rCSWYLRgW9T9eMc8H27RJOWw5PENi2og8VC3fhhCU= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jaegerreceiver v0.93.0/go.mod h1:3JtoKMpUmUcMsQ6DwJ5bKX3jzURjdOLrMoh8/vV2y8M= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kafkareceiver v0.93.0 h1:MsdFoC5HzMMIrQvdDiChCMXeAjUBlRJwKE3EL35XfC8= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kafkareceiver v0.93.0/go.mod h1:Wj8AhrBGkx0zSig+N+VXpcjnwYgCLWuS2RjWHWDMbYc= +github.com/open-telemetry/opentelemetry-collector-contrib/receiver/opencensusreceiver v0.93.0 h1:Zi3uV20lAEDGMXjTHw0oJs1aH9kUhNlZ6/4oqleQsrQ= +github.com/open-telemetry/opentelemetry-collector-contrib/receiver/opencensusreceiver v0.93.0/go.mod h1:ef6ovSvykQu4JWHHOaFSVn9i37Km5pttj9uSHq+Vj7k= +github.com/open-telemetry/opentelemetry-collector-contrib/receiver/syslogreceiver v0.93.0 h1:mX00QlM/1+peBhDijBdygK8Ac/H7tPMKwjho1A9MCS4= +github.com/open-telemetry/opentelemetry-collector-contrib/receiver/syslogreceiver v0.93.0/go.mod h1:rwsJDxHjMYpZzaVzGAahf69Up+VBDzP6F/0eVKMBI+Q= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/zipkinreceiver v0.93.0 h1:OHL6sqnWmVmCQeJ2qqy7GWUA60zqSFSkYMM5SB1uV8w= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/zipkinreceiver v0.93.0/go.mod h1:MyRGk5vswD6r74L1O8qHyV/abbigQztSJ4ChToj/qO4= +github.com/open-telemetry/opentelemetry-collector-contrib/testbed v0.93.0 h1:RbPc9GoEIeESpkGGQBpjwbh+hqVI0XTk6Litm+PBdRQ= +github.com/open-telemetry/opentelemetry-collector-contrib/testbed v0.93.0/go.mod h1:3QBnpJKCZYgbwylygF7mnl5DKgBzyMQBRtLqoya3QOY= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= @@ -451,8 +483,9 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c h1:NRoLoZvkBTKvR5gQLgA3e0hqjkY9u1wm+iOL45VN/qI= +github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -572,6 +605,8 @@ github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVK github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= +github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= @@ -892,6 +927,7 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -1024,11 +1060,11 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20231120223509-83a465c0220f h1:Vn+VyHU5guc9KjB5KrjI2q0wCOWEOIh0OEsleqakHJg= -google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo= -google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4 h1:DC7wcm+i+P1rN3Ff07vL+OndGg5OhNddHyTA+ocPqYE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231127180814-3a041ad873d4/go.mod h1:eJVxU6o+4G1PSczBr85xmyvSNYAKvAYgkub40YGomFM= +google.golang.org/genproto v0.0.0-20231212172506-995d672761c0 h1:YJ5pD9rF8o9Qtta0Cmy9rdBwkSjrTCT6XTiUQVOtIos= +google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 h1:rcS6EyEaoCO52hQDupoSfrxI3R6C2Tq741is7X8OvnM= +google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917/go.mod h1:CmlNWB9lSezaYELKS5Ym1r44VrrbPUa7JTvw+6MbpJ0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 h1:6G8oQ016D88m1xAKljMlBOOGWDZkes4kMhgGFlf8WcQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917/go.mod h1:xtjpI3tXFPP051KaWnhvxkiubL/6dJ18vLVf7q2pTOU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= diff --git a/plugin/storage/grpc/factory.go b/plugin/storage/grpc/factory.go index c4656c36ab4..8a9d360018a 100644 --- a/plugin/storage/grpc/factory.go +++ b/plugin/storage/grpc/factory.go @@ -60,6 +60,21 @@ func NewFactory() *Factory { return &Factory{} } +// NewFactoryWithConfig is used from jaeger(v2). +func NewFactoryWithConfig( + cfg config.Configuration, + metricsFactory metrics.Factory, + logger *zap.Logger, +) (*Factory, error) { + f := NewFactory() + f.InitFromOptions(Options{Configuration: cfg}) + err := f.Initialize(metricsFactory, logger) + if err != nil { + return nil, err + } + return f, nil +} + // AddFlags implements plugin.Configurable func (f *Factory) AddFlags(flagSet *flag.FlagSet) { f.options.AddFlags(flagSet) diff --git a/plugin/storage/grpc/factory_test.go b/plugin/storage/grpc/factory_test.go index e9b4417383b..a28553323fc 100644 --- a/plugin/storage/grpc/factory_test.go +++ b/plugin/storage/grpc/factory_test.go @@ -143,6 +143,12 @@ func TestGRPCStorageFactory(t *testing.T) { assert.Equal(t, f.store.DependencyReader(), depReader) } +func TestGRPCStorageFactoryWithConfig(t *testing.T) { + cfg := grpcConfig.Configuration{} + _, err := NewFactoryWithConfig(cfg, metrics.NullFactory, zap.NewNop()) + require.ErrorContains(t, err, "grpc-plugin builder failed to create a store: error connecting to remote storage") +} + func TestGRPCStorageFactory_Capabilities(t *testing.T) { f := NewFactory() v := viper.New() diff --git a/scripts/otel-kafka-integration-test.sh b/scripts/otel-kafka-integration-test.sh new file mode 100755 index 00000000000..3c8e21ad595 --- /dev/null +++ b/scripts/otel-kafka-integration-test.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +set -euxf -o pipefail + +bring_up_kafka() { + local tag=$1 + local image="bitnami/kafka" + + local cid + cid=$(docker run --detach \ + --name kafka \ + --publish 9092:9092 \ + --env KAFKA_CFG_NODE_ID=0 \ + --env KAFKA_CFG_PROCESS_ROLES=controller,broker \ + --env KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@localhost:9093 \ + --env KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093 \ + --env KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 \ + --env KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT \ + --env KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER \ + --env KAFKA_CFG_INTER_BROKER_LISTENER_NAME=PLAINTEXT \ + --env KAFKA_CFG_MESSAGE_MAX_BYTES=1048588 \ + "${image}:${tag}") + + wait_for_storage "9092" "${cid}" + + echo "${cid}" +} + +bring_up_remote_storage() { + local tag=$1 + local image="jaegertracing/jaeger-remote-storage" + + local cid + cid=$(docker run --detach \ + --publish 17271:17271 \ + --publish 17270:17270 \ + --env SPAN_STORAGE_TYPE=memory \ + "${image}:${tag}") + + wait_for_storage "17271" "${cid}" + + echo "${cid}" +} + +teardown_storage() { + for cid in "$@" + do + docker kill "${cid}" + done +} + +wait_for_storage() { + local port=$1 + local cid=$2 + + local counter=0 + local max_counter=30 + local interval=10 + while [[ $(! nc -z localhost "${port}") && ${counter} -lt ${max_counter} ]]; do + docker inspect "${cid}" | jq '.[].State' + echo "waiting for localhost:${port} to be up..." + sleep "${interval}" + counter=$((counter+1)) + done + + if ! nc -z localhost "${port}"; then + docker inspect "${cid}" | jq '.[].State' + echo "timed out waiting storage to start" + exit 1 + fi +} + +main() { + local kafka_version="${1:-"latest"}" + local remote_storage_version="${2:-"latest"}" + + kafka_cid=$(bring_up_kafka "${kafka_version}") + remote_storage_cid=$(bring_up_remote_storage "${remote_storage_version}") + + trap 'teardown_storage "${kafka_cid}" "${remote_storage_cid}"' EXIT + + STORAGE="otel_kafka" make otel-integration-test +} + +main "$@"