From a78a29456fbd94d07960f8b6c73209c34d872c11 Mon Sep 17 00:00:00 2001 From: dmitryk-dk Date: Tue, 10 Sep 2024 16:23:28 +0200 Subject: [PATCH 1/3] fix issue with doubled labels in the response --- CHANGELOG.md | 2 ++ pkg/plugin/response.go | 1 + pkg/plugin/response_test.go | 65 +++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f50d133..ade1146 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ * FEATURE: add support of the `$__interval` variable in queries. See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/61). Thanks to @yincongcyincong for [the pull request](https://github.com/VictoriaMetrics/victorialogs-datasource/pull/69). +* BUGFIX: fix issue collecting labels in different log lines. Removed duplicated labels from previous log lines. See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/60). + ## v0.4.0 * FEATURE: make retry attempt for datasource requests if returned error is a temporary network error. See [this issue](https://github.com/VictoriaMetrics/victoriametrics-datasource/issues/193) diff --git a/pkg/plugin/response.go b/pkg/plugin/response.go index 3712d49..82053bc 100644 --- a/pkg/plugin/response.go +++ b/pkg/plugin/response.go @@ -98,6 +98,7 @@ func parseStreamResponse(reader io.Reader) backend.DataResponse { return newResponseError(err, backend.StatusInternal) } labelsField.Append(d) + labels = data.Labels{} } // Grafana expects lineFields to be always non-empty. diff --git a/pkg/plugin/response_test.go b/pkg/plugin/response_test.go index c799b82..46d4c2e 100644 --- a/pkg/plugin/response_test.go +++ b/pkg/plugin/response_test.go @@ -350,6 +350,70 @@ func Test_parseStreamResponse(t *testing.T) { frame.Meta = &data.FrameMeta{} rsp.Frames = append(rsp.Frames, frame) + return rsp + }, + }, + { + name: "new test", + response: `{"_time": "2024-09-10T12:24:38.124811Z","_stream_id": "00000000000000002e3bd2bdc376279a6418761ca20c417c","_stream": "{path=\"/var/lib/docker/containers/c01cbe414773fa6b3e4e0976fb27c3583b1a5cd4b7007662477df66987f97f89/c01cbe414773fa6b3e4e0976fb27c3583b1a5cd4b7007662477df66987f97f89-json.log\",stream=\"stderr\"}","_msg": "1","path": "/var/lib/docker/containers/c01cbe414773fa6b3e4e0976fb27c3583b1a5cd4b7007662477df66987f97f89/c01cbe414773fa6b3e4e0976fb27c3583b1a5cd4b7007662477df66987f97f89-json.log","stream": "stderr","time": "2024-09-10T12:24:38.124811792Z"} +{"_time": "2024-09-10T12:36:10.664553169Z","_stream_id": "0000000000000000356bfe9e3c71128c750d94c15df6b908","_stream": "{stream=\"stream1\"}","_msg": "2","date": "0","stream": "stream1","log.level": "info"} +{"_time": "2024-09-10T13:06:56.45147Z","_stream_id": "00000000000000002e3bd2bdc376279a6418761ca20c417c","_stream": "{path=\"/var/lib/docker/containers/c01cbe414773fa6b3e4e0976fb27c3583b1a5cd4b7007662477df66987f97f89/c01cbe414773fa6b3e4e0976fb27c3583b1a5cd4b7007662477df66987f97f89-json.log\",stream=\"stderr\"}","_msg": "3","path": "/var/lib/docker/containers/c01cbe414773fa6b3e4e0976fb27c3583b1a5cd4b7007662477df66987f97f89/c01cbe414773fa6b3e4e0976fb27c3583b1a5cd4b7007662477df66987f97f89-json.log","stream": "stderr","time": "2024-09-10T13:06:56.451470093Z"}`, + want: func() backend.DataResponse { + labelsField := data.NewFieldFromFieldType(data.FieldTypeJSON, 0) + labelsField.Name = gLabelsField + + timeFd := data.NewFieldFromFieldType(data.FieldTypeTime, 0) + timeFd.Name = gTimeField + + lineField := data.NewFieldFromFieldType(data.FieldTypeString, 0) + lineField.Name = gLineField + + timeFd.Append(time.Date(2024, 9, 10, 12, 24, 38, 124000000, time.UTC)) + timeFd.Append(time.Date(2024, 9, 10, 12, 36, 10, 664000000, time.UTC)) + timeFd.Append(time.Date(2024, 9, 10, 13, 06, 56, 451000000, time.UTC)) + + lineField.Append("1") + + labels := data.Labels{ + "_stream_id": "00000000000000002e3bd2bdc376279a6418761ca20c417c", + "path": "/var/lib/docker/containers/c01cbe414773fa6b3e4e0976fb27c3583b1a5cd4b7007662477df66987f97f89/c01cbe414773fa6b3e4e0976fb27c3583b1a5cd4b7007662477df66987f97f89-json.log", + "stream": "stderr", + "time": "2024-09-10T12:24:38.124811792Z", + } + + b, _ := labelsToJSON(labels) + labelsField.Append(b) + + lineField.Append("2") + + labels = data.Labels{ + "_stream_id": "0000000000000000356bfe9e3c71128c750d94c15df6b908", + "date": "0", + "stream": "stream1", + "log.level": "info", + } + + b, _ = labelsToJSON(labels) + labelsField.Append(b) + + lineField.Append("3") + + labels = data.Labels{ + "_stream_id": "00000000000000002e3bd2bdc376279a6418761ca20c417c", + "path": "/var/lib/docker/containers/c01cbe414773fa6b3e4e0976fb27c3583b1a5cd4b7007662477df66987f97f89/c01cbe414773fa6b3e4e0976fb27c3583b1a5cd4b7007662477df66987f97f89-json.log", + "stream": "stderr", + "time": "2024-09-10T13:06:56.451470093Z", + } + + b, _ = labelsToJSON(labels) + labelsField.Append(b) + + frame := data.NewFrame("", timeFd, lineField, labelsField) + + rsp := backend.DataResponse{} + frame.Meta = &data.FrameMeta{} + rsp.Frames = append(rsp.Frames, frame) + return rsp }, }, @@ -369,6 +433,7 @@ func Test_parseStreamResponse(t *testing.T) { if len(resp.Frames) != 1 { t.Fatalf("expected for response to always contain 1 Frame; got %d", len(resp.Frames)) } + got := resp.Frames[0] want := w.Frames[0] expFieldsLen := got.Fields[0].Len() From 5ba7de6d506f0d4db063a9aef8b0dbd9f8a9ac8d Mon Sep 17 00:00:00 2001 From: dmitryk-dk Date: Tue, 10 Sep 2024 16:48:31 +0200 Subject: [PATCH 2/3] cleanup --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ade1146..edd3fbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ * FEATURE: add support of the `$__interval` variable in queries. See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/61). Thanks to @yincongcyincong for [the pull request](https://github.com/VictoriaMetrics/victorialogs-datasource/pull/69). -* BUGFIX: fix issue collecting labels in different log lines. Removed duplicated labels from previous log lines. See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/60). +* BUGFIX: fix the issue of collecting labels in different log lines. Removed duplicated labels from previous log lines. See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/60). ## v0.4.0 From bc2dee54dc3da9f1507abe47cf0cc92a123de53e Mon Sep 17 00:00:00 2001 From: dmitryk-dk Date: Thu, 12 Sep 2024 15:43:18 +0200 Subject: [PATCH 3/3] fix comments --- CHANGELOG.md | 2 +- pkg/plugin/response.go | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 075e450..1678a47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ Thanks to @yincongcyincong for [the pull request](https://github.com/VictoriaMetrics/victorialogs-datasource/pull/69). * BUGFIX: correctly pass time range filter when querying variable values. Before, time filter wasn't applied for `/field_values` and `/field_names` API calls. See [this](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/71) and [this](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/72) issues. -* BUGFIX: fix the issue of collecting labels in different log lines. Removed duplicated labels from previous log lines. See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/60). +* BUGFIX: fix the issue with displaying incorrect subfields when requesting logs with different set of fields. See [this issue](https://github.com/VictoriaMetrics/victorialogs-datasource/issues/60). ## v0.4.0 diff --git a/pkg/plugin/response.go b/pkg/plugin/response.go index 82053bc..115c441 100644 --- a/pkg/plugin/response.go +++ b/pkg/plugin/response.go @@ -41,8 +41,6 @@ func parseStreamResponse(reader io.Reader) backend.DataResponse { lineField := data.NewFieldFromFieldType(data.FieldTypeString, 0) lineField.Name = gLineField - labels := data.Labels{} - scanner := bufio.NewScanner(reader) for scanner.Scan() { @@ -63,6 +61,8 @@ func parseStreamResponse(reader io.Reader) backend.DataResponse { } timeFd.Append(getTime) } + + labels := data.Labels{} if value.Exists(streamField) { stream := value.GetStringBytes(streamField) expr, err := metricsql.Parse(string(stream)) @@ -98,7 +98,6 @@ func parseStreamResponse(reader io.Reader) backend.DataResponse { return newResponseError(err, backend.StatusInternal) } labelsField.Append(d) - labels = data.Labels{} } // Grafana expects lineFields to be always non-empty.