From 738ed8f68f1384dca72427cd9fae95208bb4974b Mon Sep 17 00:00:00 2001 From: Koenraad Verheyden Date: Mon, 27 Mar 2023 15:50:35 +0200 Subject: [PATCH 1/3] Extend /flush to flush a single tenant --- docs/sources/tempo/api_docs/_index.md | 6 ++++++ modules/ingester/flush.go | 20 +++++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/sources/tempo/api_docs/_index.md b/docs/sources/tempo/api_docs/_index.md index 55dc8571c3e..f4cafa40347 100644 --- a/docs/sources/tempo/api_docs/_index.md +++ b/docs/sources/tempo/api_docs/_index.md @@ -373,6 +373,12 @@ GET,POST /flush Triggers a flush of all in-memory traces to the WAL. Useful at the time of rollout restarts and unexpected crashes. +Specify the `instance` parameter to flush data of a single tenant only. + +``` +GET,POST /flush?instance=dev +``` + ### Shutdown ``` diff --git a/modules/ingester/flush.go b/modules/ingester/flush.go index e9c702fc424..be473c8f8d5 100644 --- a/modules/ingester/flush.go +++ b/modules/ingester/flush.go @@ -115,9 +115,23 @@ func (i *Ingester) ShutdownHandler(w http.ResponseWriter, _ *http.Request) { } // FlushHandler calls sweepAllInstances(true) which will force push all traces into the WAL and force -// mark all head blocks as ready to flush. -func (i *Ingester) FlushHandler(w http.ResponseWriter, _ *http.Request) { - i.sweepAllInstances(true) +// mark all head blocks as ready to flush. It will either flush all instances or if an instance is specified, +// just that one. +func (i *Ingester) FlushHandler(w http.ResponseWriter, r *http.Request) { + queryParamInstance := "instance" + + if r.URL.Query().Has(queryParamInstance) { + instance, ok := i.getInstanceByID(r.URL.Query().Get(queryParamInstance)) + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + level.Info(log.Logger).Log("msg", "flushing instance", "instance", instance.instanceID) + i.sweepInstance(instance, true) + } else { + i.sweepAllInstances(true) + } + w.WriteHeader(http.StatusNoContent) } From acbb8e972a4ca4d6b3c067392f79f6ac3e06864d Mon Sep 17 00:00:00 2001 From: Koenraad Verheyden Date: Mon, 27 Mar 2023 15:54:09 +0200 Subject: [PATCH 2/3] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89d2d17e2e4..65e84d77a1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ * [ENHANCEMENT] Add support for arbitrary arithemtic to TraceQL queries [#2146](https://github.com/grafana/tempo/pull/2146) (@joe-elliott) * [ENHANCEMENT] tempo-cli: add command to migrate a tenant [#2130](https://github.com/grafana/tempo/pull/2130) (@kvrhdn) * [ENHANCEMENT] Added the ability to multiple span metrics by an attribute such as `X-SampleRatio` [#2172](https://github.com/grafana/tempo/pull/2172) (@altanozlu) +* [ENHANCEMENT] Extend `/flush` to support flushing a single tenant [#2260](https://github.com/grafana/tempo/pull/2260) (@kvrhdn) * [BUGFIX] Apply `rate()` to bytes/s panel in tenant's dashboard. [#2081](https://github.com/grafana/tempo/pull/2081) (@mapno) * [CHANGE] Update Go to 1.20 [#2079](https://github.com/grafana/tempo/pull/2079) (@scalalang2) * [BUGFIX] Retry copy operations during compaction in GCS backend [#2111](https://github.com/grafana/tempo/pull/2111) (@mapno) From 6499f3fe737cbf87ef43fbece67fa7ed552b19df Mon Sep 17 00:00:00 2001 From: Koenraad Verheyden Date: Mon, 27 Mar 2023 17:10:00 +0200 Subject: [PATCH 3/3] Use tenant query param instead of instance --- docs/sources/tempo/api_docs/_index.md | 4 ++-- modules/ingester/flush.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/sources/tempo/api_docs/_index.md b/docs/sources/tempo/api_docs/_index.md index f4cafa40347..e7f4fa31bc8 100644 --- a/docs/sources/tempo/api_docs/_index.md +++ b/docs/sources/tempo/api_docs/_index.md @@ -373,10 +373,10 @@ GET,POST /flush Triggers a flush of all in-memory traces to the WAL. Useful at the time of rollout restarts and unexpected crashes. -Specify the `instance` parameter to flush data of a single tenant only. +Specify the `tenant` parameter to flush data of a single tenant only. ``` -GET,POST /flush?instance=dev +GET,POST /flush?tenant=dev ``` ### Shutdown diff --git a/modules/ingester/flush.go b/modules/ingester/flush.go index be473c8f8d5..1ba73f3cb38 100644 --- a/modules/ingester/flush.go +++ b/modules/ingester/flush.go @@ -118,7 +118,7 @@ func (i *Ingester) ShutdownHandler(w http.ResponseWriter, _ *http.Request) { // mark all head blocks as ready to flush. It will either flush all instances or if an instance is specified, // just that one. func (i *Ingester) FlushHandler(w http.ResponseWriter, r *http.Request) { - queryParamInstance := "instance" + queryParamInstance := "tenant" if r.URL.Query().Has(queryParamInstance) { instance, ok := i.getInstanceByID(r.URL.Query().Get(queryParamInstance))