query: add store.read-timeout parameter to avoid partial response failure when one of stores timed out #895
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -7,7 +7,6 @@ import ( | |
| "math" | ||
| "math/rand" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/fortytw2/leaktest" | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -33,6 +33,7 @@ type Client interface { | |
| TimeRange() (mint int64, maxt int64) | ||
|
|
||
| String() string | ||
|
|
||
| // Addr returns address of a Client. | ||
| Addr() string | ||
| } | ||
|
|
@@ -44,7 +45,11 @@ type ProxyStore struct { | |
| component component.StoreAPI | ||
| selectorLabels labels.Labels | ||
|
|
||
| // responseTimeout is a timeout for any GRPC operation during series query | ||
|
There was a problem hiding this comment. missing trailing period in comment, same below |
||
| responseTimeout time.Duration | ||
|
|
||
| // queryTimeout is a timeout for entire request | ||
| queryTimeout time.Duration | ||
|
|
||
| } | ||
|
|
||
| // NewProxyStore returns a new ProxyStore that uses the given clients that implements storeAPI to fan-in all series to the client. | ||
|
|
@@ -55,6 +60,7 @@ func NewProxyStore( | |
| component component.StoreAPI, | ||
| selectorLabels labels.Labels, | ||
| responseTimeout time.Duration, | ||
| queryTimeout time.Duration, | ||
| ) *ProxyStore { | ||
| if logger == nil { | ||
| logger = log.NewNopLogger() | ||
|
|
@@ -66,6 +72,7 @@ func NewProxyStore( | |
| component: component, | ||
| selectorLabels: selectorLabels, | ||
| responseTimeout: responseTimeout, | ||
| queryTimeout: queryTimeout, | ||
| } | ||
| return s | ||
| } | ||
|
|
@@ -119,6 +126,8 @@ func newRespCh(ctx context.Context, buffer int) (*ctxRespSender, <-chan *storepb | |
| return &ctxRespSender{ctx: ctx, ch: respCh}, respCh, func() { close(respCh) } | ||
| } | ||
|
|
||
| // send writes response to sender channel | ||
|
There was a problem hiding this comment. Please use full sentence comments. They are for humans (: |
||
| // or just returns if sender context was timed out | ||
| func (s ctxRespSender) send(r *storepb.SeriesResponse) { | ||
| select { | ||
| case <-s.ctx.Done(): | ||
|
|
@@ -147,6 +156,9 @@ func (s *ProxyStore) Series(r *storepb.SeriesRequest, srv storepb.Store_SeriesSe | |
| respSender, respRecv, closeFn = newRespCh(gctx, 10) | ||
| ) | ||
|
|
||
| storeCtx, cancel := s.contextWithTimeout(gctx) | ||
|
There was a problem hiding this comment. Ok, what's the difference in this timeout to the overall gRPC request timeout or gRPC server timeout? I believe Querier can control the same thing by just specifying timeout here: https://github.com/improbable-eng/thanos/blob/1cd9ddd14999d6b074f34a4328e03f7ac3b7c26a/pkg/query/querier.go#L183 I would remove this from Effect is the same, if not better, as you missed to pass this context to There was a problem hiding this comment.
Imagine that we have two stores: first is very slow, second is fast. Partial time-out is enabled. I think it doesn't matter where we have this timeout, more important that at the moment it doesn't work perfect because of reading from stores in mergedSet: reading from stores works in series there and first slow store still blocks reading from second. We need some improvements there... |
||
| defer cancel() | ||
|
|
||
| g.Go(func() error { | ||
| var ( | ||
| seriesSet []storepb.SeriesSet | ||
|
|
@@ -177,8 +189,8 @@ func (s *ProxyStore) Series(r *storepb.SeriesRequest, srv storepb.Store_SeriesSe | |
| } | ||
| storeDebugMsgs = append(storeDebugMsgs, fmt.Sprintf("store %s queried", st)) | ||
|
|
||
| // This is used to cancel this stream when one operation takes too long. | ||
| seriesCtx, closeSeries := context.WithCancel(storeCtx) | ||
| defer closeSeries() | ||
|
|
||
| sc, err := st.Series(seriesCtx, r) | ||
|
|
@@ -295,19 +307,12 @@ func startStreamSeriesSet( | |
| } | ||
|
|
||
| if ctx.Err() != nil { | ||
| s.writeWarningOrErrorResponse(partialResponse, errors.Wrapf(ctx.Err(), "receive series from %s", s.name)) | ||
|
|
||
| return | ||
| } | ||
|
|
||
| if err != nil { | ||
| s.writeWarningOrErrorResponse(partialResponse, errors.Wrapf(err, "receive series from %s", s.name)) | ||
| return | ||
| } | ||
|
|
||
|
|
@@ -356,6 +361,24 @@ func (s *streamSeriesSet) Next() (ok bool) { | |
| } | ||
| } | ||
|
|
||
| // writeWarningOrErrorResponse sends warning if partial response enabled or sets error otherwise | ||
| func (s *streamSeriesSet) writeWarningOrErrorResponse(partialResponse bool, err error) { | ||
|
There was a problem hiding this comment. We can make this return nothing since we aren't using the return values anywhere and we aren't even setting them 😄 |
||
| if partialResponse { | ||
| s.warnCh.send(storepb.NewWarnSeriesResponse(err)) | ||
| return | ||
| } | ||
|
|
||
| s.setError(err) | ||
| return | ||
| } | ||
|
|
||
| // setError sets error (thread-safe) | ||
|
|
||
| func (s *streamSeriesSet) setError(err error) { | ||
| s.errMtx.Lock() | ||
| s.err = err | ||
| s.errMtx.Unlock() | ||
| } | ||
|
|
||
| func (s *streamSeriesSet) At() ([]storepb.Label, []storepb.AggrChunk) { | ||
| if s.currSeries == nil { | ||
| return nil, nil | ||
|
|
@@ -411,11 +434,14 @@ func (s *ProxyStore) LabelValues(ctx context.Context, r *storepb.LabelValuesRequ | |
| g, gctx = errgroup.WithContext(ctx) | ||
| ) | ||
|
|
||
| storeCtx, cancel := s.contextWithTimeout(gctx) | ||
| defer cancel() | ||
|
|
||
| for _, st := range s.stores() { | ||
| store := st | ||
| g.Go(func() error { | ||
| resp, err := store.LabelValues(storeCtx, &storepb.LabelValuesRequest{ | ||
| Label: r.Label, | ||
| PartialResponseDisabled: r.PartialResponseDisabled, | ||
| }) | ||
| if err != nil { | ||
|
|
@@ -448,3 +474,11 @@ func (s *ProxyStore) LabelValues(ctx context.Context, r *storepb.LabelValuesRequ | |
| Warnings: warnings, | ||
| }, nil | ||
| } | ||
|
|
||
| func (s *ProxyStore) contextWithTimeout(ctx context.Context) (context.Context, context.CancelFunc) { | ||
| if s.queryTimeout > 0 { | ||
| return context.WithTimeout(ctx, s.queryTimeout) | ||
| } | ||
|
|
||
| return ctx, func() {} | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I would leave this as it was, reasons:
querysounds like being related to QueryAPI, that's why this name. Which always includedpromqlone. That's why I would leave like it was, the fact that query also queries something else underneath is hidden. Also it matches Prometheusqueryflags as well. And yet another thing compatibility. Even though we are not1.0this API change is major hit into compatibiltiy as suddently query timeout means something opposite to what it was ):promqlsounds like only promQL, however we know it involvesproxy StoreAPI Series()invocation. More over not only one, but sometimes more then one! PromQL can run multipleSelects per single Query.I really like this idea, but IMO it should be:
query.timeout(as it was)store.timeoutfor proxy.go client timeout.alternatively if we want to be more explicit maybe
query.select-timeout? That will also include then the deduplication process.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
agreed