Skip to content

Commit

Permalink
Merge #87914
Browse files Browse the repository at this point in the history
87914: kvserver/reports: cancel context on stopper quiescence r=irfansharif a=erikgrinaker

The replication reporter did not cancel the context on stopper quiescence. An in-flight update could therefore prevent node shutdown until it completed, which can take a long time in large clusters. This patch ensures the context is cancelled on stopper quiescence.

Resolves #87913.

Release note (bug fix): An active replication report update could prevent a node from shutting down until it completed. The report update is now cancelled on node shutdown instead.

Co-authored-by: Erik Grinaker <[email protected]>
  • Loading branch information
craig[bot] and erikgrinaker committed Sep 13, 2022
2 parents f7b7889 + e28f3fd commit 287836d
Showing 1 changed file with 11 additions and 1 deletion.
12 changes: 11 additions & 1 deletion pkg/kv/kvserver/reports/reporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,9 +127,12 @@ func (stats *Reporter) Start(ctx context.Context, stopper *stop.Stopper) {
stats.frequencyMu.interval = ReporterInterval.Get(&stats.settings.SV)
})
_ = stopper.RunAsyncTask(ctx, "stats-reporter", func(ctx context.Context) {
ctx = logtags.AddTag(ctx, "replication-reporter", nil /* value */)
ctx, cancel := stopper.WithCancelOnQuiesce(ctx)
defer cancel()

var timer timeutil.Timer
defer timer.Stop()
ctx = logtags.AddTag(ctx, "replication-reporter", nil /* value */)

replStatsSaver := makeReplicationStatsReportSaver()
constraintsSaver := makeReplicationConstraintStatusReportSaver()
Expand Down Expand Up @@ -164,6 +167,8 @@ func (stats *Reporter) Start(ctx context.Context, stopper *stop.Stopper) {
case <-timerCh:
timer.Read = true
case <-changeCh:
case <-ctx.Done():
return
case <-stopper.ShouldQuiesce():
return
}
Expand Down Expand Up @@ -588,6 +593,11 @@ func visitRanges(
break
}

// Check for context cancellation.
if err := ctx.Err(); err != nil {
return err
}

newKey, err := resolver.resolveRange(ctx, &rd, cfg)
if err != nil {
return err
Expand Down

0 comments on commit 287836d

Please sign in to comment.