query: cleanup store statuses as they come and go

Signed-off-by: Adrien Fillon <[email protected]>

pkg/query/storeset.go

@@ -22,6 +22,7 @@ import (
 const (
 	unhealthyStoreMessage = "removing store because it's unhealthy or does not exist"
 	droppingStoreMessage  = "dropping store, external labels are not unique"
+	storeUnhealthyTimeout = 5 * time.Minute
 )
 
 type StoreSpec interface {
@@ -251,6 +252,7 @@ func (s *StoreSet) Update(ctx context.Context) {
 	}
 	s.externalLabelStores = externalLabelStores
 	s.storeNodeConnections.Set(float64(len(s.stores)))
+	s.cleanUpStoreStatuses()
 }
 
 func (s *StoreSet) getHealthyStores(ctx context.Context) map[string]*storeRef {
@@ -342,15 +344,24 @@ func (s *StoreSet) updateStoreStatus(store *storeRef, err error) {
 	defer s.storesStatusesMtx.Unlock()
 
 	now := time.Now()
-	s.storeStatuses[store.addr] = &StoreStatus{
-		Name:      store.addr,
-		LastError: err,
-		LastCheck: now,
-		Labels:    store.labels,
-		StoreType: store.storeType,
-		MinTime:   store.minTime,
-		MaxTime:   store.maxTime,
+
+	status := StoreStatus{Name: store.addr}
+	prev, ok := s.storeStatuses[store.addr]
+	if ok {
+		status = *prev
+	}
+
+	status.LastError = err
+	status.LastCheck = now
+
+	if err == nil {
+		status.Labels = store.labels
+		status.StoreType = store.storeType
+		status.MinTime = store.minTime
+		status.MaxTime = store.maxTime
 	}
+
+	s.storeStatuses[store.addr] = &status
 }
 
 func (s *StoreSet) GetStoreStatus() []StoreStatus {
@@ -397,3 +408,17 @@ func (s *StoreSet) Close() {
 		st.close()
 	}
 }
+
+func (s *StoreSet) cleanUpStoreStatuses() {
+	s.storesStatusesMtx.RLock()
+	defer s.storesStatusesMtx.RUnlock()
+
+	now := time.Now()
+	for addr, status := range s.storeStatuses {
+		if _, ok := s.stores[addr]; !ok {
+			if now.Sub(status.LastCheck) >= storeUnhealthyTimeout {
+				delete(s.storeStatuses, addr)
+			}
+		}
+	}
+}