fix race on shutdown (#666)

pkg/service/process.go

@@ -70,7 +70,7 @@ func (p *Process) Gather() ([]*dto.MetricFamily, error) {
 	// Get the metrics from the handler via IPC
 	metricsResponse, err := p.ipcHandlerClient.GetMetrics(context.Background(), &ipc.MetricsRequest{})
 	if err != nil {
-		logger.Warnw("failed to obtain metrics from handler", err, "egress_id", p.req.EgressId)
+		logger.Warnw("failed to obtain metrics from handler", err, "egressID", p.req.EgressId)
 		return make([]*dto.MetricFamily, 0), nil // don't return an error, just skip this handler
 	}
 

pkg/service/service.go

@@ -203,9 +203,17 @@ func (s *Service) killProcess(egressID string, maxUsage float64) {
 }
 
 func (s *Service) Close() {
-	for s.GetRequestCount() > 0 {
+	for {
+		s.mu.RLock()
+		isIdle := len(s.activeHandlers) == 0
+		s.mu.RUnlock()
+
+		if isIdle {
+			logger.Infow("closing server")
+			s.psrpcServer.Shutdown()
+			return
+		}
+
 		time.Sleep(time.Second)
 	}
-	logger.Infow("closing server")
-	s.psrpcServer.Shutdown()
 }

pkg/service/service_rpc.go

@@ -161,6 +161,7 @@ func (s *Service) AddHandler(egressID string, p *Process) error {
 		}()
 
 	case <-time.After(10 * time.Second):
+		logger.Warnw("no response from handler", nil, "egressID", egressID)
 		_ = p.cmd.Process.Kill()
 		s.processEnded(p, errors.ErrEgressNotFound)
 	}