Count git checkout failures (argoproj#15657)

Signed-off-by: Mikołaj Przybysz <[email protected]> Co-authored-by: Mikołaj Przybysz <[email protected]> Co-authored-by: Blake Pettersson <[email protected]>
mkieweg · Jun 11, 2024 · 51170fb · 51170fb
1 parent a00fdb4
commit 51170fb
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 1 deletion.
diff --git a/docs/operator-manual/metrics.md b/docs/operator-manual/metrics.md
@@ -81,6 +81,7 @@ Scraped at the `argocd-repo-server:8084/metrics` endpoint.
 |--------|:----:|-------------|
 | `argocd_git_request_duration_seconds` | histogram | Git requests duration seconds. |
 | `argocd_git_request_total` | counter | Number of git requests performed by repo server |
+| `argocd_git_fetch_fail_total` | counter | Number of git fetch requests failures by repo server |
 | `argocd_redis_request_duration_seconds` | histogram | Redis requests duration seconds. |
 | `argocd_redis_request_total` | counter | Number of Kubernetes requests executed during application reconciliation. |
 | `argocd_repo_pending_request_total` | gauge | Number of pending requests requiring repository lock |

diff --git a/reposerver/metrics/metrics.go b/reposerver/metrics/metrics.go
@@ -12,6 +12,7 @@ import (
 
 type MetricsServer struct {
 	handler                  http.Handler
+	gitFetchFailCounter      *prometheus.CounterVec
 	gitRequestCounter        *prometheus.CounterVec
 	gitRequestHistogram      *prometheus.HistogramVec
 	repoPendingRequestsGauge *prometheus.GaugeVec
@@ -32,6 +33,15 @@ func NewMetricsServer() *MetricsServer {
 	registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
 	registry.MustRegister(collectors.NewGoCollector())
 
+	gitFetchFailCounter := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "argocd_git_fetch_fail_total",
+			Help: "Number of git fetch requests failures by repo server",
+		},
+		[]string{"repo", "revision"},
+	)
+	registry.MustRegister(gitFetchFailCounter)
+
 	gitRequestCounter := prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "argocd_git_request_total",
@@ -81,6 +91,7 @@ func NewMetricsServer() *MetricsServer {
 
 	return &MetricsServer{
 		handler:                  promhttp.HandlerFor(registry, promhttp.HandlerOpts{}),
+		gitFetchFailCounter:      gitFetchFailCounter,
 		gitRequestCounter:        gitRequestCounter,
 		gitRequestHistogram:      gitRequestHistogram,
 		repoPendingRequestsGauge: repoPendingRequestsGauge,
@@ -93,6 +104,10 @@ func (m *MetricsServer) GetHandler() http.Handler {
 	return m.handler
 }
 
+func (m *MetricsServer) IncGitFetchFail(repo string, revision string) {
+	m.gitFetchFailCounter.WithLabelValues(repo, revision).Inc()
+}
+
 // IncGitRequest increments the git requests counter
 func (m *MetricsServer) IncGitRequest(repo string, requestType GitRequestType) {
 	m.gitRequestCounter.WithLabelValues(repo, string(requestType)).Inc()

diff --git a/reposerver/repository/repository.go b/reposerver/repository/repository.go
@@ -2397,7 +2397,11 @@ func directoryPermissionInitializer(rootPath string) goio.Closer {
 // nolint:unparam
 func (s *Service) checkoutRevision(gitClient git.Client, revision string, submoduleEnabled bool) (goio.Closer, error) {
 	closer := s.gitRepoInitializer(gitClient.Root())
-	return closer, checkoutRevision(gitClient, revision, submoduleEnabled)
+	err := checkoutRevision(gitClient, revision, submoduleEnabled)
+	if err != nil {
+		s.metricsServer.IncGitFetchFail(gitClient.Root(), revision)
+	}
+	return closer, err
 }
 
 func checkoutRevision(gitClient git.Client, revision string, submoduleEnabled bool) error {