Cache statistics and provide estimation methods

Currently whenever the prometheus metrics endpoint or `/admin` endpoint are viewed the statistics are recalculated immediately - using COUNT rather than a less expensive method. This PR provides a mechanism to cache these statistics, avoids generating all of the metrics on the admin page and provides an estimation method for the plain table counts. Fix go-gitea#17506 Signed-off-by: Andrew Thornton <[email protected]>
zeripath · Apr 23, 2022 · 2892d1c · 2892d1c
1 parent 1f05417
commit 2892d1c
Show file tree

Hide file tree

Showing 9 changed files with 165 additions and 76 deletions.
diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md
@@ -197,6 +197,8 @@ The following configuration set `Content-Type: application/vnd.android.package-a
 - `REPO_PAGING_NUM`: **50**: Number of repos that are shown in one page.
 - `NOTICE_PAGING_NUM`: **25**: Number of notices that are shown in one page.
 - `ORG_PAGING_NUM`: **50**: Number of organizations that are shown in one page.
+- `ESTIMATE_COUNTS`: **false**: Estimate counts for summary statistics instead counting directly.
+- `STATISTICS_TTL`: **5m**: Cache summary statistics for this period of time.
 
 ### UI - Metadata (`ui.meta`)
 
@@ -974,6 +976,8 @@ Default templates for project boards:
 - `ENABLED_ISSUE_BY_LABEL`: **false**: Enable issue by label metrics with format `gitea_issues_by_label{label="bug"} 2`.
 - `ENABLED_ISSUE_BY_REPOSITORY`: **false**: Enable issue by repository metrics with format `gitea_issues_by_repository{repository="org/repo"} 5`.
 - `TOKEN`: **\<empty\>**: You need to specify the token, if you want to include in the authorization the metrics . The same token need to be used in prometheus parameters `bearer_token` or `bearer_token_file`.
+- `ESTIMATE_COUNTS`: **false**: Estimate counts for statistics instead counting directly.
+- `STATISTICS_TTL`: **5m**: Cache summary statistics for this period of time.
 
 ## API (`api`)
 

diff --git a/models/db/context.go b/models/db/context.go
@@ -11,6 +11,7 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 
 	"xorm.io/builder"
+	"xorm.io/xorm/schemas"
 )
 
 // DefaultContext is the default context to run xorm queries in
@@ -168,3 +169,24 @@ func CountByBean(ctx context.Context, bean interface{}) (int64, error) {
 func TableName(bean interface{}) string {
 	return x.TableName(bean)
 }
+
+// EstimateTotal returns an estimate of total number of rows in table
+func EstimateTotal(bean interface{}) (int64, error) {
+	tablename := x.TableName(bean)
+	switch x.Dialect().URI().DBType {
+	case schemas.MYSQL:
+		var rows int64
+		_, err := x.SQL("SELECT table_rows FROM information_schema.tables WHERE tables.table_name = ? AND tables.table_schema = ?;", tablename, x.Dialect().URI().DBName).Get(&rows)
+		return rows, err
+	case schemas.POSTGRES:
+		var rows int64
+		_, err := x.SQL("SELECT reltuples AS estimate FROM pg_class WHERE relname = ?;", tablename).Get(&rows)
+		return rows, err
+	case schemas.MSSQL:
+		var rows int64
+		_, err := x.SQL("sp_spaceused ?;", tablename).Get(&rows)
+		return rows, err
+	default:
+		return x.Count(tablename)
+	}
+}
diff --git a/models/statistic.go b/models/statistic.go
@@ -5,6 +5,8 @@
 package models
 
 import (
+	"time"
+
 	asymkey_model "code.gitea.io/gitea/models/asymkey"
 	"code.gitea.io/gitea/models/auth"
 	"code.gitea.io/gitea/models/db"
@@ -31,6 +33,7 @@ type Statistic struct {
 		IssueByLabel      []IssueByLabelCount
 		IssueByRepository []IssueByRepositoryCount
 	}
+	Time time.Time
 }
 
 // IssueByLabelCount contains the number of issue group by label
@@ -47,23 +50,31 @@ type IssueByRepositoryCount struct {
 }
 
 // GetStatistic returns the database statistics
-func GetStatistic() (stats Statistic) {
+func GetStatistic(estimate, metrics bool) (stats Statistic) {
 	e := db.GetEngine(db.DefaultContext)
 	stats.Counter.User = user_model.CountUsers()
 	stats.Counter.Org = organization.CountOrganizations()
-	stats.Counter.PublicKey, _ = e.Count(new(asymkey_model.PublicKey))
 	stats.Counter.Repo = repo_model.CountRepositories(true)
-	stats.Counter.Watch, _ = e.Count(new(repo_model.Watch))
-	stats.Counter.Star, _ = e.Count(new(repo_model.Star))
-	stats.Counter.Action, _ = e.Count(new(Action))
-	stats.Counter.Access, _ = e.Count(new(Access))
+	if estimate {
+		stats.Counter.PublicKey, _ = db.EstimateTotal(new(asymkey_model.PublicKey))
+		stats.Counter.Watch, _ = db.EstimateTotal(new(repo_model.Watch))
+		stats.Counter.Star, _ = db.EstimateTotal(new(repo_model.Star))
+		stats.Counter.Action, _ = db.EstimateTotal(new(Action))
+		stats.Counter.Access, _ = db.EstimateTotal(new(Access))
+	} else {
+		stats.Counter.PublicKey, _ = e.Count(new(asymkey_model.PublicKey))
+		stats.Counter.Watch, _ = e.Count(new(repo_model.Watch))
+		stats.Counter.Star, _ = e.Count(new(repo_model.Star))
+		stats.Counter.Action, _ = e.Count(new(Action))
+		stats.Counter.Access, _ = e.Count(new(Access))
+	}
 
 	type IssueCount struct {
 		Count    int64
 		IsClosed bool
 	}
 
-	if setting.Metrics.EnabledIssueByLabel {
+	if metrics && setting.Metrics.EnabledIssueByLabel {
 		stats.Counter.IssueByLabel = []IssueByLabelCount{}
 
 		_ = e.Select("COUNT(*) AS count, l.name AS label").
@@ -73,7 +84,7 @@ func GetStatistic() (stats Statistic) {
 			Find(&stats.Counter.IssueByLabel)
 	}
 
-	if setting.Metrics.EnabledIssueByRepository {
+	if metrics && setting.Metrics.EnabledIssueByRepository {
 		stats.Counter.IssueByRepository = []IssueByRepositoryCount{}
 
 		_ = e.Select("COUNT(*) AS count, r.owner_name, r.name AS repository").
@@ -110,5 +121,6 @@ func GetStatistic() (stats Statistic) {
 	stats.Counter.Attachment, _ = e.Count(new(repo_model.Attachment))
 	stats.Counter.Project, _ = e.Count(new(project_model.Project))
 	stats.Counter.ProjectBoard, _ = e.Count(new(project_model.Board))
+	stats.Time = time.Now()
 	return
 }
diff --git a/modules/metrics/collector.go b/modules/metrics/collector.go
@@ -5,7 +5,7 @@
 package metrics
 
 import (
-	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/setting"
 
 	"github.com/prometheus/client_golang/prometheus"
 )
@@ -43,6 +43,8 @@ type Collector struct {
 	Users              *prometheus.Desc
 	Watches            *prometheus.Desc
 	Webhooks           *prometheus.Desc
+
+	StatisticsTime *prometheus.Desc
 }
 
 // NewCollector returns a new Collector with all prometheus.Desc initialized
@@ -225,152 +227,152 @@ func (c Collector) Describe(ch chan<- *prometheus.Desc) {
 
 // Collect returns the metrics with values
 func (c Collector) Collect(ch chan<- prometheus.Metric) {
-	stats := models.GetStatistic()
+	stats := GetStatistic(setting.Metrics.EstimateCounts, setting.Metrics.StatisticTTL)
 
-	ch <- prometheus.MustNewConstMetric(
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Accesses,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Access),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Actions,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Action),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Attachments,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Attachment),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Comments,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Comment),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Follows,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Follow),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.HookTasks,
 		prometheus.GaugeValue,
 		float64(stats.Counter.HookTask),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Issues,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Issue),
-	)
+	))
 	for _, il := range stats.Counter.IssueByLabel {
-		ch <- prometheus.MustNewConstMetric(
+		ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 			c.IssuesByLabel,
 			prometheus.GaugeValue,
 			float64(il.Count),
 			il.Label,
-		)
+		))
 	}
 	for _, ir := range stats.Counter.IssueByRepository {
-		ch <- prometheus.MustNewConstMetric(
+		ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 			c.IssuesByRepository,
 			prometheus.GaugeValue,
 			float64(ir.Count),
 			ir.OwnerName+"/"+ir.Repository,
-		)
+		))
 	}
-	ch <- prometheus.MustNewConstMetric(
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.IssuesClosed,
 		prometheus.GaugeValue,
 		float64(stats.Counter.IssueClosed),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.IssuesOpen,
 		prometheus.GaugeValue,
 		float64(stats.Counter.IssueOpen),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Labels,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Label),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.LoginSources,
 		prometheus.GaugeValue,
 		float64(stats.Counter.AuthSource),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Milestones,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Milestone),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Mirrors,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Mirror),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Oauths,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Oauth),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Organizations,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Org),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Projects,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Project),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.ProjectBoards,
 		prometheus.GaugeValue,
 		float64(stats.Counter.ProjectBoard),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.PublicKeys,
 		prometheus.GaugeValue,
 		float64(stats.Counter.PublicKey),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Releases,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Release),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Repositories,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Repo),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Stars,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Star),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Teams,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Team),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.UpdateTasks,
 		prometheus.GaugeValue,
 		float64(stats.Counter.UpdateTask),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Users,
 		prometheus.GaugeValue,
 		float64(stats.Counter.User),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Watches,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Watch),
-	)
-	ch <- prometheus.MustNewConstMetric(
+	))
+	ch <- prometheus.NewMetricWithTimestamp(stats.Time, prometheus.MustNewConstMetric(
 		c.Webhooks,
 		prometheus.GaugeValue,
 		float64(stats.Counter.Webhook),
-	)
+	))
 }
diff --git a/modules/metrics/statistics.go b/modules/metrics/statistics.go
@@ -0,0 +1,38 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package metrics
+
+import (
+	"strconv"
+	"sync"
+	"time"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/cache"
+	"code.gitea.io/gitea/modules/setting"
+)
+
+var statisticsLock sync.Mutex
+
+func GetStatistic(estimate bool, statisticsTTL time.Duration, metrics bool) models.Statistic {
+	if statisticsTTL > 0 {
+		c := cache.GetCache()
+		if c != nil {
+			statisticsLock.Lock()
+			defer statisticsLock.Unlock()
+			cacheKey := "models/statistic.Statistic." + strconv.FormatBool(estimate) + strconv.FormatBool(metrics)
+
+			if stats, ok := c.Get(cacheKey).(*models.Statistic); ok {
+				return *stats
+			}
+
+			stats := models.GetStatistic(estimate, metrics)
+			c.Put(cacheKey, &stats, setting.DurationToCacheTTL(statisticsTTL))
+			return stats
+		}
+	}
+
+	return models.GetStatistic(estimate, metrics)
+}