feat: optimized stats on home page

crawlab-team · Jul 2, 2023 · 7375718 · 7375718
1 parent 0fc4a66
commit 7375718
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 18 deletions.
diff --git a/controllers/stats.go b/controllers/stats.go
@@ -8,6 +8,7 @@ import (
 	"go.mongodb.org/mongo-driver/bson"
 	"go.uber.org/dig"
 	"net/http"
+	"time"
 )
 
 var StatsController ActionController
@@ -34,11 +35,12 @@ func getStatsActions() []Action {
 }
 
 type statsContext struct {
-	statsSvc interfaces.StatsService
+	statsSvc     interfaces.StatsService
+	defaultQuery bson.M
 }
 
 func (svc *statsContext) getOverview(c *gin.Context) {
-	data, err := svc.statsSvc.GetOverviewStats(bson.M{})
+	data, err := svc.statsSvc.GetOverviewStats(svc.defaultQuery)
 	if err != nil {
 		HandleErrorInternalServerError(c, err)
 		return
@@ -47,7 +49,7 @@ func (svc *statsContext) getOverview(c *gin.Context) {
 }
 
 func (svc *statsContext) getDaily(c *gin.Context) {
-	data, err := svc.statsSvc.GetDailyStats(bson.M{})
+	data, err := svc.statsSvc.GetDailyStats(svc.defaultQuery)
 	if err != nil {
 		HandleErrorInternalServerError(c, err)
 		return
@@ -56,7 +58,7 @@ func (svc *statsContext) getDaily(c *gin.Context) {
 }
 
 func (svc *statsContext) getTasks(c *gin.Context) {
-	data, err := svc.statsSvc.GetTaskStats(bson.M{})
+	data, err := svc.statsSvc.GetTaskStats(svc.defaultQuery)
 	if err != nil {
 		HandleErrorInternalServerError(c, err)
 		return
@@ -66,7 +68,13 @@ func (svc *statsContext) getTasks(c *gin.Context) {
 
 func newStatsContext() *statsContext {
 	// context
-	ctx := &statsContext{}
+	ctx := &statsContext{
+		defaultQuery: bson.M{
+			"create_ts": bson.M{
+				"$gte": time.Now().Add(-30 * 24 * time.Hour),
+			},
+		},
+	}
 
 	// dependency injection
 	c := dig.New()

diff --git a/models/common/index_service.go b/models/common/index_service.go
@@ -59,6 +59,11 @@ func CreateIndexes() {
 		{Keys: bson.M{"has_sub": 1}},
 	})
 
+	// task stats
+	mongo.GetMongoCol(interfaces.ModelColNameTaskStat).MustCreateIndexes([]mongo2.IndexModel{
+		{Keys: bson.M{"create_ts": 1}},
+	})
+
 	// schedules
 	mongo.GetMongoCol(interfaces.ModelColNameSchedule).MustCreateIndexes([]mongo2.IndexModel{
 		{Keys: bson.M{"name": 1}},

diff --git a/stats/service.go b/stats/service.go
@@ -124,7 +124,9 @@ func (svc *Service) GetTaskStats(query bson.M) (data interface{}, err error) {
 
 func (svc *Service) getDailyTasksStats(query bson.M) (data interface{}, err error) {
 	pipeline := mongo2.Pipeline{
-		{{"$match", query}},
+		{{
+			"$match", query,
+		}},
 		{{
 			"$addFields",
 			bson.M{
@@ -137,22 +139,12 @@ func (svc *Service) getDailyTasksStats(query bson.M) (data interface{}, err erro
 				},
 			},
 		}},
-		{{
-			"$lookup",
-			bson.M{
-				"from":         interfaces.ModelColNameTaskStat,
-				"localField":   "_id",
-				"foreignField": "_id",
-				"as":           "_ts",
-			},
-		}},
-		{{"$addFields", bson.M{"ts": bson.M{"$arrayElemAt": bson.A{"$_ts", 0}}}}},
 		{{
 			"$group",
 			bson.M{
 				"_id":     "$date",
 				"tasks":   bson.M{"$sum": 1},
-				"results": bson.M{"$sum": "$ts.result_count"},
+				"results": bson.M{"$sum": "$result_count"},
 			},
 		}},
 		{{
@@ -161,7 +153,7 @@ func (svc *Service) getDailyTasksStats(query bson.M) (data interface{}, err erro
 		}},
 	}
 	var results []entity.StatsDailyItem
-	if err := mongo.GetMongoCol(interfaces.ModelColNameTask).Aggregate(pipeline, nil).All(&results); err != nil {
+	if err := mongo.GetMongoCol(interfaces.ModelColNameTaskStat).Aggregate(pipeline, nil).All(&results); err != nil {
 		return nil, err
 	}
 	return results, nil