Skip to content

Commit

Permalink
feat: add prometheus metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
garethgeorge committed Sep 7, 2024
1 parent 5293631 commit 5bdc951
Show file tree
Hide file tree
Showing 16 changed files with 159 additions and 7 deletions.
2 changes: 2 additions & 0 deletions cmd/backrest/backrest.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"github.com/garethgeorge/backrest/internal/config"
"github.com/garethgeorge/backrest/internal/env"
"github.com/garethgeorge/backrest/internal/logwriter"
"github.com/garethgeorge/backrest/internal/metric"
"github.com/garethgeorge/backrest/internal/oplog"
"github.com/garethgeorge/backrest/internal/oplog/bboltstore"
"github.com/garethgeorge/backrest/internal/orchestrator"
Expand Down Expand Up @@ -115,6 +116,7 @@ func main() {
mux.Handle(backrestHandlerPath, auth.RequireAuthentication(backrestHandler, authenticator))
mux.Handle("/", webui.Handler())
mux.Handle("/download/", http.StripPrefix("/download", api.NewDownloadHandler(oplog)))
mux.Handle("/metrics", metric.GetRegistry().Handler())

// Serve the HTTP gateway
server := &http.Server{
Expand Down
20 changes: 14 additions & 6 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,19 @@ require (
github.com/ncruces/zenity v0.10.12
go.etcd.io/bbolt v1.3.10
go.uber.org/zap v1.27.0
golang.org/x/crypto v0.23.0
golang.org/x/net v0.25.0
golang.org/x/crypto v0.24.0
golang.org/x/net v0.26.0
golang.org/x/sync v0.7.0
google.golang.org/genproto/googleapis/api v0.0.0-20240521202816-d264139d666e
google.golang.org/grpc v1.64.0
google.golang.org/protobuf v1.34.1
google.golang.org/protobuf v1.34.2
gopkg.in/natefinch/lumberjack.v2 v2.2.1
)

require (
github.com/akavel/rsrc v0.10.2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dchest/jsmin v0.0.0-20220218165748-59f39799265f // indirect
github.com/fatih/color v1.17.0 // indirect
github.com/getlantern/context v0.0.0-20220418194847-3d5e7a086201 // indirect
Expand All @@ -43,16 +45,22 @@ require (
github.com/go-stack/stack v1.8.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/josephspurrier/goversioninfo v1.4.0 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect
github.com/prometheus/client_golang v1.20.3 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.55.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/randall77/makefat v0.0.0-20210315173500-7ddd0e42c844 // indirect
go.opentelemetry.io/otel v1.27.0 // indirect
go.opentelemetry.io/otel/metric v1.27.0 // indirect
go.opentelemetry.io/otel/trace v1.27.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/image v0.16.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/text v0.15.0 // indirect
golang.org/x/tools v0.20.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240521202816-d264139d666e // indirect
)
28 changes: 28 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@ github.com/akavel/rsrc v0.10.2/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxk
github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0=
github.com/alessio/shellescape v1.4.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/containrrr/shoutrrr v0.8.0 h1:mfG2ATzIS7NR2Ec6XL+xyoHzN97H8WPjir8aYzJUSec=
github.com/containrrr/shoutrrr v0.8.0/go.mod h1:ioyQAyu1LJY6sILuNyKaQaw+9Ttik5QePU8atnAdO2o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand Down Expand Up @@ -76,6 +80,8 @@ github.com/jarcoal/httpmock v1.3.0 h1:2RJ8GP0IIaWwcC9Fp2BmVi8Kog3v2Hn7VXM3fTd+nu
github.com/jarcoal/httpmock v1.3.0/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg=
github.com/josephspurrier/goversioninfo v1.4.0 h1:Puhl12NSHUSALHSuzYwPYQkqa2E1+7SrtAPJorKK0C8=
github.com/josephspurrier/goversioninfo v1.4.0/go.mod h1:JWzv5rKQr+MmW+LvM412ToT/IkYDZjaclF2pKDss8IY=
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
Expand All @@ -86,6 +92,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A=
github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM=
github.com/ncruces/zenity v0.10.12 h1:o4SErDa0kQijlqG6W4OYYzO6kA0fGu34uegvJGcMLBI=
Expand All @@ -99,6 +107,14 @@ github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwU
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4=
github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE=
github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc=
github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8=
github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
github.com/randall77/makefat v0.0.0-20210315173500-7ddd0e42c844 h1:GranzK4hv1/pqTIhMTXt2X8MmMOuH3hMeUR0o9SP5yc=
github.com/randall77/makefat v0.0.0-20210315173500-7ddd0e42c844/go.mod h1:T1TLSfyWVBRXVGzWd0o9BI4kfoO9InEgfQe4NV3mLz8=
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
Expand Down Expand Up @@ -136,6 +152,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
golang.org/x/image v0.16.0 h1:9kloLAKhUufZhA12l5fwnx2NZW39/we1UhBesW433jw=
golang.org/x/image v0.16.0/go.mod h1:ugSZItdV4nOxyqp56HmXwH0Ry0nBCpjnZdpDaIHdoPs=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
Expand All @@ -146,6 +164,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
Expand All @@ -162,17 +182,23 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY=
golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand All @@ -184,6 +210,8 @@ google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY=
google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg=
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
gopkg.in/Knetic/govaluate.v3 v3.0.0/go.mod h1:csKLBORsPbafmSCGTEh3U7Ozmsuq8ZSIlKk1bcqph0E=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
Expand Down
1 change: 1 addition & 0 deletions internal/hook/hook.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ func newOneoffRunHookTask(title, instanceID, repoID, planID string, parentOp *v1
return &tasks.GenericOneoffTask{
OneoffTask: tasks.OneoffTask{
BaseTask: tasks.BaseTask{
TaskType: "hook",
TaskName: fmt.Sprintf("run hook %v", title),
TaskRepoID: repoID,
TaskPlanID: planID,
Expand Down
90 changes: 90 additions & 0 deletions internal/metric/metric.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
package metric

import (
"net/http"
"slices"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
globalRegistry = initRegistry()
)

func initRegistry() *Registry {

commonDims := []string{"repo_id", "plan_id"}

registry := &Registry{
reg: prometheus.NewRegistry(),
backupBytesProcessed: prometheus.NewSummaryVec(prometheus.SummaryOpts{
Name: "backrest_backup_bytes_processed",
Help: "The total number of bytes processed during a backup",
}, commonDims),
backupBytesAdded: prometheus.NewSummaryVec(prometheus.SummaryOpts{
Name: "backrest_backup_bytes_added",
Help: "The total number of bytes added during a backup",
}, commonDims),
backupFileWarnings: prometheus.NewSummaryVec(prometheus.SummaryOpts{
Name: "backrest_backup_file_warnings",
Help: "The total number of file warnings during a backup",
}, commonDims),
tasksDuration: prometheus.NewSummaryVec(prometheus.SummaryOpts{
Name: "backrest_tasks_duration_secs",
Help: "The duration of a task in seconds",
}, append(slices.Clone(commonDims), "task_type")),
tasksRun: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "backrest_tasks_run_total",
Help: "The total number of tasks run",
}, append(slices.Clone(commonDims), "task_type", "status")),
tasksErrors: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "backrest_tasks_errors_total",
Help: "The total number of tasks that errored",
}, append(slices.Clone(commonDims), "task_type")),
}

registry.reg.MustRegister(registry.backupBytesProcessed)
registry.reg.MustRegister(registry.backupBytesAdded)
registry.reg.MustRegister(registry.backupFileWarnings)
registry.reg.MustRegister(registry.tasksDuration)
registry.reg.MustRegister(registry.tasksRun)
registry.reg.MustRegister(registry.tasksErrors)

return registry
}

func GetRegistry() *Registry {
return globalRegistry
}

type Registry struct {
reg *prometheus.Registry
backupBytesProcessed *prometheus.SummaryVec
backupBytesAdded *prometheus.SummaryVec
backupFileWarnings *prometheus.SummaryVec
tasksDuration *prometheus.SummaryVec
tasksRun *prometheus.CounterVec
tasksErrors *prometheus.CounterVec
}

func (r *Registry) Handler() http.Handler {
return promhttp.HandlerFor(r.reg, promhttp.HandlerOpts{})
}

func (r *Registry) RecordTaskRun(repoID, planID, taskType string, duration_secs float64, status string) {
if repoID == "" {
repoID = "_unassociated_"
}
if planID == "" {
planID = "_unassociated_"
}
r.tasksRun.WithLabelValues(repoID, planID, taskType, status).Inc()
r.tasksDuration.WithLabelValues(repoID, planID, taskType).Observe(duration_secs)
}

func (r *Registry) RecordBackupSummary(repoID, planID string, bytesProcessed, bytesAdded int64, fileWarnings int64) {
r.backupBytesProcessed.WithLabelValues(repoID, planID).Observe(float64(bytesProcessed))
r.backupBytesAdded.WithLabelValues(repoID, planID).Observe(float64(bytesAdded))
r.backupFileWarnings.WithLabelValues(repoID, planID).Observe(float64(fileWarnings))
}
2 changes: 2 additions & 0 deletions internal/orchestrator/orchestrator.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/config"
"github.com/garethgeorge/backrest/internal/logwriter"
"github.com/garethgeorge/backrest/internal/metric"
"github.com/garethgeorge/backrest/internal/oplog"
"github.com/garethgeorge/backrest/internal/orchestrator/logging"
"github.com/garethgeorge/backrest/internal/orchestrator/repo"
Expand Down Expand Up @@ -426,6 +427,7 @@ func (o *Orchestrator) RunTask(ctx context.Context, st tasks.ScheduledTask) erro
runner.Logger(ctx).Error("task failed", zap.Error(err), zap.Duration("duration", time.Since(start)))
} else {
runner.Logger(ctx).Info("task finished", zap.Duration("duration", time.Since(start)))
metric.GetRegistry().RecordTaskRun(st.Task.RepoID(), st.Task.PlanID(), st.Task.Type(), time.Since(start).Seconds(), "success")
}

if op != nil {
Expand Down
8 changes: 7 additions & 1 deletion internal/orchestrator/tasks/task.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,18 +86,24 @@ func (s ScheduledTask) Less(other ScheduledTask) bool {
// Task is a task that can be scheduled to run at a specific time.
type Task interface {
Name() string // human readable name for this task.
Type() string // simple string 'type' for this task.
Next(now time.Time, runner TaskRunner) (ScheduledTask, error) // returns the next scheduled task.
Run(ctx context.Context, st ScheduledTask, runner TaskRunner) error // run the task.
PlanID() string // the ID of the plan this task is associated with.
RepoID() string // the ID of the repo this task is associated with.
}

type BaseTask struct {
TaskType string
TaskName string
TaskPlanID string
TaskRepoID string
}

func (b BaseTask) Type() string {
return b.TaskType
}

func (b BaseTask) Name() string {
return b.TaskName
}
Expand Down Expand Up @@ -164,7 +170,7 @@ type testTaskRunner struct {

var _ TaskRunner = &testTaskRunner{}

func newTestTaskRunner(t testing.TB, config *v1.Config, oplog *oplog.OpLog) *testTaskRunner {
func newTestTaskRunner(_ testing.TB, config *v1.Config, oplog *oplog.OpLog) *testTaskRunner {
return &testTaskRunner{
config: config,
oplog: oplog,
Expand Down
7 changes: 7 additions & 0 deletions internal/orchestrator/tasks/taskbackup.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"time"

v1 "github.com/garethgeorge/backrest/gen/go/v1"
"github.com/garethgeorge/backrest/internal/metric"
"github.com/garethgeorge/backrest/internal/oplog"
"github.com/garethgeorge/backrest/internal/protoutil"
"github.com/garethgeorge/backrest/pkg/restic"
Expand All @@ -29,6 +30,7 @@ var _ Task = &BackupTask{}
func NewScheduledBackupTask(plan *v1.Plan) *BackupTask {
return &BackupTask{
BaseTask: BaseTask{
TaskType: "backup",
TaskName: fmt.Sprintf("backup for plan %q", plan.Id),
TaskRepoID: plan.Repo,
TaskPlanID: plan.Id,
Expand All @@ -39,6 +41,7 @@ func NewScheduledBackupTask(plan *v1.Plan) *BackupTask {
func NewOneoffBackupTask(plan *v1.Plan, at time.Time) *BackupTask {
return &BackupTask{
BaseTask: BaseTask{
TaskType: "backup",
TaskName: fmt.Sprintf("backup for plan %q", plan.Id),
TaskRepoID: plan.Repo,
TaskPlanID: plan.Id,
Expand Down Expand Up @@ -132,6 +135,7 @@ func (t *BackupTask) Run(ctx context.Context, st ScheduledTask, runner TaskRunne
var sendWg sync.WaitGroup
lastSent := time.Now() // debounce progress updates, these can endup being very frequent.
var lastFiles []string
fileErrorCount := 0
summary, err := repo.Backup(ctx, plan, func(entry *restic.BackupProgressEntry) {
sendWg.Wait()
if entry.MessageType == "status" {
Expand All @@ -145,6 +149,7 @@ func (t *BackupTask) Run(ctx context.Context, st ScheduledTask, runner TaskRunne
backupOp.OperationBackup.LastStatus = protoutil.BackupProgressEntryToProto(entry)
} else if entry.MessageType == "error" {
l.Sugar().Warnf("an unknown error was encountered in processing item: %v", entry.Item)
fileErrorCount++
backupError, err := protoutil.BackupProgressEntryToBackupError(entry)
if err != nil {
l.Sugar().Errorf("failed to convert backup progress entry to backup error: %v", err)
Expand Down Expand Up @@ -180,6 +185,8 @@ func (t *BackupTask) Run(ctx context.Context, st ScheduledTask, runner TaskRunne
summary = &restic.BackupProgressEntry{}
}

metric.GetRegistry().RecordBackupSummary(t.RepoID(), t.PlanID(), summary.TotalBytesProcessed, summary.DataAdded, int64(fileErrorCount))

vars := HookVars{
Task: t.Name(),
SnapshotStats: summary,
Expand Down
1 change: 1 addition & 0 deletions internal/orchestrator/tasks/taskcheck.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ type CheckTask struct {
func NewCheckTask(repoID, planID string, force bool) Task {
return &CheckTask{
BaseTask: BaseTask{
TaskType: "check",
TaskName: fmt.Sprintf("check for repo %q", repoID),
TaskRepoID: repoID,
TaskPlanID: planID,
Expand Down
1 change: 1 addition & 0 deletions internal/orchestrator/tasks/taskcollectgarbage.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ type CollectGarbageTask struct {
func NewCollectGarbageTask() *CollectGarbageTask {
return &CollectGarbageTask{
BaseTask: BaseTask{
TaskType: "collect_garbage",
TaskName: "collect garbage",
},
}
Expand Down
Loading

0 comments on commit 5bdc951

Please sign in to comment.