diff --git a/api/log.go b/api/log.go index 105ae1337..fe593c40d 100644 --- a/api/log.go +++ b/api/log.go @@ -83,7 +83,9 @@ func GetBuildLogs(c *gin.Context) { }).Infof("reading logs for build %s", entry) // send API call to capture the list of logs for the build - l, err := database.FromContext(c).GetBuildLogs(b.GetID()) + // + // TODO: add page and per_page query parameters + l, t, err := database.FromContext(c).ListLogsForBuild(b, 1, 100) if err != nil { retErr := fmt.Errorf("unable to get logs for build %s: %w", entry, err) @@ -92,6 +94,15 @@ func GetBuildLogs(c *gin.Context) { return } + // create pagination object + pagination := Pagination{ + Page: 1, + PerPage: 100, + Total: t, + } + // set pagination headers + pagination.SetHeaderLink(c) + c.JSON(http.StatusOK, l) } @@ -200,7 +211,7 @@ func CreateServiceLog(c *gin.Context) { } // send API call to capture the created log - l, _ := database.FromContext(c).GetServiceLog(s.GetID()) + l, _ := database.FromContext(c).GetLogForService(s) c.JSON(http.StatusCreated, l) } @@ -270,7 +281,7 @@ func GetServiceLog(c *gin.Context) { }).Infof("reading logs for service %s", entry) // send API call to capture the service logs - l, err := database.FromContext(c).GetServiceLog(s.GetID()) + l, err := database.FromContext(c).GetLogForService(s) if err != nil { retErr := fmt.Errorf("unable to get logs for service %s: %w", entry, err) @@ -358,7 +369,7 @@ func UpdateServiceLog(c *gin.Context) { }).Infof("updating logs for service %s", entry) // send API call to capture the service logs - l, err := database.FromContext(c).GetServiceLog(s.GetID()) + l, err := database.FromContext(c).GetLogForService(s) if err != nil { retErr := fmt.Errorf("unable to get logs for service %s: %w", entry, err) @@ -396,7 +407,7 @@ func UpdateServiceLog(c *gin.Context) { } // send API call to capture the updated log - l, _ = database.FromContext(c).GetServiceLog(s.GetID()) + l, _ = database.FromContext(c).GetLogForService(s) c.JSON(http.StatusOK, l) } @@ -444,8 +455,6 @@ func UpdateServiceLog(c *gin.Context) { // DeleteServiceLog represents the API handler to remove // the logs for a service from the configured backend. -// -//nolint:dupl // ignore similar code with step func DeleteServiceLog(c *gin.Context) { // capture middleware values b := build.Retrieve(c) @@ -467,8 +476,18 @@ func DeleteServiceLog(c *gin.Context) { "user": u.GetName(), }).Infof("deleting logs for service %s", entry) + // send API call to capture the service logs + l, err := database.FromContext(c).GetLogForService(s) + if err != nil { + retErr := fmt.Errorf("unable to get logs for service %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + // send API call to remove the log - err := database.FromContext(c).DeleteLog(s.GetID()) + err = database.FromContext(c).DeleteLog(l) if err != nil { retErr := fmt.Errorf("unable to delete logs for service %s: %w", entry, err) @@ -585,7 +604,7 @@ func CreateStepLog(c *gin.Context) { } // send API call to capture the created log - l, _ := database.FromContext(c).GetStepLog(s.GetID()) + l, _ := database.FromContext(c).GetLogForStep(s) c.JSON(http.StatusCreated, l) } @@ -656,7 +675,7 @@ func GetStepLog(c *gin.Context) { }).Infof("reading logs for step %s", entry) // send API call to capture the step logs - l, err := database.FromContext(c).GetStepLog(s.GetID()) + l, err := database.FromContext(c).GetLogForStep(s) if err != nil { retErr := fmt.Errorf("unable to get logs for step %s: %w", entry, err) @@ -744,7 +763,7 @@ func UpdateStepLog(c *gin.Context) { }).Infof("updating logs for step %s", entry) // send API call to capture the step logs - l, err := database.FromContext(c).GetStepLog(s.GetID()) + l, err := database.FromContext(c).GetLogForStep(s) if err != nil { retErr := fmt.Errorf("unable to get logs for step %s: %w", entry, err) @@ -782,7 +801,7 @@ func UpdateStepLog(c *gin.Context) { } // send API call to capture the updated log - l, _ = database.FromContext(c).GetStepLog(s.GetID()) + l, _ = database.FromContext(c).GetLogForStep(s) c.JSON(http.StatusOK, l) } @@ -830,8 +849,6 @@ func UpdateStepLog(c *gin.Context) { // DeleteStepLog represents the API handler to remove // the logs for a step from the configured backend. -// -//nolint:dupl // ignore similar code with service func DeleteStepLog(c *gin.Context) { // capture middleware values b := build.Retrieve(c) @@ -853,8 +870,18 @@ func DeleteStepLog(c *gin.Context) { "user": u.GetName(), }).Infof("deleting logs for step %s", entry) + // send API call to capture the step logs + l, err := database.FromContext(c).GetLogForStep(s) + if err != nil { + retErr := fmt.Errorf("unable to get logs for step %s: %w", entry, err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + // send API call to remove the log - err := database.FromContext(c).DeleteLog(s.GetID()) + err = database.FromContext(c).DeleteLog(l) if err != nil { retErr := fmt.Errorf("unable to delete logs for step %s: %w", entry, err) diff --git a/api/stream.go b/api/stream.go index c3425d517..701758f14 100644 --- a/api/stream.go +++ b/api/stream.go @@ -114,7 +114,7 @@ func PostServiceStream(c *gin.Context) { defer close(done) // send API call to capture the service logs - _log, err := database.FromContext(c).GetServiceLog(s.GetID()) + _log, err := database.FromContext(c).GetLogForService(s) if err != nil { retErr := fmt.Errorf("unable to get logs for service %s/%d: %w", entry, s.GetNumber(), err) @@ -269,7 +269,7 @@ func PostStepStream(c *gin.Context) { defer close(done) // send API call to capture the step logs - _log, err := database.FromContext(c).GetStepLog(s.GetID()) + _log, err := database.FromContext(c).GetLogForStep(s) if err != nil { retErr := fmt.Errorf("unable to get logs for step %s/%d: %w", entry, s.GetNumber(), err) diff --git a/database/log/count.go b/database/log/count.go new file mode 100644 index 000000000..e4ec570fe --- /dev/null +++ b/database/log/count.go @@ -0,0 +1,25 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "github.com/go-vela/types/constants" +) + +// CountLogs gets the count of all logs from the database. +func (e *engine) CountLogs() (int64, error) { + e.logger.Tracef("getting count of all logs from the database") + + // variable to store query results + var l int64 + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableLog). + Count(&l). + Error + + return l, err +} diff --git a/database/log/count_build.go b/database/log/count_build.go new file mode 100644 index 000000000..6a7fb74fb --- /dev/null +++ b/database/log/count_build.go @@ -0,0 +1,27 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/library" +) + +// CountLogsForBuild gets the count of logs by build ID from the database. +func (e *engine) CountLogsForBuild(b *library.Build) (int64, error) { + e.logger.Tracef("getting count of logs for build %d from the database", b.GetID()) + + // variable to store query results + var l int64 + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableLog). + Where("build_id = ?", b.GetID()). + Count(&l). + Error + + return l, err +} diff --git a/database/log/count_build_test.go b/database/log/count_build_test.go new file mode 100644 index 000000000..d462eedf0 --- /dev/null +++ b/database/log/count_build_test.go @@ -0,0 +1,99 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestLog_Engine_CountLogsForBuild(t *testing.T) { + // setup types + _service := testLog() + _service.SetID(1) + _service.SetRepoID(1) + _service.SetBuildID(1) + _service.SetServiceID(1) + + _step := testLog() + _step.SetID(2) + _step.SetRepoID(1) + _step.SetBuildID(1) + _step.SetStepID(1) + + _build := testBuild() + _build.SetID(1) + _build.SetID(1) + _build.SetRepoID(1) + _build.SetNumber(1) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "logs" WHERE build_id = $1`).WithArgs(1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateLog(_service) + if err != nil { + t.Errorf("unable to create test service log for sqlite: %v", err) + } + + err = _sqlite.CreateLog(_step) + if err != nil { + t.Errorf("unable to create test step log for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want int64 + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: 2, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: 2, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CountLogsForBuild(_build) + + if test.failure { + if err == nil { + t.Errorf("CountLogsForBuild for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CountLogsForBuild for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("CountLogsForBuild for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/log/count_test.go b/database/log/count_test.go new file mode 100644 index 000000000..99e75a767 --- /dev/null +++ b/database/log/count_test.go @@ -0,0 +1,93 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestLog_Engine_CountLogs(t *testing.T) { + // setup types + _service := testLog() + _service.SetID(1) + _service.SetRepoID(1) + _service.SetBuildID(1) + _service.SetServiceID(1) + + _step := testLog() + _step.SetID(2) + _step.SetRepoID(1) + _step.SetBuildID(1) + _step.SetStepID(1) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "logs"`).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateLog(_service) + if err != nil { + t.Errorf("unable to create test service log for sqlite: %v", err) + } + + err = _sqlite.CreateLog(_step) + if err != nil { + t.Errorf("unable to create test step log for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want int64 + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: 2, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: 2, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.CountLogs() + + if test.failure { + if err == nil { + t.Errorf("CountLogs for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CountLogs for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("CountLogs for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/log/create.go b/database/log/create.go new file mode 100644 index 000000000..978da9a1f --- /dev/null +++ b/database/log/create.go @@ -0,0 +1,57 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +//nolint:dupl // ignore similar code with create.go +package log + +import ( + "fmt" + + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +// CreateLog creates a new log in the database. +func (e *engine) CreateLog(l *library.Log) error { + // check what the log entry is for + switch { + case l.GetServiceID() > 0: + e.logger.Tracef("creating log for service %d for build %d in the database", l.GetServiceID(), l.GetBuildID()) + case l.GetStepID() > 0: + e.logger.Tracef("creating log for step %d for build %d in the database", l.GetStepID(), l.GetBuildID()) + } + + // cast the library type to database type + // + // https://pkg.go.dev/github.com/go-vela/types/database#LogFromLibrary + log := database.LogFromLibrary(l) + + // validate the necessary fields are populated + // + // https://pkg.go.dev/github.com/go-vela/types/database#Log.Validate + err := log.Validate() + if err != nil { + return err + } + + // compress log data for the resource + // + // https://pkg.go.dev/github.com/go-vela/types/database#Log.Compress + err = log.Compress(e.config.CompressionLevel) + if err != nil { + switch { + case l.GetServiceID() > 0: + return fmt.Errorf("unable to compress log for service %d for build %d: %w", l.GetServiceID(), l.GetBuildID(), err) + case l.GetStepID() > 0: + return fmt.Errorf("unable to compress log for step %d for build %d: %w", l.GetStepID(), l.GetBuildID(), err) + } + } + + // send query to the database + return e.client. + Table(constants.TableLog). + Create(log). + Error +} diff --git a/database/log/create_test.go b/database/log/create_test.go new file mode 100644 index 000000000..1574e88a8 --- /dev/null +++ b/database/log/create_test.go @@ -0,0 +1,92 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestLog_Engine_CreateLog(t *testing.T) { + // setup types + _service := testLog() + _service.SetID(1) + _service.SetRepoID(1) + _service.SetBuildID(1) + _service.SetServiceID(1) + + _step := testLog() + _step.SetID(2) + _step.SetRepoID(1) + _step.SetBuildID(1) + _step.SetStepID(1) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"id"}).AddRow(1) + + // ensure the mock expects the service query + _mock.ExpectQuery(`INSERT INTO "logs" +("build_id","repo_id","service_id","step_id","data","id") +VALUES ($1,$2,$3,$4,$5,$6) RETURNING "id"`). + WithArgs(1, 1, 1, nil, AnyArgument{}, 1). + WillReturnRows(_rows) + + // ensure the mock expects the step query + _mock.ExpectQuery(`INSERT INTO "logs" +("build_id","repo_id","service_id","step_id","data","id") +VALUES ($1,$2,$3,$4,$5,$6) RETURNING "id"`). + WithArgs(1, 1, nil, 1, AnyArgument{}, 2). + WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + logs []*library.Log + }{ + { + failure: false, + name: "postgres", + database: _postgres, + logs: []*library.Log{_service, _step}, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + logs: []*library.Log{_service, _step}, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + for _, log := range test.logs { + err := test.database.CreateLog(log) + + if test.failure { + if err == nil { + t.Errorf("CreateLog for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateLog for %s returned err: %v", test.name, err) + } + } + }) + } +} diff --git a/database/log/delete.go b/database/log/delete.go new file mode 100644 index 000000000..255e6213b --- /dev/null +++ b/database/log/delete.go @@ -0,0 +1,33 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +// DeleteLog deletes an existing log from the database. +func (e *engine) DeleteLog(l *library.Log) error { + // check what the log entry is for + switch { + case l.GetServiceID() > 0: + e.logger.Tracef("deleting log for service %d for build %d in the database", l.GetServiceID(), l.GetBuildID()) + case l.GetStepID() > 0: + e.logger.Tracef("deleting log for step %d for build %d in the database", l.GetStepID(), l.GetBuildID()) + } + + // cast the library type to database type + // + // https://pkg.go.dev/github.com/go-vela/types/database#LogFromLibrary + log := database.LogFromLibrary(l) + + // send query to the database + return e.client. + Table(constants.TableLog). + Delete(log). + Error +} diff --git a/database/log/delete_test.go b/database/log/delete_test.go new file mode 100644 index 000000000..15329a0af --- /dev/null +++ b/database/log/delete_test.go @@ -0,0 +1,73 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestLog_Engine_DeleteLog(t *testing.T) { + // setup types + _log := testLog() + _log.SetID(1) + _log.SetRepoID(1) + _log.SetBuildID(1) + _log.SetStepID(1) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // ensure the mock expects the query + _mock.ExpectExec(`DELETE FROM "logs" WHERE "logs"."id" = $1`). + WithArgs(1). + WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateLog(_log) + if err != nil { + t.Errorf("unable to create test log for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err = test.database.DeleteLog(_log) + + if test.failure { + if err == nil { + t.Errorf("DeleteLog for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("DeleteLog for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/log/get.go b/database/log/get.go new file mode 100644 index 000000000..d91a6687d --- /dev/null +++ b/database/log/get.go @@ -0,0 +1,48 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +// GetLog gets a log by ID from the database. +func (e *engine) GetLog(id int64) (*library.Log, error) { + e.logger.Tracef("getting log %d from the database", id) + + // variable to store query results + l := new(database.Log) + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableLog). + Where("id = ?", id). + Take(l). + Error + if err != nil { + return nil, err + } + + // decompress log data + // + // https://pkg.go.dev/github.com/go-vela/types/database#Log.Decompress + err = l.Decompress() + if err != nil { + // ensures that the change is backwards compatible + // by logging the error instead of returning it + // which allowing us to fetch uncompressed logs + e.logger.Errorf("unable to decompress log %d: %v", id, err) + + // return the uncompressed log + return l.ToLibrary(), nil + } + + // return the log + // + // https://pkg.go.dev/github.com/go-vela/types/database#Log.ToLibrary + return l.ToLibrary(), nil +} diff --git a/database/log/get_service.go b/database/log/get_service.go new file mode 100644 index 000000000..2d03bf127 --- /dev/null +++ b/database/log/get_service.go @@ -0,0 +1,49 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +//nolint:dupl // ignore similar code with get_step.go +package log + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +// GetLogForService gets a log by service ID from the database. +func (e *engine) GetLogForService(s *library.Service) (*library.Log, error) { + e.logger.Tracef("getting log for service %d for build %d from the database", s.GetID(), s.GetBuildID()) + + // variable to store query results + l := new(database.Log) + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableLog). + Where("service_id = ?", s.GetID()). + Take(l). + Error + if err != nil { + return nil, err + } + + // decompress log data for the service + // + // https://pkg.go.dev/github.com/go-vela/types/database#Log.Decompress + err = l.Decompress() + if err != nil { + // ensures that the change is backwards compatible + // by logging the error instead of returning it + // which allowing us to fetch uncompressed logs + e.logger.Errorf("unable to decompress log for service %d for build %d: %v", s.GetID(), s.GetBuildID(), err) + + // return the uncompressed log + return l.ToLibrary(), nil + } + + // return the log + // + // https://pkg.go.dev/github.com/go-vela/types/database#Log.ToLibrary + return l.ToLibrary(), nil +} diff --git a/database/log/get_service_test.go b/database/log/get_service_test.go new file mode 100644 index 000000000..26f42813c --- /dev/null +++ b/database/log/get_service_test.go @@ -0,0 +1,93 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestLog_Engine_GetLogForService(t *testing.T) { + // setup types + _log := testLog() + _log.SetID(1) + _log.SetRepoID(1) + _log.SetBuildID(1) + _log.SetServiceID(1) + _log.SetData([]byte{}) + + _service := testService() + _service.SetID(1) + _service.SetID(1) + _service.SetRepoID(1) + _service.SetBuildID(1) + _service.SetNumber(1) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows( + []string{"id", "build_id", "repo_id", "service_id", "step_id", "data"}). + AddRow(1, 1, 1, 1, 0, []byte{}) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "logs" WHERE service_id = $1 LIMIT 1`).WithArgs(1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateLog(_log) + if err != nil { + t.Errorf("unable to create test log for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want *library.Log + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: _log, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: _log, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.GetLogForService(_service) + + if test.failure { + if err == nil { + t.Errorf("GetLogForService for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("GetLogForService for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("GetLogForService for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/log/get_step.go b/database/log/get_step.go new file mode 100644 index 000000000..eb1b95c7f --- /dev/null +++ b/database/log/get_step.go @@ -0,0 +1,49 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +//nolint:dupl // ignore similar code with get_service.go +package log + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +// GetLogForStep gets a log by step ID from the database. +func (e *engine) GetLogForStep(s *library.Step) (*library.Log, error) { + e.logger.Tracef("getting log for step %d for build %d from the database", s.GetID(), s.GetBuildID()) + + // variable to store query results + l := new(database.Log) + + // send query to the database and store result in variable + err := e.client. + Table(constants.TableLog). + Where("step_id = ?", s.GetID()). + Take(l). + Error + if err != nil { + return nil, err + } + + // decompress log data for the step + // + // https://pkg.go.dev/github.com/go-vela/types/database#Log.Decompress + err = l.Decompress() + if err != nil { + // ensures that the change is backwards compatible + // by logging the error instead of returning it + // which allowing us to fetch uncompressed logs + e.logger.Errorf("unable to decompress log for step %d for build %d: %v", s.GetID(), s.GetBuildID(), err) + + // return the uncompressed log + return l.ToLibrary(), nil + } + + // return the log + // + // https://pkg.go.dev/github.com/go-vela/types/database#Log.ToLibrary + return l.ToLibrary(), nil +} diff --git a/database/log/get_step_test.go b/database/log/get_step_test.go new file mode 100644 index 000000000..39f019039 --- /dev/null +++ b/database/log/get_step_test.go @@ -0,0 +1,93 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestLog_Engine_GetLogForStep(t *testing.T) { + // setup types + _log := testLog() + _log.SetID(1) + _log.SetRepoID(1) + _log.SetBuildID(1) + _log.SetStepID(1) + _log.SetData([]byte{}) + + _step := testStep() + _step.SetID(1) + _step.SetID(1) + _step.SetRepoID(1) + _step.SetBuildID(1) + _step.SetNumber(1) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows( + []string{"id", "build_id", "repo_id", "step_id", "step_id", "data"}). + AddRow(1, 1, 1, 0, 1, []byte{}) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "logs" WHERE step_id = $1 LIMIT 1`).WithArgs(1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateLog(_log) + if err != nil { + t.Errorf("unable to create test log for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want *library.Log + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: _log, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: _log, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.GetLogForStep(_step) + + if test.failure { + if err == nil { + t.Errorf("GetLogForStep for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("GetLogForStep for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("GetLogForStep for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/log/get_test.go b/database/log/get_test.go new file mode 100644 index 000000000..31325e3ac --- /dev/null +++ b/database/log/get_test.go @@ -0,0 +1,86 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestLog_Engine_GetLog(t *testing.T) { + // setup types + _log := testLog() + _log.SetID(1) + _log.SetRepoID(1) + _log.SetBuildID(1) + _log.SetStepID(1) + _log.SetData([]byte{}) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows( + []string{"id", "build_id", "repo_id", "service_id", "step_id", "data"}). + AddRow(1, 1, 1, 0, 1, []byte{}) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "logs" WHERE id = $1 LIMIT 1`).WithArgs(1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateLog(_log) + if err != nil { + t.Errorf("unable to create test log for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want *library.Log + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: _log, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: _log, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.GetLog(1) + + if test.failure { + if err == nil { + t.Errorf("GetLog for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("GetLog for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("GetLog for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/log/index.go b/database/log/index.go new file mode 100644 index 000000000..3ff3cdbc5 --- /dev/null +++ b/database/log/index.go @@ -0,0 +1,24 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +const ( + // CreateBuildIDIndex represents a query to create an + // index on the logs table for the build_id column. + CreateBuildIDIndex = ` +CREATE INDEX +IF NOT EXISTS +logs_build_id +ON logs (build_id); +` +) + +// CreateLogIndexes creates the indexes for the logs table in the database. +func (e *engine) CreateLogIndexes() error { + e.logger.Tracef("creating indexes for logs table in the database") + + // create the hostname and address columns index for the logs table + return e.client.Exec(CreateBuildIDIndex).Error +} diff --git a/database/log/index_test.go b/database/log/index_test.go new file mode 100644 index 000000000..26c0045db --- /dev/null +++ b/database/log/index_test.go @@ -0,0 +1,59 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestLog_Engine_CreateLogIndexes(t *testing.T) { + // setup types + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + _mock.ExpectExec(CreateBuildIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.CreateLogIndexes() + + if test.failure { + if err == nil { + t.Errorf("CreateLogIndexes for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateLogIndexes for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/log/list.go b/database/log/list.go new file mode 100644 index 000000000..703155a37 --- /dev/null +++ b/database/log/list.go @@ -0,0 +1,65 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +// ListLogs gets a list of all logs from the database. +func (e *engine) ListLogs() ([]*library.Log, error) { + e.logger.Trace("listing all logs from the database") + + // variables to store query results and return value + count := int64(0) + h := new([]database.Log) + logs := []*library.Log{} + + // count the results + count, err := e.CountLogs() + if err != nil { + return nil, err + } + + // short-circuit if there are no results + if count == 0 { + return logs, nil + } + + // send query to the database and store result in variable + err = e.client. + Table(constants.TableLog). + Find(&h). + Error + if err != nil { + return nil, err + } + + // iterate through all query results + for _, log := range *h { + // https://golang.org/doc/faq#closures_and_goroutines + tmp := log + + // decompress log data + // + // https://pkg.go.dev/github.com/go-vela/types/database#Log.Decompress + err = tmp.Decompress() + if err != nil { + // ensures that the change is backwards compatible + // by logging the error instead of returning it + // which allows us to fetch uncompressed logs + e.logger.Errorf("unable to decompress logs: %v", err) + } + + // convert query result to library type + // + // https://pkg.go.dev/github.com/go-vela/types/database#Log.ToLibrary + logs = append(logs, tmp.ToLibrary()) + } + + return logs, nil +} diff --git a/database/log/list_build.go b/database/log/list_build.go new file mode 100644 index 000000000..58ca12111 --- /dev/null +++ b/database/log/list_build.go @@ -0,0 +1,72 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +// ListLogsForBuild gets a list of logs by build ID from the database. +func (e *engine) ListLogsForBuild(b *library.Build, page, perPage int) ([]*library.Log, int64, error) { + e.logger.Tracef("listing logs for build %d from the database", b.GetID()) + + // variables to store query results and return value + count := int64(0) + l := new([]database.Log) + logs := []*library.Log{} + + // count the results + count, err := e.CountLogsForBuild(b) + if err != nil { + return nil, 0, err + } + + // short-circuit if there are no results + if count == 0 { + return logs, 0, nil + } + + // calculate offset for pagination through results + offset := perPage * (page - 1) + + // send query to the database and store result in variable + err = e.client. + Table(constants.TableLog). + Where("build_id = ?", b.GetID()). + Order("step_id ASC"). + Limit(perPage). + Offset(offset). + Find(&l). + Error + if err != nil { + return nil, count, err + } + + // iterate through all query results + for _, log := range *l { + // https://golang.org/doc/faq#closures_and_goroutines + tmp := log + + // decompress log data for the build + // + // https://pkg.go.dev/github.com/go-vela/types/database#Log.Decompress + err = tmp.Decompress() + if err != nil { + // ensures that the change is backwards compatible + // by logging the error instead of returning it + // which allows us to fetch uncompressed logs + e.logger.Errorf("unable to decompress logs for build %d: %v", b.GetID(), err) + } + + // convert query result to library type + // + // https://pkg.go.dev/github.com/go-vela/types/database#Log.ToLibrary + logs = append(logs, tmp.ToLibrary()) + } + + return logs, count, nil +} diff --git a/database/log/list_build_test.go b/database/log/list_build_test.go new file mode 100644 index 000000000..fb08236e9 --- /dev/null +++ b/database/log/list_build_test.go @@ -0,0 +1,110 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestLog_Engine_ListLogsForBuild(t *testing.T) { + // setup types + _service := testLog() + _service.SetID(1) + _service.SetRepoID(1) + _service.SetBuildID(1) + _service.SetServiceID(1) + _service.SetData([]byte{}) + + _step := testLog() + _step.SetID(2) + _step.SetRepoID(1) + _step.SetBuildID(1) + _step.SetStepID(1) + _step.SetData([]byte{}) + + _build := testBuild() + _build.SetID(1) + _build.SetID(1) + _build.SetRepoID(1) + _build.SetNumber(1) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "logs" WHERE build_id = $1`).WithArgs(1).WillReturnRows(_rows) + + // create expected result in mock + _rows = sqlmock.NewRows( + []string{"id", "build_id", "repo_id", "service_id", "step_id", "data"}). + AddRow(1, 1, 1, 1, 0, []byte{}).AddRow(2, 1, 1, 0, 1, []byte{}) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "logs" WHERE build_id = $1 ORDER BY step_id ASC LIMIT 10`).WithArgs(1).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateLog(_service) + if err != nil { + t.Errorf("unable to create test service log for sqlite: %v", err) + } + + err = _sqlite.CreateLog(_step) + if err != nil { + t.Errorf("unable to create test step log for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want []*library.Log + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: []*library.Log{_service, _step}, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: []*library.Log{_service, _step}, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, _, err := test.database.ListLogsForBuild(_build, 1, 10) + + if test.failure { + if err == nil { + t.Errorf("ListLogsForBuild for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("ListLogsForBuild for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ListLogsForBuild for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/log/list_test.go b/database/log/list_test.go new file mode 100644 index 000000000..0cf420255 --- /dev/null +++ b/database/log/list_test.go @@ -0,0 +1,104 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestLog_Engine_ListLogs(t *testing.T) { + // setup types + _service := testLog() + _service.SetID(1) + _service.SetRepoID(1) + _service.SetBuildID(1) + _service.SetServiceID(1) + _service.SetData([]byte{}) + + _step := testLog() + _step.SetID(2) + _step.SetRepoID(1) + _step.SetBuildID(1) + _step.SetStepID(1) + _step.SetData([]byte{}) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // create expected result in mock + _rows := sqlmock.NewRows([]string{"count"}).AddRow(2) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT count(*) FROM "logs"`).WillReturnRows(_rows) + + // create expected result in mock + _rows = sqlmock.NewRows( + []string{"id", "build_id", "repo_id", "service_id", "step_id", "data"}). + AddRow(1, 1, 1, 1, 0, []byte{}).AddRow(2, 1, 1, 0, 1, []byte{}) + + // ensure the mock expects the query + _mock.ExpectQuery(`SELECT * FROM "logs"`).WillReturnRows(_rows) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateLog(_service) + if err != nil { + t.Errorf("unable to create test service log for sqlite: %v", err) + } + + err = _sqlite.CreateLog(_step) + if err != nil { + t.Errorf("unable to create test step log for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + want []*library.Log + }{ + { + failure: false, + name: "postgres", + database: _postgres, + want: []*library.Log{_service, _step}, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + want: []*library.Log{_service, _step}, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := test.database.ListLogs() + + if test.failure { + if err == nil { + t.Errorf("ListLogs for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("ListLogs for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ListLogs for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} diff --git a/database/log/log.go b/database/log/log.go new file mode 100644 index 000000000..22604e411 --- /dev/null +++ b/database/log/log.go @@ -0,0 +1,82 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "fmt" + + "github.com/go-vela/types/constants" + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +type ( + // config represents the settings required to create the engine that implements the LogService interface. + config struct { + // specifies the level of compression to use for the Log engine + CompressionLevel int + // specifies to skip creating tables and indexes for the Log engine + SkipCreation bool + } + + // engine represents the log functionality that implements the LogService interface. + engine struct { + // engine configuration settings used in log functions + config *config + + // gorm.io/gorm database client used in log functions + // + // https://pkg.go.dev/gorm.io/gorm#DB + client *gorm.DB + + // sirupsen/logrus logger used in log functions + // + // https://pkg.go.dev/github.com/sirupsen/logrus#Entry + logger *logrus.Entry + } +) + +// New creates and returns a Vela service for integrating with logs in the database. +// +//nolint:revive // ignore returning unexported engine +func New(opts ...EngineOpt) (*engine, error) { + // create new Log engine + e := new(engine) + + // create new fields + e.client = new(gorm.DB) + e.config = new(config) + e.logger = new(logrus.Entry) + + // apply all provided configuration options + for _, opt := range opts { + err := opt(e) + if err != nil { + return nil, err + } + } + + // check if we should skip creating log database objects + if e.config.SkipCreation { + e.logger.Warning("skipping creation of logs table and indexes in the database") + + return e, nil + } + + // create the logs table + err := e.CreateLogTable(e.client.Config.Dialector.Name()) + if err != nil { + return nil, fmt.Errorf("unable to create %s table: %w", constants.TableLog, err) + } + + // create the indexes for the logs table + err = e.CreateLogIndexes() + if err != nil { + return nil, fmt.Errorf("unable to create indexes for %s table: %w", constants.TableLog, err) + } + + return e, nil +} diff --git a/database/log/log_test.go b/database/log/log_test.go new file mode 100644 index 000000000..69e3835d5 --- /dev/null +++ b/database/log/log_test.go @@ -0,0 +1,276 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "database/sql/driver" + "reflect" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" + "github.com/sirupsen/logrus" + + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +func TestLog_New(t *testing.T) { + // setup types + logger := logrus.NewEntry(logrus.StandardLogger()) + + _sql, _mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + defer _sql.Close() + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(CreateBuildIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + + _config := &gorm.Config{SkipDefaultTransaction: true} + + _postgres, err := gorm.Open(postgres.New(postgres.Config{Conn: _sql}), _config) + if err != nil { + t.Errorf("unable to create new postgres database: %v", err) + } + + _sqlite, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), _config) + if err != nil { + t.Errorf("unable to create new sqlite database: %v", err) + } + + defer func() { _sql, _ := _sqlite.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + client *gorm.DB + key string + logger *logrus.Entry + skipCreation bool + want *engine + }{ + { + failure: false, + name: "postgres", + client: _postgres, + logger: logger, + skipCreation: false, + want: &engine{ + client: _postgres, + config: &config{SkipCreation: false}, + logger: logger, + }, + }, + { + failure: false, + name: "sqlite3", + client: _sqlite, + logger: logger, + skipCreation: false, + want: &engine{ + client: _sqlite, + config: &config{SkipCreation: false}, + logger: logger, + }, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := New( + WithClient(test.client), + WithLogger(test.logger), + WithSkipCreation(test.skipCreation), + ) + + if test.failure { + if err == nil { + t.Errorf("New for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("New for %s returned err: %v", test.name, err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("New for %s is %v, want %v", test.name, got, test.want) + } + }) + } +} + +// testPostgres is a helper function to create a Postgres engine for testing. +func testPostgres(t *testing.T) (*engine, sqlmock.Sqlmock) { + // create the new mock sql database + // + // https://pkg.go.dev/github.com/DATA-DOG/go-sqlmock#New + _sql, _mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Errorf("unable to create new SQL mock: %v", err) + } + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(CreateBuildIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + + // create the new mock Postgres database client + // + // https://pkg.go.dev/gorm.io/gorm#Open + _postgres, err := gorm.Open( + postgres.New(postgres.Config{Conn: _sql}), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new postgres database: %v", err) + } + + _engine, err := New( + WithClient(_postgres), + WithLogger(logrus.NewEntry(logrus.StandardLogger())), + WithSkipCreation(false), + ) + if err != nil { + t.Errorf("unable to create new postgres log engine: %v", err) + } + + return _engine, _mock +} + +// testSqlite is a helper function to create a Sqlite engine for testing. +func testSqlite(t *testing.T) *engine { + _sqlite, err := gorm.Open( + sqlite.Open("file::memory:?cache=shared"), + &gorm.Config{SkipDefaultTransaction: true}, + ) + if err != nil { + t.Errorf("unable to create new sqlite database: %v", err) + } + + _engine, err := New( + WithClient(_sqlite), + WithLogger(logrus.NewEntry(logrus.StandardLogger())), + WithSkipCreation(false), + ) + if err != nil { + t.Errorf("unable to create new sqlite log engine: %v", err) + } + + return _engine +} + +// This will be used with the github.com/DATA-DOG/go-sqlmock +// library to compare values that are otherwise not easily +// compared. These typically would be values generated before +// adding or updating them in the database. +// +// https://github.com/DATA-DOG/go-sqlmock#matching-arguments-like-timetime +type AnyArgument struct{} + +// Match satisfies sqlmock.Argument interface. +func (a AnyArgument) Match(v driver.Value) bool { + return true +} + +// testBuild is a test helper function to create a library +// Build type with all fields set to their zero values. +func testBuild() *library.Build { + return &library.Build{ + ID: new(int64), + RepoID: new(int64), + PipelineID: new(int64), + Number: new(int), + Parent: new(int), + Event: new(string), + EventAction: new(string), + Status: new(string), + Error: new(string), + Enqueued: new(int64), + Created: new(int64), + Started: new(int64), + Finished: new(int64), + Deploy: new(string), + Clone: new(string), + Source: new(string), + Title: new(string), + Message: new(string), + Commit: new(string), + Sender: new(string), + Author: new(string), + Email: new(string), + Link: new(string), + Branch: new(string), + Ref: new(string), + BaseRef: new(string), + HeadRef: new(string), + Host: new(string), + Runtime: new(string), + Distribution: new(string), + } +} + +// testLog is a test helper function to create a library +// Log type with all fields set to their zero values. +func testLog() *library.Log { + return &library.Log{ + ID: new(int64), + RepoID: new(int64), + BuildID: new(int64), + ServiceID: new(int64), + StepID: new(int64), + Data: new([]byte), + } +} + +// testService is a test helper function to create a library +// Service type with all fields set to their zero values. +func testService() *library.Service { + return &library.Service{ + ID: new(int64), + BuildID: new(int64), + RepoID: new(int64), + Number: new(int), + Name: new(string), + Image: new(string), + Status: new(string), + Error: new(string), + ExitCode: new(int), + Created: new(int64), + Started: new(int64), + Finished: new(int64), + Host: new(string), + Runtime: new(string), + Distribution: new(string), + } +} + +// testStep is a test helper function to create a library +// Step type with all fields set to their zero values. +func testStep() *library.Step { + return &library.Step{ + ID: new(int64), + BuildID: new(int64), + RepoID: new(int64), + Number: new(int), + Name: new(string), + Image: new(string), + Stage: new(string), + Status: new(string), + Error: new(string), + ExitCode: new(int), + Created: new(int64), + Started: new(int64), + Finished: new(int64), + Host: new(string), + Runtime: new(string), + Distribution: new(string), + } +} diff --git a/database/log/opts.go b/database/log/opts.go new file mode 100644 index 000000000..ea1897ba9 --- /dev/null +++ b/database/log/opts.go @@ -0,0 +1,54 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +// EngineOpt represents a configuration option to initialize the database engine for Logs. +type EngineOpt func(*engine) error + +// WithClient sets the gorm.io/gorm client in the database engine for Logs. +func WithClient(client *gorm.DB) EngineOpt { + return func(e *engine) error { + // set the gorm.io/gorm client in the log engine + e.client = client + + return nil + } +} + +// WithCompressionLevel sets the compression level in the database engine for Logs. +func WithCompressionLevel(level int) EngineOpt { + return func(e *engine) error { + // set the compression level in the log engine + e.config.CompressionLevel = level + + return nil + } +} + +// WithLogger sets the github.com/sirupsen/logrus logger in the database engine for Logs. +func WithLogger(logger *logrus.Entry) EngineOpt { + return func(e *engine) error { + // set the github.com/sirupsen/logrus logger in the log engine + e.logger = logger + + return nil + } +} + +// WithSkipCreation sets the skip creation logic in the database engine for Logs. +func WithSkipCreation(skipCreation bool) EngineOpt { + return func(e *engine) error { + // set to skip creating tables and indexes in the log engine + e.config.SkipCreation = skipCreation + + return nil + } +} diff --git a/database/log/opts_test.go b/database/log/opts_test.go new file mode 100644 index 000000000..c35dbaa48 --- /dev/null +++ b/database/log/opts_test.go @@ -0,0 +1,216 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "reflect" + "testing" + + "github.com/sirupsen/logrus" + + "gorm.io/gorm" +) + +func TestLog_EngineOpt_WithClient(t *testing.T) { + // setup types + e := &engine{client: new(gorm.DB)} + + // setup tests + tests := []struct { + failure bool + name string + client *gorm.DB + want *gorm.DB + }{ + { + failure: false, + name: "client set to new database", + client: new(gorm.DB), + want: new(gorm.DB), + }, + { + failure: false, + name: "client set to nil", + client: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithClient(test.client)(e) + + if test.failure { + if err == nil { + t.Errorf("WithClient for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithClient returned err: %v", err) + } + + if !reflect.DeepEqual(e.client, test.want) { + t.Errorf("WithClient is %v, want %v", e.client, test.want) + } + }) + } +} + +func TestLog_EngineOpt_WithCompressionLevel(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + level int + want int + }{ + { + failure: false, + name: "compression level set to -1", + level: -1, + want: -1, + }, + { + failure: false, + name: "compression level set to 0", + level: 0, + want: 0, + }, + { + failure: false, + name: "compression level set to 1", + level: 1, + want: 1, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithCompressionLevel(test.level)(e) + + if test.failure { + if err == nil { + t.Errorf("WithCompressionLevel for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithCompressionLevel returned err: %v", err) + } + + if !reflect.DeepEqual(e.config.CompressionLevel, test.want) { + t.Errorf("WithCompressionLevel is %v, want %v", e.config.CompressionLevel, test.want) + } + }) + } +} + +func TestLog_EngineOpt_WithLogger(t *testing.T) { + // setup types + e := &engine{logger: new(logrus.Entry)} + + // setup tests + tests := []struct { + failure bool + name string + logger *logrus.Entry + want *logrus.Entry + }{ + { + failure: false, + name: "logger set to new entry", + logger: new(logrus.Entry), + want: new(logrus.Entry), + }, + { + failure: false, + name: "logger set to nil", + logger: nil, + want: nil, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithLogger(test.logger)(e) + + if test.failure { + if err == nil { + t.Errorf("WithLogger for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithLogger returned err: %v", err) + } + + if !reflect.DeepEqual(e.logger, test.want) { + t.Errorf("WithLogger is %v, want %v", e.logger, test.want) + } + }) + } +} + +func TestLog_EngineOpt_WithSkipCreation(t *testing.T) { + // setup types + e := &engine{config: new(config)} + + // setup tests + tests := []struct { + failure bool + name string + skipCreation bool + want bool + }{ + { + failure: false, + name: "skip creation set to true", + skipCreation: true, + want: true, + }, + { + failure: false, + name: "skip creation set to false", + skipCreation: false, + want: false, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := WithSkipCreation(test.skipCreation)(e) + + if test.failure { + if err == nil { + t.Errorf("WithSkipCreation for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("WithSkipCreation returned err: %v", err) + } + + if !reflect.DeepEqual(e.config.SkipCreation, test.want) { + t.Errorf("WithSkipCreation is %v, want %v", e.config.SkipCreation, test.want) + } + }) + } +} diff --git a/database/log/service.go b/database/log/service.go new file mode 100644 index 000000000..00e686ee9 --- /dev/null +++ b/database/log/service.go @@ -0,0 +1,49 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "github.com/go-vela/types/library" +) + +// LogService represents the Vela interface for log +// functions with the supported Database backends. +// +//nolint:revive // ignore name stutter +type LogService interface { + // Log Data Definition Language Functions + // + // https://en.wikipedia.org/wiki/Data_definition_language + + // CreateLogIndexes defines a function that creates the indexes for the logs table. + CreateLogIndexes() error + // CreateLogTable defines a function that creates the logs table. + CreateLogTable(string) error + + // Log Data Manipulation Language Functions + // + // https://en.wikipedia.org/wiki/Data_manipulation_language + + // CountLogs defines a function that gets the count of all logs. + CountLogs() (int64, error) + // CountLogsForBuild defines a function that gets the count of logs by build ID. + CountLogsForBuild(*library.Build) (int64, error) + // CreateLog defines a function that creates a new log. + CreateLog(*library.Log) error + // DeleteLog defines a function that deletes an existing log. + DeleteLog(*library.Log) error + // GetLog defines a function that gets a log by ID. + GetLog(int64) (*library.Log, error) + // GetLogForService defines a function that gets a log by service ID. + GetLogForService(*library.Service) (*library.Log, error) + // GetLogForStep defines a function that gets a log by step ID. + GetLogForStep(*library.Step) (*library.Log, error) + // ListLogs defines a function that gets a list of all logs. + ListLogs() ([]*library.Log, error) + // ListLogsForBuild defines a function that gets a list of logs by build ID. + ListLogsForBuild(*library.Build, int, int) ([]*library.Log, int64, error) + // UpdateLog defines a function that updates an existing log. + UpdateLog(*library.Log) error +} diff --git a/database/log/table.go b/database/log/table.go new file mode 100644 index 000000000..2d5e5e0c5 --- /dev/null +++ b/database/log/table.go @@ -0,0 +1,60 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "github.com/go-vela/types/constants" +) + +const ( + // CreatePostgresTable represents a query to create the Postgres logs table. + CreatePostgresTable = ` +CREATE TABLE +IF NOT EXISTS +logs ( + id SERIAL PRIMARY KEY, + build_id INTEGER, + repo_id INTEGER, + service_id INTEGER, + step_id INTEGER, + data BYTEA, + UNIQUE(step_id), + UNIQUE(service_id) +); +` + + // CreateSqliteTable represents a query to create the Sqlite logs table. + CreateSqliteTable = ` +CREATE TABLE +IF NOT EXISTS +logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + build_id INTEGER, + repo_id INTEGER, + service_id INTEGER, + step_id INTEGER, + data BLOB, + UNIQUE(step_id), + UNIQUE(service_id) +); +` +) + +// CreateLogTable creates the logs table in the database. +func (e *engine) CreateLogTable(driver string) error { + e.logger.Tracef("creating logs table in the database") + + // handle the driver provided to create the table + switch driver { + case constants.DriverPostgres: + // create the logs table for Postgres + return e.client.Exec(CreatePostgresTable).Error + case constants.DriverSqlite: + fallthrough + default: + // create the logs table for Sqlite + return e.client.Exec(CreateSqliteTable).Error + } +} diff --git a/database/log/table_test.go b/database/log/table_test.go new file mode 100644 index 000000000..066d9f38a --- /dev/null +++ b/database/log/table_test.go @@ -0,0 +1,59 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func TestLog_Engine_CreateLogTable(t *testing.T) { + // setup types + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + _mock.ExpectExec(CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + // setup tests + tests := []struct { + failure bool + name string + database *engine + }{ + { + failure: false, + name: "postgres", + database: _postgres, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := test.database.CreateLogTable(test.name) + + if test.failure { + if err == nil { + t.Errorf("CreateLogTable for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("CreateLogTable for %s returned err: %v", test.name, err) + } + }) + } +} diff --git a/database/log/update.go b/database/log/update.go new file mode 100644 index 000000000..fb7165004 --- /dev/null +++ b/database/log/update.go @@ -0,0 +1,57 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +//nolint:dupl // ignore similar code with update.go +package log + +import ( + "fmt" + + "github.com/go-vela/types/constants" + "github.com/go-vela/types/database" + "github.com/go-vela/types/library" +) + +// UpdateLog updates an existing log in the database. +func (e *engine) UpdateLog(l *library.Log) error { + // check what the log entry is for + switch { + case l.GetServiceID() > 0: + e.logger.Tracef("updating log for service %d for build %d in the database", l.GetServiceID(), l.GetBuildID()) + case l.GetStepID() > 0: + e.logger.Tracef("updating log for step %d for build %d in the database", l.GetStepID(), l.GetBuildID()) + } + + // cast the library type to database type + // + // https://pkg.go.dev/github.com/go-vela/types/database#LogFromLibrary + log := database.LogFromLibrary(l) + + // validate the necessary fields are populated + // + // https://pkg.go.dev/github.com/go-vela/types/database#Log.Validate + err := log.Validate() + if err != nil { + return err + } + + // compress log data for the resource + // + // https://pkg.go.dev/github.com/go-vela/types/database#Log.Compress + err = log.Compress(e.config.CompressionLevel) + if err != nil { + switch { + case l.GetServiceID() > 0: + return fmt.Errorf("unable to compress log for service %d for build %d: %w", l.GetServiceID(), l.GetBuildID(), err) + case l.GetStepID() > 0: + return fmt.Errorf("unable to compress log for step %d for build %d: %w", l.GetStepID(), l.GetBuildID(), err) + } + } + + // send query to the database + return e.client. + Table(constants.TableLog). + Save(log). + Error +} diff --git a/database/log/update_test.go b/database/log/update_test.go new file mode 100644 index 000000000..0b4e8e127 --- /dev/null +++ b/database/log/update_test.go @@ -0,0 +1,101 @@ +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. +// +// Use of this source code is governed by the LICENSE file in this repository. + +package log + +import ( + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/go-vela/types/library" +) + +func TestLog_Engine_UpdateLog(t *testing.T) { + // setup types + _service := testLog() + _service.SetID(1) + _service.SetRepoID(1) + _service.SetBuildID(1) + _service.SetServiceID(1) + _service.SetData([]byte{}) + + _step := testLog() + _step.SetID(2) + _step.SetRepoID(1) + _step.SetBuildID(1) + _step.SetStepID(1) + _step.SetData([]byte{}) + + _postgres, _mock := testPostgres(t) + defer func() { _sql, _ := _postgres.client.DB(); _sql.Close() }() + + // ensure the mock expects the service query + _mock.ExpectExec(`UPDATE "logs" +SET "build_id"=$1,"repo_id"=$2,"service_id"=$3,"step_id"=$4,"data"=$5 +WHERE "id" = $6`). + WithArgs(1, 1, 1, nil, AnyArgument{}, 1). + WillReturnResult(sqlmock.NewResult(1, 1)) + + // ensure the mock expects the step query + _mock.ExpectExec(`UPDATE "logs" +SET "build_id"=$1,"repo_id"=$2,"service_id"=$3,"step_id"=$4,"data"=$5 +WHERE "id" = $6`). + WithArgs(1, 1, nil, 1, AnyArgument{}, 2). + WillReturnResult(sqlmock.NewResult(1, 1)) + + _sqlite := testSqlite(t) + defer func() { _sql, _ := _sqlite.client.DB(); _sql.Close() }() + + err := _sqlite.CreateLog(_service) + if err != nil { + t.Errorf("unable to create test service log for sqlite: %v", err) + } + + err = _sqlite.CreateLog(_step) + if err != nil { + t.Errorf("unable to create test step log for sqlite: %v", err) + } + + // setup tests + tests := []struct { + failure bool + name string + database *engine + logs []*library.Log + }{ + { + failure: false, + name: "postgres", + database: _postgres, + logs: []*library.Log{_service, _step}, + }, + { + failure: false, + name: "sqlite3", + database: _sqlite, + logs: []*library.Log{_service, _step}, + }, + } + + // run tests + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + for _, log := range test.logs { + err = test.database.UpdateLog(log) + + if test.failure { + if err == nil { + t.Errorf("UpdateLog for %s should have returned err", test.name) + } + + return + } + + if err != nil { + t.Errorf("UpdateLog for %s returned err: %v", test.name, err) + } + } + }) + } +} diff --git a/database/postgres/ddl/log.go b/database/postgres/ddl/log.go deleted file mode 100644 index f9c9d7bd9..000000000 --- a/database/postgres/ddl/log.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package ddl - -const ( - // CreateLogTable represents a query to - // create the logs table for Vela. - CreateLogTable = ` -CREATE TABLE -IF NOT EXISTS -logs ( - id SERIAL PRIMARY KEY, - build_id INTEGER, - repo_id INTEGER, - service_id INTEGER, - step_id INTEGER, - data BYTEA, - UNIQUE(step_id), - UNIQUE(service_id) -); -` - - // CreateLogBuildIDIndex represents a query to create an - // index on the logs table for the build_id column. - CreateLogBuildIDIndex = ` -CREATE INDEX -IF NOT EXISTS -logs_build_id -ON logs (build_id); -` -) diff --git a/database/postgres/dml/log.go b/database/postgres/dml/log.go deleted file mode 100644 index e3e904cc4..000000000 --- a/database/postgres/dml/log.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package dml - -const ( - // ListLogs represents a query to - // list all logs in the database. - ListLogs = ` -SELECT * -FROM logs; -` - - // ListBuildLogs represents a query to list - // all logs for a build_id in the database. - ListBuildLogs = ` -SELECT * -FROM logs -WHERE build_id = ? -ORDER BY step_id ASC; -` - - // SelectStepLog represents a query to select - // a log for a step_id in the database. - SelectStepLog = ` -SELECT * -FROM logs -WHERE step_id = ? -LIMIT 1; -` - - // SelectServiceLog represents a query to select - // a log for a service_id in the database. - SelectServiceLog = ` -SELECT * -FROM logs -WHERE service_id = ? -LIMIT 1; -` - - // DeleteLog represents a query to - // remove a log from the database. - DeleteLog = ` -DELETE -FROM logs -WHERE id = ?; -` -) diff --git a/database/postgres/log.go b/database/postgres/log.go deleted file mode 100644 index fe23549eb..000000000 --- a/database/postgres/log.go +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "errors" - "fmt" - - "github.com/go-vela/server/database/postgres/dml" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" - - "gorm.io/gorm" -) - -// GetBuildLogs gets a collection of logs for a build by unique ID from the database. -func (c *client) GetBuildLogs(id int64) ([]*library.Log, error) { - c.Logger.Tracef("listing logs for build %d from the database", id) - - // variable to store query results - l := new([]database.Log) - - // send query to the database and store result in variable - err := c.Postgres. - Table(constants.TableLog). - Raw(dml.ListBuildLogs, id). - Scan(l).Error - if err != nil { - return nil, err - } - - // variable we want to return - logs := []*library.Log{} - // iterate through all query results - for _, log := range *l { - // https://golang.org/doc/faq#closures_and_goroutines - tmp := log - - // decompress log data for the step - // - // https://pkg.go.dev/github.com/go-vela/types/database#Log.Decompress - err = tmp.Decompress() - if err != nil { - // ensures that the change is backwards compatible - // by logging the error instead of returning it - // which allows us to fetch uncompressed logs - c.Logger.Errorf("unable to decompress logs for build %d: %v", id, err) - } - - // convert query result to library type - logs = append(logs, tmp.ToLibrary()) - } - - return logs, nil -} - -// GetStepLog gets a log by unique ID from the database. -// -//nolint:dupl // ignore similar code with service -func (c *client) GetStepLog(id int64) (*library.Log, error) { - c.Logger.Tracef("getting log for step %d from the database", id) - - // variable to store query results - l := new(database.Log) - - // send query to the database and store result in variable - result := c.Postgres. - Table(constants.TableLog). - Raw(dml.SelectStepLog, id). - Scan(l) - - // check if the query returned a record not found error or no rows were returned - if errors.Is(result.Error, gorm.ErrRecordNotFound) || result.RowsAffected == 0 { - return nil, gorm.ErrRecordNotFound - } - - // decompress log data for the step - // - // https://pkg.go.dev/github.com/go-vela/types/database#Log.Decompress - err := l.Decompress() - if err != nil { - // ensures that the change is backwards compatible - // by logging the error instead of returning it - // which allows us to fetch uncompressed logs - c.Logger.Errorf("unable to decompress logs for step %d: %v", id, err) - - // return the uncompressed log - return l.ToLibrary(), result.Error - } - - // return the decompressed log - return l.ToLibrary(), result.Error -} - -// GetServiceLog gets a log by unique ID from the database. -// -//nolint:dupl // ignore similar code with step -func (c *client) GetServiceLog(id int64) (*library.Log, error) { - c.Logger.Tracef("getting log for service %d from the database", id) - - // variable to store query results - l := new(database.Log) - - // send query to the database and store result in variable - result := c.Postgres. - Table(constants.TableLog). - Raw(dml.SelectServiceLog, id). - Scan(l) - - // check if the query returned a record not found error or no rows were returned - if errors.Is(result.Error, gorm.ErrRecordNotFound) || result.RowsAffected == 0 { - return nil, gorm.ErrRecordNotFound - } - - // decompress log data for the service - // - // https://pkg.go.dev/github.com/go-vela/types/database#Log.Decompress - err := l.Decompress() - if err != nil { - // ensures that the change is backwards compatible - // by logging the error instead of returning it - // which allowing us to fetch uncompressed logs - c.Logger.Errorf("unable to decompress logs for service %d: %v", id, err) - - // return the uncompressed log - return l.ToLibrary(), result.Error - } - - // return the decompressed log - return l.ToLibrary(), result.Error -} - -// CreateLog creates a new log in the database. -// -//nolint:dupl // ignore false positive of duplicate code -func (c *client) CreateLog(l *library.Log) error { - // check if the log entry is for a step - if l.GetStepID() > 0 { - c.Logger.Tracef("creating log for step %d in the database", l.GetStepID()) - } else { - c.Logger.Tracef("creating log for service %d in the database", l.GetServiceID()) - } - - // cast to database type - log := database.LogFromLibrary(l) - - // validate the necessary fields are populated - err := log.Validate() - if err != nil { - return err - } - - // compress log data for the resource - // - // https://pkg.go.dev/github.com/go-vela/types/database#Log.Compress - err = log.Compress(c.config.CompressionLevel) - if err != nil { - return fmt.Errorf("unable to compress logs for step %d: %w", l.GetStepID(), err) - } - - // send query to the database - return c.Postgres. - Table(constants.TableLog). - Create(log).Error -} - -// UpdateLog updates a log in the database. -// -//nolint:dupl // ignore false positive of duplicate code -func (c *client) UpdateLog(l *library.Log) error { - // check if the log entry is for a step - if l.GetStepID() > 0 { - c.Logger.Tracef("updating log for step %d in the database", l.GetStepID()) - } else { - c.Logger.Tracef("updating log for service %d in the database", l.GetServiceID()) - } - - // cast to database type - log := database.LogFromLibrary(l) - - // validate the necessary fields are populated - err := log.Validate() - if err != nil { - return err - } - - // compress log data for the resource - // - // https://pkg.go.dev/github.com/go-vela/types/database#Log.Compress - err = log.Compress(c.config.CompressionLevel) - if err != nil { - return fmt.Errorf("unable to compress logs for step %d: %w", l.GetStepID(), err) - } - - // send query to the database - return c.Postgres. - Table(constants.TableLog). - Save(log).Error -} - -// DeleteLog deletes a log by unique ID from the database. -func (c *client) DeleteLog(id int64) error { - c.Logger.Tracef("deleting log %d from the database", id) - - // send query to the database - return c.Postgres. - Table(constants.TableLog). - Exec(dml.DeleteLog, id).Error -} diff --git a/database/postgres/log_test.go b/database/postgres/log_test.go deleted file mode 100644 index 7173cecbf..000000000 --- a/database/postgres/log_test.go +++ /dev/null @@ -1,388 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package postgres - -import ( - "reflect" - "testing" - - sqlmock "github.com/DATA-DOG/go-sqlmock" - - "github.com/go-vela/server/database/postgres/dml" - "github.com/go-vela/types/library" - - "gorm.io/gorm" -) - -func TestPostgres_Client_GetBuildLogs(t *testing.T) { - // setup types - _logOne := testLog() - _logOne.SetID(1) - _logOne.SetStepID(1) - _logOne.SetBuildID(1) - _logOne.SetRepoID(1) - _logOne.SetData([]byte{}) - - _logTwo := testLog() - _logTwo.SetID(2) - _logTwo.SetServiceID(1) - _logTwo.SetBuildID(1) - _logTwo.SetRepoID(1) - _logTwo.SetData([]byte{}) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.ListBuildLogs, 1).Statement - - // create expected return in mock - _rows := sqlmock.NewRows( - []string{"id", "build_id", "repo_id", "service_id", "step_id", "data"}, - ).AddRow(1, 1, 1, 0, 1, []byte{}).AddRow(2, 1, 1, 1, 0, []byte{}) - - // ensure the mock expects the query - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - want []*library.Log - }{ - { - failure: false, - want: []*library.Log{_logOne, _logTwo}, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetBuildLogs(1) - - if test.failure { - if err == nil { - t.Errorf("GetBuildLogs should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetBuildLogs returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetBuildLogs is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_GetStepLog(t *testing.T) { - // setup types - _log := testLog() - _log.SetID(1) - _log.SetStepID(1) - _log.SetBuildID(1) - _log.SetRepoID(1) - _log.SetData([]byte{}) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.SelectStepLog, 1).Statement - - // create expected return in mock - _rows := sqlmock.NewRows( - []string{"id", "build_id", "repo_id", "service_id", "step_id", "data"}, - ).AddRow(1, 1, 1, 0, 1, []byte{}) - - // ensure the mock expects the query for test case 1 - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - // ensure the mock expects the error for test case 2 - _mock.ExpectQuery(_query.SQL.String()).WillReturnError(gorm.ErrRecordNotFound) - - // setup tests - tests := []struct { - failure bool - want *library.Log - }{ - { - failure: false, - want: _log, - }, - { - failure: true, - want: nil, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetStepLog(1) - - if test.failure { - if err == nil { - t.Errorf("GetStepLog should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetStepLog returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetStepLog is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_GetServiceLog(t *testing.T) { - // setup types - _log := testLog() - _log.SetID(1) - _log.SetServiceID(1) - _log.SetBuildID(1) - _log.SetRepoID(1) - _log.SetData([]byte{}) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Raw(dml.SelectServiceLog, 1).Statement - - // create expected return in mock - _rows := sqlmock.NewRows( - []string{"id", "build_id", "repo_id", "service_id", "step_id", "data"}, - ).AddRow(1, 1, 1, 1, 0, []byte{}) - - // ensure the mock expects the query for test case 1 - _mock.ExpectQuery(_query.SQL.String()).WillReturnRows(_rows) - // ensure the mock expects the error for test case 2 - _mock.ExpectQuery(_query.SQL.String()).WillReturnError(gorm.ErrRecordNotFound) - - // setup tests - tests := []struct { - failure bool - want *library.Log - }{ - { - failure: false, - want: _log, - }, - { - failure: true, - want: nil, - }, - } - - // run tests - for _, test := range tests { - got, err := _database.GetServiceLog(1) - - if test.failure { - if err == nil { - t.Errorf("GetServiceLog should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetServiceLog returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetServiceLog is %v, want %v", got, test.want) - } - } -} - -func TestPostgres_Client_CreateLog(t *testing.T) { - // setup types - _log := testLog() - _log.SetID(1) - _log.SetStepID(1) - _log.SetBuildID(1) - _log.SetRepoID(1) - _log.SetData([]byte{}) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // create expected return in mock - _rows := sqlmock.NewRows([]string{"id"}).AddRow(1) - - // ensure the mock expects the query - _mock.ExpectQuery(`INSERT INTO "logs" ("build_id","repo_id","service_id","step_id","data","id") VALUES ($1,$2,$3,$4,$5,$6) RETURNING "id"`). - WithArgs(1, 1, nil, 1, AnyArgument{}, 1). - WillReturnRows(_rows) - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - err := _database.CreateLog(_log) - - if test.failure { - if err == nil { - t.Errorf("CreateLog should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("CreateLog returned err: %v", err) - } - } -} - -func TestPostgres_Client_UpdateLog(t *testing.T) { - // setup types - _log := testLog() - _log.SetID(1) - _log.SetStepID(1) - _log.SetBuildID(1) - _log.SetRepoID(1) - _log.SetData([]byte{}) - - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // ensure the mock expects the query - _mock.ExpectExec(`UPDATE "logs" SET "build_id"=$1,"repo_id"=$2,"service_id"=$3,"step_id"=$4,"data"=$5 WHERE "id" = $6`). - WithArgs(1, 1, nil, 1, AnyArgument{}, 1). - WillReturnResult(sqlmock.NewResult(1, 1)) - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - err := _database.UpdateLog(_log) - - if test.failure { - if err == nil { - t.Errorf("UpdateLog should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("UpdateLog returned err: %v", err) - } - } -} - -func TestPostgres_Client_DeleteLog(t *testing.T) { - // setup types - // setup the test database client - _database, _mock, err := NewTest() - if err != nil { - t.Errorf("unable to create new postgres test database: %v", err) - } - - defer func() { _sql, _ := _database.Postgres.DB(); _sql.Close() }() - - // capture the current expected SQL query - // - // https://gorm.io/docs/sql_builder.html#DryRun-Mode - _query := _database.Postgres.Session(&gorm.Session{DryRun: true}).Exec(dml.DeleteLog, 1).Statement - - // ensure the mock expects the query - _mock.ExpectExec(_query.SQL.String()).WillReturnResult(sqlmock.NewResult(1, 1)) - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - err := _database.DeleteLog(1) - - if test.failure { - if err == nil { - t.Errorf("DeleteLog should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("DeleteLog returned err: %v", err) - } - } -} - -// testLog is a test helper function to create a -// library Log type with all fields set to their -// zero values. -func testLog() *library.Log { - i64 := int64(0) - b := []byte{} - - return &library.Log{ - ID: &i64, - BuildID: &i64, - RepoID: &i64, - ServiceID: &i64, - StepID: &i64, - Data: &b, - } -} diff --git a/database/postgres/postgres.go b/database/postgres/postgres.go index a4793ca56..a4199f316 100644 --- a/database/postgres/postgres.go +++ b/database/postgres/postgres.go @@ -10,6 +10,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" "github.com/go-vela/server/database/hook" + "github.com/go-vela/server/database/log" "github.com/go-vela/server/database/pipeline" "github.com/go-vela/server/database/postgres/ddl" "github.com/go-vela/server/database/repo" @@ -48,6 +49,8 @@ type ( Logger *logrus.Entry // https://pkg.go.dev/github.com/go-vela/server/database/hook#HookService hook.HookService + // https://pkg.go.dev/github.com/go-vela/server/database/log#LogService + log.LogService // https://pkg.go.dev/github.com/go-vela/server/database/pipeline#PipelineService pipeline.PipelineService // https://pkg.go.dev/github.com/go-vela/server/database/repo#RepoService @@ -152,14 +155,22 @@ func NewTest() (*client, sqlmock.Sqlmock, error) { return nil, nil, err } + // ensure the mock expects the hook queries _mock.ExpectExec(hook.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(hook.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the log queries + _mock.ExpectExec(log.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(log.CreateBuildIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the pipeline queries _mock.ExpectExec(pipeline.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(pipeline.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the repo queries _mock.ExpectExec(repo.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(repo.CreateOrgNameIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the user queries _mock.ExpectExec(user.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(user.CreateUserRefreshIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the worker queries _mock.ExpectExec(worker.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(worker.CreateHostnameAddressIndex).WillReturnResult(sqlmock.NewResult(1, 1)) @@ -248,12 +259,6 @@ func createTables(c *client) error { return fmt.Errorf("unable to create %s table: %w", constants.TableBuild, err) } - // create the logs table - err = c.Postgres.Exec(ddl.CreateLogTable).Error - if err != nil { - return fmt.Errorf("unable to create %s table: %w", constants.TableLog, err) - } - // create the secrets table err = c.Postgres.Exec(ddl.CreateSecretTable).Error if err != nil { @@ -304,12 +309,6 @@ func createIndexes(c *client) error { return fmt.Errorf("unable to create builds_source index for the %s table: %w", constants.TableBuild, err) } - // create the logs_build_id index for the logs table - err = c.Postgres.Exec(ddl.CreateLogBuildIDIndex).Error - if err != nil { - return fmt.Errorf("unable to create logs_build_id index for the %s table: %w", constants.TableLog, err) - } - // create the secrets_type_org_repo index for the secrets table err = c.Postgres.Exec(ddl.CreateSecretTypeOrgRepo).Error if err != nil { @@ -347,6 +346,19 @@ func createServices(c *client) error { return err } + // create the database agnostic log service + // + // https://pkg.go.dev/github.com/go-vela/server/database/log#New + c.LogService, err = log.New( + log.WithClient(c.Postgres), + log.WithCompressionLevel(c.config.CompressionLevel), + log.WithLogger(c.Logger), + log.WithSkipCreation(c.config.SkipCreation), + ) + if err != nil { + return err + } + // create the database agnostic pipeline service // // https://pkg.go.dev/github.com/go-vela/server/database/pipeline#New diff --git a/database/postgres/postgres_test.go b/database/postgres/postgres_test.go index d00c540f3..97bb1c46c 100644 --- a/database/postgres/postgres_test.go +++ b/database/postgres/postgres_test.go @@ -11,6 +11,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" "github.com/go-vela/server/database/hook" + "github.com/go-vela/server/database/log" "github.com/go-vela/server/database/pipeline" "github.com/go-vela/server/database/postgres/ddl" "github.com/go-vela/server/database/repo" @@ -79,7 +80,6 @@ func TestPostgres_setupDatabase(t *testing.T) { // ensure the mock expects the table queries _mock.ExpectExec(ddl.CreateBuildTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(ddl.CreateLogTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateSecretTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateServiceTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateStepTable).WillReturnResult(sqlmock.NewResult(1, 1)) @@ -89,7 +89,6 @@ func TestPostgres_setupDatabase(t *testing.T) { _mock.ExpectExec(ddl.CreateBuildStatusIndex).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateBuildCreatedIndex).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateBuildSourceIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(ddl.CreateLogBuildIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateSecretTypeOrgRepo).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateSecretTypeOrgTeam).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateSecretTypeOrg).WillReturnResult(sqlmock.NewResult(1, 1)) @@ -97,6 +96,9 @@ func TestPostgres_setupDatabase(t *testing.T) { // ensure the mock expects the hook queries _mock.ExpectExec(hook.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(hook.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the log queries + _mock.ExpectExec(log.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(log.CreateBuildIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the pipeline queries _mock.ExpectExec(pipeline.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(pipeline.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) @@ -170,7 +172,6 @@ func TestPostgres_createTables(t *testing.T) { // ensure the mock expects the table queries _mock.ExpectExec(ddl.CreateBuildTable).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(ddl.CreateLogTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateSecretTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateServiceTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateStepTable).WillReturnResult(sqlmock.NewResult(1, 1)) @@ -216,7 +217,6 @@ func TestPostgres_createIndexes(t *testing.T) { _mock.ExpectExec(ddl.CreateBuildStatusIndex).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateBuildCreatedIndex).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateBuildSourceIndex).WillReturnResult(sqlmock.NewResult(1, 1)) - _mock.ExpectExec(ddl.CreateLogBuildIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateSecretTypeOrgRepo).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateSecretTypeOrgTeam).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(ddl.CreateSecretTypeOrg).WillReturnResult(sqlmock.NewResult(1, 1)) @@ -260,6 +260,9 @@ func TestPostgres_createServices(t *testing.T) { // ensure the mock expects the hook queries _mock.ExpectExec(hook.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(hook.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) + // ensure the mock expects the log queries + _mock.ExpectExec(log.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) + _mock.ExpectExec(log.CreateBuildIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) // ensure the mock expects the pipeline queries _mock.ExpectExec(pipeline.CreatePostgresTable).WillReturnResult(sqlmock.NewResult(1, 1)) _mock.ExpectExec(pipeline.CreateRepoIDIndex).WillReturnResult(sqlmock.NewResult(1, 1)) diff --git a/database/service.go b/database/service.go index 01ab1cf58..33517b484 100644 --- a/database/service.go +++ b/database/service.go @@ -1,4 +1,4 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. +// Copyright (c) 2023 Target Brands, Inc. All rights reserved. // // Use of this source code is governed by the LICENSE file in this repository. @@ -6,6 +6,7 @@ package database import ( "github.com/go-vela/server/database/hook" + "github.com/go-vela/server/database/log" "github.com/go-vela/server/database/pipeline" "github.com/go-vela/server/database/repo" "github.com/go-vela/server/database/user" @@ -77,26 +78,9 @@ type Service interface { // related to hooks stored in the database. hook.HookService - // Log Database Interface Functions - - // GetStepLog defines a function that - // gets a step log by unique ID. - GetStepLog(int64) (*library.Log, error) - // GetServiceLog defines a function that - // gets a service log by unique ID. - GetServiceLog(int64) (*library.Log, error) - // GetBuildLogs defines a function that - // gets a list of logs by build ID. - GetBuildLogs(int64) ([]*library.Log, error) - // CreateLog defines a function that - // creates a new log. - CreateLog(*library.Log) error - // UpdateLog defines a function that - // updates a log. - UpdateLog(*library.Log) error - // DeleteLog defines a function that - // deletes a log by unique ID. - DeleteLog(int64) error + // LogService provides the interface for functionality + // related to logs stored in the database. + log.LogService // PipelineService provides the interface for functionality // related to pipelines stored in the database. diff --git a/database/sqlite/ddl/log.go b/database/sqlite/ddl/log.go deleted file mode 100644 index 225155bfb..000000000 --- a/database/sqlite/ddl/log.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package ddl - -const ( - // CreateLogTable represents a query to - // create the logs table for Vela. - CreateLogTable = ` -CREATE TABLE -IF NOT EXISTS -logs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - build_id INTEGER, - repo_id INTEGER, - service_id INTEGER, - step_id INTEGER, - data BLOB, - UNIQUE(step_id), - UNIQUE(service_id) -); -` - - // CreateLogBuildIDIndex represents a query to create an - // index on the logs table for the build_id column. - CreateLogBuildIDIndex = ` -CREATE INDEX -IF NOT EXISTS -logs_build_id -ON logs (build_id); -` -) diff --git a/database/sqlite/dml/log.go b/database/sqlite/dml/log.go deleted file mode 100644 index e3e904cc4..000000000 --- a/database/sqlite/dml/log.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package dml - -const ( - // ListLogs represents a query to - // list all logs in the database. - ListLogs = ` -SELECT * -FROM logs; -` - - // ListBuildLogs represents a query to list - // all logs for a build_id in the database. - ListBuildLogs = ` -SELECT * -FROM logs -WHERE build_id = ? -ORDER BY step_id ASC; -` - - // SelectStepLog represents a query to select - // a log for a step_id in the database. - SelectStepLog = ` -SELECT * -FROM logs -WHERE step_id = ? -LIMIT 1; -` - - // SelectServiceLog represents a query to select - // a log for a service_id in the database. - SelectServiceLog = ` -SELECT * -FROM logs -WHERE service_id = ? -LIMIT 1; -` - - // DeleteLog represents a query to - // remove a log from the database. - DeleteLog = ` -DELETE -FROM logs -WHERE id = ?; -` -) diff --git a/database/sqlite/log.go b/database/sqlite/log.go deleted file mode 100644 index 037bdcc08..000000000 --- a/database/sqlite/log.go +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "errors" - "fmt" - - "github.com/go-vela/server/database/sqlite/dml" - "github.com/go-vela/types/constants" - "github.com/go-vela/types/database" - "github.com/go-vela/types/library" - - "gorm.io/gorm" -) - -// GetBuildLogs gets a collection of logs for a build by unique ID from the database. -func (c *client) GetBuildLogs(id int64) ([]*library.Log, error) { - c.Logger.Tracef("listing logs for build %d from the database", id) - - // variable to store query results - l := new([]database.Log) - - // send query to the database and store result in variable - err := c.Sqlite. - Table(constants.TableLog). - Raw(dml.ListBuildLogs, id). - Scan(l).Error - if err != nil { - return nil, err - } - - // variable we want to return - logs := []*library.Log{} - // iterate through all query results - for _, log := range *l { - // https://golang.org/doc/faq#closures_and_goroutines - tmp := log - - // decompress log data for the step - // - // https://pkg.go.dev/github.com/go-vela/types/database#Log.Decompress - err = tmp.Decompress() - if err != nil { - // ensures that the change is backwards compatible - // by logging the error instead of returning it - // which allows us to fetch uncompressed logs - c.Logger.Errorf("unable to decompress logs for build %d: %v", id, err) - } - - // convert query result to library type - logs = append(logs, tmp.ToLibrary()) - } - - return logs, nil -} - -// GetStepLog gets a log by unique ID from the database. -// -//nolint:dupl // ignore similar code with service -func (c *client) GetStepLog(id int64) (*library.Log, error) { - c.Logger.Tracef("getting log for step %d from the database", id) - - // variable to store query results - l := new(database.Log) - - // send query to the database and store result in variable - result := c.Sqlite. - Table(constants.TableLog). - Raw(dml.SelectStepLog, id). - Scan(l) - - // check if the query returned a record not found error or no rows were returned - if errors.Is(result.Error, gorm.ErrRecordNotFound) || result.RowsAffected == 0 { - return nil, gorm.ErrRecordNotFound - } - - // decompress log data for the step - // - // https://pkg.go.dev/github.com/go-vela/types/database#Log.Decompress - err := l.Decompress() - if err != nil { - // ensures that the change is backwards compatible - // by logging the error instead of returning it - // which allows us to fetch uncompressed logs - c.Logger.Errorf("unable to decompress logs for step %d: %v", id, err) - - // return the uncompressed log - return l.ToLibrary(), result.Error - } - - // return the decompressed log - return l.ToLibrary(), result.Error -} - -// GetServiceLog gets a log by unique ID from the database. -// -//nolint:dupl // ignore similar code with step -func (c *client) GetServiceLog(id int64) (*library.Log, error) { - c.Logger.Tracef("getting log for service %d from the database", id) - - // variable to store query results - l := new(database.Log) - - // send query to the database and store result in variable - result := c.Sqlite. - Table(constants.TableLog). - Raw(dml.SelectServiceLog, id). - Scan(l) - - // check if the query returned a record not found error or no rows were returned - if errors.Is(result.Error, gorm.ErrRecordNotFound) || result.RowsAffected == 0 { - return nil, gorm.ErrRecordNotFound - } - - // decompress log data for the service - // - // https://pkg.go.dev/github.com/go-vela/types/database#Log.Decompress - err := l.Decompress() - if err != nil { - // ensures that the change is backwards compatible - // by logging the error instead of returning it - // which allowing us to fetch uncompressed logs - c.Logger.Errorf("unable to decompress logs for service %d: %v", id, err) - - // return the uncompressed log - return l.ToLibrary(), result.Error - } - - // return the decompressed log - return l.ToLibrary(), result.Error -} - -// CreateLog creates a new log in the database. -// -//nolint:dupl // ignore false positive of duplicate code -func (c *client) CreateLog(l *library.Log) error { - // check if the log entry is for a step - if l.GetStepID() > 0 { - c.Logger.Tracef("creating log for step %d in the database", l.GetStepID()) - } else { - c.Logger.Tracef("creating log for service %d in the database", l.GetServiceID()) - } - - // cast to database type - log := database.LogFromLibrary(l) - - // validate the necessary fields are populated - err := log.Validate() - if err != nil { - return err - } - - // compress log data for the resource - // - // https://pkg.go.dev/github.com/go-vela/types/database#Log.Compress - err = log.Compress(c.config.CompressionLevel) - if err != nil { - return fmt.Errorf("unable to compress logs for step %d: %w", l.GetStepID(), err) - } - - // send query to the database - return c.Sqlite. - Table(constants.TableLog). - Create(log).Error -} - -// UpdateLog updates a log in the database. -// -//nolint:dupl // ignore false positive of duplicate code -func (c *client) UpdateLog(l *library.Log) error { - // check if the log entry is for a step - if l.GetStepID() > 0 { - c.Logger.Tracef("updating log for step %d in the database", l.GetStepID()) - } else { - c.Logger.Tracef("updating log for service %d in the database", l.GetServiceID()) - } - - // cast to database type - log := database.LogFromLibrary(l) - - // validate the necessary fields are populated - err := log.Validate() - if err != nil { - return err - } - - // compress log data for the resource - // - // https://pkg.go.dev/github.com/go-vela/types/database#Log.Compress - err = log.Compress(c.config.CompressionLevel) - if err != nil { - return fmt.Errorf("unable to compress logs for step %d: %w", l.GetStepID(), err) - } - - // send query to the database - return c.Sqlite. - Table(constants.TableLog). - Save(log).Error -} - -// DeleteLog deletes a log by unique ID from the database. -func (c *client) DeleteLog(id int64) error { - c.Logger.Tracef("deleting log %d from the database", id) - - // send query to the database - return c.Sqlite. - Table(constants.TableLog). - Exec(dml.DeleteLog, id).Error -} diff --git a/database/sqlite/log_test.go b/database/sqlite/log_test.go deleted file mode 100644 index a1a52a502..000000000 --- a/database/sqlite/log_test.go +++ /dev/null @@ -1,380 +0,0 @@ -// Copyright (c) 2022 Target Brands, Inc. All rights reserved. -// -// Use of this source code is governed by the LICENSE file in this repository. - -package sqlite - -import ( - "reflect" - "testing" - - "github.com/go-vela/types/library" -) - -func TestSqlite_Client_GetBuildLogs(t *testing.T) { - // setup types - _logOne := testLog() - _logOne.SetID(1) - _logOne.SetStepID(1) - _logOne.SetBuildID(1) - _logOne.SetRepoID(1) - _logOne.SetData([]byte{}) - - _logTwo := testLog() - _logTwo.SetID(2) - _logTwo.SetServiceID(1) - _logTwo.SetBuildID(1) - _logTwo.SetRepoID(1) - _logTwo.SetData([]byte{}) - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want []*library.Log - }{ - { - failure: false, - want: []*library.Log{_logTwo, _logOne}, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the logs table - defer _database.Sqlite.Exec("delete from logs;") - - for _, log := range test.want { - // create the log in the database - err := _database.CreateLog(log) - if err != nil { - t.Errorf("unable to create test log: %v", err) - } - } - - got, err := _database.GetBuildLogs(1) - - if test.failure { - if err == nil { - t.Errorf("GetBuildLogs should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetBuildLogs returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetBuildLogs is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_GetStepLog(t *testing.T) { - // setup types - _log := testLog() - _log.SetID(1) - _log.SetStepID(1) - _log.SetBuildID(1) - _log.SetRepoID(1) - _log.SetData([]byte{}) - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want *library.Log - }{ - { - failure: false, - want: _log, - }, - { - failure: true, - want: nil, - }, - } - - // run tests - for _, test := range tests { - if test.want != nil { - // create the log in the database - err := _database.CreateLog(test.want) - if err != nil { - t.Errorf("unable to create test log: %v", err) - } - } - - got, err := _database.GetStepLog(1) - - // cleanup the logs table - _ = _database.Sqlite.Exec("DELETE FROM logs;") - - if test.failure { - if err == nil { - t.Errorf("GetStepLog should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetStepLog returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetStepLog is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_GetServiceLog(t *testing.T) { - // setup types - _log := testLog() - _log.SetID(1) - _log.SetServiceID(1) - _log.SetBuildID(1) - _log.SetRepoID(1) - _log.SetData([]byte{}) - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - want *library.Log - }{ - { - failure: false, - want: _log, - }, - { - failure: true, - want: nil, - }, - } - - // run tests - for _, test := range tests { - if test.want != nil { - // create the log in the database - err := _database.CreateLog(test.want) - if err != nil { - t.Errorf("unable to create test log: %v", err) - } - } - - got, err := _database.GetServiceLog(1) - - // cleanup the logs table - _ = _database.Sqlite.Exec("DELETE FROM logs;") - - if test.failure { - if err == nil { - t.Errorf("GetServiceLog should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("GetServiceLog returned err: %v", err) - } - - if !reflect.DeepEqual(got, test.want) { - t.Errorf("GetServiceLog is %v, want %v", got, test.want) - } - } -} - -func TestSqlite_Client_CreateLog(t *testing.T) { - // setup types - _log := testLog() - _log.SetID(1) - _log.SetStepID(1) - _log.SetBuildID(1) - _log.SetRepoID(1) - _log.SetData([]byte{}) - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the logs table - defer _database.Sqlite.Exec("delete from logs;") - - err := _database.CreateLog(_log) - - if test.failure { - if err == nil { - t.Errorf("CreateLog should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("CreateLog returned err: %v", err) - } - } -} - -func TestSqlite_Client_UpdateLog(t *testing.T) { - // setup types - _log := testLog() - _log.SetID(1) - _log.SetStepID(1) - _log.SetBuildID(1) - _log.SetRepoID(1) - _log.SetData([]byte{}) - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the logs table - defer _database.Sqlite.Exec("delete from logs;") - - // create the log in the database - err := _database.CreateLog(_log) - if err != nil { - t.Errorf("unable to create test log: %v", err) - } - - err = _database.UpdateLog(_log) - - if test.failure { - if err == nil { - t.Errorf("UpdateLog should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("UpdateLog returned err: %v", err) - } - } -} - -func TestSqlite_Client_DeleteLog(t *testing.T) { - // setup types - _log := testLog() - _log.SetID(1) - _log.SetStepID(1) - _log.SetBuildID(1) - _log.SetRepoID(1) - _log.SetData([]byte{}) - - // setup the test database client - _database, err := NewTest() - if err != nil { - t.Errorf("unable to create new sqlite test database: %v", err) - } - - defer func() { _sql, _ := _database.Sqlite.DB(); _sql.Close() }() - - // setup tests - tests := []struct { - failure bool - }{ - { - failure: false, - }, - } - - // run tests - for _, test := range tests { - // defer cleanup of the logs table - defer _database.Sqlite.Exec("delete from logs;") - - // create the log in the database - err := _database.CreateLog(_log) - if err != nil { - t.Errorf("unable to create test log: %v", err) - } - - err = _database.DeleteLog(1) - - if test.failure { - if err == nil { - t.Errorf("DeleteLog should have returned err") - } - - continue - } - - if err != nil { - t.Errorf("DeleteLog returned err: %v", err) - } - } -} - -// testLog is a test helper function to create a -// library Log type with all fields set to their -// zero values. -func testLog() *library.Log { - i64 := int64(0) - b := []byte{} - - return &library.Log{ - ID: &i64, - BuildID: &i64, - RepoID: &i64, - ServiceID: &i64, - StepID: &i64, - Data: &b, - } -} diff --git a/database/sqlite/sqlite.go b/database/sqlite/sqlite.go index d89a0ee2f..5018e8548 100644 --- a/database/sqlite/sqlite.go +++ b/database/sqlite/sqlite.go @@ -9,6 +9,7 @@ import ( "time" "github.com/go-vela/server/database/hook" + "github.com/go-vela/server/database/log" "github.com/go-vela/server/database/pipeline" "github.com/go-vela/server/database/repo" "github.com/go-vela/server/database/sqlite/ddl" @@ -47,6 +48,8 @@ type ( Logger *logrus.Entry // https://pkg.go.dev/github.com/go-vela/server/database/hook#HookService hook.HookService + // https://pkg.go.dev/github.com/go-vela/server/database/log#LogService + log.LogService // https://pkg.go.dev/github.com/go-vela/server/database/pipeline#PipelineService pipeline.PipelineService // https://pkg.go.dev/github.com/go-vela/server/database/repo#RepoService @@ -236,12 +239,6 @@ func createTables(c *client) error { return fmt.Errorf("unable to create %s table: %w", constants.TableBuild, err) } - // create the logs table - err = c.Sqlite.Exec(ddl.CreateLogTable).Error - if err != nil { - return fmt.Errorf("unable to create %s table: %w", constants.TableLog, err) - } - // create the secrets table err = c.Sqlite.Exec(ddl.CreateSecretTable).Error if err != nil { @@ -292,12 +289,6 @@ func createIndexes(c *client) error { return fmt.Errorf("unable to create builds_source index for the %s table: %w", constants.TableBuild, err) } - // create the logs_build_id index for the logs table - err = c.Sqlite.Exec(ddl.CreateLogBuildIDIndex).Error - if err != nil { - return fmt.Errorf("unable to create logs_build_id index for the %s table: %w", constants.TableLog, err) - } - // create the secrets_type_org_repo index for the secrets table err = c.Sqlite.Exec(ddl.CreateSecretTypeOrgRepo).Error if err != nil { @@ -335,6 +326,19 @@ func createServices(c *client) error { return err } + // create the database agnostic log service + // + // https://pkg.go.dev/github.com/go-vela/server/database/log#New + c.LogService, err = log.New( + log.WithClient(c.Sqlite), + log.WithCompressionLevel(c.config.CompressionLevel), + log.WithLogger(c.Logger), + log.WithSkipCreation(c.config.SkipCreation), + ) + if err != nil { + return err + } + // create the database agnostic pipeline service // // https://pkg.go.dev/github.com/go-vela/server/database/pipeline#New