From 31183cf08b163c474261e1af7acff70a75b0026b Mon Sep 17 00:00:00 2001 From: Don Browne Date: Mon, 17 Jun 2024 09:20:23 +0100 Subject: [PATCH 1/3] Log evaluation history in new tables This adds an interface and implementation for logging rule evaluation statuses. The logic flow is described in the design doc. Note that this does not wire in the logic into the engine yet, nor does it track remediations/alerts. These will be added in future PRs. Relates to: #3556 --- database/mock/store.go | 88 ++++++++++++ database/query/eval_history.sql | 67 +++++++++ database/query/rule_instances.sql | 8 +- internal/db/eval_history.sql.go | 165 ++++++++++++++++++++++ internal/db/querier.go | 19 +++ internal/db/rule_instances.sql.go | 20 +++ internal/engine/executor.go | 8 +- internal/history/service.go | 188 +++++++++++++++++++++++++ internal/history/service_test.go | 224 ++++++++++++++++++++++++++++++ 9 files changed, 782 insertions(+), 5 deletions(-) create mode 100644 database/query/eval_history.sql create mode 100644 internal/db/eval_history.sql.go create mode 100644 internal/history/service.go create mode 100644 internal/history/service_test.go diff --git a/database/mock/store.go b/database/mock/store.go index 2d8dbb37c2..0939a45f09 100644 --- a/database/mock/store.go +++ b/database/mock/store.go @@ -733,6 +733,21 @@ func (mr *MockStoreMockRecorder) GetFeatureInProject(arg0, arg1 any) *gomock.Cal return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFeatureInProject", reflect.TypeOf((*MockStore)(nil).GetFeatureInProject), arg0, arg1) } +// GetIDByProfileEntityName mocks base method. +func (m *MockStore) GetIDByProfileEntityName(arg0 context.Context, arg1 db.GetIDByProfileEntityNameParams) (uuid.UUID, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetIDByProfileEntityName", arg0, arg1) + ret0, _ := ret[0].(uuid.UUID) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetIDByProfileEntityName indicates an expected call of GetIDByProfileEntityName. +func (mr *MockStoreMockRecorder) GetIDByProfileEntityName(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIDByProfileEntityName", reflect.TypeOf((*MockStore)(nil).GetIDByProfileEntityName), arg0, arg1) +} + // GetImmediateChildrenProjects mocks base method. func (m *MockStore) GetImmediateChildrenProjects(arg0 context.Context, arg1 uuid.UUID) ([]db.Project, error) { m.ctrl.T.Helper() @@ -838,6 +853,21 @@ func (mr *MockStoreMockRecorder) GetInvitationsByEmail(arg0, arg1 any) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInvitationsByEmail", reflect.TypeOf((*MockStore)(nil).GetInvitationsByEmail), arg0, arg1) } +// GetLatestEvalStateForRuleEntity mocks base method. +func (m *MockStore) GetLatestEvalStateForRuleEntity(arg0 context.Context, arg1 db.GetLatestEvalStateForRuleEntityParams) (db.EvaluationStatus, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLatestEvalStateForRuleEntity", arg0, arg1) + ret0, _ := ret[0].(db.EvaluationStatus) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetLatestEvalStateForRuleEntity indicates an expected call of GetLatestEvalStateForRuleEntity. +func (mr *MockStoreMockRecorder) GetLatestEvalStateForRuleEntity(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLatestEvalStateForRuleEntity", reflect.TypeOf((*MockStore)(nil).GetLatestEvalStateForRuleEntity), arg0, arg1) +} + // GetParentProjects mocks base method. func (m *MockStore) GetParentProjects(arg0 context.Context, arg1 uuid.UUID) ([]uuid.UUID, error) { m.ctrl.T.Helper() @@ -1362,6 +1392,36 @@ func (mr *MockStoreMockRecorder) GlobalListProvidersByClass(arg0, arg1 any) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GlobalListProvidersByClass", reflect.TypeOf((*MockStore)(nil).GlobalListProvidersByClass), arg0, arg1) } +// InsertEvaluationRuleEntity mocks base method. +func (m *MockStore) InsertEvaluationRuleEntity(arg0 context.Context, arg1 db.InsertEvaluationRuleEntityParams) (uuid.UUID, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InsertEvaluationRuleEntity", arg0, arg1) + ret0, _ := ret[0].(uuid.UUID) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// InsertEvaluationRuleEntity indicates an expected call of InsertEvaluationRuleEntity. +func (mr *MockStoreMockRecorder) InsertEvaluationRuleEntity(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertEvaluationRuleEntity", reflect.TypeOf((*MockStore)(nil).InsertEvaluationRuleEntity), arg0, arg1) +} + +// InsertEvaluationStatus mocks base method. +func (m *MockStore) InsertEvaluationStatus(arg0 context.Context, arg1 db.InsertEvaluationStatusParams) (uuid.UUID, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InsertEvaluationStatus", arg0, arg1) + ret0, _ := ret[0].(uuid.UUID) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// InsertEvaluationStatus indicates an expected call of InsertEvaluationStatus. +func (mr *MockStoreMockRecorder) InsertEvaluationStatus(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertEvaluationStatus", reflect.TypeOf((*MockStore)(nil).InsertEvaluationStatus), arg0, arg1) +} + // ListArtifactsByRepoID mocks base method. func (m *MockStore) ListArtifactsByRepoID(arg0 context.Context, arg1 uuid.NullUUID) ([]db.Artifact, error) { m.ctrl.T.Helper() @@ -1733,6 +1793,20 @@ func (mr *MockStoreMockRecorder) UpdateEncryptedSecret(arg0, arg1 any) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateEncryptedSecret", reflect.TypeOf((*MockStore)(nil).UpdateEncryptedSecret), arg0, arg1) } +// UpdateEvaluationTimes mocks base method. +func (m *MockStore) UpdateEvaluationTimes(arg0 context.Context, arg1 db.UpdateEvaluationTimesParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateEvaluationTimes", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateEvaluationTimes indicates an expected call of UpdateEvaluationTimes. +func (mr *MockStoreMockRecorder) UpdateEvaluationTimes(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateEvaluationTimes", reflect.TypeOf((*MockStore)(nil).UpdateEvaluationTimes), arg0, arg1) +} + // UpdateInvitation mocks base method. func (m *MockStore) UpdateInvitation(arg0 context.Context, arg1 string) (db.UserInvite, error) { m.ctrl.T.Helper() @@ -1894,6 +1968,20 @@ func (mr *MockStoreMockRecorder) UpsertInstallationID(arg0, arg1 any) *gomock.Ca return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertInstallationID", reflect.TypeOf((*MockStore)(nil).UpsertInstallationID), arg0, arg1) } +// UpsertLatestEvaluationStatus mocks base method. +func (m *MockStore) UpsertLatestEvaluationStatus(arg0 context.Context, arg1 db.UpsertLatestEvaluationStatusParams) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpsertLatestEvaluationStatus", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpsertLatestEvaluationStatus indicates an expected call of UpsertLatestEvaluationStatus. +func (mr *MockStoreMockRecorder) UpsertLatestEvaluationStatus(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertLatestEvaluationStatus", reflect.TypeOf((*MockStore)(nil).UpsertLatestEvaluationStatus), arg0, arg1) +} + // UpsertProfileForEntity mocks base method. func (m *MockStore) UpsertProfileForEntity(arg0 context.Context, arg1 db.UpsertProfileForEntityParams) (db.EntityProfile, error) { m.ctrl.T.Helper() diff --git a/database/query/eval_history.sql b/database/query/eval_history.sql new file mode 100644 index 0000000000..44bbf5dda9 --- /dev/null +++ b/database/query/eval_history.sql @@ -0,0 +1,67 @@ +-- Copyright 2024 Stacklok, Inc +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- name: GetLatestEvalStateForRuleEntity :one +SELECT eh.* FROM evaluation_rule_entities AS re +JOIN latest_evaluation_statuses AS les ON les.rule_entity_id = re.id +JOIN evaluation_statuses AS eh ON les.evaluation_history_id = eh.id +WHERE re.rule_id = $1 +AND ( + re.repository_id = $2 + OR re.pull_request_id = $3 + OR re.artifact_id = $4 +) +FOR UPDATE; + +-- name: InsertEvaluationRuleEntity :one +INSERT INTO evaluation_rule_entities( + rule_id, + repository_id, + pull_request_id, + artifact_id +) VALUES ( + $1, + $2, + $3, + $4 +) +RETURNING id; + +-- name: InsertEvaluationStatus :one +INSERT INTO evaluation_statuses( + rule_entity_id, + status, + details +) VALUES ( + $1, + $2, + $3 +) +RETURNING id; + +-- name: UpdateEvaluationTimes :exec +UPDATE evaluation_statuses +SET evaluation_times = $1 +WHERE id = $2; + +-- name: UpsertLatestEvaluationStatus :exec +INSERT INTO latest_evaluation_statuses( + rule_entity_id, + evaluation_history_id +) VALUES ( + $1, + $2 +) +ON CONFLICT (rule_entity_id, evaluation_history_id) DO UPDATE +SET evaluation_history_id = $2; \ No newline at end of file diff --git a/database/query/rule_instances.sql b/database/query/rule_instances.sql index 196245e74b..f27a8557e6 100644 --- a/database/query/rule_instances.sql +++ b/database/query/rule_instances.sql @@ -51,4 +51,10 @@ SELECT * FROM rule_instances WHERE profile_id = $1 AND entity_type = $2; DELETE FROM rule_instances WHERE profile_id = $1 AND entity_type = $2 -AND NOT id = ANY(sqlc.arg(updated_ids)::UUID[]); \ No newline at end of file +AND NOT id = ANY(sqlc.arg(updated_ids)::UUID[]); + +-- name: GetIDByProfileEntityName :one +SELECT id FROM rule_instances +WHERE profile_id = $1 +AND entity_type = $2 +AND name = $3; \ No newline at end of file diff --git a/internal/db/eval_history.sql.go b/internal/db/eval_history.sql.go new file mode 100644 index 0000000000..916e63a8fc --- /dev/null +++ b/internal/db/eval_history.sql.go @@ -0,0 +1,165 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.26.0 +// source: eval_history.sql + +package db + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/lib/pq" +) + +const getLatestEvalStateForRuleEntity = `-- name: GetLatestEvalStateForRuleEntity :one + +SELECT eh.id, eh.rule_entity_id, eh.status, eh.details, eh.evaluation_times, eh.most_recent_evaluation FROM evaluation_rule_entities AS re +JOIN latest_evaluation_statuses AS les ON les.rule_entity_id = re.id +JOIN evaluation_statuses AS eh ON les.evaluation_history_id = eh.id +WHERE re.rule_id = $1 +AND ( + re.repository_id = $2 + OR re.pull_request_id = $3 + OR re.artifact_id = $4 +) +FOR UPDATE +` + +type GetLatestEvalStateForRuleEntityParams struct { + RuleID uuid.UUID `json:"rule_id"` + RepositoryID uuid.NullUUID `json:"repository_id"` + PullRequestID uuid.NullUUID `json:"pull_request_id"` + ArtifactID uuid.NullUUID `json:"artifact_id"` +} + +// Copyright 2024 Stacklok, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +func (q *Queries) GetLatestEvalStateForRuleEntity(ctx context.Context, arg GetLatestEvalStateForRuleEntityParams) (EvaluationStatus, error) { + row := q.db.QueryRowContext(ctx, getLatestEvalStateForRuleEntity, + arg.RuleID, + arg.RepositoryID, + arg.PullRequestID, + arg.ArtifactID, + ) + var i EvaluationStatus + err := row.Scan( + &i.ID, + &i.RuleEntityID, + &i.Status, + &i.Details, + pq.Array(&i.EvaluationTimes), + &i.MostRecentEvaluation, + ) + return i, err +} + +const insertEvaluationRuleEntity = `-- name: InsertEvaluationRuleEntity :one +INSERT INTO evaluation_rule_entities( + rule_id, + repository_id, + pull_request_id, + artifact_id +) VALUES ( + $1, + $2, + $3, + $4 +) +RETURNING id +` + +type InsertEvaluationRuleEntityParams struct { + RuleID uuid.UUID `json:"rule_id"` + RepositoryID uuid.NullUUID `json:"repository_id"` + PullRequestID uuid.NullUUID `json:"pull_request_id"` + ArtifactID uuid.NullUUID `json:"artifact_id"` +} + +func (q *Queries) InsertEvaluationRuleEntity(ctx context.Context, arg InsertEvaluationRuleEntityParams) (uuid.UUID, error) { + row := q.db.QueryRowContext(ctx, insertEvaluationRuleEntity, + arg.RuleID, + arg.RepositoryID, + arg.PullRequestID, + arg.ArtifactID, + ) + var id uuid.UUID + err := row.Scan(&id) + return id, err +} + +const insertEvaluationStatus = `-- name: InsertEvaluationStatus :one +INSERT INTO evaluation_statuses( + rule_entity_id, + status, + details +) VALUES ( + $1, + $2, + $3 +) +RETURNING id +` + +type InsertEvaluationStatusParams struct { + RuleEntityID uuid.UUID `json:"rule_entity_id"` + Status EvalStatusTypes `json:"status"` + Details string `json:"details"` +} + +func (q *Queries) InsertEvaluationStatus(ctx context.Context, arg InsertEvaluationStatusParams) (uuid.UUID, error) { + row := q.db.QueryRowContext(ctx, insertEvaluationStatus, arg.RuleEntityID, arg.Status, arg.Details) + var id uuid.UUID + err := row.Scan(&id) + return id, err +} + +const updateEvaluationTimes = `-- name: UpdateEvaluationTimes :exec +UPDATE evaluation_statuses +SET evaluation_times = $1 +WHERE id = $2 +` + +type UpdateEvaluationTimesParams struct { + EvaluationTimes []time.Time `json:"evaluation_times"` + ID uuid.UUID `json:"id"` +} + +func (q *Queries) UpdateEvaluationTimes(ctx context.Context, arg UpdateEvaluationTimesParams) error { + _, err := q.db.ExecContext(ctx, updateEvaluationTimes, pq.Array(arg.EvaluationTimes), arg.ID) + return err +} + +const upsertLatestEvaluationStatus = `-- name: UpsertLatestEvaluationStatus :exec +INSERT INTO latest_evaluation_statuses( + rule_entity_id, + evaluation_history_id +) VALUES ( + $1, + $2 +) +ON CONFLICT (rule_entity_id, evaluation_history_id) DO UPDATE +SET evaluation_history_id = $2 +` + +type UpsertLatestEvaluationStatusParams struct { + RuleEntityID uuid.UUID `json:"rule_entity_id"` + EvaluationHistoryID uuid.UUID `json:"evaluation_history_id"` +} + +func (q *Queries) UpsertLatestEvaluationStatus(ctx context.Context, arg UpsertLatestEvaluationStatusParams) error { + _, err := q.db.ExecContext(ctx, upsertLatestEvaluationStatus, arg.RuleEntityID, arg.EvaluationHistoryID) + return err +} diff --git a/internal/db/querier.go b/internal/db/querier.go index 12bd7ec7a8..2b68609be1 100644 --- a/internal/db/querier.go +++ b/internal/db/querier.go @@ -73,6 +73,7 @@ type Querier interface { // GetFeatureInProject verifies if a feature is available for a specific project. // It returns the settings for the feature if it is available. GetFeatureInProject(ctx context.Context, arg GetFeatureInProjectParams) (json.RawMessage, error) + GetIDByProfileEntityName(ctx context.Context, arg GetIDByProfileEntityNameParams) (uuid.UUID, error) // GetImmediateChildrenProjects is a query that returns all the immediate children of a project. GetImmediateChildrenProjects(ctx context.Context, parentID uuid.UUID) ([]Project, error) GetInstallationIDByAppID(ctx context.Context, appInstallationID int64) (ProviderGithubAppInstallation, error) @@ -92,6 +93,20 @@ type Querier interface { // Note that this requires that the destination email address matches the email // address of the logged in user in the external identity service / auth token. GetInvitationsByEmail(ctx context.Context, email string) ([]UserInvite, error) + // Copyright 2024 Stacklok, Inc + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. + // You may obtain a copy of the License at + // + // http://www.apache.org/licenses/LICENSE-2.0 + // + // Unless required by applicable law or agreed to in writing, software + // distributed under the License is distributed on an "AS IS" BASIS, + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + // See the License for the specific language governing permissions and + // limitations under the License. + GetLatestEvalStateForRuleEntity(ctx context.Context, arg GetLatestEvalStateForRuleEntityParams) (EvaluationStatus, error) GetParentProjects(ctx context.Context, id uuid.UUID) ([]uuid.UUID, error) GetParentProjectsUntil(ctx context.Context, arg GetParentProjectsUntilParams) ([]uuid.UUID, error) GetProfileByID(ctx context.Context, arg GetProfileByIDParams) (Profile, error) @@ -132,6 +147,8 @@ type Querier interface { GetUserBySubject(ctx context.Context, identitySubject string) (User, error) GlobalListProviders(ctx context.Context) ([]Provider, error) GlobalListProvidersByClass(ctx context.Context, class ProviderClass) ([]Provider, error) + InsertEvaluationRuleEntity(ctx context.Context, arg InsertEvaluationRuleEntityParams) (uuid.UUID, error) + InsertEvaluationStatus(ctx context.Context, arg InsertEvaluationStatusParams) (uuid.UUID, error) ListArtifactsByRepoID(ctx context.Context, repositoryID uuid.NullUUID) ([]Artifact, error) ListFlushCache(ctx context.Context) ([]FlushCache, error) // ListInvitationsForProject collects the information visible to project @@ -192,6 +209,7 @@ type Querier interface { RepositoryExistsAfterID(ctx context.Context, id uuid.UUID) (bool, error) SetCurrentVersion(ctx context.Context, arg SetCurrentVersionParams) error UpdateEncryptedSecret(ctx context.Context, arg UpdateEncryptedSecretParams) error + UpdateEvaluationTimes(ctx context.Context, arg UpdateEvaluationTimesParams) error // UpdateInvitation updates an invitation by its code. This is intended to be // called by a user who has issued an invitation and then decided to bump its // expiration. @@ -220,6 +238,7 @@ type Querier interface { // Bundles -- UpsertBundle(ctx context.Context, arg UpsertBundleParams) error UpsertInstallationID(ctx context.Context, arg UpsertInstallationIDParams) (ProviderGithubAppInstallation, error) + UpsertLatestEvaluationStatus(ctx context.Context, arg UpsertLatestEvaluationStatusParams) error UpsertProfileForEntity(ctx context.Context, arg UpsertProfileForEntityParams) (EntityProfile, error) UpsertPullRequest(ctx context.Context, arg UpsertPullRequestParams) (PullRequest, error) UpsertRuleDetailsAlert(ctx context.Context, arg UpsertRuleDetailsAlertParams) (uuid.UUID, error) diff --git a/internal/db/rule_instances.sql.go b/internal/db/rule_instances.sql.go index 84aef26403..4e334399ed 100644 --- a/internal/db/rule_instances.sql.go +++ b/internal/db/rule_instances.sql.go @@ -31,6 +31,26 @@ func (q *Queries) DeleteNonUpdatedRules(ctx context.Context, arg DeleteNonUpdate return err } +const getIDByProfileEntityName = `-- name: GetIDByProfileEntityName :one +SELECT id FROM rule_instances +WHERE profile_id = $1 +AND entity_type = $2 +AND name = $3 +` + +type GetIDByProfileEntityNameParams struct { + ProfileID uuid.UUID `json:"profile_id"` + EntityType Entities `json:"entity_type"` + Name string `json:"name"` +} + +func (q *Queries) GetIDByProfileEntityName(ctx context.Context, arg GetIDByProfileEntityNameParams) (uuid.UUID, error) { + row := q.db.QueryRowContext(ctx, getIDByProfileEntityName, arg.ProfileID, arg.EntityType, arg.Name) + var id uuid.UUID + err := row.Scan(&id) + return id, err +} + const getRuleInstancesForProfile = `-- name: GetRuleInstancesForProfile :many SELECT id, profile_id, rule_type_id, name, entity_type, def, params, created_at, updated_at, project_id FROM rule_instances WHERE profile_id = $1 ` diff --git a/internal/engine/executor.go b/internal/engine/executor.go index e30f4f10bc..3d915f9fe8 100644 --- a/internal/engine/executor.go +++ b/internal/engine/executor.go @@ -191,7 +191,7 @@ func (e *Executor) evalEntityEvent(ctx context.Context, inf *entities.EntityInfo // Let's evaluate all the rules for this profile err = profiles.TraverseRules(relevant, func(rule *pb.Profile_Rule) error { // Get the engine evaluator for this rule type - evalParams, ruleEngine, actions, err := e.getEvaluator( + evalParams, ruleEngine, actionEngine, err := e.getEvaluator( ctx, inf, provider, profile, rule, hierarchy, ingestCache) if err != nil { return err @@ -204,11 +204,11 @@ func (e *Executor) evalEntityEvent(ctx context.Context, inf *entities.EntityInfo evalErr := ruleEngine.Eval(ctx, inf, evalParams) evalParams.SetEvalErr(evalErr) - // Perform actions, if any - actionsErr := actions.DoActions(ctx, inf.Entity, evalParams) + // Perform actionEngine, if any + actionsErr := actionEngine.DoActions(ctx, inf.Entity, evalParams) + evalParams.SetActionsErr(ctx, actionsErr) // Log the evaluation - evalParams.SetActionsErr(ctx, actionsErr) logEval(ctx, inf, evalParams) // Create or update the evaluation status diff --git a/internal/history/service.go b/internal/history/service.go new file mode 100644 index 0000000000..35e096436e --- /dev/null +++ b/internal/history/service.go @@ -0,0 +1,188 @@ +// Copyright 2024 Stacklok, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package history contains logic for tracking evaluation history +package history + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + + "github.com/stacklok/minder/internal/db" + evalerrors "github.com/stacklok/minder/internal/engine/errors" +) + +// EvaluationHistoryService contains methods to access the eval history log +type EvaluationHistoryService interface { + StoreEvaluationStatus( + ctx context.Context, + qtx db.Querier, + ruleID uuid.UUID, + entityType db.Entities, + entityID uuid.UUID, + evalError error, + ) error +} + +// NewEvaluationHistoryService creates a new instance of EvaluationHistoryService +func NewEvaluationHistoryService() EvaluationHistoryService { + return &evaluationHistoryService{} +} + +type evaluationHistoryService struct{} + +func (e *evaluationHistoryService) StoreEvaluationStatus( + ctx context.Context, + qtx db.Querier, + ruleID uuid.UUID, + entityType db.Entities, + entityID uuid.UUID, + evalError error, +) error { + var ruleEntityID, evaluationID uuid.UUID + status := evalerrors.ErrorAsEvalStatus(evalError) + details := evalerrors.ErrorAsEvalDetails(evalError) + + params, err := paramsFromEntity(ruleID, entityID, entityType) + if err != nil { + return err + } + + // find the latest record for this rule/entity pair + latestRecord, err := qtx.GetLatestEvalStateForRuleEntity(ctx, + db.GetLatestEvalStateForRuleEntityParams{ + RuleID: params.RuleID, + RepositoryID: params.RepositoryID, + PullRequestID: params.PullRequestID, + ArtifactID: params.ArtifactID, + }, + ) + if err != nil { + // if we find nothing, create a new rule/entity record + if errors.Is(err, sql.ErrNoRows) { + ruleEntityID, err = qtx.InsertEvaluationRuleEntity(ctx, + db.InsertEvaluationRuleEntityParams{ + RuleID: params.RuleID, + RepositoryID: params.RepositoryID, + PullRequestID: params.PullRequestID, + ArtifactID: params.ArtifactID, + }, + ) + if err != nil { + return fmt.Errorf("error while creating new rule/entity in database: %w", err) + } + } else { + return fmt.Errorf("error while querying DB: %w", err) + } + } else { + ruleEntityID = latestRecord.RuleEntityID + evaluationID = latestRecord.ID + } + + previousDetails := latestRecord.Details + previousStatus := latestRecord.Status + + if evaluationID == uuid.Nil || previousDetails != details || previousStatus != status { + // if there is no prior state for this rule/entity, or the previous state + // differs from the current one, create a new status record. + if err = e.createNewStatus(ctx, qtx, ruleEntityID, status, details); err != nil { + return fmt.Errorf("error while creating new evaluation status for rule/entity %s: %w", ruleEntityID, err) + } + } else { + if err = e.updateExistingStatus(ctx, qtx, entityID, latestRecord.EvaluationTimes); err != nil { + return fmt.Errorf("error while updating existing evaluation status for rule/entity %s: %w", ruleEntityID, err) + } + } + + return nil +} + +func (_ *evaluationHistoryService) createNewStatus( + ctx context.Context, + qtx db.Querier, + ruleEntityID uuid.UUID, + status db.EvalStatusTypes, + details string, +) error { + newEvaluationID, err := qtx.InsertEvaluationStatus(ctx, + db.InsertEvaluationStatusParams{ + RuleEntityID: ruleEntityID, + Status: status, + Details: details, + }, + ) + if err != nil { + return err + } + + // mark this as the latest status for this rule/entity + return qtx.UpsertLatestEvaluationStatus(ctx, + db.UpsertLatestEvaluationStatusParams{ + RuleEntityID: ruleEntityID, + EvaluationHistoryID: newEvaluationID, + }, + ) +} + +func (_ *evaluationHistoryService) updateExistingStatus( + ctx context.Context, + qtx db.Querier, + evaluationID uuid.UUID, + times []time.Time, +) error { + // if the status is repeated, then just append the current timestamp to it + times = append(times, time.Now()) + return qtx.UpdateEvaluationTimes(ctx, db.UpdateEvaluationTimesParams{ + EvaluationTimes: times, + ID: evaluationID, + }) +} + +func paramsFromEntity( + ruleID uuid.UUID, + entityID uuid.UUID, + entityType db.Entities, +) (*ruleEntityParams, error) { + params := ruleEntityParams{RuleID: ruleID} + + nullableEntityID := uuid.NullUUID{ + UUID: entityID, + Valid: true, + } + + switch entityType { + case db.EntitiesRepository: + params.RepositoryID = nullableEntityID + case db.EntitiesPullRequest: + params.PullRequestID = nullableEntityID + case db.EntitiesArtifact: + params.ArtifactID = nullableEntityID + case db.EntitiesBuildEnvironment: + default: + return nil, fmt.Errorf("unknown entity %s", entityType) + } + return ¶ms, nil +} + +type ruleEntityParams struct { + RuleID uuid.UUID + RepositoryID uuid.NullUUID + ArtifactID uuid.NullUUID + PullRequestID uuid.NullUUID +} diff --git a/internal/history/service_test.go b/internal/history/service_test.go new file mode 100644 index 0000000000..446e68e090 --- /dev/null +++ b/internal/history/service_test.go @@ -0,0 +1,224 @@ +// Copyright 2024 Stacklok, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package history_test + +import ( + "context" + "database/sql" + "errors" + "testing" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/stacklok/minder/internal/db" + dbf "github.com/stacklok/minder/internal/db/fixtures" + engerr "github.com/stacklok/minder/internal/engine/errors" + "github.com/stacklok/minder/internal/history" +) + +func TestStoreEvaluationStatus(t *testing.T) { + t.Parallel() + + scenarios := []struct { + Name string + EntityType db.Entities + DBSetup dbf.DBMockBuilder + ExpectedError string + }{ + { + Name: "StoreEvaluationStatus rejects invalid entity type", + EntityType: "I'm a little teapot", + ExpectedError: "unknown entity", + }, + { + Name: "StoreEvaluationStatus returns error when unable to query previous status", + EntityType: db.EntitiesArtifact, + DBSetup: dbf.NewDBMock(withGetLatestEval(emptyLatestResult, errTest)), + ExpectedError: "error while querying DB", + }, + { + Name: "StoreEvaluationStatus returns error when unable to create new rule/entity", + EntityType: db.EntitiesPullRequest, + DBSetup: dbf.NewDBMock( + withGetLatestEval(emptyLatestResult, sql.ErrNoRows), + withInsertEvaluationRuleEntity(uuid.Nil, errTest), + ), + ExpectedError: "error while creating new rule/entity in database", + }, + { + Name: "StoreEvaluationStatus returns error when unable to create new evaluation status", + EntityType: db.EntitiesRepository, + DBSetup: dbf.NewDBMock( + withGetLatestEval(emptyLatestResult, sql.ErrNoRows), + withInsertEvaluationRuleEntity(ruleEntityID, nil), + withInsertEvaluationStatus(uuid.Nil, errTest), + ), + ExpectedError: "error while creating new evaluation status for rule/entity", + }, + { + Name: "StoreEvaluationStatus returns error when unable to set latest status", + EntityType: db.EntitiesRepository, + DBSetup: dbf.NewDBMock( + withGetLatestEval(emptyLatestResult, sql.ErrNoRows), + withInsertEvaluationRuleEntity(ruleEntityID, nil), + withInsertEvaluationStatus(evaluationID, nil), + withUpsertLatestEvaluationStatus(errTest), + ), + ExpectedError: "error while creating new evaluation status for rule/entity", + }, + { + Name: "StoreEvaluationStatus returns error when unable to update status with timestamp", + EntityType: db.EntitiesRepository, + DBSetup: dbf.NewDBMock( + withGetLatestEval(sameState, nil), + withUpdateEvaluationTimes(errTest), + ), + ExpectedError: "error while updating existing evaluation status for rule/entity", + }, + { + Name: "StoreEvaluationStatus creates new status for new rule/entity", + EntityType: db.EntitiesRepository, + DBSetup: dbf.NewDBMock( + withGetLatestEval(emptyLatestResult, sql.ErrNoRows), + withInsertEvaluationRuleEntity(ruleEntityID, nil), + withInsertEvaluationStatus(evaluationID, nil), + withUpsertLatestEvaluationStatus(nil), + ), + }, + { + Name: "StoreEvaluationStatus creates new status for state change", + EntityType: db.EntitiesRepository, + DBSetup: dbf.NewDBMock( + withGetLatestEval(differentState, nil), + withInsertEvaluationStatus(evaluationID, nil), + withUpsertLatestEvaluationStatus(nil), + ), + }, + { + Name: "StoreEvaluationStatus creates new status when status is the same, but details differ", + EntityType: db.EntitiesRepository, + DBSetup: dbf.NewDBMock( + withGetLatestEval(differentDetails, nil), + withInsertEvaluationStatus(evaluationID, nil), + withUpsertLatestEvaluationStatus(nil), + ), + }, + { + Name: "StoreEvaluationStatus adds timestamp when state does not change", + EntityType: db.EntitiesRepository, + DBSetup: dbf.NewDBMock( + withGetLatestEval(sameState, nil), + withUpdateEvaluationTimes(nil), + ), + }, + } + + for _, scenario := range scenarios { + t.Run(scenario.Name, func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + ctx := context.Background() + + var store db.Store + if scenario.DBSetup != nil { + store = scenario.DBSetup(ctrl) + } + + service := history.NewEvaluationHistoryService() + err := service.StoreEvaluationStatus(ctx, store, ruleID, scenario.EntityType, entityID, errTest) + if scenario.ExpectedError == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, scenario.ExpectedError) + } + }) + } +} + +var ( + ruleID = uuid.New() + entityID = uuid.New() + ruleEntityID = uuid.New() + evaluationID = uuid.New() + + emptyLatestResult = db.EvaluationStatus{} + sameState = db.EvaluationStatus{ + ID: evaluationID, + RuleEntityID: ruleEntityID, + Status: db.EvalStatusTypesError, + Details: errTest.Error(), + EvaluationTimes: []time.Time{time.Now()}, + } + differentDetails = db.EvaluationStatus{ + ID: evaluationID, + RuleEntityID: ruleEntityID, + Status: db.EvalStatusTypesError, + Details: "something went wrong", + EvaluationTimes: []time.Time{time.Now()}, + } + differentState = db.EvaluationStatus{ + ID: evaluationID, + RuleEntityID: ruleEntityID, + Status: db.EvalStatusTypesSkipped, + Details: engerr.ErrEvaluationSkipped.Error(), + EvaluationTimes: []time.Time{time.Now()}, + } + errTest = errors.New("oh no") +) + +func withGetLatestEval(result db.EvaluationStatus, err error) func(dbf.DBMock) { + return func(mock dbf.DBMock) { + mock.EXPECT(). + GetLatestEvalStateForRuleEntity(gomock.Any(), gomock.Any()). + Return(result, err) + } +} + +func withInsertEvaluationRuleEntity(id uuid.UUID, err error) func(dbf.DBMock) { + return func(mock dbf.DBMock) { + mock.EXPECT(). + InsertEvaluationRuleEntity(gomock.Any(), gomock.Any()). + Return(id, err) + } +} + +func withInsertEvaluationStatus(id uuid.UUID, err error) func(dbf.DBMock) { + return func(mock dbf.DBMock) { + mock.EXPECT(). + InsertEvaluationStatus(gomock.Any(), gomock.Any()). + Return(id, err) + } +} + +func withUpdateEvaluationTimes(err error) func(dbf.DBMock) { + return func(mock dbf.DBMock) { + mock.EXPECT(). + UpdateEvaluationTimes(gomock.Any(), gomock.Any()). + Return(err) + } +} + +func withUpsertLatestEvaluationStatus(err error) func(dbf.DBMock) { + return func(mock dbf.DBMock) { + mock.EXPECT(). + UpsertLatestEvaluationStatus(gomock.Any(), gomock.Any()). + Return(err) + } +} From a083155b64a9584c023e54d0a034392a4b01e162 Mon Sep 17 00:00:00 2001 From: Don Browne Date: Wed, 19 Jun 2024 16:04:27 +0100 Subject: [PATCH 2/3] set most_recent_evaluation timestamp --- database/query/eval_history.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/database/query/eval_history.sql b/database/query/eval_history.sql index 44bbf5dda9..b3481cc8ce 100644 --- a/database/query/eval_history.sql +++ b/database/query/eval_history.sql @@ -52,7 +52,9 @@ RETURNING id; -- name: UpdateEvaluationTimes :exec UPDATE evaluation_statuses -SET evaluation_times = $1 +SET + evaluation_times = $1, + most_recent_evaluation = NOW() WHERE id = $2; -- name: UpsertLatestEvaluationStatus :exec From 0021713c722586038c688600c8c3423961c52191 Mon Sep 17 00:00:00 2001 From: Don Browne Date: Wed, 19 Jun 2024 16:08:30 +0100 Subject: [PATCH 3/3] codegen --- internal/db/eval_history.sql.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/db/eval_history.sql.go b/internal/db/eval_history.sql.go index 916e63a8fc..cc0bda6003 100644 --- a/internal/db/eval_history.sql.go +++ b/internal/db/eval_history.sql.go @@ -128,7 +128,9 @@ func (q *Queries) InsertEvaluationStatus(ctx context.Context, arg InsertEvaluati const updateEvaluationTimes = `-- name: UpdateEvaluationTimes :exec UPDATE evaluation_statuses -SET evaluation_times = $1 +SET + evaluation_times = $1, + most_recent_evaluation = NOW() WHERE id = $2 `