diff --git a/database/migrations/000102_properties_upstream_id_index.down.sql b/database/migrations/000102_properties_upstream_id_index.down.sql new file mode 100644 index 0000000000..fb51aa01f9 --- /dev/null +++ b/database/migrations/000102_properties_upstream_id_index.down.sql @@ -0,0 +1,19 @@ +-- Copyright 2023 Stacklok, Inc +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +BEGIN; + +DROP INDEX IF EXISTS idx_properties_value_gin; + +COMMIT; \ No newline at end of file diff --git a/database/migrations/000102_properties_upstream_id_index.up.sql b/database/migrations/000102_properties_upstream_id_index.up.sql new file mode 100644 index 0000000000..767c4ef950 --- /dev/null +++ b/database/migrations/000102_properties_upstream_id_index.up.sql @@ -0,0 +1,20 @@ +-- Copyright 2023 Stacklok, Inc +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +BEGIN; + +-- Create index on properties for upstream_id +CREATE INDEX idx_properties_value_gin ON properties USING GIN (value jsonb_path_ops); + +COMMIT; \ No newline at end of file diff --git a/database/mock/store.go b/database/mock/store.go index c455488311..31d0afd46c 100644 --- a/database/mock/store.go +++ b/database/mock/store.go @@ -1718,6 +1718,36 @@ func (mr *MockStoreMockRecorder) GetSubscriptionByProjectBundle(arg0, arg1 any) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSubscriptionByProjectBundle", reflect.TypeOf((*MockStore)(nil).GetSubscriptionByProjectBundle), arg0, arg1) } +// GetTypedEntitiesByProperty mocks base method. +func (m *MockStore) GetTypedEntitiesByProperty(arg0 context.Context, arg1 db.GetTypedEntitiesByPropertyParams) ([]db.EntityInstance, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTypedEntitiesByProperty", arg0, arg1) + ret0, _ := ret[0].([]db.EntityInstance) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTypedEntitiesByProperty indicates an expected call of GetTypedEntitiesByProperty. +func (mr *MockStoreMockRecorder) GetTypedEntitiesByProperty(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTypedEntitiesByProperty", reflect.TypeOf((*MockStore)(nil).GetTypedEntitiesByProperty), arg0, arg1) +} + +// GetTypedEntitiesByPropertyV1 mocks base method. +func (m *MockStore) GetTypedEntitiesByPropertyV1(arg0 context.Context, arg1 uuid.UUID, arg2 db.Entities, arg3 string, arg4 any) ([]db.EntityInstance, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetTypedEntitiesByPropertyV1", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].([]db.EntityInstance) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetTypedEntitiesByPropertyV1 indicates an expected call of GetTypedEntitiesByPropertyV1. +func (mr *MockStoreMockRecorder) GetTypedEntitiesByPropertyV1(arg0, arg1, arg2, arg3, arg4 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTypedEntitiesByPropertyV1", reflect.TypeOf((*MockStore)(nil).GetTypedEntitiesByPropertyV1), arg0, arg1, arg2, arg3, arg4) +} + // GetUnclaimedInstallationsByUser mocks base method. func (m *MockStore) GetUnclaimedInstallationsByUser(arg0 context.Context, arg1 sql.NullString) ([]db.ProviderGithubAppInstallation, error) { m.ctrl.T.Helper() diff --git a/database/query/entities.sql b/database/query/entities.sql index 057e39190c..6d5e3684a4 100644 --- a/database/query/entities.sql +++ b/database/query/entities.sql @@ -99,4 +99,13 @@ WHERE entity_id = $1; -- name: DeleteAllPropertiesForEntity :exec DELETE FROM properties -WHERE entity_id = $1; \ No newline at end of file +WHERE entity_id = $1; + +-- name: GetTypedEntitiesByProperty :many +SELECT ei.* +FROM entity_instances ei + JOIN properties p ON ei.id = p.entity_id +WHERE ei.entity_type = sqlc.arg(entity_type) + AND (sqlc.arg(project_id)::uuid = '00000000-0000-0000-0000-000000000000'::uuid OR ei.project_id = sqlc.arg(project_id)) + AND p.key = sqlc.arg(key) + AND p.value @> sqlc.arg(value)::jsonb; \ No newline at end of file diff --git a/internal/db/entities.sql.go b/internal/db/entities.sql.go index 5930cc916d..98de247cc5 100644 --- a/internal/db/entities.sql.go +++ b/internal/db/entities.sql.go @@ -364,6 +364,59 @@ func (q *Queries) GetProperty(ctx context.Context, arg GetPropertyParams) (Prope return i, err } +const getTypedEntitiesByProperty = `-- name: GetTypedEntitiesByProperty :many +SELECT ei.id, ei.entity_type, ei.name, ei.project_id, ei.provider_id, ei.created_at, ei.originated_from +FROM entity_instances ei + JOIN properties p ON ei.id = p.entity_id +WHERE ei.entity_type = $1 + AND ($2::uuid = '00000000-0000-0000-0000-000000000000'::uuid OR ei.project_id = $2) + AND p.key = $3 + AND p.value @> $4::jsonb +` + +type GetTypedEntitiesByPropertyParams struct { + EntityType Entities `json:"entity_type"` + ProjectID uuid.UUID `json:"project_id"` + Key string `json:"key"` + Value json.RawMessage `json:"value"` +} + +func (q *Queries) GetTypedEntitiesByProperty(ctx context.Context, arg GetTypedEntitiesByPropertyParams) ([]EntityInstance, error) { + rows, err := q.db.QueryContext(ctx, getTypedEntitiesByProperty, + arg.EntityType, + arg.ProjectID, + arg.Key, + arg.Value, + ) + if err != nil { + return nil, err + } + defer rows.Close() + items := []EntityInstance{} + for rows.Next() { + var i EntityInstance + if err := rows.Scan( + &i.ID, + &i.EntityType, + &i.Name, + &i.ProjectID, + &i.ProviderID, + &i.CreatedAt, + &i.OriginatedFrom, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const upsertProperty = `-- name: UpsertProperty :one INSERT INTO properties ( entity_id, diff --git a/internal/db/entities_test.go b/internal/db/entities_test.go index 56115b5f45..9993bcdefd 100644 --- a/internal/db/entities_test.go +++ b/internal/db/entities_test.go @@ -202,6 +202,76 @@ func Test_PropertyCrud(t *testing.T) { require.NoError(t, err) require.Equal(t, anotherKeyVal.Value, "anothervalue") }) + + t.Run("GetTypedEntitiesByPropertyV1", func(t *testing.T) { + t.Parallel() + + const testRepoName = "testorg/testrepo_getbyprops" + const testArtifactName = "testorg/testartifact_getbyprops" + + for i := 0; i < 50000; i++ { + createRandomEntity(t, proj.ID, prov.ID, EntitiesRepository) + } + + repo, err := testQueries.CreateEntity(context.Background(), CreateEntityParams{ + EntityType: EntitiesRepository, + Name: testRepoName, + ProjectID: proj.ID, + ProviderID: prov.ID, + OriginatedFrom: uuid.NullUUID{}, + }) + require.NoError(t, err) + require.NotEmpty(t, repo) + + _, err = testQueries.UpsertPropertyValueV1(context.Background(), UpsertPropertyValueV1Params{ + EntityID: repo.ID, + Key: "sharedkey", + Value: "sharedvalue", + }) + require.NoError(t, err) + + _, err = testQueries.UpsertPropertyValueV1(context.Background(), UpsertPropertyValueV1Params{ + EntityID: repo.ID, + Key: "repokey", + Value: "repovalue", + }) + require.NoError(t, err) + + art, err := testQueries.CreateEntity(context.Background(), CreateEntityParams{ + EntityType: EntitiesArtifact, + Name: testArtifactName, + ProjectID: proj.ID, + ProviderID: prov.ID, + OriginatedFrom: uuid.NullUUID{}, + }) + require.NoError(t, err) + require.NotEmpty(t, art) + + _, err = testQueries.UpsertPropertyValueV1(context.Background(), UpsertPropertyValueV1Params{ + EntityID: art.ID, + Key: "sharedkey", + Value: "sharedvalue", + }) + require.NoError(t, err) + + getEnt, err := testQueries.GetTypedEntitiesByPropertyV1( + context.Background(), proj.ID, EntitiesRepository, "sharedkey", "sharedvalue") + require.NoError(t, err) + require.Len(t, getEnt, 1) + require.Equal(t, getEnt[0].ID, repo.ID) + + getEnt, err = testQueries.GetTypedEntitiesByPropertyV1( + context.Background(), proj.ID, EntitiesArtifact, "sharedkey", "sharedvalue") + require.NoError(t, err) + require.Len(t, getEnt, 1) + require.Equal(t, getEnt[0].ID, art.ID) + + getEnt, err = testQueries.GetTypedEntitiesByPropertyV1( + context.Background(), proj.ID, EntitiesRepository, "repokey", "repovalue") + require.NoError(t, err) + require.Len(t, getEnt, 1) + require.Equal(t, getEnt[0].ID, repo.ID) + }) } func propertyByKey(t *testing.T, props []PropertyValueV1, key string) PropertyValueV1 { diff --git a/internal/db/entity_helpers.go b/internal/db/entity_helpers.go index 1e5f22636c..c5d7d9d63c 100644 --- a/internal/db/entity_helpers.go +++ b/internal/db/entity_helpers.go @@ -152,3 +152,20 @@ func (q *Queries) GetAllPropertyValuesV1(ctx context.Context, entityID uuid.UUID return props, nil } + +// GetTypedEntitiesByPropertyV1 retrieves all entities with a property value +func (q *Queries) GetTypedEntitiesByPropertyV1( + ctx context.Context, project uuid.UUID, entType Entities, key string, value any, +) ([]EntityInstance, error) { + jsonVal, err := PropValueToDbV1(value) + if err != nil { + return nil, err + } + + return q.GetTypedEntitiesByProperty(ctx, GetTypedEntitiesByPropertyParams{ + EntityType: entType, + ProjectID: project, + Key: key, + Value: jsonVal, + }) +} diff --git a/internal/db/projects_test.go b/internal/db/projects_test.go index 8663555b12..bcf7be41f5 100644 --- a/internal/db/projects_test.go +++ b/internal/db/projects_test.go @@ -30,6 +30,45 @@ import ( "github.com/stacklok/minder/internal/util/rand" ) +func createRandomEntity(t *testing.T, project uuid.UUID, provider uuid.UUID, entType Entities) { + t.Helper() + + seed := time.Now().UnixNano() + + ent, err := testQueries.CreateEntity(context.Background(), CreateEntityParams{ + EntityType: entType, + Name: rand.RandomName(seed), + ProjectID: project, + ProviderID: provider, + OriginatedFrom: uuid.NullUUID{}, + }) + require.NoError(t, err) + + prop, err := testQueries.UpsertPropertyValueV1(context.Background(), UpsertPropertyValueV1Params{ + EntityID: ent.ID, + Key: "testkey1", + Value: rand.RandomName(seed), + }) + require.NoError(t, err) + require.NotEmpty(t, prop) + + prop, err = testQueries.UpsertPropertyValueV1(context.Background(), UpsertPropertyValueV1Params{ + EntityID: ent.ID, + Key: "testkey1", + Value: rand.RandomName(seed), + }) + require.NoError(t, err) + require.NotEmpty(t, prop) + + prop, err = testQueries.UpsertPropertyValueV1(context.Background(), UpsertPropertyValueV1Params{ + EntityID: ent.ID, + Key: "upstream_id", + Value: rand.RandomName(seed), + }) + require.NoError(t, err) + require.NotEmpty(t, prop) +} + func createRandomProject(t *testing.T, orgID uuid.UUID) Project { t.Helper() diff --git a/internal/db/querier.go b/internal/db/querier.go index e068401746..d94078034f 100644 --- a/internal/db/querier.go +++ b/internal/db/querier.go @@ -179,6 +179,7 @@ type Querier interface { GetSelectorByID(ctx context.Context, id uuid.UUID) (ProfileSelector, error) GetSelectorsByProfileID(ctx context.Context, profileID uuid.UUID) ([]ProfileSelector, error) GetSubscriptionByProjectBundle(ctx context.Context, arg GetSubscriptionByProjectBundleParams) (Subscription, error) + GetTypedEntitiesByProperty(ctx context.Context, arg GetTypedEntitiesByPropertyParams) ([]EntityInstance, error) GetUnclaimedInstallationsByUser(ctx context.Context, ghID sql.NullString) ([]ProviderGithubAppInstallation, error) GetUserByID(ctx context.Context, id int32) (User, error) GetUserBySubject(ctx context.Context, identitySubject string) (User, error) diff --git a/internal/db/store.go b/internal/db/store.go index 987bfd828b..1a7d37cd2f 100644 --- a/internal/db/store.go +++ b/internal/db/store.go @@ -31,6 +31,9 @@ type ExtendQuerier interface { UpsertPropertyValueV1(ctx context.Context, params UpsertPropertyValueV1Params) (Property, error) GetPropertyValueV1(ctx context.Context, entityID uuid.UUID, key string) (PropertyValueV1, error) GetAllPropertyValuesV1(ctx context.Context, entityID uuid.UUID) ([]PropertyValueV1, error) + GetTypedEntitiesByPropertyV1( + ctx context.Context, project uuid.UUID, entType Entities, key string, value any, + ) ([]EntityInstance, error) } // Store provides all functions to execute db queries and transactions