From 6b97042e26e76cafd1f5d50edd1969a97bd26dac Mon Sep 17 00:00:00 2001 From: shaniah868 Date: Wed, 26 Jan 2022 09:18:55 -0400 Subject: [PATCH 01/11] feature: WEOS-1134 Update list method on projection to accept filter map -Added test for each filter operator -Added the filter properties --- controllers/rest/dtos.go | 8 ++ projections/projections_test.go | 151 ++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+) diff --git a/controllers/rest/dtos.go b/controllers/rest/dtos.go index d4d6b9c0..f0300e64 100644 --- a/controllers/rest/dtos.go +++ b/controllers/rest/dtos.go @@ -55,3 +55,11 @@ type ListApiResponse struct { Page int `json:"page"` Items []map[string]interface{} `json:"items"` } + +//FilterProperties is the properties need to use filters +type FilterProperties struct { + Field string + Operator string + Value string + Values []string +} diff --git a/projections/projections_test.go b/projections/projections_test.go index 51efbd24..7ecdb595 100644 --- a/projections/projections_test.go +++ b/projections/projections_test.go @@ -2217,3 +2217,154 @@ components: }) } +func TestProjections_Filters(t *testing.T) { + openAPI := `openapi: 3.0.3 +info: + title: Blog + description: Blog example + version: 1.0.0 +servers: + - url: https://prod1.weos.sh/blog/dev + description: WeOS Dev + - url: https://prod1.weos.sh/blog/v1 +x-weos-config: + logger: + level: warn + report-caller: true + formatter: json + database: + driver: sqlite3 + database: test.db + event-source: + - title: default + driver: service + endpoint: https://prod1.weos.sh/events/v1 + - title: event + driver: sqlite3 + database: test.db + databases: + - title: default + driver: sqlite3 + database: test.db + rest: + middleware: + - RequestID + - Recover + - ZapLogger +components: + schemas: + Blog: + type: object + properties: + title: + type: string + description: blog title + description: + type: string + required: + - title + Post: + type: object + properties: + title: + type: string + description: post title + description: + type: string + blog: + $ref: "#/components/schemas/Blog" +` + loader := openapi3.NewSwaggerLoader() + swagger, err := loader.LoadSwaggerFromData([]byte(openAPI)) + if err != nil { + t.Fatal(err) + } + + schemes := rest.CreateSchema(context.Background(), echo.New(), swagger) + p, err := projections.NewProjection(context.Background(), app, schemes) + if err != nil { + t.Fatal(err) + } + + err = p.Migrate(context.Background()) + if err != nil { + t.Fatal(err) + } + + gormDB := app.DB() + if !gormDB.Migrator().HasTable("Blog") { + t.Fatal("expected to get a table 'Blog'") + } + + if !gormDB.Migrator().HasTable("Post") { + t.Fatal("expected to get a table 'Post'") + } + blogWeosID := "abc123" + blogWeosID1 := "abc1234" + blogWeosID2 := "abc12345" + blogWeosID3 := "abc123456" + blogWeosID4 := "abc1234567" + + blog := map[string]interface{}{"weos_id": blogWeosID, "title": "hugs1", "sequence_no": int64(1)} + blog1 := map[string]interface{}{"weos_id": blogWeosID1, "title": "hugs2", "sequence_no": int64(1)} + blog2 := map[string]interface{}{"weos_id": blogWeosID2, "title": "hugs3", "sequence_no": int64(1)} + blog3 := map[string]interface{}{"weos_id": blogWeosID3, "title": "morehugs4", "sequence_no": int64(1)} + blog4 := map[string]interface{}{"weos_id": blogWeosID4, "title": "morehugs5", "sequence_no": int64(1)} + + gormDB.Table("Blog").Create(blog) + gormDB.Table("Blog").Create(blog1) + gormDB.Table("Blog").Create(blog2) + gormDB.Table("Blog").Create(blog3) + gormDB.Table("Blog").Create(blog4) + + t.Run("testing filters with the eq operator", func(t *testing.T) { + page := 1 + limit := 2 + sortOptions := map[string]string{ + "id": "asc", + } + ctxt := context.Background() + name := "Blog" + scheme := swagger.Components.Schemas[name] + ctxt = context.WithValue(ctxt, weosContext.CONTENT_TYPE, &weosContext.ContentType{ + Name: strings.Title(name), + Schema: scheme.Value, + }) + filter := &rest.FilterProperties{ + Field: "title", + Operator: "eq", + Value: "hugs1", + Values: nil, + } + filters := []*rest.FilterProperties{filter} + //TODO figure out if u want a map or an array being passed in + results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) + if err != nil { + t.Errorf("error getting content entities: %s", err) + } + if results == nil || len(results) == 0 { + t.Errorf("expected to get results but got nil") + } + if total != int64(2) { + t.Errorf("expected total to be %d got %d", int64(2), total) + } + }) + t.Run("testing filters with the ne operator", func(t *testing.T) { + + }) + t.Run("testing filters with the like operator", func(t *testing.T) { + + }) + t.Run("testing filters with the in operator with a single value", func(t *testing.T) { + + }) + t.Run("testing filters with the eq operator with multiple values", func(t *testing.T) { + + }) + t.Run("testing filters with the lt operator", func(t *testing.T) { + + }) + t.Run("testing filters with the gt operator", func(t *testing.T) { + + }) +} From 47f5c6c9443c963a0a0a2a2ad22e7abfffd463cb Mon Sep 17 00:00:00 2001 From: shaniah868 Date: Thu, 27 Jan 2022 12:05:16 -0400 Subject: [PATCH 02/11] feature: WEOS-1134 Update list method on projection to accept filter map -Fleshed out tests for all the operators -Added helper functions for filter --- controllers/rest/dtos.go | 8 +- projections/gorm.go | 80 +++++++++++- projections/projections_test.go | 210 ++++++++++++++++++++++++++++++-- 3 files changed, 281 insertions(+), 17 deletions(-) diff --git a/controllers/rest/dtos.go b/controllers/rest/dtos.go index f0300e64..0fc1dd4e 100644 --- a/controllers/rest/dtos.go +++ b/controllers/rest/dtos.go @@ -58,8 +58,8 @@ type ListApiResponse struct { //FilterProperties is the properties need to use filters type FilterProperties struct { - Field string - Operator string - Value string - Values []string + Field string `json:"field"` + Operator string `json:"operator"` + Value string `json:"value"` + Values []string `json:"values"` } diff --git a/projections/gorm.go b/projections/gorm.go index e9eb755b..3212fb37 100644 --- a/projections/gorm.go +++ b/projections/gorm.go @@ -3,6 +3,7 @@ package projections import ( "encoding/json" "fmt" + "github.com/wepala/weos/controllers/rest" "strings" ds "github.com/ompluscator/dynamic-struct" @@ -21,6 +22,12 @@ type GORMProjection struct { Schema map[string]ds.Builder } +type filterProperty struct { + Operator string `json:"operator"` + Value string `json:"value"` + Values []string `json:"values"` +} + func (p *GORMProjection) GetByKey(ctxt context.Context, contentType weosContext.ContentType, identifiers map[string]interface{}) (map[string]interface{}, error) { if s, ok := p.Schema[strings.Title(contentType.Name)]; ok { //pulling the primary keys from the schema in order to match with the keys given for searching @@ -233,6 +240,10 @@ func (p *GORMProjection) GetContentEntities(ctx context.Context, page int, limit var result *gorm.DB var schemes interface{} contentType := weosContext.GetContentType(ctx) + prop := p.convertProperties(filterOptions, p.Schema, contentType) + if prop != nil { + + } if s, ok := p.Schema[strings.Title(contentType.Name)]; ok { schemes = s.Build().NewSliceOfStructs() scheme := s.Build().New() @@ -248,7 +259,7 @@ func (p *GORMProjection) GetContentEntities(ctx context.Context, page int, limit return entities, count, result.Error } -//paginate to query results +//paginate is used for querying results func paginate(page int, limit int) func(db *gorm.DB) *gorm.DB { return func(db *gorm.DB) *gorm.DB { actualLimit := limit @@ -263,7 +274,7 @@ func paginate(page int, limit int) func(db *gorm.DB) *gorm.DB { } } -// function that sorts the query results +//sort is used to sort the query results func sort(order map[string]string) func(db *gorm.DB) *gorm.DB { return func(db *gorm.DB) *gorm.DB { for key, value := range order { @@ -278,10 +289,65 @@ func sort(order map[string]string) func(db *gorm.DB) *gorm.DB { } } +// filter that filters query results +func filter(filter map[string]interface{}) func(db *gorm.DB) *gorm.DB { + return func(db *gorm.DB) *gorm.DB { + if filter != nil { + return db.Where(filter) + } + return db + } +} + +//filterStringBuilder is used to build the query strings +func filterStringBuilder(dbName string, contentType *weosContext.ContentType, properties *rest.FilterProperties) string { + var query string + switch properties.Operator { + case "eq": + query = contentType.Name + "." + properties.Field + " = " + properties.Value + + } + return query +} + +//convertProperties is used to convert the filter properties into key (field) , value pairs +func (p *GORMProjection) convertProperties(properties map[string]interface{}, schema map[string]ds.Builder, contentType *weosContext.ContentType) map[string]interface{} { + filters := map[string]interface{}{} + + if properties == nil { + return nil + } + + for field, filterProperty := range properties { + if filterProperty.Value != "" && (filterProperty == nil || len(filterProperty.Values) == 0) { + columns, _ := p.db.Migrator().ColumnTypes(contentType.Name) + + for _, c := range columns { + if c.Name() == field { + cType := c.DatabaseTypeName() + if cType == "string" { + + } + } + + } + filters[field] = filterProperty.Value + } + if filterProperty.Value == "" && (filterProperty != nil && len(filterProperty.Values) > 0) { + filters[field] = filterProperty.Values + } + + } + + return filters +} + //query modifier for making queries to the database type QueryModifier func() func(db *gorm.DB) *gorm.DB +type QueryFilterModifier func(query string) func(db *gorm.DB) *gorm.DB var ContentQuery QueryModifier +var FilterQuery QueryFilterModifier //NewProjection creates an instance of the projection func NewProjection(ctx context.Context, application weos.Service, schemas map[string]ds.Builder) (*GORMProjection, error) { @@ -293,6 +359,16 @@ func NewProjection(ctx context.Context, application weos.Service, schemas map[st } application.AddProjection(projection) + FilterQuery = func(query string) func(db *gorm.DB) *gorm.DB { + return func(db *gorm.DB) *gorm.DB { + if query != "" { + + return db.Where(query, func(tx *gorm.DB) *gorm.DB { return tx.Omit("weos_id, sequence_no, table") }) + } + return db + } + } + ContentQuery = func() func(db *gorm.DB) *gorm.DB { return func(db *gorm.DB) *gorm.DB { if projection.db.Dialector.Name() == "sqlite" { diff --git a/projections/projections_test.go b/projections/projections_test.go index 7ecdb595..1d6fb09b 100644 --- a/projections/projections_test.go +++ b/projections/projections_test.go @@ -2317,7 +2317,7 @@ components: gormDB.Table("Blog").Create(blog3) gormDB.Table("Blog").Create(blog4) - t.Run("testing filters with the eq operator", func(t *testing.T) { + t.Run("testing a filter with the eq operator", func(t *testing.T) { page := 1 limit := 2 sortOptions := map[string]string{ @@ -2336,8 +2336,7 @@ components: Value: "hugs1", Values: nil, } - filters := []*rest.FilterProperties{filter} - //TODO figure out if u want a map or an array being passed in + filters := map[string]*rest.FilterProperties{filter.Field: filter} results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) if err != nil { t.Errorf("error getting content entities: %s", err) @@ -2345,26 +2344,215 @@ components: if results == nil || len(results) == 0 { t.Errorf("expected to get results but got nil") } - if total != int64(2) { + if total != int64(1) { t.Errorf("expected total to be %d got %d", int64(2), total) } + if results[0]["id"] != 1 { + t.Errorf("expected result id to be %d got %d", 1, results[0]["id"]) + } }) t.Run("testing filters with the ne operator", func(t *testing.T) { - + page := 1 + limit := 2 + sortOptions := map[string]string{ + "id": "asc", + } + ctxt := context.Background() + name := "Blog" + scheme := swagger.Components.Schemas[name] + ctxt = context.WithValue(ctxt, weosContext.CONTENT_TYPE, &weosContext.ContentType{ + Name: strings.Title(name), + Schema: scheme.Value, + }) + filter := &rest.FilterProperties{ + Field: "title", + Operator: "ne", + Value: "hugs1", + Values: nil, + } + filters := map[string]*rest.FilterProperties{filter.Field: filter} + results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) + if err != nil { + t.Errorf("error getting content entities: %s", err) + } + if results == nil || len(results) == 0 { + t.Errorf("expected to get results but got nil") + } + if total != int64(1) { + t.Errorf("expected total to be %d got %d", int64(2), total) + } + if len(results) != 4 { + t.Errorf("expected length of results to be %d got %d", 4, len(results)) + } }) t.Run("testing filters with the like operator", func(t *testing.T) { - + page := 1 + limit := 2 + sortOptions := map[string]string{ + "id": "asc", + } + ctxt := context.Background() + name := "Blog" + scheme := swagger.Components.Schemas[name] + ctxt = context.WithValue(ctxt, weosContext.CONTENT_TYPE, &weosContext.ContentType{ + Name: strings.Title(name), + Schema: scheme.Value, + }) + filter := &rest.FilterProperties{ + Field: "title", + Operator: "like", + Value: "morehugs", + Values: nil, + } + filters := map[string]*rest.FilterProperties{filter.Field: filter} + results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) + if err != nil { + t.Errorf("error getting content entities: %s", err) + } + if results == nil || len(results) == 0 { + t.Errorf("expected to get results but got nil") + } + if total != int64(1) { + t.Errorf("expected total to be %d got %d", int64(2), total) + } + if len(results) != 2 { + t.Errorf("expected length of results to be %d got %d", 2, len(results)) + } }) t.Run("testing filters with the in operator with a single value", func(t *testing.T) { - + page := 1 + limit := 2 + sortOptions := map[string]string{ + "id": "asc", + } + ctxt := context.Background() + name := "Blog" + scheme := swagger.Components.Schemas[name] + ctxt = context.WithValue(ctxt, weosContext.CONTENT_TYPE, &weosContext.ContentType{ + Name: strings.Title(name), + Schema: scheme.Value, + }) + filter := &rest.FilterProperties{ + Field: "title", + Operator: "in", + Value: "hugs2", + Values: nil, + } + filters := map[string]*rest.FilterProperties{filter.Field: filter} + results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) + if err != nil { + t.Errorf("error getting content entities: %s", err) + } + if results == nil || len(results) == 0 { + t.Errorf("expected to get results but got nil") + } + if total != int64(1) { + t.Errorf("expected total to be %d got %d", int64(2), total) + } + if len(results) != 1 { + t.Errorf("expected length of results to be %d got %d", 1, len(results)) + } }) - t.Run("testing filters with the eq operator with multiple values", func(t *testing.T) { - + t.Run("testing filters with the in operator with multiple values", func(t *testing.T) { + page := 1 + limit := 2 + sortOptions := map[string]string{ + "id": "asc", + } + ctxt := context.Background() + name := "Blog" + scheme := swagger.Components.Schemas[name] + ctxt = context.WithValue(ctxt, weosContext.CONTENT_TYPE, &weosContext.ContentType{ + Name: strings.Title(name), + Schema: scheme.Value, + }) + arrValues := []string{"hugs1", "hugs3"} + filter := &rest.FilterProperties{ + Field: "title", + Operator: "ne", + Values: arrValues, + } + filters := map[string]*rest.FilterProperties{filter.Field: filter} + results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) + if err != nil { + t.Errorf("error getting content entities: %s", err) + } + if results == nil || len(results) == 0 { + t.Errorf("expected to get results but got nil") + } + if total != int64(1) { + t.Errorf("expected total to be %d got %d", int64(2), total) + } + if len(results) != 2 { + t.Errorf("expected length of results to be %d got %d", 2, len(results)) + } }) t.Run("testing filters with the lt operator", func(t *testing.T) { - + page := 1 + limit := 2 + sortOptions := map[string]string{ + "id": "asc", + } + ctxt := context.Background() + name := "Blog" + scheme := swagger.Components.Schemas[name] + ctxt = context.WithValue(ctxt, weosContext.CONTENT_TYPE, &weosContext.ContentType{ + Name: strings.Title(name), + Schema: scheme.Value, + }) + filter := &rest.FilterProperties{ + Field: "id", + Operator: "lt", + Value: "2", + Values: nil, + } + filters := map[string]*rest.FilterProperties{filter.Field: filter} + results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) + if err != nil { + t.Errorf("error getting content entities: %s", err) + } + if results == nil || len(results) == 0 { + t.Errorf("expected to get results but got nil") + } + if total != int64(1) { + t.Errorf("expected total to be %d got %d", int64(2), total) + } + if len(results) != 1 { + t.Errorf("expected length of results to be %d got %d", 1, len(results)) + } }) t.Run("testing filters with the gt operator", func(t *testing.T) { - + page := 1 + limit := 2 + sortOptions := map[string]string{ + "id": "asc", + } + ctxt := context.Background() + name := "Blog" + scheme := swagger.Components.Schemas[name] + ctxt = context.WithValue(ctxt, weosContext.CONTENT_TYPE, &weosContext.ContentType{ + Name: strings.Title(name), + Schema: scheme.Value, + }) + filter := &rest.FilterProperties{ + Field: "id", + Operator: "gt", + Value: "3", + Values: nil, + } + filters := map[string]*rest.FilterProperties{filter.Field: filter} + results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) + if err != nil { + t.Errorf("error getting content entities: %s", err) + } + if results == nil || len(results) == 0 { + t.Errorf("expected to get results but got nil") + } + if total != int64(1) { + t.Errorf("expected total to be %d got %d", int64(2), total) + } + if len(results) != 2 { + t.Errorf("expected length of results to be %d got %d", 2, len(results)) + } }) } From 273045e7eb92fba5a025ab5e01dba249dc665389 Mon Sep 17 00:00:00 2001 From: shaniah868 Date: Fri, 28 Jan 2022 07:27:25 -0400 Subject: [PATCH 03/11] feature: WEOS-1134 Update list method on projection to accept filter map -Added functionality to convert the filters into the types that are in the db --- projections/gorm.go | 52 ++++++++++++++++++++++----------- projections/projections_test.go | 30 +++++++++---------- 2 files changed, 50 insertions(+), 32 deletions(-) diff --git a/projections/gorm.go b/projections/gorm.go index 3212fb37..e245d88b 100644 --- a/projections/gorm.go +++ b/projections/gorm.go @@ -3,7 +3,7 @@ package projections import ( "encoding/json" "fmt" - "github.com/wepala/weos/controllers/rest" + "strconv" "strings" ds "github.com/ompluscator/dynamic-struct" @@ -22,7 +22,8 @@ type GORMProjection struct { Schema map[string]ds.Builder } -type filterProperty struct { +type FilterProperty struct { + Field string `json:"field"` Operator string `json:"operator"` Value string `json:"value"` Values []string `json:"values"` @@ -239,15 +240,19 @@ func (p *GORMProjection) GetContentEntities(ctx context.Context, page int, limit var count int64 var result *gorm.DB var schemes interface{} + var filtersProp map[string]FilterProperty + props, _ := json.Marshal(filterOptions) + json.Unmarshal(props, &filtersProp) contentType := weosContext.GetContentType(ctx) - prop := p.convertProperties(filterOptions, p.Schema, contentType) - if prop != nil { - } if s, ok := p.Schema[strings.Title(contentType.Name)]; ok { schemes = s.Build().NewSliceOfStructs() scheme := s.Build().New() - + if len(filterOptions) != 0 { + filters := p.convertProperties(filtersProp, contentType) + queryString := filterStringBuilder(p.db.Dialector.Name(), contentType, filtersProp) + result = p.db.Table(contentType.Name).Scopes(FilterQuery(queryString)).Model(&scheme).Omit("weos_id, sequence_no, table").Count(&count).Scopes(paginate(page, limit), sort(sortOptions), filter(filters)).Find(schemes) + } result = p.db.Table(contentType.Name).Scopes(ContentQuery()).Model(&scheme).Omit("weos_id, sequence_no, table").Count(&count).Scopes(paginate(page, limit), sort(sortOptions)).Find(schemes) } bytes, err := json.Marshal(schemes) @@ -300,18 +305,26 @@ func filter(filter map[string]interface{}) func(db *gorm.DB) *gorm.DB { } //filterStringBuilder is used to build the query strings -func filterStringBuilder(dbName string, contentType *weosContext.ContentType, properties *rest.FilterProperties) string { +func filterStringBuilder(dbName string, contentType *weosContext.ContentType, properties map[string]FilterProperty) string { var query string - switch properties.Operator { - case "eq": - query = contentType.Name + "." + properties.Field + " = " + properties.Value + for _, prop := range properties { + + switch prop.Operator { + case "eq": + if query != "" { + query += " AND " + prop.Field + " = '" + prop.Value + "'" + } else { + query += prop.Field + " = '" + prop.Value + "'" + } + } } + return query } //convertProperties is used to convert the filter properties into key (field) , value pairs -func (p *GORMProjection) convertProperties(properties map[string]interface{}, schema map[string]ds.Builder, contentType *weosContext.ContentType) map[string]interface{} { +func (p *GORMProjection) convertProperties(properties map[string]FilterProperty, contentType *weosContext.ContentType) map[string]interface{} { filters := map[string]interface{}{} if properties == nil { @@ -319,21 +332,27 @@ func (p *GORMProjection) convertProperties(properties map[string]interface{}, sc } for field, filterProperty := range properties { - if filterProperty.Value != "" && (filterProperty == nil || len(filterProperty.Values) == 0) { + if filterProperty.Value != "" && (filterProperty.Values == nil || len(filterProperty.Values) == 0) { columns, _ := p.db.Migrator().ColumnTypes(contentType.Name) - for _, c := range columns { if c.Name() == field { cType := c.DatabaseTypeName() - if cType == "string" { + switch cType { + case "text": + filters[field] = filterProperty.Value + case "INTEGER": + case "integer": + v, err := strconv.Atoi(filterProperty.Value) + if err == nil { + filters[field] = v + } } } } - filters[field] = filterProperty.Value } - if filterProperty.Value == "" && (filterProperty != nil && len(filterProperty.Values) > 0) { + if filterProperty.Value == "" && len(filterProperty.Values) > 0 { filters[field] = filterProperty.Values } @@ -362,7 +381,6 @@ func NewProjection(ctx context.Context, application weos.Service, schemas map[st FilterQuery = func(query string) func(db *gorm.DB) *gorm.DB { return func(db *gorm.DB) *gorm.DB { if query != "" { - return db.Where(query, func(tx *gorm.DB) *gorm.DB { return tx.Omit("weos_id, sequence_no, table") }) } return db diff --git a/projections/projections_test.go b/projections/projections_test.go index 6c1df028..28e49e3b 100644 --- a/projections/projections_test.go +++ b/projections/projections_test.go @@ -2222,7 +2222,7 @@ components: }) } -func TestProjections_Filters(t *testing.T) { +func TestProjections_ListFilters(t *testing.T) { openAPI := `openapi: 3.0.3 info: title: Blog @@ -2335,13 +2335,13 @@ components: Name: strings.Title(name), Schema: scheme.Value, }) - filter := &rest.FilterProperties{ + filter := &projections.FilterProperty{ Field: "title", Operator: "eq", Value: "hugs1", Values: nil, } - filters := map[string]*rest.FilterProperties{filter.Field: filter} + filters := map[string]interface{}{filter.Field: filter} results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) if err != nil { t.Errorf("error getting content entities: %s", err) @@ -2369,13 +2369,13 @@ components: Name: strings.Title(name), Schema: scheme.Value, }) - filter := &rest.FilterProperties{ + filter := &projections.FilterProperty{ Field: "title", Operator: "ne", Value: "hugs1", Values: nil, } - filters := map[string]*rest.FilterProperties{filter.Field: filter} + filters := map[string]interface{}{filter.Field: filter} results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) if err != nil { t.Errorf("error getting content entities: %s", err) @@ -2403,13 +2403,13 @@ components: Name: strings.Title(name), Schema: scheme.Value, }) - filter := &rest.FilterProperties{ + filter := &projections.FilterProperty{ Field: "title", Operator: "like", Value: "morehugs", Values: nil, } - filters := map[string]*rest.FilterProperties{filter.Field: filter} + filters := map[string]interface{}{filter.Field: filter} results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) if err != nil { t.Errorf("error getting content entities: %s", err) @@ -2437,13 +2437,13 @@ components: Name: strings.Title(name), Schema: scheme.Value, }) - filter := &rest.FilterProperties{ + filter := &projections.FilterProperty{ Field: "title", Operator: "in", Value: "hugs2", Values: nil, } - filters := map[string]*rest.FilterProperties{filter.Field: filter} + filters := map[string]interface{}{filter.Field: filter} results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) if err != nil { t.Errorf("error getting content entities: %s", err) @@ -2472,12 +2472,12 @@ components: Schema: scheme.Value, }) arrValues := []string{"hugs1", "hugs3"} - filter := &rest.FilterProperties{ + filter := &projections.FilterProperty{ Field: "title", Operator: "ne", Values: arrValues, } - filters := map[string]*rest.FilterProperties{filter.Field: filter} + filters := map[string]interface{}{filter.Field: filter} results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) if err != nil { t.Errorf("error getting content entities: %s", err) @@ -2505,13 +2505,13 @@ components: Name: strings.Title(name), Schema: scheme.Value, }) - filter := &rest.FilterProperties{ + filter := &projections.FilterProperty{ Field: "id", Operator: "lt", Value: "2", Values: nil, } - filters := map[string]*rest.FilterProperties{filter.Field: filter} + filters := map[string]interface{}{filter.Field: filter} results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) if err != nil { t.Errorf("error getting content entities: %s", err) @@ -2539,13 +2539,13 @@ components: Name: strings.Title(name), Schema: scheme.Value, }) - filter := &rest.FilterProperties{ + filter := &projections.FilterProperty{ Field: "id", Operator: "gt", Value: "3", Values: nil, } - filters := map[string]*rest.FilterProperties{filter.Field: filter} + filters := map[string]interface{}{filter.Field: filter} results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) if err != nil { t.Errorf("error getting content entities: %s", err) From 713ef03b216f9b917e41310dda0b86b441eaefe2 Mon Sep 17 00:00:00 2001 From: shaniah868 Date: Fri, 28 Jan 2022 07:54:02 -0400 Subject: [PATCH 04/11] feature: WEOS-1134 Update list method on projection to accept filter map -Fixing projection test --- projections/projections_test.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/projections/projections_test.go b/projections/projections_test.go index 28e49e3b..6269d7a7 100644 --- a/projections/projections_test.go +++ b/projections/projections_test.go @@ -2324,7 +2324,7 @@ components: t.Run("testing a filter with the eq operator", func(t *testing.T) { page := 1 - limit := 2 + limit := 0 sortOptions := map[string]string{ "id": "asc", } @@ -2350,7 +2350,7 @@ components: t.Errorf("expected to get results but got nil") } if total != int64(1) { - t.Errorf("expected total to be %d got %d", int64(2), total) + t.Errorf("expected total to be %d got %d", int64(1), total) } if results[0]["id"] != 1 { t.Errorf("expected result id to be %d got %d", 1, results[0]["id"]) @@ -2384,10 +2384,10 @@ components: t.Errorf("expected to get results but got nil") } if total != int64(1) { - t.Errorf("expected total to be %d got %d", int64(2), total) + t.Errorf("expected total to be %d got %d", int64(1), total) } if len(results) != 4 { - t.Errorf("expected length of results to be %d got %d", 4, len(results)) + t.Errorf("expected length of results to be %d got %d", 4, len(results)) } }) t.Run("testing filters with the like operator", func(t *testing.T) { @@ -2418,7 +2418,7 @@ components: t.Errorf("expected to get results but got nil") } if total != int64(1) { - t.Errorf("expected total to be %d got %d", int64(2), total) + t.Errorf("expected total to be %d got %d", int64(1), total) } if len(results) != 2 { t.Errorf("expected length of results to be %d got %d", 2, len(results)) @@ -2426,7 +2426,7 @@ components: }) t.Run("testing filters with the in operator with a single value", func(t *testing.T) { page := 1 - limit := 2 + limit := 0 sortOptions := map[string]string{ "id": "asc", } @@ -2452,7 +2452,7 @@ components: t.Errorf("expected to get results but got nil") } if total != int64(1) { - t.Errorf("expected total to be %d got %d", int64(2), total) + t.Errorf("expected total to be %d got %d", int64(1), total) } if len(results) != 1 { t.Errorf("expected length of results to be %d got %d", 1, len(results)) @@ -2460,7 +2460,7 @@ components: }) t.Run("testing filters with the in operator with multiple values", func(t *testing.T) { page := 1 - limit := 2 + limit := 0 sortOptions := map[string]string{ "id": "asc", } @@ -2474,7 +2474,7 @@ components: arrValues := []string{"hugs1", "hugs3"} filter := &projections.FilterProperty{ Field: "title", - Operator: "ne", + Operator: "in", Values: arrValues, } filters := map[string]interface{}{filter.Field: filter} @@ -2485,7 +2485,7 @@ components: if results == nil || len(results) == 0 { t.Errorf("expected to get results but got nil") } - if total != int64(1) { + if total != int64(2) { t.Errorf("expected total to be %d got %d", int64(2), total) } if len(results) != 2 { @@ -2494,7 +2494,7 @@ components: }) t.Run("testing filters with the lt operator", func(t *testing.T) { page := 1 - limit := 2 + limit := 0 sortOptions := map[string]string{ "id": "asc", } @@ -2520,7 +2520,7 @@ components: t.Errorf("expected to get results but got nil") } if total != int64(1) { - t.Errorf("expected total to be %d got %d", int64(2), total) + t.Errorf("expected total to be %d got %d", int64(1), total) } if len(results) != 1 { t.Errorf("expected length of results to be %d got %d", 1, len(results)) @@ -2528,7 +2528,7 @@ components: }) t.Run("testing filters with the gt operator", func(t *testing.T) { page := 1 - limit := 2 + limit := 0 sortOptions := map[string]string{ "id": "asc", } @@ -2553,7 +2553,7 @@ components: if results == nil || len(results) == 0 { t.Errorf("expected to get results but got nil") } - if total != int64(1) { + if total != int64(2) { t.Errorf("expected total to be %d got %d", int64(2), total) } if len(results) != 2 { From 2909d29c754fffb5e40b4d2eb6acc16ff67cd48a Mon Sep 17 00:00:00 2001 From: shaniah868 Date: Fri, 28 Jan 2022 09:41:51 -0400 Subject: [PATCH 05/11] feature: WEOS-1134 Update List Controller to call the projection with the filter map -Fix test --- projections/gorm.go | 5 +++-- projections/projections_test.go | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/projections/gorm.go b/projections/gorm.go index e245d88b..a80d3fe7 100644 --- a/projections/gorm.go +++ b/projections/gorm.go @@ -252,8 +252,9 @@ func (p *GORMProjection) GetContentEntities(ctx context.Context, page int, limit filters := p.convertProperties(filtersProp, contentType) queryString := filterStringBuilder(p.db.Dialector.Name(), contentType, filtersProp) result = p.db.Table(contentType.Name).Scopes(FilterQuery(queryString)).Model(&scheme).Omit("weos_id, sequence_no, table").Count(&count).Scopes(paginate(page, limit), sort(sortOptions), filter(filters)).Find(schemes) + } else { + result = p.db.Table(contentType.Name).Scopes(ContentQuery()).Model(&scheme).Omit("weos_id, sequence_no, table").Count(&count).Scopes(paginate(page, limit), sort(sortOptions)).Find(schemes) } - result = p.db.Table(contentType.Name).Scopes(ContentQuery()).Model(&scheme).Omit("weos_id, sequence_no, table").Count(&count).Scopes(paginate(page, limit), sort(sortOptions)).Find(schemes) } bytes, err := json.Marshal(schemes) if err != nil { @@ -381,7 +382,7 @@ func NewProjection(ctx context.Context, application weos.Service, schemas map[st FilterQuery = func(query string) func(db *gorm.DB) *gorm.DB { return func(db *gorm.DB) *gorm.DB { if query != "" { - return db.Where(query, func(tx *gorm.DB) *gorm.DB { return tx.Omit("weos_id, sequence_no, table") }) + return db.Where(query).Omit("weos_id, sequence_no, table") } return db } diff --git a/projections/projections_test.go b/projections/projections_test.go index 6269d7a7..db8e192a 100644 --- a/projections/projections_test.go +++ b/projections/projections_test.go @@ -2352,8 +2352,8 @@ components: if total != int64(1) { t.Errorf("expected total to be %d got %d", int64(1), total) } - if results[0]["id"] != 1 { - t.Errorf("expected result id to be %d got %d", 1, results[0]["id"]) + if int(results[0]["id"].(float64)) != 1 { + t.Errorf("expected result id to be %d got %d", 1, int(results[0]["id"].(float64))) } }) t.Run("testing filters with the ne operator", func(t *testing.T) { From d72060bfd46e1e63f0ee5c7ed48de774470c599b Mon Sep 17 00:00:00 2001 From: shaniah868 Date: Sun, 30 Jan 2022 13:42:00 -0400 Subject: [PATCH 06/11] feature: WEOS-1134 Update list method on projection to accept filter map -Fix test for all the filters that is supported -Added a test for using mixed filters --- projections/gorm.go | 96 +++++++++++++++------------------ projections/projections_test.go | 86 ++++++++++++++++++++++------- 2 files changed, 111 insertions(+), 71 deletions(-) diff --git a/projections/gorm.go b/projections/gorm.go index a80d3fe7..16cfe42b 100644 --- a/projections/gorm.go +++ b/projections/gorm.go @@ -3,7 +3,6 @@ package projections import ( "encoding/json" "fmt" - "strconv" "strings" ds "github.com/ompluscator/dynamic-struct" @@ -249,9 +248,8 @@ func (p *GORMProjection) GetContentEntities(ctx context.Context, page int, limit schemes = s.Build().NewSliceOfStructs() scheme := s.Build().New() if len(filterOptions) != 0 { - filters := p.convertProperties(filtersProp, contentType) - queryString := filterStringBuilder(p.db.Dialector.Name(), contentType, filtersProp) - result = p.db.Table(contentType.Name).Scopes(FilterQuery(queryString)).Model(&scheme).Omit("weos_id, sequence_no, table").Count(&count).Scopes(paginate(page, limit), sort(sortOptions), filter(filters)).Find(schemes) + queryString := filterStringBuilder(p.db.Dialector.Name(), filtersProp) + result = p.db.Table(contentType.Name).Scopes(FilterQuery(queryString)).Model(&scheme).Omit("weos_id, sequence_no, table").Count(&count).Scopes(paginate(page, limit), sort(sortOptions)).Find(schemes) } else { result = p.db.Table(contentType.Name).Scopes(ContentQuery()).Model(&scheme).Omit("weos_id, sequence_no, table").Count(&count).Scopes(paginate(page, limit), sort(sortOptions)).Find(schemes) } @@ -295,18 +293,8 @@ func sort(order map[string]string) func(db *gorm.DB) *gorm.DB { } } -// filter that filters query results -func filter(filter map[string]interface{}) func(db *gorm.DB) *gorm.DB { - return func(db *gorm.DB) *gorm.DB { - if filter != nil { - return db.Where(filter) - } - return db - } -} - //filterStringBuilder is used to build the query strings -func filterStringBuilder(dbName string, contentType *weosContext.ContentType, properties map[string]FilterProperty) string { +func filterStringBuilder(dbName string, properties map[string]FilterProperty) string { var query string for _, prop := range properties { @@ -317,49 +305,51 @@ func filterStringBuilder(dbName string, contentType *weosContext.ContentType, pr } else { query += prop.Field + " = '" + prop.Value + "'" } - - } - } - - return query -} - -//convertProperties is used to convert the filter properties into key (field) , value pairs -func (p *GORMProjection) convertProperties(properties map[string]FilterProperty, contentType *weosContext.ContentType) map[string]interface{} { - filters := map[string]interface{}{} - - if properties == nil { - return nil - } - - for field, filterProperty := range properties { - if filterProperty.Value != "" && (filterProperty.Values == nil || len(filterProperty.Values) == 0) { - columns, _ := p.db.Migrator().ColumnTypes(contentType.Name) - for _, c := range columns { - if c.Name() == field { - cType := c.DatabaseTypeName() - switch cType { - case "text": - filters[field] = filterProperty.Value - case "INTEGER": - case "integer": - v, err := strconv.Atoi(filterProperty.Value) - if err == nil { - filters[field] = v - } - - } + case "ne": + if query != "" { + query += " AND " + prop.Field + " != '" + prop.Value + "'" + } else { + query += prop.Field + " != '" + prop.Value + "'" + } + case "like": + if dbName == "postgres" { + if query != "" { + query += " AND " + prop.Field + " ILIKE '%" + prop.Value + "%'" + } else { + query += prop.Field + " ILIKE '%" + prop.Value + "%'" + } + } else { + if query != "" { + query += " AND " + prop.Field + " LIKE '%" + prop.Value + "%'" + } else { + query += prop.Field + " LIKE '%" + prop.Value + "%'" } - + } + case "in": + vals := "'" + if query != "" { + vals += strings.Join(prop.Values, "','") + "'" + query += " AND " + prop.Field + " in '(" + vals + ")'" + } else { + vals += strings.Join(prop.Values, "','") + "'" + query += prop.Field + " in (" + vals + ")" + } + case "lt": + if query != "" { + query += " AND " + prop.Field + " < '" + prop.Value + "'" + } else { + query += prop.Field + " < '" + prop.Value + "'" + } + case "gt": + if query != "" { + query += " AND " + prop.Field + " > '" + prop.Value + "'" + } else { + query += prop.Field + " > '" + prop.Value + "'" } } - if filterProperty.Value == "" && len(filterProperty.Values) > 0 { - filters[field] = filterProperty.Values - } - } - return filters + return query } //query modifier for making queries to the database diff --git a/projections/projections_test.go b/projections/projections_test.go index db8e192a..699dce72 100644 --- a/projections/projections_test.go +++ b/projections/projections_test.go @@ -2175,7 +2175,10 @@ components: if found != limit { t.Errorf("expected to find %d blogs got %d", limit, found) } - + err = gormDB.Migrator().DropTable("Blog") + if err != nil { + t.Errorf("error removing table '%s' '%s'", "Blog", err) + } }) t.Run("get a basic list with the foreign key returned", func(t *testing.T) { @@ -2222,6 +2225,7 @@ components: }) } + func TestProjections_ListFilters(t *testing.T) { openAPI := `openapi: 3.0.3 info: @@ -2310,11 +2314,11 @@ components: blogWeosID3 := "abc123456" blogWeosID4 := "abc1234567" - blog := map[string]interface{}{"weos_id": blogWeosID, "title": "hugs1", "sequence_no": int64(1)} - blog1 := map[string]interface{}{"weos_id": blogWeosID1, "title": "hugs2", "sequence_no": int64(1)} - blog2 := map[string]interface{}{"weos_id": blogWeosID2, "title": "hugs3", "sequence_no": int64(1)} + blog := map[string]interface{}{"weos_id": blogWeosID, "title": "hugs1", "description": "first blog", "sequence_no": int64(1)} + blog1 := map[string]interface{}{"weos_id": blogWeosID1, "title": "hugs2", "description": "first blog", "sequence_no": int64(1)} + blog2 := map[string]interface{}{"weos_id": blogWeosID2, "title": "hugs3", "description": "third blog", "sequence_no": int64(1)} blog3 := map[string]interface{}{"weos_id": blogWeosID3, "title": "morehugs4", "sequence_no": int64(1)} - blog4 := map[string]interface{}{"weos_id": blogWeosID4, "title": "morehugs5", "sequence_no": int64(1)} + blog4 := map[string]interface{}{"weos_id": blogWeosID4, "id": uint(123), "title": "morehugs5", "description": "last blog", "sequence_no": int64(1)} gormDB.Table("Blog").Create(blog) gormDB.Table("Blog").Create(blog1) @@ -2322,7 +2326,7 @@ components: gormDB.Table("Blog").Create(blog3) gormDB.Table("Blog").Create(blog4) - t.Run("testing a filter with the eq operator", func(t *testing.T) { + t.Run("testing filter with the eq operator on 2 fields", func(t *testing.T) { page := 1 limit := 0 sortOptions := map[string]string{ @@ -2341,7 +2345,13 @@ components: Value: "hugs1", Values: nil, } - filters := map[string]interface{}{filter.Field: filter} + filter2 := &projections.FilterProperty{ + Field: "description", + Operator: "eq", + Value: "first blog", + Values: nil, + } + filters := map[string]interface{}{filter.Field: filter, filter2.Field: filter2} results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) if err != nil { t.Errorf("error getting content entities: %s", err) @@ -2358,7 +2368,7 @@ components: }) t.Run("testing filters with the ne operator", func(t *testing.T) { page := 1 - limit := 2 + limit := 0 sortOptions := map[string]string{ "id": "asc", } @@ -2370,9 +2380,9 @@ components: Schema: scheme.Value, }) filter := &projections.FilterProperty{ - Field: "title", + Field: "id", Operator: "ne", - Value: "hugs1", + Value: "1", Values: nil, } filters := map[string]interface{}{filter.Field: filter} @@ -2383,8 +2393,8 @@ components: if results == nil || len(results) == 0 { t.Errorf("expected to get results but got nil") } - if total != int64(1) { - t.Errorf("expected total to be %d got %d", int64(1), total) + if total != int64(4) { + t.Errorf("expected total to be %d got %d", int64(4), total) } if len(results) != 4 { t.Errorf("expected length of results to be %d got %d", 4, len(results)) @@ -2404,9 +2414,9 @@ components: Schema: scheme.Value, }) filter := &projections.FilterProperty{ - Field: "title", + Field: "id", Operator: "like", - Value: "morehugs", + Value: "1", Values: nil, } filters := map[string]interface{}{filter.Field: filter} @@ -2417,8 +2427,8 @@ components: if results == nil || len(results) == 0 { t.Errorf("expected to get results but got nil") } - if total != int64(1) { - t.Errorf("expected total to be %d got %d", int64(1), total) + if total != int64(2) { + t.Errorf("expected total to be %d got %d", int64(2), total) } if len(results) != 2 { t.Errorf("expected length of results to be %d got %d", 2, len(results)) @@ -2437,11 +2447,11 @@ components: Name: strings.Title(name), Schema: scheme.Value, }) + vals := []string{"hugs2"} filter := &projections.FilterProperty{ Field: "title", Operator: "in", - Value: "hugs2", - Values: nil, + Values: vals, } filters := map[string]interface{}{filter.Field: filter} results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) @@ -2560,4 +2570,44 @@ components: t.Errorf("expected length of results to be %d got %d", 2, len(results)) } }) + t.Run("testing filters with the multiple operators", func(t *testing.T) { + page := 1 + limit := 0 + sortOptions := map[string]string{ + "id": "asc", + } + ctxt := context.Background() + name := "Blog" + scheme := swagger.Components.Schemas[name] + ctxt = context.WithValue(ctxt, weosContext.CONTENT_TYPE, &weosContext.ContentType{ + Name: strings.Title(name), + Schema: scheme.Value, + }) + filter := &projections.FilterProperty{ + Field: "id", + Operator: "like", + Value: "1", + Values: nil, + } + filter2 := &projections.FilterProperty{ + Field: "title", + Operator: "ne", + Value: "hugs1", + Values: nil, + } + filters := map[string]interface{}{filter.Field: filter, filter2.Field: filter2} + results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) + if err != nil { + t.Errorf("error getting content entities: %s", err) + } + if results == nil || len(results) == 0 { + t.Errorf("expected to get results but got nil") + } + if total != int64(1) { + t.Errorf("expected total to be %d got %d", int64(1), total) + } + if len(results) != 1 { + t.Errorf("expected length of results to be %d got %d", 1, len(results)) + } + }) } From 6468fbc744a0309aca3a7b8c708ee6e637854f5f Mon Sep 17 00:00:00 2001 From: shaniah868 Date: Wed, 9 Feb 2022 14:04:49 -0400 Subject: [PATCH 07/11] feature: WEOS-1134 Update list method on projection to accept filter map -Made changes requested on pr -Updated test --- projections/gorm.go | 91 ++----- projections/projections_test.go | 423 +------------------------------- 2 files changed, 24 insertions(+), 490 deletions(-) diff --git a/projections/gorm.go b/projections/gorm.go index f351b334..41174ec2 100644 --- a/projections/gorm.go +++ b/projections/gorm.go @@ -294,65 +294,6 @@ func sort(order map[string]string) func(db *gorm.DB) *gorm.DB { } } -//filterStringBuilder is used to build the query strings -func filterStringBuilder(dbName string, properties map[string]FilterProperty) string { - var query string - for _, prop := range properties { - - switch prop.Operator { - case "eq": - if query != "" { - query += " AND " + prop.Field + " = '" + prop.Value + "'" - } else { - query += prop.Field + " = '" + prop.Value + "'" - } - case "ne": - if query != "" { - query += " AND " + prop.Field + " != '" + prop.Value + "'" - } else { - query += prop.Field + " != '" + prop.Value + "'" - } - case "like": - if dbName == "postgres" { - if query != "" { - query += " AND " + prop.Field + " ILIKE '%" + prop.Value + "%'" - } else { - query += prop.Field + " ILIKE '%" + prop.Value + "%'" - } - } else { - if query != "" { - query += " AND " + prop.Field + " LIKE '%" + prop.Value + "%'" - } else { - query += prop.Field + " LIKE '%" + prop.Value + "%'" - } - } - case "in": - vals := "'" - if query != "" { - vals += strings.Join(prop.Values, "','") + "'" - query += " AND " + prop.Field + " in '(" + vals + ")'" - } else { - vals += strings.Join(prop.Values, "','") + "'" - query += prop.Field + " in (" + vals + ")" - } - case "lt": - if query != "" { - query += " AND " + prop.Field + " < '" + prop.Value + "'" - } else { - query += prop.Field + " < '" + prop.Value + "'" - } - case "gt": - if query != "" { - query += " AND " + prop.Field + " > '" + prop.Value + "'" - } else { - query += prop.Field + " > '" + prop.Value + "'" - } - } - } - - return query -} - //query modifier for making queries to the database type QueryModifier func() func(db *gorm.DB) *gorm.DB type QueryFilterModifier func(options map[string]FilterProperty) func(db *gorm.DB) *gorm.DB @@ -376,18 +317,33 @@ func NewProjection(ctx context.Context, db *gorm.DB, logger weos.Log) (*GORMProj case "gt": operator = ">" case "lt": + operator = "<" case "ne": + operator = "!=" case "like": - //TODO check db + if projection.db.Dialector.Name() == "postgres" { + operator = "ILIKE" + } else { + operator = " LIKE" + } case "in": + operator = "IN" } if len(filter.Values) == 0 { - db.Where(filter.Field+operator+"?", filter.Value) + if filter.Operator == "like" { + db.Where(filter.Field+" "+operator+"?", "%"+filter.Value.(string)+"%") + } else { + db.Where(filter.Field+" "+operator+"?", filter.Value) + } + } else { - // TODO add a for loop to generate the number of ? needed - db.Where(filter.Field+operator+"?", filter.Values...) + //queryString := "" + //for i := 0; i < len(filter.Values); i++ { + // queryString += " ?" + //} + db.Where(filter.Field+" "+operator+" ?", filter.Values) } } @@ -395,15 +351,6 @@ func NewProjection(ctx context.Context, db *gorm.DB, logger weos.Log) (*GORMProj } } - FilterQuery = func(query string) func(db *gorm.DB) *gorm.DB { - return func(db *gorm.DB) *gorm.DB { - if query != "" { - return db.Where(query).Omit("weos_id, sequence_no, table") - } - return db - } - } - ContentQuery = func() func(db *gorm.DB) *gorm.DB { return func(db *gorm.DB) *gorm.DB { if projection.db.Dialector.Name() == "sqlite" { diff --git a/projections/projections_test.go b/projections/projections_test.go index 60035831..55c64c63 100644 --- a/projections/projections_test.go +++ b/projections/projections_test.go @@ -2354,7 +2354,7 @@ components: filter := &projections.FilterProperty{ Field: "id", Operator: "ne", - Value: "1", + Value: uint(1), Values: nil, } filters := map[string]interface{}{filter.Field: filter} @@ -2407,7 +2407,7 @@ components: "id": "asc", } ctxt := context.Background() - vals := []string{"hugs2"} + vals := []interface{}{"hugs2"} filter := &projections.FilterProperty{ Field: "title", Operator: "in", @@ -2435,7 +2435,7 @@ components: "id": "asc", } ctxt := context.Background() - arrValues := []string{"hugs1", "hugs3"} + arrValues := []interface{}{"hugs1", "hugs3"} filter := &projections.FilterProperty{ Field: "title", Operator: "in", @@ -2466,7 +2466,7 @@ components: filter := &projections.FilterProperty{ Field: "id", Operator: "lt", - Value: "2", + Value: uint(2), Values: nil, } filters := map[string]interface{}{filter.Field: filter} @@ -2494,7 +2494,7 @@ components: filter := &projections.FilterProperty{ Field: "id", Operator: "gt", - Value: "3", + Value: uint(3), Values: nil, } filters := map[string]interface{}{filter.Field: filter} @@ -2614,33 +2614,6 @@ components: t.Errorf("expected length of results to be %d got %d", 1, len(results)) } }) - t.Run("testing invalid date time format on filter ", func(t *testing.T) { - page := 1 - limit := 0 - sortOptions := map[string]string{ - "id": "asc", - } - ctxt := context.Background() - filter := &projections.FilterProperty{ - Field: "last_updated", - Operator: "lt", - Value: "2006-01-02T15:04:00Z+dsujhsd", - Values: nil, - } - - filters := map[string]interface{}{filter.Field: filter} - results, total, err := p.GetContentEntities(ctxt, blogEntityFactory, page, limit, "", sortOptions, filters) - if err == nil { - t.Fatalf("expected a date time error but got nil") - } - if results != nil { - t.Errorf("unexpect error expected results to be nil ") - } - if total != int64(0) { - t.Errorf("expecter total to be 0 got %d", total) - } - - }) } func TestProjections_Delete(t *testing.T) { @@ -2750,389 +2723,3 @@ components: } }) } - -func TestProjections_ListFilters(t *testing.T) { - openAPI := `openapi: 3.0.3 -info: - title: Blog - description: Blog example - version: 1.0.0 -servers: - - url: https://prod1.weos.sh/blog/dev - description: WeOS Dev - - url: https://prod1.weos.sh/blog/v1 -x-weos-config: - logger: - level: warn - report-caller: true - formatter: json - database: - driver: sqlite3 - database: test.db - event-source: - - title: default - driver: service - endpoint: https://prod1.weos.sh/events/v1 - - title: event - driver: sqlite3 - database: test.db - databases: - - title: default - driver: sqlite3 - database: test.db - rest: - middleware: - - RequestID - - Recover - - ZapLogger -components: - schemas: - Blog: - type: object - properties: - title: - type: string - description: blog title - description: - type: string - required: - - title - Post: - type: object - properties: - title: - type: string - description: post title - description: - type: string - blog: - $ref: "#/components/schemas/Blog" -` - loader := openapi3.NewSwaggerLoader() - swagger, err := loader.LoadSwaggerFromData([]byte(openAPI)) - if err != nil { - t.Fatal(err) - } - - schemes := rest.CreateSchema(context.Background(), echo.New(), swagger) - p, err := projections.NewProjection(context.Background(), app, schemes) - if err != nil { - t.Fatal(err) - } - - err = p.Migrate(context.Background()) - if err != nil { - t.Fatal(err) - } - - gormDB := app.DB() - if !gormDB.Migrator().HasTable("Blog") { - t.Fatal("expected to get a table 'Blog'") - } - - if !gormDB.Migrator().HasTable("Post") { - t.Fatal("expected to get a table 'Post'") - } - blogWeosID := "abc123" - blogWeosID1 := "abc1234" - blogWeosID2 := "abc12345" - blogWeosID3 := "abc123456" - blogWeosID4 := "abc1234567" - - blog := map[string]interface{}{"weos_id": blogWeosID, "title": "hugs1", "description": "first blog", "sequence_no": int64(1)} - blog1 := map[string]interface{}{"weos_id": blogWeosID1, "title": "hugs2", "description": "first blog", "sequence_no": int64(1)} - blog2 := map[string]interface{}{"weos_id": blogWeosID2, "title": "hugs3", "description": "third blog", "sequence_no": int64(1)} - blog3 := map[string]interface{}{"weos_id": blogWeosID3, "title": "morehugs4", "sequence_no": int64(1)} - blog4 := map[string]interface{}{"weos_id": blogWeosID4, "id": uint(123), "title": "morehugs5", "description": "last blog", "sequence_no": int64(1)} - - gormDB.Table("Blog").Create(blog) - gormDB.Table("Blog").Create(blog1) - gormDB.Table("Blog").Create(blog2) - gormDB.Table("Blog").Create(blog3) - gormDB.Table("Blog").Create(blog4) - - t.Run("testing filter with the eq operator on 2 fields", func(t *testing.T) { - page := 1 - limit := 0 - sortOptions := map[string]string{ - "id": "asc", - } - ctxt := context.Background() - name := "Blog" - scheme := swagger.Components.Schemas[name] - ctxt = context.WithValue(ctxt, weosContext.CONTENT_TYPE, &weosContext.ContentType{ - Name: strings.Title(name), - Schema: scheme.Value, - }) - filter := &projections.FilterProperty{ - Field: "title", - Operator: "eq", - Value: "hugs1", - Values: nil, - } - filter2 := &projections.FilterProperty{ - Field: "description", - Operator: "eq", - Value: "first blog", - Values: nil, - } - filters := map[string]interface{}{filter.Field: filter, filter2.Field: filter2} - results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) - if err != nil { - t.Errorf("error getting content entities: %s", err) - } - if results == nil || len(results) == 0 { - t.Errorf("expected to get results but got nil") - } - if total != int64(1) { - t.Errorf("expected total to be %d got %d", int64(1), total) - } - if int(results[0]["id"].(float64)) != 1 { - t.Errorf("expected result id to be %d got %d", 1, int(results[0]["id"].(float64))) - } - }) - t.Run("testing filters with the ne operator", func(t *testing.T) { - page := 1 - limit := 0 - sortOptions := map[string]string{ - "id": "asc", - } - ctxt := context.Background() - name := "Blog" - scheme := swagger.Components.Schemas[name] - ctxt = context.WithValue(ctxt, weosContext.CONTENT_TYPE, &weosContext.ContentType{ - Name: strings.Title(name), - Schema: scheme.Value, - }) - filter := &projections.FilterProperty{ - Field: "id", - Operator: "ne", - Value: "1", - Values: nil, - } - filters := map[string]interface{}{filter.Field: filter} - results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) - if err != nil { - t.Errorf("error getting content entities: %s", err) - } - if results == nil || len(results) == 0 { - t.Errorf("expected to get results but got nil") - } - if total != int64(4) { - t.Errorf("expected total to be %d got %d", int64(4), total) - } - if len(results) != 4 { - t.Errorf("expected length of results to be %d got %d", 4, len(results)) - } - }) - t.Run("testing filters with the like operator", func(t *testing.T) { - page := 1 - limit := 2 - sortOptions := map[string]string{ - "id": "asc", - } - ctxt := context.Background() - name := "Blog" - scheme := swagger.Components.Schemas[name] - ctxt = context.WithValue(ctxt, weosContext.CONTENT_TYPE, &weosContext.ContentType{ - Name: strings.Title(name), - Schema: scheme.Value, - }) - filter := &projections.FilterProperty{ - Field: "id", - Operator: "like", - Value: "1", - Values: nil, - } - filters := map[string]interface{}{filter.Field: filter} - results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) - if err != nil { - t.Errorf("error getting content entities: %s", err) - } - if results == nil || len(results) == 0 { - t.Errorf("expected to get results but got nil") - } - if total != int64(2) { - t.Errorf("expected total to be %d got %d", int64(2), total) - } - if len(results) != 2 { - t.Errorf("expected length of results to be %d got %d", 2, len(results)) - } - }) - t.Run("testing filters with the in operator with a single value", func(t *testing.T) { - page := 1 - limit := 0 - sortOptions := map[string]string{ - "id": "asc", - } - ctxt := context.Background() - name := "Blog" - scheme := swagger.Components.Schemas[name] - ctxt = context.WithValue(ctxt, weosContext.CONTENT_TYPE, &weosContext.ContentType{ - Name: strings.Title(name), - Schema: scheme.Value, - }) - vals := []string{"hugs2"} - filter := &projections.FilterProperty{ - Field: "title", - Operator: "in", - Values: vals, - } - filters := map[string]interface{}{filter.Field: filter} - results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) - if err != nil { - t.Errorf("error getting content entities: %s", err) - } - if results == nil || len(results) == 0 { - t.Errorf("expected to get results but got nil") - } - if total != int64(1) { - t.Errorf("expected total to be %d got %d", int64(1), total) - } - if len(results) != 1 { - t.Errorf("expected length of results to be %d got %d", 1, len(results)) - } - }) - t.Run("testing filters with the in operator with multiple values", func(t *testing.T) { - page := 1 - limit := 0 - sortOptions := map[string]string{ - "id": "asc", - } - ctxt := context.Background() - name := "Blog" - scheme := swagger.Components.Schemas[name] - ctxt = context.WithValue(ctxt, weosContext.CONTENT_TYPE, &weosContext.ContentType{ - Name: strings.Title(name), - Schema: scheme.Value, - }) - arrValues := []string{"hugs1", "hugs3"} - filter := &projections.FilterProperty{ - Field: "title", - Operator: "in", - Values: arrValues, - } - filters := map[string]interface{}{filter.Field: filter} - results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) - if err != nil { - t.Errorf("error getting content entities: %s", err) - } - if results == nil || len(results) == 0 { - t.Errorf("expected to get results but got nil") - } - if total != int64(2) { - t.Errorf("expected total to be %d got %d", int64(2), total) - } - if len(results) != 2 { - t.Errorf("expected length of results to be %d got %d", 2, len(results)) - } - }) - t.Run("testing filters with the lt operator", func(t *testing.T) { - page := 1 - limit := 0 - sortOptions := map[string]string{ - "id": "asc", - } - ctxt := context.Background() - name := "Blog" - scheme := swagger.Components.Schemas[name] - ctxt = context.WithValue(ctxt, weosContext.CONTENT_TYPE, &weosContext.ContentType{ - Name: strings.Title(name), - Schema: scheme.Value, - }) - filter := &projections.FilterProperty{ - Field: "id", - Operator: "lt", - Value: "2", - Values: nil, - } - filters := map[string]interface{}{filter.Field: filter} - results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) - if err != nil { - t.Errorf("error getting content entities: %s", err) - } - if results == nil || len(results) == 0 { - t.Errorf("expected to get results but got nil") - } - if total != int64(1) { - t.Errorf("expected total to be %d got %d", int64(1), total) - } - if len(results) != 1 { - t.Errorf("expected length of results to be %d got %d", 1, len(results)) - } - }) - t.Run("testing filters with the gt operator", func(t *testing.T) { - page := 1 - limit := 0 - sortOptions := map[string]string{ - "id": "asc", - } - ctxt := context.Background() - name := "Blog" - scheme := swagger.Components.Schemas[name] - ctxt = context.WithValue(ctxt, weosContext.CONTENT_TYPE, &weosContext.ContentType{ - Name: strings.Title(name), - Schema: scheme.Value, - }) - filter := &projections.FilterProperty{ - Field: "id", - Operator: "gt", - Value: "3", - Values: nil, - } - filters := map[string]interface{}{filter.Field: filter} - results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) - if err != nil { - t.Errorf("error getting content entities: %s", err) - } - if results == nil || len(results) == 0 { - t.Errorf("expected to get results but got nil") - } - if total != int64(2) { - t.Errorf("expected total to be %d got %d", int64(2), total) - } - if len(results) != 2 { - t.Errorf("expected length of results to be %d got %d", 2, len(results)) - } - }) - t.Run("testing filters with the multiple operators", func(t *testing.T) { - page := 1 - limit := 0 - sortOptions := map[string]string{ - "id": "asc", - } - ctxt := context.Background() - name := "Blog" - scheme := swagger.Components.Schemas[name] - ctxt = context.WithValue(ctxt, weosContext.CONTENT_TYPE, &weosContext.ContentType{ - Name: strings.Title(name), - Schema: scheme.Value, - }) - filter := &projections.FilterProperty{ - Field: "id", - Operator: "like", - Value: "1", - Values: nil, - } - filter2 := &projections.FilterProperty{ - Field: "title", - Operator: "ne", - Value: "hugs1", - Values: nil, - } - filters := map[string]interface{}{filter.Field: filter, filter2.Field: filter2} - results, total, err := p.GetContentEntities(ctxt, page, limit, "", sortOptions, filters) - if err != nil { - t.Errorf("error getting content entities: %s", err) - } - if results == nil || len(results) == 0 { - t.Errorf("expected to get results but got nil") - } - if total != int64(1) { - t.Errorf("expected total to be %d got %d", int64(1), total) - } - if len(results) != 1 { - t.Errorf("expected length of results to be %d got %d", 1, len(results)) - } - }) -} From cd9851f2ff242e01debd47bd244d8fcc33a40769 Mon Sep 17 00:00:00 2001 From: shaniah868 Date: Wed, 9 Feb 2022 14:08:56 -0400 Subject: [PATCH 08/11] feature: WEOS-1134 Update list method on projection to accept filter map -Made changes requested on pr -Updated test --- controllers/rest/dtos.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/controllers/rest/dtos.go b/controllers/rest/dtos.go index 0fc1dd4e..45beb539 100644 --- a/controllers/rest/dtos.go +++ b/controllers/rest/dtos.go @@ -58,8 +58,8 @@ type ListApiResponse struct { //FilterProperties is the properties need to use filters type FilterProperties struct { - Field string `json:"field"` - Operator string `json:"operator"` - Value string `json:"value"` - Values []string `json:"values"` + Field string `json:"field"` + Operator string `json:"operator"` + Value interface{} `json:"value"` + Values []interface{} `json:"values"` } From 9b6b302702d241c899e5b6ca79257980887417d2 Mon Sep 17 00:00:00 2001 From: shaniah868 Date: Wed, 9 Feb 2022 14:10:56 -0400 Subject: [PATCH 09/11] feature: WEOS-1134 Update list method on projection to accept filter map -Change the in operator code to just take one ? --- projections/gorm.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/projections/gorm.go b/projections/gorm.go index 41174ec2..95934135 100644 --- a/projections/gorm.go +++ b/projections/gorm.go @@ -339,10 +339,6 @@ func NewProjection(ctx context.Context, db *gorm.DB, logger weos.Log) (*GORMProj } } else { - //queryString := "" - //for i := 0; i < len(filter.Values); i++ { - // queryString += " ?" - //} db.Where(filter.Field+" "+operator+" ?", filter.Values) } From 28c16cbb635c7b3df049de855c84ecf0e9186ac9 Mon Sep 17 00:00:00 2001 From: shaniah868 Date: Wed, 9 Feb 2022 14:40:13 -0400 Subject: [PATCH 10/11] feature: WEOS-1134 Update list method on projection to accept filter map -Fixed test on different databases --- projections/gorm.go | 50 +++++++++++++++++---------------- projections/projections_test.go | 8 +++--- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/projections/gorm.go b/projections/gorm.go index 95934135..1b3e678a 100644 --- a/projections/gorm.go +++ b/projections/gorm.go @@ -311,37 +311,39 @@ func NewProjection(ctx context.Context, db *gorm.DB, logger weos.Log) (*GORMProj FilterQuery = func(options map[string]FilterProperty) func(db *gorm.DB) *gorm.DB { return func(db *gorm.DB) *gorm.DB { - for _, filter := range options { - operator := "=" - switch filter.Operator { - case "gt": - operator = ">" - case "lt": - operator = "<" - case "ne": - operator = "!=" - case "like": - if projection.db.Dialector.Name() == "postgres" { - operator = "ILIKE" - } else { - operator = " LIKE" + if options != nil { + for _, filter := range options { + operator := "=" + switch filter.Operator { + case "gt": + operator = ">" + case "lt": + operator = "<" + case "ne": + operator = "!=" + case "like": + if projection.db.Dialector.Name() == "postgres" { + operator = "ILIKE" + } else { + operator = " LIKE" + } + case "in": + operator = "IN" + } - case "in": - operator = "IN" - } + if len(filter.Values) == 0 { + if filter.Operator == "like" { + db.Where(filter.Field+" "+operator+" ?", "%"+filter.Value.(string)+"%") + } else { + db.Where(filter.Field+" "+operator+" ?", filter.Value) + } - if len(filter.Values) == 0 { - if filter.Operator == "like" { - db.Where(filter.Field+" "+operator+"?", "%"+filter.Value.(string)+"%") } else { - db.Where(filter.Field+" "+operator+"?", filter.Value) + db.Where(filter.Field+" "+operator+" ?", filter.Values) } - } else { - db.Where(filter.Field+" "+operator+" ?", filter.Values) } - } return db } diff --git a/projections/projections_test.go b/projections/projections_test.go index 55c64c63..e5835203 100644 --- a/projections/projections_test.go +++ b/projections/projections_test.go @@ -2150,10 +2150,10 @@ components: if found != limit { t.Errorf("expected to find %d blogs got %d", limit, found) } - err = gormDB.Migrator().DropTable("Blog") - if err != nil { - t.Errorf("error removing table '%s' '%s'", "Blog", err) - } + //err = gormDB.Migrator().DropTable("Blog") + //if err != nil { + // t.Errorf("error removing table '%s' '%s'", "Blog", err) + //} }) t.Run("get a basic list with the foreign key returned", func(t *testing.T) { From 25d3d6d6be276164d924c7eafe2eba2a91326adb Mon Sep 17 00:00:00 2001 From: shaniah868 Date: Wed, 9 Feb 2022 15:05:48 -0400 Subject: [PATCH 11/11] feature: WEOS-1134 Update list method on projection to accept filter map -Added date check and test for it --- projections/gorm.go | 22 ++++++++++++++++++++++ projections/projections_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/projections/gorm.go b/projections/gorm.go index 1b3e678a..69f7dd41 100644 --- a/projections/gorm.go +++ b/projections/gorm.go @@ -9,6 +9,7 @@ import ( "gorm.io/gorm" "gorm.io/gorm/clause" "strings" + "time" ) //GORMProjection interface struct @@ -249,6 +250,10 @@ func (p *GORMProjection) GetContentEntities(ctx context.Context, entityFactory w var filtersProp map[string]FilterProperty props, _ := json.Marshal(filterOptions) json.Unmarshal(props, &filtersProp) + filtersProp, err := DateTimeCheck(entityFactory, filtersProp) + if err != nil { + return nil, int64(0), err + } builder := entityFactory.DynamicStruct(ctx) if builder != nil { schemes = builder.NewSliceOfStructs() @@ -264,6 +269,23 @@ func (p *GORMProjection) GetContentEntities(ctx context.Context, entityFactory w return entities, count, result.Error } +//DateTimeChecks checks to make sure the format is correctly as well as it manipulates the date +func DateTimeCheck(entityFactory weos.EntityFactory, properties map[string]FilterProperty) (map[string]FilterProperty, error) { + var err error + schema := entityFactory.Schema() + for key, value := range properties { + if schema.Properties[key] != nil && schema.Properties[key].Value.Format == "date-time" { + _, err := time.Parse(time.RFC3339, value.Value.(string)) + if err != nil { + return nil, err + } + + } + } + + return properties, err +} + //paginate is used for querying results func paginate(page int, limit int) func(db *gorm.DB) *gorm.DB { return func(db *gorm.DB) *gorm.DB { diff --git a/projections/projections_test.go b/projections/projections_test.go index e5835203..08dd0809 100644 --- a/projections/projections_test.go +++ b/projections/projections_test.go @@ -2614,6 +2614,33 @@ components: t.Errorf("expected length of results to be %d got %d", 1, len(results)) } }) + t.Run("testing invalid date time format on filter ", func(t *testing.T) { + page := 1 + limit := 0 + sortOptions := map[string]string{ + "id": "asc", + } + ctxt := context.Background() + filter := &projections.FilterProperty{ + Field: "last_updated", + Operator: "lt", + Value: "2006-01-02T15:04:00Z+dsujhsd", + Values: nil, + } + + filters := map[string]interface{}{filter.Field: filter} + results, total, err := p.GetContentEntities(ctxt, blogEntityFactory, page, limit, "", sortOptions, filters) + if err == nil { + t.Fatalf("expected a date time error but got nil") + } + if results != nil { + t.Errorf("unexpect error expected results to be nil ") + } + if total != int64(0) { + t.Errorf("expecter total to be 0 got %d", total) + } + + }) } func TestProjections_Delete(t *testing.T) {