From 5d2c41e66f7ad034556838d9215d0e45f2a69572 Mon Sep 17 00:00:00 2001 From: andreaangiolillo Date: Wed, 24 Jul 2024 16:20:59 +0100 Subject: [PATCH 1/6] fixes the filtering logic --- tools/cli/internal/cli/split/split.go | 32 +++----------- tools/cli/internal/cli/versions/versions.go | 8 +++- tools/cli/internal/openapi/filter/filter.go | 43 ++++++++++++++++++- .../internal/openapi/filter/hidden_envs.go | 40 +++++++++++------ .../openapi/filter/hidden_envs_test.go | 28 ------------ tools/cli/internal/openapi/versions.go | 18 +++++--- tools/cli/internal/openapi/versions_test.go | 5 ++- 7 files changed, 96 insertions(+), 78 deletions(-) diff --git a/tools/cli/internal/cli/split/split.go b/tools/cli/internal/cli/split/split.go index 6264aa480..3de2745bd 100644 --- a/tools/cli/internal/cli/split/split.go +++ b/tools/cli/internal/cli/split/split.go @@ -15,7 +15,6 @@ package split import ( - "encoding/json" "fmt" "log" "strings" @@ -45,17 +44,13 @@ func (o *Opts) Run() error { return err } - oas := specInfo.Spec - versions := openapi.ExtractVersions(oas) + versions, err := openapi.ExtractVersions(specInfo.Spec, o.env) + if err != nil { + return err + } for _, version := range versions { - // make a copy of the oas to avoid modifying the original document when applying filters - versionedOas, err := duplicateOas(oas) - if err != nil { - return err - } - - filteredOAS, err := o.filter(versionedOas, version) + filteredOAS, err := o.filter(specInfo.Spec, version) if err != nil { return err } @@ -72,23 +67,6 @@ func (o *Opts) Run() error { return nil } -func duplicateOas(doc *openapi3.T) (*openapi3.T, error) { - // Marshal the original document to JSON - jsonData, err := json.Marshal(doc) - if err != nil { - return nil, fmt.Errorf("failed to marshal original OpenAPI specification: %w", err) - } - - // Unmarshal the JSON data into a new OpenAPI document - duplicateDoc := &openapi3.T{} - err = json.Unmarshal(jsonData, duplicateDoc) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal duplicated OpenAPI specification: %w", err) - } - - return duplicateDoc, nil -} - func (o *Opts) filter(oas *openapi3.T, version string) (result *openapi3.T, err error) { log.Printf("Filtering OpenAPI document by version %s", version) apiVersion, err := apiversion.New(apiversion.WithVersion(version)) diff --git a/tools/cli/internal/cli/versions/versions.go b/tools/cli/internal/cli/versions/versions.go index 1a37427c4..f95e12d17 100644 --- a/tools/cli/internal/cli/versions/versions.go +++ b/tools/cli/internal/cli/versions/versions.go @@ -32,6 +32,7 @@ type Opts struct { basePath string outputPath string format string + env string } func (o *Opts) Run() error { @@ -41,7 +42,11 @@ func (o *Opts) Run() error { return err } - versions := openapi.ExtractVersions(specInfo.Spec) + versions, err := openapi.ExtractVersions(specInfo.Spec, o.env) + if err != nil { + return err + } + if versions == nil { return fmt.Errorf("no versions found in the OpenAPI specification") } @@ -118,6 +123,7 @@ func Builder() *cobra.Command { } cmd.Flags().StringVarP(&opts.basePath, flag.Spec, flag.SpecShort, "", usage.Spec) + cmd.Flags().StringVar(&opts.env, flag.Environment, "dev", usage.Environment) cmd.Flags().StringVarP(&opts.outputPath, flag.Output, flag.OutputShort, "", usage.Output) cmd.Flags().StringVarP(&opts.format, flag.Format, flag.FormatShort, "json", usage.Format) return cmd diff --git a/tools/cli/internal/openapi/filter/filter.go b/tools/cli/internal/openapi/filter/filter.go index 16d854082..f65139d5b 100644 --- a/tools/cli/internal/openapi/filter/filter.go +++ b/tools/cli/internal/openapi/filter/filter.go @@ -14,7 +14,9 @@ package filter import ( + "encoding/json" "errors" + "fmt" "github.com/getkin/kin-openapi/openapi3" "github.com/mongodb/openapi/tools/cli/internal/apiversion" @@ -74,8 +76,30 @@ func initFilters(oas *openapi3.T, metadata *Metadata) error { return nil } +func ApplyFiltersWithInit(doc *openapi3.T, metadata *Metadata, init func(oas *openapi3.T, metadata *Metadata) []Filter) error { + // make a copy of the oas to avoid modifying the original document when applying filters + oas, err := duplicateOas(doc) + if err != nil { + return err + } + + filtersWithInit := init(oas, metadata) + for _, filter := range filtersWithInit { + if err := filter.Apply(); err != nil { + return err + } + } + return nil +} + func ApplyFilters(doc *openapi3.T, metadata *Metadata) error { - if err := initFilters(doc, metadata); err != nil { + // make a copy of the oas to avoid modifying the original document when applying filters + oas, err := duplicateOas(doc) + if err != nil { + return err + } + + if err := initFilters(oas, metadata); err != nil { return err } @@ -86,3 +110,20 @@ func ApplyFilters(doc *openapi3.T, metadata *Metadata) error { } return nil } + +func duplicateOas(doc *openapi3.T) (*openapi3.T, error) { + // Marshal the original document to JSON + jsonData, err := json.Marshal(doc) + if err != nil { + return nil, fmt.Errorf("failed to marshal original OpenAPI specification: %w", err) + } + + // Unmarshal the JSON data into a new OpenAPI document + duplicateDoc := &openapi3.T{} + err = json.Unmarshal(jsonData, duplicateDoc) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal duplicated OpenAPI specification: %w", err) + } + + return duplicateDoc, nil +} diff --git a/tools/cli/internal/openapi/filter/hidden_envs.go b/tools/cli/internal/openapi/filter/hidden_envs.go index ede0ec550..a3f7fb20c 100644 --- a/tools/cli/internal/openapi/filter/hidden_envs.go +++ b/tools/cli/internal/openapi/filter/hidden_envs.go @@ -30,6 +30,13 @@ type HiddenEnvsFilter struct { metadata *Metadata } +func InitHiddenEnvsFilter(oas *openapi3.T, metadata *Metadata) *HiddenEnvsFilter { + return &HiddenEnvsFilter{ + oas: oas, + metadata: metadata, + } +} + func (f *HiddenEnvsFilter) Apply() error { // delete hidden paths first before processing for pathName, pathItem := range f.oas.Paths.Map() { @@ -91,8 +98,15 @@ func (f *HiddenEnvsFilter) removeRequestBodyIfHiddenForEnv(operation *openapi3.O return } - for _, contentType := range operation.RequestBody.Value.Content { - f.removeContentIfHiddenForEnv(contentType) + for k, contentType := range operation.RequestBody.Value.Content { + if isContentTypeHiddenForEnv := isContentTypeHiddenForEnv(contentType, f.metadata.targetEnv); isContentTypeHiddenForEnv { + log.Printf("Removing contentType: %q because is hidden for target env: %q", contentType.Schema.Ref, f.metadata.targetEnv) + // Remove ContentType if it is hidden for the target environment + delete(operation.RequestBody.Value.Content, k) + } else if contentType.Extensions != nil { + // Remove the Hidden extension from the final OAS + delete(contentType.Extensions, hiddenEnvsExtension) + } } } @@ -109,19 +123,17 @@ func (f *HiddenEnvsFilter) removeResponseIfHiddenForEnv(operation *openapi3.Oper if response.Value == nil || response.Value.Content == nil { continue } - for _, contentType := range response.Value.Content { - f.removeContentIfHiddenForEnv(contentType) - } - } -} -func (f *HiddenEnvsFilter) removeContentIfHiddenForEnv(contentType *openapi3.MediaType) { - if isContentTypeHiddenForEnv := isContentTypeHiddenForEnv(contentType, f.metadata.targetEnv); isContentTypeHiddenForEnv { - log.Printf("Removing contentType: %q because is hidden for target env: %q", contentType.Schema.Ref, f.metadata.targetEnv) - contentType.Schema = nil // Remove ContentType if it is hidden for the target environment - } else if contentType.Extensions != nil { - // Remove the Hidden extension from the final OAS - delete(contentType.Extensions, hiddenEnvsExtension) + for k, contentType := range response.Value.Content { + if isContentTypeHiddenForEnv := isContentTypeHiddenForEnv(contentType, f.metadata.targetEnv); isContentTypeHiddenForEnv { + log.Printf("Removing contentType: %q because is hidden for target env: %q", contentType.Schema.Ref, f.metadata.targetEnv) + // Remove ContentType if it is hidden for the target environment + delete(response.Value.Content, k) + } else if contentType.Extensions != nil { + // Remove the Hidden extension from the final OAS + delete(contentType.Extensions, hiddenEnvsExtension) + } + } } } diff --git a/tools/cli/internal/openapi/filter/hidden_envs_test.go b/tools/cli/internal/openapi/filter/hidden_envs_test.go index 91e443b12..9e4f45043 100644 --- a/tools/cli/internal/openapi/filter/hidden_envs_test.go +++ b/tools/cli/internal/openapi/filter/hidden_envs_test.go @@ -607,34 +607,6 @@ func TestIsContentTypeHiddenForEnv(t *testing.T) { } } -func TestRemoveContentIfHiddenForEnv(t *testing.T) { - tests := []struct { - name string - envs string - targetEnv string - shouldBeNil bool - }{ - {"Remove if hidden for target env", "prod", "prod", true}, - {"Do not remove if no extension", "", "prod", false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - contentType := &openapi3.MediaType{ - Schema: &openapi3.SchemaRef{}, - Extensions: map[string]interface{}{ - hiddenEnvsExtension: map[string]interface{}{ - hiddenEnvsExtKey: tt.envs, - }, - }, - } - f := HiddenEnvsFilter{metadata: &Metadata{targetEnv: tt.targetEnv}} - f.removeContentIfHiddenForEnv(contentType) - assert.Equal(t, tt.shouldBeNil, contentType.Schema == nil) - }) - } -} - func TestApply(t *testing.T) { metadata := &Metadata{ targetEnv: "prod", diff --git a/tools/cli/internal/openapi/versions.go b/tools/cli/internal/openapi/versions.go index a237a0f01..d16b13ed3 100644 --- a/tools/cli/internal/openapi/versions.go +++ b/tools/cli/internal/openapi/versions.go @@ -18,12 +18,21 @@ import ( "github.com/getkin/kin-openapi/openapi3" "github.com/mongodb/openapi/tools/cli/internal/apiversion" + "github.com/mongodb/openapi/tools/cli/internal/openapi/filter" ) // ExtractVersions extracts version strings from an OpenAPI specification. -func ExtractVersions(oas *openapi3.T) []string { - versions := make(map[string]struct{}) +func ExtractVersions(oas *openapi3.T, env string) ([]string, error) { + // We need to remove the version that are hidden for the given environment + if err := filter.ApplyFiltersWithInit(oas, filter.NewMetadata(nil, env), func(oas *openapi3.T, metadata *filter.Metadata) []filter.Filter { + return []filter.Filter{ + filter.InitHiddenEnvsFilter(oas, metadata), + } + }); err != nil { + return nil, nil + } + versions := make(map[string]struct{}) for _, pathItem := range oas.Paths.Map() { if pathItem == nil { continue @@ -33,7 +42,7 @@ func ExtractVersions(oas *openapi3.T) []string { continue } for _, response := range op.Responses.Map() { - if response.Value.Content == nil { + if response.Value == nil || response.Value.Content == nil { continue } for contentType := range response.Value.Content { @@ -45,8 +54,7 @@ func ExtractVersions(oas *openapi3.T) []string { } } } - - return mapKeysToSortedSlice(versions) + return mapKeysToSortedSlice(versions), nil } // mapKeysToSortedSlice converts map keys to a sorted slice. diff --git a/tools/cli/internal/openapi/versions_test.go b/tools/cli/internal/openapi/versions_test.go index 3f669a7f4..244eaaf22 100644 --- a/tools/cli/internal/openapi/versions_test.go +++ b/tools/cli/internal/openapi/versions_test.go @@ -17,12 +17,13 @@ import ( "testing" "github.com/getkin/kin-openapi/openapi3" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestVersions(t *testing.T) { - versions := ExtractVersions(NewVersionedResponses(t)) + versions, err := ExtractVersions(NewVersionedResponses(t), "prod") + require.NoError(t, err) assert.Equal(t, []string{"2023-01-01", "2023-02-01"}, versions) } From 4f1abbc6f5f8a1e17fdabdcf30bd06eb30970697 Mon Sep 17 00:00:00 2001 From: andreaangiolillo Date: Wed, 24 Jul 2024 16:37:45 +0100 Subject: [PATCH 2/6] fixes --- tools/cli/internal/cli/split/split.go | 2 +- tools/cli/internal/openapi/filter/filter.go | 18 +++++++++--------- tools/cli/internal/openapi/versions.go | 9 ++++++--- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/tools/cli/internal/cli/split/split.go b/tools/cli/internal/cli/split/split.go index 3de2745bd..85fd03718 100644 --- a/tools/cli/internal/cli/split/split.go +++ b/tools/cli/internal/cli/split/split.go @@ -74,7 +74,7 @@ func (o *Opts) filter(oas *openapi3.T, version string) (result *openapi3.T, err return nil, err } - return oas, filter.ApplyFilters(oas, filter.NewMetadata(apiVersion, o.env)) + return filter.ApplyFilters(oas, filter.NewMetadata(apiVersion, o.env)) } func (o *Opts) saveVersionedOas(oas *openapi3.T, version string) error { diff --git a/tools/cli/internal/openapi/filter/filter.go b/tools/cli/internal/openapi/filter/filter.go index f65139d5b..dea031371 100644 --- a/tools/cli/internal/openapi/filter/filter.go +++ b/tools/cli/internal/openapi/filter/filter.go @@ -76,39 +76,39 @@ func initFilters(oas *openapi3.T, metadata *Metadata) error { return nil } -func ApplyFiltersWithInit(doc *openapi3.T, metadata *Metadata, init func(oas *openapi3.T, metadata *Metadata) []Filter) error { +func ApplyFiltersWithInit(doc *openapi3.T, metadata *Metadata, init func(oas *openapi3.T, metadata *Metadata) []Filter) (*openapi3.T, error) { // make a copy of the oas to avoid modifying the original document when applying filters oas, err := duplicateOas(doc) if err != nil { - return err + return nil, err } filtersWithInit := init(oas, metadata) for _, filter := range filtersWithInit { if err := filter.Apply(); err != nil { - return err + return nil, err } } - return nil + return oas, nil } -func ApplyFilters(doc *openapi3.T, metadata *Metadata) error { +func ApplyFilters(doc *openapi3.T, metadata *Metadata) (*openapi3.T, error) { // make a copy of the oas to avoid modifying the original document when applying filters oas, err := duplicateOas(doc) if err != nil { - return err + return nil, err } if err := initFilters(oas, metadata); err != nil { - return err + return nil, err } for _, filter := range filters { if err := filter.Apply(); err != nil { - return err + return nil, err } } - return nil + return oas, nil } func duplicateOas(doc *openapi3.T) (*openapi3.T, error) { diff --git a/tools/cli/internal/openapi/versions.go b/tools/cli/internal/openapi/versions.go index d16b13ed3..019d8c0c8 100644 --- a/tools/cli/internal/openapi/versions.go +++ b/tools/cli/internal/openapi/versions.go @@ -24,16 +24,18 @@ import ( // ExtractVersions extracts version strings from an OpenAPI specification. func ExtractVersions(oas *openapi3.T, env string) ([]string, error) { // We need to remove the version that are hidden for the given environment - if err := filter.ApplyFiltersWithInit(oas, filter.NewMetadata(nil, env), func(oas *openapi3.T, metadata *filter.Metadata) []filter.Filter { + doc, err := filter.ApplyFiltersWithInit(oas, filter.NewMetadata(nil, env), func(oas *openapi3.T, metadata *filter.Metadata) []filter.Filter { return []filter.Filter{ filter.InitHiddenEnvsFilter(oas, metadata), } - }); err != nil { + }) + + if err != nil { return nil, nil } versions := make(map[string]struct{}) - for _, pathItem := range oas.Paths.Map() { + for _, pathItem := range doc.Paths.Map() { if pathItem == nil { continue } @@ -54,6 +56,7 @@ func ExtractVersions(oas *openapi3.T, env string) ([]string, error) { } } } + return mapKeysToSortedSlice(versions), nil } From 54a93c53cc1b9f5764a4f1504f71dc38d0666c61 Mon Sep 17 00:00:00 2001 From: andreaangiolillo Date: Wed, 24 Jul 2024 17:30:57 +0100 Subject: [PATCH 3/6] Update openapi-foas-dev.json --- tools/cli/test/data/openapi-foas-dev.json | 86 ++++------------------- 1 file changed, 14 insertions(+), 72 deletions(-) diff --git a/tools/cli/test/data/openapi-foas-dev.json b/tools/cli/test/data/openapi-foas-dev.json index d2691d7c5..7d8769e7e 100644 --- a/tools/cli/test/data/openapi-foas-dev.json +++ b/tools/cli/test/data/openapi-foas-dev.json @@ -22566,14 +22566,6 @@ "tags": [ "Streams" ], - "x-xgen-hidden-env": { - "dev": "false", - "dev-gov": "true", - "prod": "false", - "prod-gov": "true", - "qa": "false", - "qa-gov": "true" - }, "x-xgen-owner-team": "Atlas Streams" }, "get": { @@ -22642,14 +22634,6 @@ "tags": [ "Streams" ], - "x-xgen-hidden-env": { - "dev": "false", - "dev-gov": "true", - "prod": "false", - "prod-gov": "true", - "qa": "false", - "qa-gov": "true" - }, "x-xgen-owner-team": "Atlas Streams" } }, @@ -22720,14 +22704,6 @@ "tags": [ "Streams" ], - "x-xgen-hidden-env": { - "dev": "false", - "dev-gov": "true", - "prod": "false", - "prod-gov": "true", - "qa": "false", - "qa-gov": "true" - }, "x-xgen-owner-team": "Atlas Streams" } }, @@ -22798,14 +22774,6 @@ "tags": [ "Streams" ], - "x-xgen-hidden-env": { - "dev": "false", - "dev-gov": "true", - "prod": "false", - "prod-gov": "true", - "qa": "false", - "qa-gov": "true" - }, "x-xgen-owner-team": "Atlas Streams" } }, @@ -24015,12 +23983,7 @@ "USS Snapshots" ], "x-xgen-hidden-env": { - "dev": "true", - "dev-gov": "true", - "prod": "true", - "prod-gov": "true", - "qa": "true", - "qa-gov": "true" + "envs": "dev,qa,stage,prod" }, "x-xgen-owner-team": "Atlas Serverless II" } @@ -24093,12 +24056,7 @@ "USS Restore Jobs" ], "x-xgen-hidden-env": { - "dev": "true", - "dev-gov": "true", - "prod": "true", - "prod-gov": "true", - "qa": "true", - "qa-gov": "true" + "envs": "dev,qa,stage,prod" }, "x-xgen-owner-team": "Atlas Serverless II" }, @@ -24177,12 +24135,7 @@ "USS Restore Jobs" ], "x-xgen-hidden-env": { - "dev": "true", - "dev-gov": "true", - "prod": "true", - "prod-gov": "true", - "qa": "true", - "qa-gov": "true" + "envs": "dev,qa,stage,prod" }, "x-xgen-owner-team": "Atlas Serverless II" } @@ -24258,12 +24211,7 @@ "USS Restore Jobs" ], "x-xgen-hidden-env": { - "dev": "true", - "dev-gov": "true", - "prod": "true", - "prod-gov": "true", - "qa": "true", - "qa-gov": "true" + "envs": "dev,qa,stage,prod" }, "x-xgen-owner-team": "Atlas Serverless II" } @@ -24336,12 +24284,7 @@ "USS Snapshots" ], "x-xgen-hidden-env": { - "dev": "true", - "dev-gov": "true", - "prod": "true", - "prod-gov": "true", - "qa": "true", - "qa-gov": "true" + "envs": "dev,qa,stage,prod" }, "x-xgen-owner-team": "Atlas Serverless II" } @@ -24415,12 +24358,7 @@ "USS Snapshots" ], "x-xgen-hidden-env": { - "dev": "true", - "dev-gov": "true", - "prod": "true", - "prod-gov": "true", - "qa": "true", - "qa-gov": "true" + "envs": "dev,qa,stage,prod" }, "x-xgen-owner-team": "Atlas Serverless II" } @@ -24529,7 +24467,11 @@ "summary": "Return general information about the MongoDB Atlas Administration API OpenAPI Specification.", "tags": [ "OpenAPI" - ] + ], + "x-xgen-hidden-env": { + "envs": "qa,stage,prod" + }, + "x-xgen-owner-team": "apix" } }, "/api/atlas/v2/orgs": { @@ -33502,7 +33444,7 @@ "description": "Specifications for scheduled policy.", "properties": { "frequencyInterval": { - "description": "Number that indicates the frequency interval for a set of snapshots. A value of `1` specifies the first instance of the corresponding `frequencyType`.\n\n- In a yearly policy item, `1` indicates that the yearly snapshot occurs on the first day of January and `12` indicates the first day of December.\n\n- In a monthly policy item, `1` indicates that the monthly snapshot occurs on the first day of the month and `40` indicates the last day of the month.\n\n- In a weekly policy item, `1` indicates that the weekly snapshot occurs on Monday and `7` indicates Sunday.\n\n- In an hourly policy item, you can set the frequency interval to `1`, `2`, `4`, `6`, `8`, or `12`. For hourly policy items for NVMe clusters, MongoDB Cloud accepts only `12` as the frequency interval value.\n\n MongoDB Cloud ignores this setting for non-hourly policy items in Backup Compliance Policy settings.", + "description": "Number that indicates the frequency interval for a set of Snapshots. A value of `1` specifies the first instance of the corresponding `frequencyType`.\n\n- In a yearly policy item, `1` indicates that the yearly Snapshot occurs on the first day of January and `12` indicates the first day of December.\n\n- In a monthly policy item, `1` indicates that the monthly Snapshot occurs on the first day of the month and `40` indicates the last day of the month.\n\n- In a weekly policy item, `1` indicates that the weekly Snapshot occurs on Monday and `7` indicates Sunday.\n\n- In an hourly policy item, you can set the frequency interval to `1`, `2`, `4`, `6`, `8`, or `12`. For hourly policy items for NVMe clusters, MongoDB Cloud accepts only `12` as the frequency interval value.\n\n MongoDB Cloud ignores this setting for non-hourly policy items in Backup Compliance Policy settings.", "enum": [ 1, 2, @@ -33557,7 +33499,7 @@ "type": "string" }, "retentionUnit": { - "description": "Unit of time in which MongoDB Cloud measures snapshot retention.", + "description": "Unit of time in which MongoDB Cloud measures Snapshot retention.", "enum": [ "days", "weeks", @@ -33567,7 +33509,7 @@ "type": "string" }, "retentionValue": { - "description": "Duration in days, weeks, months, or years that MongoDB Cloud retains the snapshot. For less frequent policy items, MongoDB Cloud requires that you specify a value greater than or equal to the value specified for more frequent policy items.\n\nFor example: If the hourly policy item specifies a retention of two days, you must specify two days or greater for the retention of the weekly policy item.", + "description": "Duration in days, weeks, months, or years that MongoDB Cloud retains the Snapshot. For less frequent policy items, MongoDB Cloud requires that you specify a value greater than or equal to the value specified for more frequent policy items.\n\nFor example: If the hourly policy item specifies a retention of two days, you must specify two days or greater for the retention of the weekly policy item.", "format": "int32", "type": "integer" } From 34fb778c17597798bfd7b88964cd112e96298fab Mon Sep 17 00:00:00 2001 From: andreaangiolillo Date: Wed, 24 Jul 2024 17:32:46 +0100 Subject: [PATCH 4/6] Update openapi-foas-dev.yaml --- tools/cli/test/data/openapi-foas-dev.yaml | 86 ++++------------------- 1 file changed, 15 insertions(+), 71 deletions(-) diff --git a/tools/cli/test/data/openapi-foas-dev.yaml b/tools/cli/test/data/openapi-foas-dev.yaml index fbd6a5e05..653c138ab 100644 --- a/tools/cli/test/data/openapi-foas-dev.yaml +++ b/tools/cli/test/data/openapi-foas-dev.yaml @@ -4110,13 +4110,13 @@ components: properties: frequencyInterval: description: |- - Number that indicates the frequency interval for a set of snapshots. A value of `1` specifies the first instance of the corresponding `frequencyType`. + Number that indicates the frequency interval for a set of Snapshots. A value of `1` specifies the first instance of the corresponding `frequencyType`. - - In a yearly policy item, `1` indicates that the yearly snapshot occurs on the first day of January and `12` indicates the first day of December. + - In a yearly policy item, `1` indicates that the yearly Snapshot occurs on the first day of January and `12` indicates the first day of December. - - In a monthly policy item, `1` indicates that the monthly snapshot occurs on the first day of the month and `40` indicates the last day of the month. + - In a monthly policy item, `1` indicates that the monthly Snapshot occurs on the first day of the month and `40` indicates the last day of the month. - - In a weekly policy item, `1` indicates that the weekly snapshot occurs on Monday and `7` indicates Sunday. + - In a weekly policy item, `1` indicates that the weekly Snapshot occurs on Monday and `7` indicates Sunday. - In an hourly policy item, you can set the frequency interval to `1`, `2`, `4`, `6`, `8`, or `12`. For hourly policy items for NVMe clusters, MongoDB Cloud accepts only `12` as the frequency interval value. @@ -4170,7 +4170,7 @@ components: readOnly: true type: string retentionUnit: - description: Unit of time in which MongoDB Cloud measures snapshot retention. + description: Unit of time in which MongoDB Cloud measures Snapshot retention. enum: - days - weeks @@ -4179,7 +4179,7 @@ components: type: string retentionValue: description: |- - Duration in days, weeks, months, or years that MongoDB Cloud retains the snapshot. For less frequent policy items, MongoDB Cloud requires that you specify a value greater than or equal to the value specified for more frequent policy items. + Duration in days, weeks, months, or years that MongoDB Cloud retains the Snapshot. For less frequent policy items, MongoDB Cloud requires that you specify a value greater than or equal to the value specified for more frequent policy items. For example: If the hourly policy item specifies a retention of two days, you must specify two days or greater for the retention of the weekly policy item. format: int32 @@ -30823,7 +30823,6 @@ info: termsOfService: https://www.mongodb.com/mongodb-management-service-terms-and-conditions title: MongoDB Atlas Administration API version: "2.0" - x-xgen-sha: "450f2b276442adec8ca18461fa6b4adc7f64bbcc" openapi: 3.0.1 paths: /api/atlas/v2: @@ -44895,13 +44894,6 @@ paths: summary: Delete One Stream Processor tags: - Streams - x-xgen-hidden-env: - dev: "false" - dev-gov: "true" - prod: "false" - prod-gov: "true" - qa: "false" - qa-gov: "true" x-xgen-owner-team: Atlas Streams get: description: Get one Stream Processor within the specified stream instance. To use this resource, the requesting API Key must have the Project Owner role or Project Stream Processing Owner role. @@ -44943,13 +44935,6 @@ paths: summary: Get One Stream Processor tags: - Streams - x-xgen-hidden-env: - dev: "false" - dev-gov: "true" - prod: "false" - prod-gov: "true" - qa: "false" - qa-gov: "true" x-xgen-owner-team: Atlas Streams /api/atlas/v2/groups/{groupId}/streams/{tenantName}/processor/{processorName}:start: post: @@ -44992,13 +44977,6 @@ paths: summary: Start One Stream Processor tags: - Streams - x-xgen-hidden-env: - dev: "false" - dev-gov: "true" - prod: "false" - prod-gov: "true" - qa: "false" - qa-gov: "true" x-xgen-owner-team: Atlas Streams /api/atlas/v2/groups/{groupId}/streams/{tenantName}/processor/{processorName}:stop: post: @@ -45041,13 +45019,6 @@ paths: summary: Stop One Stream Processor tags: - Streams - x-xgen-hidden-env: - dev: "false" - dev-gov: "true" - prod: "false" - prod-gov: "true" - qa: "false" - qa-gov: "true" x-xgen-owner-team: Atlas Streams /api/atlas/v2/groups/{groupId}/teams: get: @@ -45774,12 +45745,7 @@ paths: tags: - USS Snapshots x-xgen-hidden-env: - dev: "true" - dev-gov: "true" - prod: "true" - prod-gov: "true" - qa: "true" - qa-gov: "true" + envs: dev,qa,stage,prod x-xgen-owner-team: Atlas Serverless II /api/atlas/v2/groups/{groupId}/uss/{name}/backup/restoreJobs: get: @@ -45821,12 +45787,7 @@ paths: tags: - USS Restore Jobs x-xgen-hidden-env: - dev: "true" - dev-gov: "true" - prod: "true" - prod-gov: "true" - qa: "true" - qa-gov: "true" + envs: dev,qa,stage,prod x-xgen-owner-team: Atlas Serverless II post: description: Restores one snapshot of one USS instance from the specified project. To use this resource, the requesting API Key must have the Project Owner role. @@ -45875,12 +45836,7 @@ paths: tags: - USS Restore Jobs x-xgen-hidden-env: - dev: "true" - dev-gov: "true" - prod: "true" - prod-gov: "true" - qa: "true" - qa-gov: "true" + envs: dev,qa,stage,prod x-xgen-owner-team: Atlas Serverless II /api/atlas/v2/groups/{groupId}/uss/{name}/backup/restoreJobs/{restoreJobId}: get: @@ -45928,12 +45884,7 @@ paths: tags: - USS Restore Jobs x-xgen-hidden-env: - dev: "true" - dev-gov: "true" - prod: "true" - prod-gov: "true" - qa: "true" - qa-gov: "true" + envs: dev,qa,stage,prod x-xgen-owner-team: Atlas Serverless II /api/atlas/v2/groups/{groupId}/uss/{name}/backup/snapshots: get: @@ -45975,12 +45926,7 @@ paths: tags: - USS Snapshots x-xgen-hidden-env: - dev: "true" - dev-gov: "true" - prod: "true" - prod-gov: "true" - qa: "true" - qa-gov: "true" + envs: dev,qa,stage,prod x-xgen-owner-team: Atlas Serverless II /api/atlas/v2/groups/{groupId}/uss/{name}/backup/snapshots/{snapshotId}: get: @@ -46028,12 +45974,7 @@ paths: tags: - USS Snapshots x-xgen-hidden-env: - dev: "true" - dev-gov: "true" - prod: "true" - prod-gov: "true" - qa: "true" - qa-gov: "true" + envs: dev,qa,stage,prod x-xgen-owner-team: Atlas Serverless II /api/atlas/v2/groups/{groupId}:migrate: post: @@ -46138,6 +46079,9 @@ paths: summary: Return general information about the MongoDB Atlas Administration API OpenAPI Specification. tags: - OpenAPI + x-xgen-hidden-env: + envs: qa,stage,prod + x-xgen-owner-team: apix /api/atlas/v2/orgs: get: description: Returns all organizations to which the requesting API Key has access. To use this resource, the requesting API Key must have the Organization Member role. From 02ea64509dbf6c6e5b1487beaf05b2a1ae5b600e Mon Sep 17 00:00:00 2001 From: andreaangiolillo Date: Wed, 24 Jul 2024 18:28:45 +0100 Subject: [PATCH 5/6] Update hidden_envs_test.go --- .../openapi/filter/hidden_envs_test.go | 153 ++++++++++++++++-- 1 file changed, 137 insertions(+), 16 deletions(-) diff --git a/tools/cli/internal/openapi/filter/hidden_envs_test.go b/tools/cli/internal/openapi/filter/hidden_envs_test.go index 9e4f45043..829b554fd 100644 --- a/tools/cli/internal/openapi/filter/hidden_envs_test.go +++ b/tools/cli/internal/openapi/filter/hidden_envs_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/getkin/kin-openapi/openapi3" + "github.com/mongodb/openapi/tools/cli/internal/pointer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -579,34 +580,154 @@ func TestApplyOnPath(t *testing.T) { } } -func TestIsContentTypeHiddenForEnv(t *testing.T) { +func TestRemoveResponseIfHiddenForEnv(t *testing.T) { tests := []struct { name string - envs string targetEnv string - expected bool + operation *openapi3.Operation + expected *openapi3.Operation }{ - {"Hidden for target env", "prod", "prod", true}, - {"Not hidden, no extension", "", "prod", false}, - {"Hidden for different env", "dev", "prod", false}, + { + name: "Response Hidden", + targetEnv: "prod", + operation: &openapi3.Operation{ + OperationID: "testOperation", + Responses: openapi3.NewResponses(openapi3.WithStatus(200, + &openapi3.ResponseRef{ + Value: &openapi3.Response{ + Description: pointer.Get("A sample response"), + Extensions: map[string]interface{}{ + hiddenEnvsExtension: map[string]interface{}{ + "envs": "prod", + }, + }, + Content: map[string]*openapi3.MediaType{ + "application/json": { + Schema: &openapi3.SchemaRef{ + Value: &openapi3.Schema{}, + }, + }, + }, + }, + })), + }, + expected: &openapi3.Operation{ + OperationID: "testOperation", + Responses: openapi3.NewResponses(func(_ *openapi3.Responses) {}), + }, + }, + { + name: "Response Hidden in content", + targetEnv: "prod", + operation: &openapi3.Operation{ + OperationID: "testOperation", + Responses: openapi3.NewResponses(openapi3.WithStatus(200, + &openapi3.ResponseRef{ + Value: &openapi3.Response{ + Description: pointer.Get("A sample response"), + Content: map[string]*openapi3.MediaType{ + "application/json": { + Schema: &openapi3.SchemaRef{}, + Extensions: map[string]interface{}{ + hiddenEnvsExtension: map[string]interface{}{ + "envs": "prod", + }, + }, + }, + "application/2-json": { + Schema: &openapi3.SchemaRef{}, + Extensions: map[string]interface{}{ + hiddenEnvsExtension: map[string]interface{}{ + "envs": "dev", + }, + }, + }, + }, + }, + })), + }, + expected: &openapi3.Operation{ + OperationID: "testOperation", + Responses: openapi3.NewResponses(openapi3.WithStatus(200, + &openapi3.ResponseRef{ + Value: &openapi3.Response{ + Description: pointer.Get("A sample response"), + Content: map[string]*openapi3.MediaType{ + "application/2-json": { + Schema: &openapi3.SchemaRef{}, + Extensions: map[string]interface{}{}, + }, + }, + }, + })), + }, + }, + { + name: "Response Hidden in content with different target env", + targetEnv: "prod", + operation: &openapi3.Operation{ + OperationID: "testOperation", + Responses: openapi3.NewResponses(openapi3.WithStatus(200, + &openapi3.ResponseRef{ + Value: &openapi3.Response{ + Description: pointer.Get("A sample response"), + Content: map[string]*openapi3.MediaType{ + "application/json": { + Schema: &openapi3.SchemaRef{}, + Extensions: map[string]interface{}{ + hiddenEnvsExtension: map[string]interface{}{ + "envs": "dev", + }, + }, + }, + "application/2-json": { + Schema: &openapi3.SchemaRef{}, + Extensions: map[string]interface{}{ + hiddenEnvsExtension: map[string]interface{}{ + "envs": "dev", + }, + }, + }, + }, + }, + })), + }, + expected: &openapi3.Operation{ + OperationID: "testOperation", + Responses: openapi3.NewResponses(openapi3.WithStatus(200, + &openapi3.ResponseRef{ + Value: &openapi3.Response{ + Description: pointer.Get("A sample response"), + Content: map[string]*openapi3.MediaType{ + "application/json": { + Schema: &openapi3.SchemaRef{}, + Extensions: map[string]interface{}{}, + }, + "application/2-json": { + Schema: &openapi3.SchemaRef{}, + Extensions: map[string]interface{}{}, + }, + }, + }, + })), + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - contentType := &openapi3.MediaType{ - Extensions: map[string]interface{}{ - hiddenEnvsExtension: map[string]interface{}{ - hiddenEnvsExtKey: tt.envs, - }, - }, + filter := HiddenEnvsFilter{ + metadata: &Metadata{targetEnv: tt.targetEnv}, + } + + filter.removeResponseIfHiddenForEnv(tt.operation) + + if !reflect.DeepEqual(tt.expected, tt.operation) { + t.Errorf("expected %v, got %v", tt.expected, tt.operation) } - f := HiddenEnvsFilter{metadata: &Metadata{targetEnv: tt.targetEnv}} - result := isContentTypeHiddenForEnv(contentType, f.metadata.targetEnv) - assert.Equal(t, tt.expected, result) }) } } - func TestApply(t *testing.T) { metadata := &Metadata{ targetEnv: "prod", From ee769232ec6751f50e9d7b9400917832cc8f2991 Mon Sep 17 00:00:00 2001 From: andreaangiolillo Date: Thu, 25 Jul 2024 10:22:23 +0100 Subject: [PATCH 6/6] Addressed PR comments --- tools/cli/internal/cli/split/split.go | 4 +- tools/cli/internal/cli/versions/versions.go | 10 +++- .../internal/cli/versions/versions_test.go | 23 ++++++++ tools/cli/internal/openapi/filter/filter.go | 54 ++++++------------- .../internal/openapi/filter/hidden_envs.go | 7 --- tools/cli/internal/openapi/versions.go | 17 +++--- tools/cli/internal/openapi/versions_test.go | 2 +- 7 files changed, 58 insertions(+), 59 deletions(-) diff --git a/tools/cli/internal/cli/split/split.go b/tools/cli/internal/cli/split/split.go index 85fd03718..31691b73d 100644 --- a/tools/cli/internal/cli/split/split.go +++ b/tools/cli/internal/cli/split/split.go @@ -44,7 +44,7 @@ func (o *Opts) Run() error { return err } - versions, err := openapi.ExtractVersions(specInfo.Spec, o.env) + versions, err := openapi.ExtractVersionsWithEnv(specInfo.Spec, o.env) if err != nil { return err } @@ -74,7 +74,7 @@ func (o *Opts) filter(oas *openapi3.T, version string) (result *openapi3.T, err return nil, err } - return filter.ApplyFilters(oas, filter.NewMetadata(apiVersion, o.env)) + return filter.ApplyFilters(oas, filter.NewMetadata(apiVersion, o.env), filter.DefaultFilters) } func (o *Opts) saveVersionedOas(oas *openapi3.T, version string) error { diff --git a/tools/cli/internal/cli/versions/versions.go b/tools/cli/internal/cli/versions/versions.go index f95e12d17..3cab60519 100644 --- a/tools/cli/internal/cli/versions/versions.go +++ b/tools/cli/internal/cli/versions/versions.go @@ -42,7 +42,13 @@ func (o *Opts) Run() error { return err } - versions, err := openapi.ExtractVersions(specInfo.Spec, o.env) + var versions []string + if o.env == "" { + versions, err = openapi.ExtractVersions(specInfo.Spec) + } else { + versions, err = openapi.ExtractVersionsWithEnv(specInfo.Spec, o.env) + } + if err != nil { return err } @@ -123,7 +129,7 @@ func Builder() *cobra.Command { } cmd.Flags().StringVarP(&opts.basePath, flag.Spec, flag.SpecShort, "", usage.Spec) - cmd.Flags().StringVar(&opts.env, flag.Environment, "dev", usage.Environment) + cmd.Flags().StringVar(&opts.env, flag.Environment, "", usage.Environment) cmd.Flags().StringVarP(&opts.outputPath, flag.Output, flag.OutputShort, "", usage.Output) cmd.Flags().StringVarP(&opts.format, flag.Format, flag.FormatShort, "json", usage.Format) return cmd diff --git a/tools/cli/internal/cli/versions/versions_test.go b/tools/cli/internal/cli/versions/versions_test.go index e26396c04..0e292cdbc 100644 --- a/tools/cli/internal/cli/versions/versions_test.go +++ b/tools/cli/internal/cli/versions/versions_test.go @@ -42,3 +42,26 @@ func TestVersions(t *testing.T) { assert.NotEmpty(t, b) assert.Contains(t, string(b), "2023-02-01") } + +func TestVersionWithEnv(t *testing.T) { + fs := afero.NewMemMapFs() + opts := &Opts{ + basePath: "../../../test/data/base_spec.json", + outputPath: "foas.json", + fs: fs, + env: "staging", + } + + if err := opts.Run(); err != nil { + t.Fatalf("Run() unexpected error: %v", err) + } + + b, err := afero.ReadFile(fs, opts.outputPath) + if err != nil { + t.Fatalf("ReadFile() unexpected error: %v", err) + } + + // Check initial versions + assert.NotEmpty(t, b) + assert.Contains(t, string(b), "2023-02-01") +} diff --git a/tools/cli/internal/openapi/filter/filter.go b/tools/cli/internal/openapi/filter/filter.go index dea031371..065c858e4 100644 --- a/tools/cli/internal/openapi/filter/filter.go +++ b/tools/cli/internal/openapi/filter/filter.go @@ -32,8 +32,6 @@ type Metadata struct { targetEnv string } -var filters = []Filter{} - func NewMetadata(targetVersion *apiversion.APIVersion, targetEnv string) *Metadata { return &Metadata{ targetVersion: targetVersion, @@ -46,68 +44,48 @@ func validateMetadata(metadata *Metadata) error { return errors.New("metadata is nil") } - if metadata.targetVersion == nil { - return errors.New("target version is nil") - } - return nil } -func initFilters(oas *openapi3.T, metadata *Metadata) error { - if oas == nil { - return errors.New("openapi document is nil") - } - - if err := validateMetadata(metadata); err != nil { - return err - } - - // using an array to keep the order of filter execution - filters = append( - filters, +func DefaultFilters(oas *openapi3.T, metadata *Metadata) []Filter { + return []Filter{ &SunsetFilter{oas: oas, metadata: metadata}, &VersioningFilter{oas: oas, metadata: metadata}, &InfoFilter{oas: oas, metadata: metadata}, &HiddenEnvsFilter{oas: oas, metadata: metadata}, &TagsFilter{oas: oas}, &OperationsFilter{oas: oas}, - ) + } +} - return nil +// FiltersToGetVersions returns a list of filters to apply to the OpenAPI document to get the versions. +func FiltersToGetVersions(oas *openapi3.T, metadata *Metadata) []Filter { + return []Filter{ + &HiddenEnvsFilter{oas: oas, metadata: metadata}, + } } -func ApplyFiltersWithInit(doc *openapi3.T, metadata *Metadata, init func(oas *openapi3.T, metadata *Metadata) []Filter) (*openapi3.T, error) { - // make a copy of the oas to avoid modifying the original document when applying filters - oas, err := duplicateOas(doc) - if err != nil { - return nil, err +func ApplyFilters(doc *openapi3.T, metadata *Metadata, filters func(oas *openapi3.T, metadata *Metadata) []Filter) (*openapi3.T, error) { + if doc == nil { + return nil, errors.New("openapi document is nil") } - filtersWithInit := init(oas, metadata) - for _, filter := range filtersWithInit { - if err := filter.Apply(); err != nil { - return nil, err - } + if err := validateMetadata(metadata); err != nil { + return nil, err } - return oas, nil -} -func ApplyFilters(doc *openapi3.T, metadata *Metadata) (*openapi3.T, error) { // make a copy of the oas to avoid modifying the original document when applying filters oas, err := duplicateOas(doc) if err != nil { return nil, err } - if err := initFilters(oas, metadata); err != nil { - return nil, err - } - - for _, filter := range filters { + for _, filter := range filters(oas, metadata) { if err := filter.Apply(); err != nil { return nil, err } } + return oas, nil } diff --git a/tools/cli/internal/openapi/filter/hidden_envs.go b/tools/cli/internal/openapi/filter/hidden_envs.go index a3f7fb20c..54812566a 100644 --- a/tools/cli/internal/openapi/filter/hidden_envs.go +++ b/tools/cli/internal/openapi/filter/hidden_envs.go @@ -30,13 +30,6 @@ type HiddenEnvsFilter struct { metadata *Metadata } -func InitHiddenEnvsFilter(oas *openapi3.T, metadata *Metadata) *HiddenEnvsFilter { - return &HiddenEnvsFilter{ - oas: oas, - metadata: metadata, - } -} - func (f *HiddenEnvsFilter) Apply() error { // delete hidden paths first before processing for pathName, pathItem := range f.oas.Paths.Map() { diff --git a/tools/cli/internal/openapi/versions.go b/tools/cli/internal/openapi/versions.go index 019d8c0c8..a2b1b7293 100644 --- a/tools/cli/internal/openapi/versions.go +++ b/tools/cli/internal/openapi/versions.go @@ -21,21 +21,20 @@ import ( "github.com/mongodb/openapi/tools/cli/internal/openapi/filter" ) -// ExtractVersions extracts version strings from an OpenAPI specification. -func ExtractVersions(oas *openapi3.T, env string) ([]string, error) { +func ExtractVersionsWithEnv(oas *openapi3.T, env string) ([]string, error) { // We need to remove the version that are hidden for the given environment - doc, err := filter.ApplyFiltersWithInit(oas, filter.NewMetadata(nil, env), func(oas *openapi3.T, metadata *filter.Metadata) []filter.Filter { - return []filter.Filter{ - filter.InitHiddenEnvsFilter(oas, metadata), - } - }) - + doc, err := filter.ApplyFilters(oas, filter.NewMetadata(nil, env), filter.FiltersToGetVersions) if err != nil { return nil, nil } + return ExtractVersions(doc) +} + +// ExtractVersions extracts version strings from an OpenAPI specification. +func ExtractVersions(oas *openapi3.T) ([]string, error) { versions := make(map[string]struct{}) - for _, pathItem := range doc.Paths.Map() { + for _, pathItem := range oas.Paths.Map() { if pathItem == nil { continue } diff --git a/tools/cli/internal/openapi/versions_test.go b/tools/cli/internal/openapi/versions_test.go index 244eaaf22..610f1dc2c 100644 --- a/tools/cli/internal/openapi/versions_test.go +++ b/tools/cli/internal/openapi/versions_test.go @@ -22,7 +22,7 @@ import ( ) func TestVersions(t *testing.T) { - versions, err := ExtractVersions(NewVersionedResponses(t), "prod") + versions, err := ExtractVersionsWithEnv(NewVersionedResponses(t), "prod") require.NoError(t, err) assert.Equal(t, []string{"2023-01-01", "2023-02-01"}, versions) }