CLOUDP-267267: Update changelog creation to use the exemption file

tools/cli/internal/breakingchanges/exemptions.go

@@ -27,7 +27,7 @@ type Exemption struct {
 	BreakingChangeDescription string `yaml:"breaking_change_description"`
 	ExemptUntil               string `yaml:"exempt_until"`
 	Reason                    string `yaml:"reason"`
-	HideFromChangelog         string `yaml:"hide_from_changelog:omitempty"`
+	HideFromChangelog         string `yaml:"hide_from_changelog,omitempty"`
 }
 
 func (e *Exemption) isHidden() bool {

tools/cli/internal/changelog/changelog.go

@@ -56,18 +56,18 @@ type Changelog struct {
 }
 
 type Metadata struct {
-	Path          string
-	ActiveVersion string
+	Path          string   `json:"path,omitempty"`
+	ActiveVersion string   `json:"activeVersion,omitempty"`
 	RunDate       string   `json:"runDate"`
-	SpecRevision  string   `json:"specRevision"`
+	SpecRevision  string   `json:"specRevision,omitempty"`
 	Versions      []string `json:"versions"`
 }
 
 type Entry struct {
 	Date        string  `json:"date"`
 	Paths       []*Path `json:"paths"`
-	FromVersion string
-	ToVersion   string
+	FromVersion string  `json:"fromVersion,omitempty"`
+	ToVersion   string  `json:"toVersion,omitempty"`
 }
 
 type Path struct {
@@ -98,7 +98,7 @@ type Change struct {
 // NewEntries generates the changelog entries between the base and revision specs.
 // The returned entries includes all the changes between the base and revision specs included the one
 // marked as hidden.
-func NewEntries(basePath, revisionPath string) ([]*Entry, error) {
+func NewEntries(basePath, revisionPath, exceptionFilePath string) ([]*Entry, error) {
 	baseMetadata, err := newMetadataFromFile(basePath)
 	if err != nil {
 		return nil, err
@@ -142,7 +142,7 @@ func NewEntries(basePath, revisionPath string) ([]*Entry, error) {
 	baseMetadata.ActiveVersion = baseActiveVersionOnPreviousRunDate
 	revisionMetadata.ActiveVersion = revisionActiveVersionOnPreviousRunDate
 
-	changelog, err := newChangelog(baseMetadata, revisionMetadata, nil)
+	changelog, err := newChangelog(baseMetadata, revisionMetadata, exceptionFilePath, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -160,7 +160,7 @@ func NewEntries(basePath, revisionPath string) ([]*Entry, error) {
 		// baseActiveVersionOnPreviousRunDate with revisionActiveVersionOnPreviousRunDate)
 		baseMetadata.ActiveVersion = baseActiveVersionOnRunDate
 		revisionMetadata.ActiveVersion = revisionActiveVersionOnRunDate
-		changelog, err = newChangelog(baseMetadata, revisionMetadata, changelogEntries)
+		changelog, err = newChangelog(baseMetadata, revisionMetadata, exceptionFilePath, changelogEntries)
 		if err != nil {
 			return nil, err
 		}
@@ -193,17 +193,17 @@ func NewEntries(basePath, revisionPath string) ([]*Entry, error) {
 // NewEntriesWithoutHidden generates the changelog entries between the base and revision specs.
 // The returned entries includes the changes between the base and revision specs that are not
 // marked as hidden.
-func NewEntriesWithoutHidden(basePath, revisionPath string) ([]*Entry, error) {
-	entries, err := NewEntries(basePath, revisionPath)
+func NewEntriesWithoutHidden(basePath, revisionPath, exceptionFilePath string) ([]*Entry, error) {
+	entries, err := NewEntries(basePath, revisionPath, exceptionFilePath)
 	if err != nil {
 		return nil, err
 	}
 
-	return NewNotHiddenEntries(entries), nil
+	return NewNotHiddenEntries(entries)
 }
 
 // NewEntriesBetweenRevisionVersions generates the changelog entries between the revision versions.
-func NewEntriesBetweenRevisionVersions(revisionPath string) ([]*Entry, error) {
+func NewEntriesBetweenRevisionVersions(revisionPath, exceptionFilePath string) ([]*Entry, error) {
 	revisionMetadata, err := newMetadataFromFile(revisionPath)
 	if err != nil {
 		return nil, err
@@ -212,7 +212,7 @@ func NewEntriesBetweenRevisionVersions(revisionPath string) ([]*Entry, error) {
 	entries := []*Entry{}
 	for idx, fromVersion := range revisionMetadata.Versions {
 		for _, toVersion := range revisionMetadata.Versions[idx+1:] {
-			entry, err := newEntriesBetweenVersion(revisionMetadata, fromVersion, toVersion)
+			entry, err := newEntriesBetweenVersion(revisionMetadata, fromVersion, toVersion, exceptionFilePath)
 			if err != nil {
 				return nil, err
 			}
@@ -223,7 +223,7 @@ func NewEntriesBetweenRevisionVersions(revisionPath string) ([]*Entry, error) {
 	return newVersionEntries(entries), nil
 }
 
-func newEntriesBetweenVersion(metadata *Metadata, fromVersion, toVersion string) (*Entry, error) {
+func newEntriesBetweenVersion(metadata *Metadata, fromVersion, toVersion, exceptionFilePath string) (*Entry, error) {
 	baseMetadata := &Metadata{
 		Path:          metadata.Path,
 		ActiveVersion: fromVersion,
@@ -240,7 +240,7 @@ func newEntriesBetweenVersion(metadata *Metadata, fromVersion, toVersion string)
 		Versions:      metadata.Versions,
 	}
 
-	changelog, err := newChangelog(baseMetadata, revisionMetadata, []*Entry{})
+	changelog, err := newChangelog(baseMetadata, revisionMetadata, exceptionFilePath, []*Entry{})
 	if err != nil {
 		return nil, err
 	}
@@ -307,7 +307,7 @@ func newVersionedPaths(paths []*Path, version string) []*Path {
 	return versionedPaths
 }
 
-func newChangelog(baseMetadata, revisionMetadata *Metadata, baseChangelog []*Entry) (*Changelog, error) {
+func newChangelog(baseMetadata, revisionMetadata *Metadata, exceptionFilePath string, baseChangelog []*Entry) (*Changelog, error) {
 	var err error
 	if baseChangelog == nil {
 		baseChangelog, err = newEntriesFromPath(fmt.Sprintf("%s/%s", baseMetadata.Path, "changelog.json"))
@@ -332,7 +332,7 @@ func newChangelog(baseMetadata, revisionMetadata *Metadata, baseChangelog []*Ent
 		BaseMetadata:      baseMetadata,
 		RevisionMetadata:  revisionMetadata,
 		Config:            changelogConfig,
-		ExemptionFilePath: fmt.Sprintf("%s/%s", revisionMetadata.Path, "exemptions.yaml"),
+		ExemptionFilePath: exceptionFilePath,
 		OasDiff: openapi.NewOasDiffWithSpecInfo(baseSpec, revisionSpec, &diff.Config{
 			IncludePathParams: true,
 		}),
@@ -355,7 +355,7 @@ func newEntriesFromPath(path string) ([]*Entry, error) {
 
 func newBaseAndRevisionSpecs(baseMetadata, revisionMetadata *Metadata) (baseSpec, revisionSpec *load.SpecInfo, err error) {
 	if baseMetadata.ActiveVersion != revisionMetadata.ActiveVersion {
-		log.Printf("Base spec revision %s is different from the active version %s", baseMetadata.SpecRevision, baseMetadata.ActiveVersion)
+		log.Printf("Base spec version %s is different from the active version %s", baseMetadata.ActiveVersion, revisionMetadata.ActiveVersion)
 		log.Println("Normalizing the specs: replace versioned media-types with corresponding standard media-types")
 		baseSpec, err = openapi.CreateNormalizedOpenAPISpecFromPath(fmt.Sprintf("%s/openapi-%s.json", baseMetadata.Path, baseMetadata.ActiveVersion))
 		if err != nil {
@@ -489,9 +489,14 @@ func newStringFromStruct(data interface{}) string {
 // 2. Remove all empty versions
 // 3. Remove all empty paths
 // 4. Shift changelog entry if it turns out empty
-func NewNotHiddenEntries(changelog []*Entry) []*Entry {
+func NewNotHiddenEntries(changelog []*Entry) ([]*Entry, error) {
 	if len(changelog) == 0 {
-		return changelog
+		return changelog, nil
+	}
+
+	changelog, err := duplicateEntries(changelog)
+	if err != nil {
+		return nil, err
 	}
 
 	// Get changes only for the last date, which is what was recently merged
@@ -524,11 +529,11 @@ func NewNotHiddenEntries(changelog []*Entry) []*Entry {
 	}
 
 	if len(paths) == 0 {
-		return changelog[1:]
+		return changelog[1:], nil
 	}
 
 	changelog[0].Paths = paths
-	return changelog
+	return changelog, nil
 }
 
 func newNotHiddenChanges(changes []*Change) []*Change {

tools/cli/internal/changelog/changelog_test.go

@@ -18,6 +18,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestNewNotHiddenEntries(t *testing.T) {
@@ -232,7 +233,8 @@ func TestNewNotHiddenEntries(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			actual := NewNotHiddenEntries(tt.changelog)
+			actual, err := NewNotHiddenEntries(tt.changelog)
+			require.NoError(t, err)
 			assert.Equal(t, tt.expected, actual)
 		})
 	}
@@ -300,7 +302,7 @@ func TestMultipleHiddenEntries(t *testing.T) {
 									Changes: []*Change{
 										{
 											Code:               "request-property-removed",
-											Description:        "added 'replicationSpecs.regionConfigs' request property",
+											Description:        "removed 'replicationSpecs.regionConfigs' request property",
 											BackwardCompatible: true,
 										},
 									},
@@ -339,7 +341,7 @@ func TestMultipleHiddenEntries(t *testing.T) {
 									Changes: []*Change{
 										{
 											Code:               "request-property-removed",
-											Description:        "added 'replicationSpecs.regionConfigs' request property",
+											Description:        "removed 'replicationSpecs.regionConfigs' request property",
 											BackwardCompatible: true,
 										},
 									},
@@ -593,7 +595,8 @@ func TestMultipleHiddenEntries(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			actual := NewNotHiddenEntries(tt.changelog)
+			actual, err := NewNotHiddenEntries(tt.changelog)
+			require.NoError(t, err)
 			assert.Equal(t, tt.expected, actual)
 		})
 	}

tools/cli/internal/changelog/merge.go

@@ -128,7 +128,6 @@ func (m *Changelog) newEntryFromOasDiff() ([]*Entry, error) {
 	}
 
 	log.Printf("Found %d changes between %s and %s", len(changes), m.Base.Url, m.Revision.Url)
-	log.Printf("Changes: %s", newStringFromStruct(changes))
 
 	conf := outputfilter.NewOperationConfigs(m.Base, m.Revision)
 

tools/cli/internal/changelog/outputfilter/hide.go

@@ -32,6 +32,7 @@ func MarkHiddenEntries(entries []*OasDiffEntry, exemptionsFilePath string, fs af
 	if err != nil {
 		return nil, err
 	}
+
 	return hideByIDs(entries, hideIDs)
 }
 

tools/cli/internal/changelog/outputfilter/operationconfig_test.go

@@ -5,9 +5,61 @@ import (
 	"testing"
 
 	"github.com/getkin/kin-openapi/openapi3"
+	"github.com/mongodb/openapi/tools/cli/internal/openapi"
+	"github.com/stretchr/testify/require"
 	"github.com/tufin/oasdiff/load"
 )
 
+func TestOpenApiSpecMethods(t *testing.T) {
+	loader := openapi.NewOpenAPI3().WithExcludedPrivatePaths()
+	spec, err := loader.CreateOpenAPISpecFromPath("../../../test/data/changelog/test_spec.json")
+	require.NoError(t, err)
+
+	expectedConfig := map[string]*OperationConfig{
+		"createCluster": {
+			Path:                   "/api/atlas/v2/groups/{groupId}/clusters",
+			HTTPMethod:             "POST",
+			Tag:                    "Multi-Cloud Clusters",
+			Sunset:                 "2025-06-01",
+			ManualChangelogEntries: map[string]interface{}{},
+		},
+		"deleteCluster": {
+			Path:                   "/api/atlas/v2/groups/{groupId}/clusters/{clusterName}",
+			HTTPMethod:             "DELETE",
+			Tag:                    "Multi-Cloud Clusters",
+			Sunset:                 "2025-06-01",
+			ManualChangelogEntries: map[string]interface{}{},
+		},
+		"getCluster": {
+			Path:       "/api/atlas/v2/groups/{groupId}/clusters/{clusterName}",
+			HTTPMethod: "GET",
+			Tag:        "Multi-Cloud Clusters",
+			Sunset:     "2025-06-01",
+			ManualChangelogEntries: map[string]interface{}{
+				"2023-07-07": "July 7th changelog entry",
+				"2023-08-08": "August announcement for /api/atlas/v2/clusters",
+			},
+		},
+		"listClusters": {
+			Path:                   "/api/atlas/v2/groups/{groupId}/clusters",
+			HTTPMethod:             "GET",
+			Tag:                    "Multi-Cloud Clusters",
+			Sunset:                 "2025-06-01",
+			ManualChangelogEntries: map[string]interface{}{},
+		},
+		"updateCluster": {
+			Path:                   "/api/atlas/v2/groups/{groupId}/clusters/{clusterName}",
+			HTTPMethod:             "PATCH",
+			Tag:                    "Multi-Cloud Clusters",
+			Sunset:                 "2025-06-01",
+			ManualChangelogEntries: map[string]interface{}{},
+		},
+	}
+
+	operationConfigs := newOperationConfigFromSpec(spec)
+	require.True(t, reflect.DeepEqual(expectedConfig, operationConfigs))
+}
+
 func TestOperationConfigs_Tag(t *testing.T) {
 	tests := []struct {
 		name     string

tools/cli/internal/changelog/outputfilter/outputfilter.go

@@ -67,6 +67,12 @@ func NewChangelogEntries(checkers checker.Changes, specInfoPair *load.SpecInfoPa
 }
 
 func transformEntries(entries []*OasDiffEntry, exemptionsFilePath string) ([]*OasDiffEntry, error) {
+	fs := afero.NewOsFs()
+	entries, err := MarkHiddenEntries(entries, exemptionsFilePath, fs)
+	if err != nil {
+		return nil, err
+	}
+
 	newEntries := make([]*OasDiffEntry, 0)
 	for _, entry := range entries {
 		// only changes linked to endpoints are currently considered.
@@ -78,13 +84,7 @@ func transformEntries(entries []*OasDiffEntry, exemptionsFilePath string) ([]*Oa
 		newEntries = append(newEntries, entry)
 	}
 
-	newEntries, err := squashEntries(newEntries)
-	if err != nil {
-		return nil, err
-	}
-
-	fs := afero.NewOsFs()
-	newEntries, err = MarkHiddenEntries(newEntries, exemptionsFilePath, fs)
+	newEntries, err = squashEntries(newEntries)
 	if err != nil {
 		return nil, err
 	}

tools/cli/internal/changelog/outputfilter/squash.go

@@ -159,6 +159,7 @@ func squashEntries(entries []*OasDiffEntry) ([]*OasDiffEntry, error) {
 
 	return squashedEntries, nil
 }
+
 func sortEntriesByDescription(entries []*OasDiffEntry) []*OasDiffEntry {
 	sort.Slice(entries, func(i, j int) bool {
 		return entries[i].Text < entries[j].Text

tools/cli/internal/cli/changelog/create.go

@@ -44,13 +44,17 @@ type Opts struct {
 }
 
 func (o *Opts) Run() error {
-	entries, err := changelog.NewEntries(o.basePath, o.revisionPath)
+	entries, err := changelog.NewEntries(o.basePath, o.revisionPath, o.exceptionsPaths)
 	if err != nil {
 		return err
 	}
 
-	notHiddenEntries := changelog.NewNotHiddenEntries(entries)
-	versionedEntries, err := changelog.NewEntriesBetweenRevisionVersions(o.revisionPath)
+	notHiddenEntries, err := changelog.NewNotHiddenEntries(entries)
+	if err != nil {
+		return err
+	}
+
+	versionedEntries, err := changelog.NewEntriesBetweenRevisionVersions(o.revisionPath, o.exceptionsPaths)
 	if err != nil {
 		return err
 	}
@@ -60,14 +64,14 @@ func (o *Opts) Run() error {
 		return nil
 	}
 
-	if errSaveFile := openapi.SaveToFile(o.newOutputFilePath(changelogFileName), "", notHiddenEntries, o.fs); errSaveFile != nil {
-		return errSaveFile
-	}
-
 	if err := o.fs.MkdirAll(fmt.Sprintf("%s/%s", o.outputPath, changelogAllFolderName), 0o755); err != nil {
 		return err
 	}
 
+	if errSaveFile := openapi.SaveToFile(o.newOutputFilePath(changelogFileName), "", notHiddenEntries, o.fs); errSaveFile != nil {
+		return errSaveFile
+	}
+
 	if errSaveFile := openapi.SaveToFile(
 		o.newOutputFilePath(fmt.Sprintf("%s/%s", changelogAllFolderName, changelogAllFileName)), "", entries, o.fs); errSaveFile != nil {
 		return errSaveFile