From 773fe7a7c10445aa56bd88a7e87d5d3dd6e47bc1 Mon Sep 17 00:00:00 2001 From: Vitor Date: Wed, 2 Aug 2023 11:24:05 -0300 Subject: [PATCH 01/10] Fixed the etcd retention to delete orphaned snapshots Signed-off-by: Vitor --- pkg/etcd/etcd.go | 8 +++----- pkg/etcd/s3.go | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index ae84ca3b8e67..a80c2a829b6f 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -1429,7 +1429,7 @@ func (e *ETCD) Snapshot(ctx context.Context, config *config.Control) error { if err := e.addSnapshotData(*sf); err != nil { return errors.Wrap(err, "failed to save local snapshot data to configmap") } - + if err := snapshotRetention(e.config.EtcdSnapshotRetention, e.config.EtcdSnapshotName, snapshotDir); err != nil { return errors.Wrap(err, "failed to apply local snapshot retention policy") } @@ -2029,15 +2029,14 @@ func snapshotRetention(retention int, snapshotPrefix string, snapshotDir string) return nil } - nodeName := os.Getenv("NODE_NAME") - logrus.Infof("Applying local snapshot retention policy: retention: %d, snapshotPrefix: %s, directory: %s", retention, snapshotPrefix+"-"+nodeName, snapshotDir) + logrus.Infof("Applying local snapshot retention policy: retention: %d, snapshotPrefix: %s, directory: %s", retention, snapshotPrefix, snapshotDir) var snapshotFiles []os.FileInfo if err := filepath.Walk(snapshotDir, func(path string, info os.FileInfo, err error) error { if err != nil { return err } - if strings.HasPrefix(info.Name(), snapshotPrefix+"-"+nodeName) { + if strings.HasPrefix(info.Name(), snapshotPrefix) { snapshotFiles = append(snapshotFiles, info) } return nil @@ -2050,7 +2049,6 @@ func snapshotRetention(retention int, snapshotPrefix string, snapshotDir string) sort.Slice(snapshotFiles, func(i, j int) bool { return snapshotFiles[i].Name() < snapshotFiles[j].Name() }) - delCount := len(snapshotFiles) - retention for _, df := range snapshotFiles[:delCount] { snapshotPath := filepath.Join(snapshotDir, df.Name()) diff --git a/pkg/etcd/s3.go b/pkg/etcd/s3.go index 42b4ca838786..1acb9f1c82bd 100644 --- a/pkg/etcd/s3.go +++ b/pkg/etcd/s3.go @@ -212,8 +212,7 @@ func (s *S3) Download(ctx context.Context) error { // snapshotPrefix returns the prefix used in the // naming of the snapshots. func (s *S3) snapshotPrefix() string { - nodeName := os.Getenv("NODE_NAME") - fullSnapshotPrefix := s.config.EtcdSnapshotName + "-" + nodeName + fullSnapshotPrefix := s.config.EtcdSnapshotName var prefix string if s.config.EtcdS3Folder != "" { prefix = filepath.Join(s.config.EtcdS3Folder, fullSnapshotPrefix) From 463265775ed87fcc28303495ceb8ccaafbb5d21b Mon Sep 17 00:00:00 2001 From: Vitor Date: Wed, 2 Aug 2023 11:26:42 -0300 Subject: [PATCH 02/10] Fixed file identation Signed-off-by: Vitor --- pkg/etcd/etcd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index a80c2a829b6f..6b323befdd8e 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -1429,7 +1429,6 @@ func (e *ETCD) Snapshot(ctx context.Context, config *config.Control) error { if err := e.addSnapshotData(*sf); err != nil { return errors.Wrap(err, "failed to save local snapshot data to configmap") } - if err := snapshotRetention(e.config.EtcdSnapshotRetention, e.config.EtcdSnapshotName, snapshotDir); err != nil { return errors.Wrap(err, "failed to apply local snapshot retention policy") } @@ -2049,6 +2048,7 @@ func snapshotRetention(retention int, snapshotPrefix string, snapshotDir string) sort.Slice(snapshotFiles, func(i, j int) bool { return snapshotFiles[i].Name() < snapshotFiles[j].Name() }) + delCount := len(snapshotFiles) - retention for _, df := range snapshotFiles[:delCount] { snapshotPath := filepath.Join(snapshotDir, df.Name()) From 6c5af353c0b78f61e2a62b0f82be152949036cea Mon Sep 17 00:00:00 2001 From: Vitor Date: Wed, 2 Aug 2023 11:56:43 -0300 Subject: [PATCH 03/10] Readded prefix for s3 Signed-off-by: Vitor --- pkg/etcd/s3.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/etcd/s3.go b/pkg/etcd/s3.go index 1acb9f1c82bd..42b4ca838786 100644 --- a/pkg/etcd/s3.go +++ b/pkg/etcd/s3.go @@ -212,7 +212,8 @@ func (s *S3) Download(ctx context.Context) error { // snapshotPrefix returns the prefix used in the // naming of the snapshots. func (s *S3) snapshotPrefix() string { - fullSnapshotPrefix := s.config.EtcdSnapshotName + nodeName := os.Getenv("NODE_NAME") + fullSnapshotPrefix := s.config.EtcdSnapshotName + "-" + nodeName var prefix string if s.config.EtcdS3Folder != "" { prefix = filepath.Join(s.config.EtcdS3Folder, fullSnapshotPrefix) From c9103b2eeea82a85d8ca1e5aa2e609430ba38b2d Mon Sep 17 00:00:00 2001 From: Vitor Date: Wed, 2 Aug 2023 11:58:13 -0300 Subject: [PATCH 04/10] Deleted node name prefix s3 Signed-off-by: Vitor --- pkg/etcd/s3.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/etcd/s3.go b/pkg/etcd/s3.go index 42b4ca838786..1acb9f1c82bd 100644 --- a/pkg/etcd/s3.go +++ b/pkg/etcd/s3.go @@ -212,8 +212,7 @@ func (s *S3) Download(ctx context.Context) error { // snapshotPrefix returns the prefix used in the // naming of the snapshots. func (s *S3) snapshotPrefix() string { - nodeName := os.Getenv("NODE_NAME") - fullSnapshotPrefix := s.config.EtcdSnapshotName + "-" + nodeName + fullSnapshotPrefix := s.config.EtcdSnapshotName var prefix string if s.config.EtcdS3Folder != "" { prefix = filepath.Join(s.config.EtcdS3Folder, fullSnapshotPrefix) From 988189bd175cb76797e9633c8a5c675e4665979a Mon Sep 17 00:00:00 2001 From: Vitor Date: Fri, 11 Aug 2023 22:54:29 -0300 Subject: [PATCH 05/10] Fix retention using name instead of date Signed-off-by: Vitor --- pkg/etcd/etcd.go | 4 +++- pkg/etcd/s3.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index 6b323befdd8e..cb94b73b2729 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -2046,7 +2046,9 @@ func snapshotRetention(retention int, snapshotPrefix string, snapshotDir string) return nil } sort.Slice(snapshotFiles, func(i, j int) bool { - return snapshotFiles[i].Name() < snapshotFiles[j].Name() + stringI, stringJ := strings.Split(snapshotFiles[i].Name(), "-"), strings.Split(snapshotFiles[j].Name(), "-") + dateI, dateJ := stringI[len(stringI) - 1], stringJ[len(stringJ) - 1] + return dateI < dateJ }) delCount := len(snapshotFiles) - retention diff --git a/pkg/etcd/s3.go b/pkg/etcd/s3.go index 1acb9f1c82bd..42b9824f2289 100644 --- a/pkg/etcd/s3.go +++ b/pkg/etcd/s3.go @@ -250,7 +250,9 @@ func (s *S3) snapshotRetention(ctx context.Context) error { } sort.Slice(snapshotFiles, func(i, j int) bool { - return snapshotFiles[i].Key < snapshotFiles[j].Key + stringI, stringJ := strings.Split(snapshotFiles[i].Key, "-"), strings.Split(snapshotFiles[j].Key, "-") + dateI, dateJ := stringI[len(stringI) - 1], stringJ[len(stringJ) - 1] + return dateI < dateJ }) delCount := len(snapshotFiles) - s.config.EtcdSnapshotRetention From f10e0226e42fefcb6a38f82f9b27fd78cc4dca40 Mon Sep 17 00:00:00 2001 From: Vitor Date: Fri, 11 Aug 2023 23:38:43 -0300 Subject: [PATCH 06/10] Formating PR Signed-off-by: Vitor --- pkg/etcd/etcd.go | 2 +- pkg/etcd/s3.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index 69353f7b04f0..c4c0d0ae3359 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -2048,7 +2048,7 @@ func snapshotRetention(retention int, snapshotPrefix string, snapshotDir string) } sort.Slice(snapshotFiles, func(i, j int) bool { stringI, stringJ := strings.Split(snapshotFiles[i].Name(), "-"), strings.Split(snapshotFiles[j].Name(), "-") - dateI, dateJ := stringI[len(stringI) - 1], stringJ[len(stringJ) - 1] + dateI, dateJ := stringI[len(stringI)-1], stringJ[len(stringJ)-1] return dateI < dateJ }) diff --git a/pkg/etcd/s3.go b/pkg/etcd/s3.go index 42b9824f2289..646272d18a6d 100644 --- a/pkg/etcd/s3.go +++ b/pkg/etcd/s3.go @@ -251,7 +251,7 @@ func (s *S3) snapshotRetention(ctx context.Context) error { sort.Slice(snapshotFiles, func(i, j int) bool { stringI, stringJ := strings.Split(snapshotFiles[i].Key, "-"), strings.Split(snapshotFiles[j].Key, "-") - dateI, dateJ := stringI[len(stringI) - 1], stringJ[len(stringJ) - 1] + dateI, dateJ := stringI[len(stringI)-1], stringJ[len(stringJ)-1] return dateI < dateJ }) From 6acf1f2723d7ed431c6a8163c4e5b26b3cd76bfe Mon Sep 17 00:00:00 2001 From: Vitor Date: Mon, 14 Aug 2023 11:39:52 -0300 Subject: [PATCH 07/10] Added code comment for the changes to make it clearer Signed-off-by: Vitor --- pkg/etcd/etcd.go | 1 + pkg/etcd/s3.go | 1 + 2 files changed, 2 insertions(+) diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index c4c0d0ae3359..3d5efe75d40a 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -2047,6 +2047,7 @@ func snapshotRetention(retention int, snapshotPrefix string, snapshotDir string) return nil } sort.Slice(snapshotFiles, func(i, j int) bool { + // it takes the key from the file ex: etcd-snapshot-example-{date}, makes the split using "-" to find the date, takes the date and sort by date stringI, stringJ := strings.Split(snapshotFiles[i].Name(), "-"), strings.Split(snapshotFiles[j].Name(), "-") dateI, dateJ := stringI[len(stringI)-1], stringJ[len(stringJ)-1] return dateI < dateJ diff --git a/pkg/etcd/s3.go b/pkg/etcd/s3.go index 646272d18a6d..ae83910fecd4 100644 --- a/pkg/etcd/s3.go +++ b/pkg/etcd/s3.go @@ -250,6 +250,7 @@ func (s *S3) snapshotRetention(ctx context.Context) error { } sort.Slice(snapshotFiles, func(i, j int) bool { + // it takes the key from the file ex: etcd-snapshot-example-{date}, makes the split using "-" to find the date, takes the date and sort by date stringI, stringJ := strings.Split(snapshotFiles[i].Key, "-"), strings.Split(snapshotFiles[j].Key, "-") dateI, dateJ := stringI[len(stringI)-1], stringJ[len(stringJ)-1] return dateI < dateJ From 1640304a138e2812591700a610ab5fc26de27c5d Mon Sep 17 00:00:00 2001 From: Vitor Date: Mon, 14 Aug 2023 13:07:28 -0300 Subject: [PATCH 08/10] Changed the code comment Signed-off-by: Vitor --- pkg/etcd/s3.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/etcd/s3.go b/pkg/etcd/s3.go index ae83910fecd4..57245a7ad41e 100644 --- a/pkg/etcd/s3.go +++ b/pkg/etcd/s3.go @@ -249,11 +249,11 @@ func (s *S3) snapshotRetention(ctx context.Context) error { return nil } - sort.Slice(snapshotFiles, func(i, j int) bool { - // it takes the key from the file ex: etcd-snapshot-example-{date}, makes the split using "-" to find the date, takes the date and sort by date - stringI, stringJ := strings.Split(snapshotFiles[i].Key, "-"), strings.Split(snapshotFiles[j].Key, "-") - dateI, dateJ := stringI[len(stringI)-1], stringJ[len(stringJ)-1] - return dateI < dateJ + sort.Slice(snapshotFiles, func(firstIndex, secondIndex int) bool { + // it takes the name from the file ex: etcd-snapshot-example-{date}, makes the split using "-" to find the date, takes the date and sort by date + nameSplitedFirstIndex, nameSplitedSecondIndex := strings.Split(snapshotFiles[firstIndex].Key, "-"), strings.Split(snapshotFiles[secondIndex].Key, "-") + dateFirstIndex, dateSecondIndex := nameSplitedFirstIndex[len(nameSplitedFirstIndex)-1], nameSplitedSecondIndex[len(nameSplitedSecondIndex)-1] + return dateFirstIndex < dateSecondIndex }) delCount := len(snapshotFiles) - s.config.EtcdSnapshotRetention From e206258b7ba5f55326c88bbd2b1e7b7f857ce205 Mon Sep 17 00:00:00 2001 From: Vitor Date: Mon, 14 Aug 2023 13:23:33 -0300 Subject: [PATCH 09/10] Changed the name of the variables to avoid hungarian notation Signed-off-by: Vitor --- pkg/etcd/etcd.go | 10 +++++----- pkg/etcd/s3.go | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index 3d5efe75d40a..643ed6f27f19 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -2046,11 +2046,11 @@ func snapshotRetention(retention int, snapshotPrefix string, snapshotDir string) if len(snapshotFiles) <= retention { return nil } - sort.Slice(snapshotFiles, func(i, j int) bool { - // it takes the key from the file ex: etcd-snapshot-example-{date}, makes the split using "-" to find the date, takes the date and sort by date - stringI, stringJ := strings.Split(snapshotFiles[i].Name(), "-"), strings.Split(snapshotFiles[j].Name(), "-") - dateI, dateJ := stringI[len(stringI)-1], stringJ[len(stringJ)-1] - return dateI < dateJ + sort.Slice(snapshotFiles, func(firstSnapshot, secondSnapshot int) bool { + // it takes the name from the snapshot file ex: etcd-snapshot-example-{date}, makes the split using "-" to find the date, takes the date and sort by date + firstSnapshotNameSplited, secondSnapshotNameSplited := strings.Split(snapshotFiles[firstSnapshot].Name(), "-"), strings.Split(snapshotFiles[secondSnapshot].Name(), "-") + firstSnapshotDate, secondSnapshotDate := firstSnapshotNameSplited[len(firstSnapshotNameSplited)-1], secondSnapshotNameSplited[len(secondSnapshotNameSplited)-1] + return firstSnapshotDate < secondSnapshotDate }) delCount := len(snapshotFiles) - retention diff --git a/pkg/etcd/s3.go b/pkg/etcd/s3.go index 57245a7ad41e..606cac684e04 100644 --- a/pkg/etcd/s3.go +++ b/pkg/etcd/s3.go @@ -249,11 +249,11 @@ func (s *S3) snapshotRetention(ctx context.Context) error { return nil } - sort.Slice(snapshotFiles, func(firstIndex, secondIndex int) bool { - // it takes the name from the file ex: etcd-snapshot-example-{date}, makes the split using "-" to find the date, takes the date and sort by date - nameSplitedFirstIndex, nameSplitedSecondIndex := strings.Split(snapshotFiles[firstIndex].Key, "-"), strings.Split(snapshotFiles[secondIndex].Key, "-") - dateFirstIndex, dateSecondIndex := nameSplitedFirstIndex[len(nameSplitedFirstIndex)-1], nameSplitedSecondIndex[len(nameSplitedSecondIndex)-1] - return dateFirstIndex < dateSecondIndex + sort.Slice(snapshotFiles, func(firstSnapshot, secondSnapshot int) bool { + // it takes the key from the snapshot file ex: etcd-snapshot-example-{date}, makes the split using "-" to find the date, takes the date and sort by date + firstSnapshotNameSplited, secondSnapshotNameSplited := strings.Split(snapshotFiles[firstSnapshot].Key, "-"), strings.Split(snapshotFiles[secondSnapshot].Key, "-") + firstSnapshotDate, secondSnapshotDate := firstSnapshotNameSplited[len(firstSnapshotNameSplited)-1], secondSnapshotNameSplited[len(secondSnapshotNameSplited)-1] + return firstSnapshotDate < secondSnapshotDate }) delCount := len(snapshotFiles) - s.config.EtcdSnapshotRetention From 44d67233c6eacc1fd7180f530a1545e76efe64c4 Mon Sep 17 00:00:00 2001 From: Vitor Date: Mon, 14 Aug 2023 13:40:12 -0300 Subject: [PATCH 10/10] Changed spelling error in variable name Signed-off-by: Vitor --- pkg/etcd/etcd.go | 4 ++-- pkg/etcd/s3.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/etcd/etcd.go b/pkg/etcd/etcd.go index 643ed6f27f19..249ea4d0c51c 100644 --- a/pkg/etcd/etcd.go +++ b/pkg/etcd/etcd.go @@ -2048,8 +2048,8 @@ func snapshotRetention(retention int, snapshotPrefix string, snapshotDir string) } sort.Slice(snapshotFiles, func(firstSnapshot, secondSnapshot int) bool { // it takes the name from the snapshot file ex: etcd-snapshot-example-{date}, makes the split using "-" to find the date, takes the date and sort by date - firstSnapshotNameSplited, secondSnapshotNameSplited := strings.Split(snapshotFiles[firstSnapshot].Name(), "-"), strings.Split(snapshotFiles[secondSnapshot].Name(), "-") - firstSnapshotDate, secondSnapshotDate := firstSnapshotNameSplited[len(firstSnapshotNameSplited)-1], secondSnapshotNameSplited[len(secondSnapshotNameSplited)-1] + firstSnapshotName, secondSnapshotName := strings.Split(snapshotFiles[firstSnapshot].Name(), "-"), strings.Split(snapshotFiles[secondSnapshot].Name(), "-") + firstSnapshotDate, secondSnapshotDate := firstSnapshotName[len(firstSnapshotName)-1], secondSnapshotName[len(secondSnapshotName)-1] return firstSnapshotDate < secondSnapshotDate }) diff --git a/pkg/etcd/s3.go b/pkg/etcd/s3.go index 606cac684e04..fe15f8f1f129 100644 --- a/pkg/etcd/s3.go +++ b/pkg/etcd/s3.go @@ -251,8 +251,8 @@ func (s *S3) snapshotRetention(ctx context.Context) error { sort.Slice(snapshotFiles, func(firstSnapshot, secondSnapshot int) bool { // it takes the key from the snapshot file ex: etcd-snapshot-example-{date}, makes the split using "-" to find the date, takes the date and sort by date - firstSnapshotNameSplited, secondSnapshotNameSplited := strings.Split(snapshotFiles[firstSnapshot].Key, "-"), strings.Split(snapshotFiles[secondSnapshot].Key, "-") - firstSnapshotDate, secondSnapshotDate := firstSnapshotNameSplited[len(firstSnapshotNameSplited)-1], secondSnapshotNameSplited[len(secondSnapshotNameSplited)-1] + firstSnapshotName, secondSnapshotName := strings.Split(snapshotFiles[firstSnapshot].Key, "-"), strings.Split(snapshotFiles[secondSnapshot].Key, "-") + firstSnapshotDate, secondSnapshotDate := firstSnapshotName[len(firstSnapshotName)-1], secondSnapshotName[len(secondSnapshotName)-1] return firstSnapshotDate < secondSnapshotDate })