From 7840a3caf9673798c5d0976d5e7ef97ca3d72d72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Bir=C3=B3?= Date: Fri, 8 Dec 2023 17:56:57 +0100 Subject: [PATCH 1/9] bump go sdk --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 64ce2e9..a69aa4f 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/sprig v2.22.0+incompatible - github.com/aws/aws-sdk-go v1.37.20 + github.com/aws/aws-sdk-go v1.48.15 github.com/go-logr/logr v0.1.0 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect github.com/hashicorp/golang-lru v0.5.4 From 9e79bb1e091383305d0feef74e2ce5b37764e315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Bir=C3=B3?= Date: Fri, 8 Dec 2023 18:25:25 +0100 Subject: [PATCH 2/9] bump go version --- Dockerfile | 2 +- go.mod | 64 +++++++++++++++++++++++++++++++++++++++++++++++------- go.sum | 26 ++++++++++++---------- 3 files changed, 71 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index f55aaf5..c719817 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.14 as base +FROM golang:1.20 as base WORKDIR /workspace # Copy the Go Modules manifests diff --git a/go.mod b/go.mod index a69aa4f..9b9e4a7 100644 --- a/go.mod +++ b/go.mod @@ -1,26 +1,74 @@ module github.com/contentful-labs/kube-secret-syncer -go 1.14 +go 1.20 require ( - github.com/Masterminds/goutils v1.1.1 // indirect - github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/sprig v2.22.0+incompatible github.com/aws/aws-sdk-go v1.48.15 github.com/go-logr/logr v0.1.0 - github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect github.com/hashicorp/golang-lru v0.5.4 - github.com/huandu/xstrings v1.3.0 // indirect - github.com/mitchellh/copystructure v1.0.0 // indirect github.com/onsi/ginkgo v1.14.2 github.com/onsi/gomega v1.10.5 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.11.1 go.uber.org/zap v1.16.0 - golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect - golang.org/x/tools v0.0.0-20200103221440-774c71fcf114 // indirect k8s.io/api v0.18.6 k8s.io/apimachinery v0.18.6 k8s.io/client-go v0.18.6 sigs.k8s.io/controller-runtime v0.6.4 ) + +require ( + cloud.google.com/go v0.38.0 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/Masterminds/semver v1.5.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/evanphx/json-patch v4.9.0+incompatible // indirect + github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/go-logr/zapr v0.1.0 // indirect + github.com/gogo/protobuf v1.3.1 // indirect + github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect + github.com/golang/protobuf v1.4.3 // indirect + github.com/google/go-cmp v0.5.5 // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/google/uuid v1.1.1 // indirect + github.com/googleapis/gnostic v0.3.1 // indirect + github.com/huandu/xstrings v1.3.0 // indirect + github.com/imdario/mergo v0.3.9 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/json-iterator/go v1.1.11 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/mitchellh/copystructure v1.0.0 // indirect + github.com/mitchellh/reflectwalk v1.0.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/nxadm/tail v1.4.4 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.26.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.uber.org/atomic v1.6.0 // indirect + go.uber.org/multierr v1.5.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect + gomodules.xyz/jsonpatch/v2 v2.0.1 // indirect + google.golang.org/appengine v1.5.0 // indirect + google.golang.org/protobuf v1.26.0-rc.1 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect + gopkg.in/yaml.v2 v2.3.0 // indirect + k8s.io/apiextensions-apiserver v0.18.6 // indirect + k8s.io/klog v1.0.0 // indirect + k8s.io/klog/v2 v2.0.0 // indirect + k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 // indirect + k8s.io/utils v0.0.0-20200603063816-c1c6865ac451 // indirect + sigs.k8s.io/structured-merge-diff/v3 v3.0.0 // indirect + sigs.k8s.io/yaml v1.2.0 // indirect +) diff --git a/go.sum b/go.sum index 805fcbc..276ecb3 100644 --- a/go.sum +++ b/go.sum @@ -34,8 +34,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.37.20 h1:CJCXpMYmBJrRH8YwoSE0oB9S3J5ax+62F14sYlDCztg= -github.com/aws/aws-sdk-go v1.37.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.48.15 h1:Gad2C4pLzuZDd5CA0Rvkfko6qUDDTOYru145gkO7w/Y= +github.com/aws/aws-sdk-go v1.48.15/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -366,10 +366,10 @@ golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -378,7 +378,7 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -398,9 +398,9 @@ golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -437,14 +437,18 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -466,10 +470,8 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114 h1:DnSr2mCsxyCE6ZgIkmcWUQY2R5cH/6wL7eIxEmQOMSE= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.0.1 h1:xyiBuvkD2g5n7cYzx6u2sxQvsAy4QJsZFCzGVdzOXZ0= From 4a552cf1714dd9094ff47ac59ff1b1be85e47afc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Bir=C3=B3?= Date: Fri, 8 Dec 2023 18:36:52 +0100 Subject: [PATCH 3/9] get deleted secrets too --- pkg/secretsmanager/poller.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/secretsmanager/poller.go b/pkg/secretsmanager/poller.go index 5f67a68..8382d22 100644 --- a/pkg/secretsmanager/poller.go +++ b/pkg/secretsmanager/poller.go @@ -99,8 +99,10 @@ func (p *Poller) fetchSecrets() (Secrets, error) { fetchedSecrets := make(Secrets) allSecrets := []*secretsmanager.SecretListEntry{} + includePlannedDeletion := true input := &secretsmanager.ListSecretsInput{ - MaxResults: aws.Int64(100), + MaxResults: aws.Int64(100), + IncludePlannedDeletion: &includePlannedDeletion, } smClient, err := p.getSMClient(p.defaultSearchRole) From 8c08ee25aca07b9a30a656f2a88527df7733e9b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Bir=C3=B3?= Date: Fri, 8 Dec 2023 19:22:54 +0100 Subject: [PATCH 4/9] extend polled secret meta type; store deleted status --- pkg/secretsmanager/poller.go | 6 ++---- pkg/secretsmanager/poller_test.go | 5 +++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/secretsmanager/poller.go b/pkg/secretsmanager/poller.go index 8382d22..5e08f07 100644 --- a/pkg/secretsmanager/poller.go +++ b/pkg/secretsmanager/poller.go @@ -35,6 +35,7 @@ type PolledSecretMeta struct { Tags map[string]string CurrentVersionID string UpdatedAt time.Time + Deleted bool } // New creates a new poller, will send polling or other non critical errors through the errs channel @@ -123,10 +124,6 @@ func (p *Poller) fetchSecrets() (Secrets, error) { } for _, secret := range allSecrets { - if secret.DeletedDate != nil { - continue - } - versionID, err := getCurrentVersion(secret.SecretVersionsToStages) if err != nil { continue @@ -141,6 +138,7 @@ func (p *Poller) fetchSecrets() (Secrets, error) { Tags: secretTags, CurrentVersionID: versionID, UpdatedAt: *secret.LastChangedDate, + Deleted: secret.DeletedDate != nil, } } diff --git a/pkg/secretsmanager/poller_test.go b/pkg/secretsmanager/poller_test.go index b157f2d..7d9fef4 100644 --- a/pkg/secretsmanager/poller_test.go +++ b/pkg/secretsmanager/poller_test.go @@ -116,11 +116,13 @@ func TestFetchSecret(t *testing.T) { CurrentVersionID: "002", UpdatedAt: now.AddDate(0, 0, -2), Tags: map[string]string{}, + Deleted: false, }, "random/aws/secret003": PolledSecretMeta{ CurrentVersionID: "005", UpdatedAt: now.AddDate(0, 0, -3), Tags: map[string]string{}, + Deleted: false, }, }, }, { @@ -153,6 +155,7 @@ func TestFetchSecret(t *testing.T) { CurrentVersionID: "randomuuid", UpdatedAt: now.AddDate(0, 0, -2), Tags: map[string]string{}, + Deleted: false, }, }, }, { @@ -283,6 +286,7 @@ func TestPoll(t *testing.T) { CurrentVersionID: "randomuuid", UpdatedAt: now.AddDate(0, 0, -2), Tags: map[string]string{}, + Deleted: false, }, }, }, @@ -317,6 +321,7 @@ func TestPoll(t *testing.T) { CurrentVersionID: "randomuuid", UpdatedAt: now.AddDate(0, 0, -2), Tags: map[string]string{}, + Deleted: false, }, }, }, From bb2ac7d5c945eb19336eada7db023b28a57dcb9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Bir=C3=B3?= Date: Fri, 8 Dec 2023 19:41:39 +0100 Subject: [PATCH 5/9] fixed test running --- pkg/secretsmanager/poller_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/secretsmanager/poller_test.go b/pkg/secretsmanager/poller_test.go index 7d9fef4..3e7ec89 100644 --- a/pkg/secretsmanager/poller_test.go +++ b/pkg/secretsmanager/poller_test.go @@ -188,7 +188,7 @@ func TestFetchSecret(t *testing.T) { t.Errorf("test %s returned error %s", test.name, err) } if !reflect.DeepEqual(got, test.want) { - t.Errorf("test %s, wanted %s got %s", test.name, test.want, got) + t.Errorf("test %s, wanted %v got %v", test.name, test.want, got) } } } @@ -227,7 +227,7 @@ func TestFetchSecretError(t *testing.T) { t.Errorf("test %s should have returned an error, did not", test.name) } if got != nil { - t.Errorf("test %s, wanted %s got %s", test.name, test.want, got) + t.Errorf("test %s, wanted %v got %v", test.name, test.want, got) } } } From de776afd00249a3834ea1850f497952a3ddba68c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Bir=C3=B3?= Date: Fri, 8 Dec 2023 20:06:35 +0100 Subject: [PATCH 6/9] test for polling deleted secret --- pkg/secretsmanager/poller_test.go | 56 ++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/pkg/secretsmanager/poller_test.go b/pkg/secretsmanager/poller_test.go index 3e7ec89..29c2722 100644 --- a/pkg/secretsmanager/poller_test.go +++ b/pkg/secretsmanager/poller_test.go @@ -84,7 +84,7 @@ func TestFetchSecret(t *testing.T) { want Secrets }{ { - name: "test 1", + name: "two current secrets", have: mockSecretsManagerClient{ Resp: secretsmanager.ListSecretsOutput{ SecretList: []*secretsmanager.SecretListEntry{ @@ -126,7 +126,7 @@ func TestFetchSecret(t *testing.T) { }, }, }, { - name: "test 2", + name: "one outdated secret, one current", have: mockSecretsManagerClient{ Resp: secretsmanager.ListSecretsOutput{ SecretList: []*secretsmanager.SecretListEntry{ @@ -159,7 +159,7 @@ func TestFetchSecret(t *testing.T) { }, }, }, { - name: "test 3", + name: "only one outdated secret", have: mockSecretsManagerClient{ Resp: secretsmanager.ListSecretsOutput{ SecretList: []*secretsmanager.SecretListEntry{ @@ -176,6 +176,46 @@ func TestFetchSecret(t *testing.T) { }, }, want: Secrets{}, + }, { + name: "one current secret, one deleted", + have: mockSecretsManagerClient{ + Resp: secretsmanager.ListSecretsOutput{ + SecretList: []*secretsmanager.SecretListEntry{ + { + Name: _s("random/aws/secret"), + LastChangedDate: _t(now.AddDate(0, 0, -2)), + SecretVersionsToStages: map[string][]*string{ + "randomuuid": { + _s("AWSCURRENT"), + }, + }, + }, { + Name: _s("random/aws/secretdeleted"), + LastChangedDate: _t(now.AddDate(0, 0, -2)), + SecretVersionsToStages: map[string][]*string{ + "randomuuid_deleted": { + _s("AWSCURRENT"), + }, + }, + DeletedDate: _t(now.AddDate(0, 0, -1)), + }, + }, + }, + }, + want: Secrets{ + "random/aws/secret": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secretdeleted": PolledSecretMeta{ + CurrentVersionID: "randomuuid_deleted", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: true, + }, + }, }, } { p := Poller{ @@ -257,7 +297,7 @@ func TestPoll(t *testing.T) { want Secrets }{ { - name: "test 2", + name: "one current secret, one outdated", have: mockWorkingThenFailingSecretsManagerClient{ Resp: secretsmanager.ListSecretsOutput{ SecretList: []*secretsmanager.SecretListEntry{ @@ -291,7 +331,7 @@ func TestPoll(t *testing.T) { }, }, { - name: "test 3", + name: "one current secret, one deleted", have: mockWorkingThenFailingSecretsManagerClient{ Resp: secretsmanager.ListSecretsOutput{ SecretList: []*secretsmanager.SecretListEntry{ @@ -323,6 +363,12 @@ func TestPoll(t *testing.T) { Tags: map[string]string{}, Deleted: false, }, + "random/aws/deletedsecret": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: true, + }, }, }, } { From 058bcae0aa16260f83bb9404ff140f8aaf61ef19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Bir=C3=B3?= Date: Fri, 8 Dec 2023 20:22:31 +0100 Subject: [PATCH 7/9] deleted secrets shouldnt be fetched --- pkg/secretsmanager/secrets.go | 2 +- pkg/secretsmanager/secrets_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/pkg/secretsmanager/secrets.go b/pkg/secretsmanager/secrets.go index 8d9ff15..e71775a 100644 --- a/pkg/secretsmanager/secrets.go +++ b/pkg/secretsmanager/secrets.go @@ -58,7 +58,7 @@ func (p *Poller) fetchCurrentSecretCache(secretID *string, role string) (*secret secretValuesByRole := cachedElem.(map[string]secretsmanager.GetSecretValueOutput) if secretValueOut, ok := secretValuesByRole[role]; ok { polledSecretMeta, found := p.PolledSecrets[*secretID] - if found && polledSecretMeta.CurrentVersionID == *secretValueOut.VersionId { + if found && !polledSecretMeta.Deleted && polledSecretMeta.CurrentVersionID == *secretValueOut.VersionId { return &secretValueOut, found } } diff --git a/pkg/secretsmanager/secrets_test.go b/pkg/secretsmanager/secrets_test.go index 5e4c964..d498613 100644 --- a/pkg/secretsmanager/secrets_test.go +++ b/pkg/secretsmanager/secrets_test.go @@ -33,6 +33,7 @@ func TestFetchCurrentSecret(t *testing.T) { "cf/secret/test": PolledSecretMeta{ CurrentVersionID: "present", UpdatedAt: time.Now().AddDate(0, 0, -2), + Deleted: false, }, }, }, @@ -58,6 +59,7 @@ func TestFetchCurrentSecret(t *testing.T) { "cf/secret/test": PolledSecretMeta{ CurrentVersionID: "present", UpdatedAt: time.Now().AddDate(0, 0, -2), + Deleted: false, }, }, }, @@ -97,6 +99,32 @@ func TestFetchCurrentSecret(t *testing.T) { found: false, }, }, + { + name: "when the secret is deleted", + have: Have{ + poller: &Poller{ + PolledSecrets: Secrets{ + "cf/secret/test": PolledSecretMeta{ + CurrentVersionID: "present", + UpdatedAt: time.Now().AddDate(0, 0, -2), + Deleted: true, + }, + }, + }, + secretID: "cf/secret/test", + lruElements: map[string]map[string]secretsmanager.GetSecretValueOutput{ + "cf/secret/test": { + "": { + VersionId: _s("present"), + }, + }, + }, + }, + want: Want{ + resp: nil, + found: false, + }, + }, } { test.have.poller.cachedSecretValuesByRole, _ = lru.New2Q(10) for k, v := range test.have.lruElements { From f14db2292962b1fa54dcde3a1505db11a1328dc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Bir=C3=B3?= Date: Fri, 8 Dec 2023 21:05:14 +0100 Subject: [PATCH 8/9] update polled secrets instead of replacing them --- pkg/secretsmanager/poller.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/secretsmanager/poller.go b/pkg/secretsmanager/poller.go index 5e08f07..f026c67 100644 --- a/pkg/secretsmanager/poller.go +++ b/pkg/secretsmanager/poller.go @@ -86,7 +86,7 @@ func (p *Poller) poll(ticker *time.Ticker) { if err != nil { p.errs <- errors.WithMessagef(err, "failed polling secrets") } else { - p.PolledSecrets = polledSecrets + p.updatePolledSecrets(&polledSecrets) } case <-p.quit: @@ -96,6 +96,17 @@ func (p *Poller) poll(ticker *time.Ticker) { } } +func (p *Poller) updatePolledSecrets(fetchedSecrets *Secrets) { + if p.PolledSecrets == nil { + p.PolledSecrets = *fetchedSecrets + return + } + + for name, fetchedSecret := range *fetchedSecrets { + p.PolledSecrets[name] = fetchedSecret + } +} + func (p *Poller) fetchSecrets() (Secrets, error) { fetchedSecrets := make(Secrets) From 2113278b0b333f3b9a06384685238064cd6a7c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Bir=C3=B3?= Date: Fri, 8 Dec 2023 22:32:13 +0100 Subject: [PATCH 9/9] dont put deleted secrets in local list --- pkg/secretsmanager/poller.go | 6 +- pkg/secretsmanager/poller_test.go | 291 ++++++++++++++++++++++++++++- pkg/secretsmanager/secrets.go | 2 +- pkg/secretsmanager/secrets_test.go | 26 --- 4 files changed, 291 insertions(+), 34 deletions(-) diff --git a/pkg/secretsmanager/poller.go b/pkg/secretsmanager/poller.go index f026c67..5ed1199 100644 --- a/pkg/secretsmanager/poller.go +++ b/pkg/secretsmanager/poller.go @@ -103,7 +103,11 @@ func (p *Poller) updatePolledSecrets(fetchedSecrets *Secrets) { } for name, fetchedSecret := range *fetchedSecrets { - p.PolledSecrets[name] = fetchedSecret + if fetchedSecret.Deleted { + delete(p.PolledSecrets, name) + } else { + p.PolledSecrets[name] = fetchedSecret + } } } diff --git a/pkg/secretsmanager/poller_test.go b/pkg/secretsmanager/poller_test.go index 29c2722..a09d7c0 100644 --- a/pkg/secretsmanager/poller_test.go +++ b/pkg/secretsmanager/poller_test.go @@ -363,12 +363,6 @@ func TestPoll(t *testing.T) { Tags: map[string]string{}, Deleted: false, }, - "random/aws/deletedsecret": PolledSecretMeta{ - CurrentVersionID: "randomuuid", - UpdatedAt: now.AddDate(0, 0, -2), - Tags: map[string]string{}, - Deleted: true, - }, }, }, } { @@ -410,3 +404,288 @@ func TestPoll(t *testing.T) { } } } + +func TestUpdatePolledSecrets(t *testing.T) { + + now := time.Now() + + for _, test := range []struct { + name string + poller Poller + fetched Secrets + want Secrets + }{ + {name: "empty list is populated", + poller: Poller{ + PolledSecrets: make(Secrets), + }, + fetched: Secrets{ + "random/aws/secret": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + }, + want: Secrets{ + "random/aws/secret": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + }, + }, + {name: "no diff means no new elements", + poller: Poller{ + PolledSecrets: Secrets{ + "random/aws/secret1": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret2": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret3": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + }, + }, + fetched: Secrets{ + "random/aws/secret1": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret2": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret3": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + }, + want: Secrets{ + "random/aws/secret1": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret2": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret3": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + }, + }, + {name: "changing valaues updated", + poller: Poller{ + PolledSecrets: Secrets{ + "random/aws/secret1": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret2": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret3": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + }, + }, + fetched: Secrets{ + "random/aws/secret1": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret2": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret3": PolledSecretMeta{ + CurrentVersionID: "randomuuid_other", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + }, + want: Secrets{ + "random/aws/secret1": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret2": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret3": PolledSecretMeta{ + CurrentVersionID: "randomuuid_other", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + }, + }, + {name: "new list added", + poller: Poller{ + PolledSecrets: Secrets{ + "random/aws/secret1": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret2": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret3": PolledSecretMeta{ + CurrentVersionID: "randomuuid_other", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + }, + }, + fetched: Secrets{ + "random/aws/secret1": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret2": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + }, + want: Secrets{ + "random/aws/secret1": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret2": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret3": PolledSecretMeta{ + CurrentVersionID: "randomuuid_other", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + }, + }, + {name: "deleted secret removed", + poller: Poller{ + PolledSecrets: Secrets{ + "random/aws/secret1": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret2": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret3": PolledSecretMeta{ + CurrentVersionID: "randomuuid_other", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + }, + }, + fetched: Secrets{ + "random/aws/secret1": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret2": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret3": PolledSecretMeta{ + CurrentVersionID: "randomuuid_other", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: true, + }, + }, + want: Secrets{ + "random/aws/secret1": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + "random/aws/secret2": PolledSecretMeta{ + CurrentVersionID: "randomuuid", + UpdatedAt: now.AddDate(0, 0, -2), + Tags: map[string]string{}, + Deleted: false, + }, + }, + }, + } { + test.poller.updatePolledSecrets(&test.fetched) + + if !reflect.DeepEqual(test.want, test.poller.PolledSecrets) { + t.Errorf("test %s, wanted %v got %v", test.name, test.want, test.poller.PolledSecrets) + } + + } +} diff --git a/pkg/secretsmanager/secrets.go b/pkg/secretsmanager/secrets.go index e71775a..8d9ff15 100644 --- a/pkg/secretsmanager/secrets.go +++ b/pkg/secretsmanager/secrets.go @@ -58,7 +58,7 @@ func (p *Poller) fetchCurrentSecretCache(secretID *string, role string) (*secret secretValuesByRole := cachedElem.(map[string]secretsmanager.GetSecretValueOutput) if secretValueOut, ok := secretValuesByRole[role]; ok { polledSecretMeta, found := p.PolledSecrets[*secretID] - if found && !polledSecretMeta.Deleted && polledSecretMeta.CurrentVersionID == *secretValueOut.VersionId { + if found && polledSecretMeta.CurrentVersionID == *secretValueOut.VersionId { return &secretValueOut, found } } diff --git a/pkg/secretsmanager/secrets_test.go b/pkg/secretsmanager/secrets_test.go index d498613..02d7168 100644 --- a/pkg/secretsmanager/secrets_test.go +++ b/pkg/secretsmanager/secrets_test.go @@ -99,32 +99,6 @@ func TestFetchCurrentSecret(t *testing.T) { found: false, }, }, - { - name: "when the secret is deleted", - have: Have{ - poller: &Poller{ - PolledSecrets: Secrets{ - "cf/secret/test": PolledSecretMeta{ - CurrentVersionID: "present", - UpdatedAt: time.Now().AddDate(0, 0, -2), - Deleted: true, - }, - }, - }, - secretID: "cf/secret/test", - lruElements: map[string]map[string]secretsmanager.GetSecretValueOutput{ - "cf/secret/test": { - "": { - VersionId: _s("present"), - }, - }, - }, - }, - want: Want{ - resp: nil, - found: false, - }, - }, } { test.have.poller.cachedSecretValuesByRole, _ = lru.New2Q(10) for k, v := range test.have.lruElements {