diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index b5545049c..6271560d6 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -50,13 +50,13 @@ import ( "github.com/fluxcd/pkg/runtime/patch" "github.com/fluxcd/pkg/runtime/predicates" + "github.com/fluxcd/pkg/sourceignore" sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" serror "github.com/fluxcd/source-controller/internal/error" sreconcile "github.com/fluxcd/source-controller/internal/reconcile" "github.com/fluxcd/source-controller/internal/reconcile/summarize" "github.com/fluxcd/source-controller/pkg/gcp" "github.com/fluxcd/source-controller/pkg/minio" - "github.com/fluxcd/source-controller/pkg/sourceignore" ) // maxConcurrentBucketFetches is the upper bound on the goroutines used to diff --git a/controllers/gitrepository_controller.go b/controllers/gitrepository_controller.go index 64f651efa..d4efc708b 100644 --- a/controllers/gitrepository_controller.go +++ b/controllers/gitrepository_controller.go @@ -49,6 +49,7 @@ import ( "github.com/fluxcd/pkg/runtime/patch" "github.com/fluxcd/pkg/runtime/predicates" + "github.com/fluxcd/pkg/sourceignore" sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" serror "github.com/fluxcd/source-controller/internal/error" "github.com/fluxcd/source-controller/internal/features" @@ -57,7 +58,6 @@ import ( "github.com/fluxcd/source-controller/internal/util" "github.com/fluxcd/source-controller/pkg/git" "github.com/fluxcd/source-controller/pkg/git/strategy" - "github.com/fluxcd/source-controller/pkg/sourceignore" ) // gitRepositoryReadyCondition contains the information required to summarize a diff --git a/controllers/storage.go b/controllers/storage.go index c5fd586f0..ce7c6841d 100644 --- a/controllers/storage.go +++ b/controllers/storage.go @@ -39,9 +39,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kerrors "k8s.io/apimachinery/pkg/util/errors" + "github.com/fluxcd/pkg/sourceignore" sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" sourcefs "github.com/fluxcd/source-controller/internal/fs" - "github.com/fluxcd/source-controller/pkg/sourceignore" ) const GarbageCountLimit = 1000 diff --git a/go.mod b/go.mod index 1991567ee..877fff249 100644 --- a/go.mod +++ b/go.mod @@ -38,8 +38,9 @@ require ( github.com/fluxcd/pkg/helmtestserver v0.7.4 github.com/fluxcd/pkg/lockedfile v0.1.0 github.com/fluxcd/pkg/masktoken v0.0.1 - github.com/fluxcd/pkg/oci v0.3.0 + github.com/fluxcd/pkg/oci v0.5.0 github.com/fluxcd/pkg/runtime v0.16.2 + github.com/fluxcd/pkg/sourceignore v0.1.0 github.com/fluxcd/pkg/ssh v0.5.0 github.com/fluxcd/pkg/testserver v0.2.0 github.com/fluxcd/pkg/untar v0.1.0 diff --git a/go.sum b/go.sum index 84326655b..a5c89d3dc 100644 --- a/go.sum +++ b/go.sum @@ -401,10 +401,12 @@ github.com/fluxcd/pkg/lockedfile v0.1.0 h1:YsYFAkd6wawMCcD74ikadAKXA4s2sukdxrn7w github.com/fluxcd/pkg/lockedfile v0.1.0/go.mod h1:EJLan8t9MiOcgTs8+puDjbE6I/KAfHbdvIy9VUgIjm8= github.com/fluxcd/pkg/masktoken v0.0.1 h1:egWR/ibTzf4L3PxE8TauKO1srD1Ye/aalgQRQuKKRdU= github.com/fluxcd/pkg/masktoken v0.0.1/go.mod h1:sQmMtX4s5RwdGlByJazzNasWFFgBdmtNcgeZcGBI72Y= -github.com/fluxcd/pkg/oci v0.3.0 h1:GFn6JZeg5fV2K4vsQ0s5lJFid6qrpA4RybLXL+7qUbQ= -github.com/fluxcd/pkg/oci v0.3.0/go.mod h1:c1pj9E/G5927gSa6ooACAyZe+HwjgmPk9johL7oXDHw= +github.com/fluxcd/pkg/oci v0.5.0 h1:ghVKxvDmHoMXohRRsyDIISZPAXbA6hxz7IGX/EyvUZw= +github.com/fluxcd/pkg/oci v0.5.0/go.mod h1:qEzBvOJvWKrIKIhCKw1Nlbgod9ClD8C6A7F2cylXpsA= github.com/fluxcd/pkg/runtime v0.16.2 h1:CexfMmJK+r12sHTvKWyAax0pcPomjd6VnaHXcxjUrRY= github.com/fluxcd/pkg/runtime v0.16.2/go.mod h1:OHSKsrO+T+Ym8WZRS2oidrnauWRARuE2nfm8ewevm7M= +github.com/fluxcd/pkg/sourceignore v0.1.0 h1:v36Rqp6FDB7Ntjy7NakdwscOfxFPk14peKa+VMBNugo= +github.com/fluxcd/pkg/sourceignore v0.1.0/go.mod h1:m9/q+YLMNSWjXns1n/5q3ucwzSSddti+D6ExbNaCo6s= github.com/fluxcd/pkg/ssh v0.5.0 h1:jE9F2XvUXC2mgseeXMATvO014fLqdB30/VzlPLKsk20= github.com/fluxcd/pkg/ssh v0.5.0/go.mod h1:KGgOUOy1uI6RC6+qxIBLvP1AeOOs/nLB25Ca6TZMIXE= github.com/fluxcd/pkg/testserver v0.2.0 h1:Mj0TapmKaywI6Fi5wvt1LAZpakUHmtzWQpJNKQ0Krt4= diff --git a/pkg/minio/minio_test.go b/pkg/minio/minio_test.go index 4b8798cc0..8e46a617f 100644 --- a/pkg/minio/minio_test.go +++ b/pkg/minio/minio_test.go @@ -26,8 +26,8 @@ import ( "testing" "github.com/fluxcd/pkg/apis/meta" + "github.com/fluxcd/pkg/sourceignore" sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" - "github.com/fluxcd/source-controller/pkg/sourceignore" "github.com/google/uuid" miniov7 "github.com/minio/minio-go/v7" diff --git a/pkg/sourceignore/sourceignore.go b/pkg/sourceignore/sourceignore.go deleted file mode 100644 index 38327d38a..000000000 --- a/pkg/sourceignore/sourceignore.go +++ /dev/null @@ -1,126 +0,0 @@ -/* -Copyright 2021 The Flux authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package sourceignore - -import ( - "bufio" - "io" - "os" - "path/filepath" - "strings" - - "github.com/go-git/go-git/v5/plumbing/format/gitignore" -) - -const ( - IgnoreFile = ".sourceignore" - ExcludeVCS = ".git/,.gitignore,.gitmodules,.gitattributes" - ExcludeExt = "*.jpg,*.jpeg,*.gif,*.png,*.wmv,*.flv,*.tar.gz,*.zip" - ExcludeCI = ".github/,.circleci/,.travis.yml,.gitlab-ci.yml,appveyor.yml,.drone.yml,cloudbuild.yaml,codeship-services.yml,codeship-steps.yml" - ExcludeExtra = "**/.goreleaser.yml,**/.sops.yaml,**/.flux.yaml" -) - -// NewMatcher returns a gitignore.Matcher for the given gitignore.Pattern -// slice. It mainly exists to compliment the API. -func NewMatcher(ps []gitignore.Pattern) gitignore.Matcher { - return gitignore.NewMatcher(ps) -} - -// NewDefaultMatcher returns a gitignore.Matcher with the DefaultPatterns -// as lowest priority patterns. -func NewDefaultMatcher(ps []gitignore.Pattern, domain []string) gitignore.Matcher { - var defaultPs []gitignore.Pattern - defaultPs = append(defaultPs, VCSPatterns(domain)...) - defaultPs = append(defaultPs, DefaultPatterns(domain)...) - ps = append(defaultPs, ps...) - return gitignore.NewMatcher(ps) -} - -// VCSPatterns returns a gitignore.Pattern slice with ExcludeVCS -// patterns. -func VCSPatterns(domain []string) []gitignore.Pattern { - var ps []gitignore.Pattern - for _, p := range strings.Split(ExcludeVCS, ",") { - ps = append(ps, gitignore.ParsePattern(p, domain)) - } - return ps -} - -// DefaultPatterns returns a gitignore.Pattern slice with the default -// ExcludeExt, ExcludeCI, ExcludeExtra patterns. -func DefaultPatterns(domain []string) []gitignore.Pattern { - all := strings.Join([]string{ExcludeExt, ExcludeCI, ExcludeExtra}, ",") - var ps []gitignore.Pattern - for _, p := range strings.Split(all, ",") { - ps = append(ps, gitignore.ParsePattern(p, domain)) - } - return ps -} - -// ReadPatterns collects ignore patterns from the given reader and -// returns them as a gitignore.Pattern slice. -// If a domain is supplied, this is used as the scope of the read -// patterns. -func ReadPatterns(reader io.Reader, domain []string) []gitignore.Pattern { - var ps []gitignore.Pattern - scanner := bufio.NewScanner(reader) - for scanner.Scan() { - s := scanner.Text() - if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 { - ps = append(ps, gitignore.ParsePattern(s, domain)) - } - } - return ps -} - -// ReadIgnoreFile attempts to read the file at the given path and -// returns the read patterns. -func ReadIgnoreFile(path string, domain []string) ([]gitignore.Pattern, error) { - var ps []gitignore.Pattern - if f, err := os.Open(path); err == nil { - defer f.Close() - ps = append(ps, ReadPatterns(f, domain)...) - } else if !os.IsNotExist(err) { - return nil, err - } - return ps, nil -} - -// LoadIgnorePatterns recursively loads the IgnoreFile patterns found -// in the directory. -func LoadIgnorePatterns(dir string, domain []string) ([]gitignore.Pattern, error) { - ps, err := ReadIgnoreFile(filepath.Join(dir, IgnoreFile), domain) - if err != nil { - return nil, err - } - fis, err := os.ReadDir(dir) - if err != nil { - return nil, err - } - for _, fi := range fis { - if fi.IsDir() && fi.Name() != ".git" { - var subps []gitignore.Pattern - if subps, err = LoadIgnorePatterns(filepath.Join(dir, fi.Name()), append(domain, fi.Name())); err != nil { - return nil, err - } - if len(subps) > 0 { - ps = append(ps, subps...) - } - } - } - return ps, nil -} diff --git a/pkg/sourceignore/sourceignore_test.go b/pkg/sourceignore/sourceignore_test.go deleted file mode 100644 index 5ba78cda8..000000000 --- a/pkg/sourceignore/sourceignore_test.go +++ /dev/null @@ -1,256 +0,0 @@ -/* -Copyright 2021 The Flux authors - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package sourceignore - -import ( - "os" - "path/filepath" - "reflect" - "strings" - "testing" - - "github.com/go-git/go-git/v5/plumbing/format/gitignore" - "gotest.tools/assert" -) - -func TestReadPatterns(t *testing.T) { - tests := []struct { - name string - ignore string - domain []string - matches []string - mismatches []string - }{ - { - name: "simple", - ignore: `ignore-dir/* -!ignore-dir/include -`, - matches: []string{"ignore-dir/file.yaml"}, - mismatches: []string{"file.yaml", "ignore-dir/include"}, - }, - { - name: "with comments", - ignore: `ignore-dir/* -# !ignore-dir/include`, - matches: []string{"ignore-dir/file.yaml", "ignore-dir/include"}, - }, - { - name: "domain scoped", - domain: []string{"domain", "scoped"}, - ignore: "ignore-dir/*", - matches: []string{"domain/scoped/ignore-dir/file.yaml"}, - mismatches: []string{"ignore-dir/file.yaml"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - reader := strings.NewReader(tt.ignore) - ps := ReadPatterns(reader, tt.domain) - matcher := NewMatcher(ps) - for _, m := range tt.matches { - assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), true, "expected %s to match", m) - } - for _, m := range tt.mismatches { - assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), false, "expected %s to not match", m) - } - }) - } -} - -func TestReadIgnoreFile(t *testing.T) { - f, err := os.CreateTemp("", IgnoreFile) - if err != nil { - t.Fatal(err) - } - defer os.Remove(f.Name()) - if _, err = f.Write([]byte(`# .sourceignore -ignore-this.txt`)); err != nil { - t.Fatal(err) - } - f.Close() - - tests := []struct { - name string - path string - domain []string - want []gitignore.Pattern - }{ - { - name: IgnoreFile, - path: f.Name(), - want: []gitignore.Pattern{ - gitignore.ParsePattern("ignore-this.txt", nil), - }, - }, - { - name: "with domain", - path: f.Name(), - domain: strings.Split(filepath.Dir(f.Name()), string(filepath.Separator)), - want: []gitignore.Pattern{ - gitignore.ParsePattern("ignore-this.txt", strings.Split(filepath.Dir(f.Name()), string(filepath.Separator))), - }, - }, - { - name: "non existing", - path: "", - want: nil, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := ReadIgnoreFile(tt.path, tt.domain) - if err != nil { - t.Error(err) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("ReadIgnoreFile() got = %d, want %#v", got, tt.want) - } - }) - } -} - -func TestVCSPatterns(t *testing.T) { - tests := []struct { - name string - domain []string - patterns []gitignore.Pattern - matches []string - mismatches []string - }{ - { - name: "simple matches", - matches: []string{".git/config", ".gitignore"}, - mismatches: []string{"workload.yaml", "workload.yml", "simple.txt"}, - }, - { - name: "domain scoped matches", - domain: []string{"directory"}, - matches: []string{"directory/.git/config", "directory/.gitignore"}, - mismatches: []string{"other/.git/config"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - matcher := NewDefaultMatcher(tt.patterns, tt.domain) - for _, m := range tt.matches { - assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), true, "expected %s to match", m) - } - for _, m := range tt.mismatches { - assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), false, "expected %s to not match", m) - } - }) - } -} - -func TestDefaultPatterns(t *testing.T) { - tests := []struct { - name string - domain []string - patterns []gitignore.Pattern - matches []string - mismatches []string - }{ - { - name: "simple matches", - matches: []string{"image.jpg", "archive.tar.gz", ".github/workflows/workflow.yaml", "subdir/.flux.yaml", "subdir2/.sops.yaml"}, - mismatches: []string{"workload.yaml", "workload.yml", "simple.txt"}, - }, - { - name: "domain scoped matches", - domain: []string{"directory"}, - matches: []string{"directory/image.jpg", "directory/archive.tar.gz"}, - mismatches: []string{"other/image.jpg", "other/archive.tar.gz"}, - }, - { - name: "patterns", - patterns: []gitignore.Pattern{gitignore.ParsePattern("!*.jpg", nil)}, - mismatches: []string{"image.jpg"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - matcher := NewDefaultMatcher(tt.patterns, tt.domain) - for _, m := range tt.matches { - assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), true, "expected %s to match", m) - } - for _, m := range tt.mismatches { - assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), false, "expected %s to not match", m) - } - }) - } -} - -func TestLoadExcludePatterns(t *testing.T) { - tmpDir := t.TempDir() - files := map[string]string{ - ".sourceignore": "root.txt", - "d/.gitignore": "ignored", - "z/.sourceignore": "last.txt", - "a/b/.sourceignore": "subdir.txt", - } - for n, c := range files { - if err := os.MkdirAll(filepath.Join(tmpDir, filepath.Dir(n)), 0o750); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(tmpDir, n), []byte(c), 0o640); err != nil { - t.Fatal(err) - } - } - tests := []struct { - name string - dir string - domain []string - want []gitignore.Pattern - }{ - { - name: "traverse loads", - dir: tmpDir, - want: []gitignore.Pattern{ - gitignore.ParsePattern("root.txt", nil), - gitignore.ParsePattern("subdir.txt", []string{"a", "b"}), - gitignore.ParsePattern("last.txt", []string{"z"}), - }, - }, - { - name: "domain", - dir: tmpDir, - domain: strings.Split(tmpDir, string(filepath.Separator)), - want: []gitignore.Pattern{ - gitignore.ParsePattern("root.txt", strings.Split(tmpDir, string(filepath.Separator))), - gitignore.ParsePattern("subdir.txt", append(strings.Split(tmpDir, string(filepath.Separator)), "a", "b")), - gitignore.ParsePattern("last.txt", append(strings.Split(tmpDir, string(filepath.Separator)), "z")), - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := LoadIgnorePatterns(tt.dir, tt.domain) - if err != nil { - t.Error(err) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("LoadIgnorePatterns() got = %#v, want %#v", got, tt.want) - for _, v := range got { - t.Error(v) - } - } - }) - } -}