From 742c642c5e6676f166df737c0c633d8e5a528f5a Mon Sep 17 00:00:00 2001 From: bitgully <32452884+bitgully@users.noreply.github.com> Date: Tue, 2 Apr 2024 08:13:44 +0200 Subject: [PATCH 1/5] renamed key for dependency-mirror bindings from 'uri' to 'default' --- dependency_cache.go | 6 +++--- dependency_cache_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dependency_cache.go b/dependency_cache.go index 5dda859..1a54da6 100644 --- a/dependency_cache.go +++ b/dependency_cache.go @@ -154,7 +154,7 @@ func customizeHttpClientTimeouts() (*HttpClientTimeouts, error) { } // Returns the URI of a dependency mirror (optional). -// Such mirror location can be defined in a binding of type 'dependency-mirror' with filename 'uri' +// Such mirror location can be defined in a binding of type 'dependency-mirror' with filename 'default' // or using the environment variable 'BP_DEPENDENCY_MIRROR'. The latter takes precedence in case both are found. func getDependencyMirror(bindings libcnb.Bindings) (string, error) { dependencyMirror := sherpa.GetEnvWithDefault("BP_DEPENDENCY_MIRROR", "") @@ -162,8 +162,8 @@ func getDependencyMirror(bindings libcnb.Bindings) (string, error) { if dependencyMirror == "" { dependencyMirrorBindings, err := filterBindingsByType(bindings, "dependency-mirror") if err == nil { - // Use the content of the file named "uri" as the mirror's URI. - dependencyMirror = dependencyMirrorBindings["uri"] + // Use the content of the file named "default" as the mirror's URI. + dependencyMirror = dependencyMirrorBindings["default"] } else { return "", err } diff --git a/dependency_cache_test.go b/dependency_cache_test.go index fd9ce7a..693471f 100644 --- a/dependency_cache_test.go +++ b/dependency_cache_test.go @@ -168,7 +168,7 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { ctx.Platform.Bindings = append(ctx.Platform.Bindings, libcnb.Binding{ Type: "dependency-mirror", Secret: map[string]string{ - "uri": "https://bindings-mirror.acme.com", + "default": "https://bindings-mirror.acme.com", }, }) }) From 35739f0fe87df8997ea27e59e81fc8909b6b959f Mon Sep 17 00:00:00 2001 From: bitgully <32452884+bitgully@users.noreply.github.com> Date: Tue, 2 Apr 2024 19:21:56 +0200 Subject: [PATCH 2/5] hostname-specific dependency mirrors added --- dependency_cache.go | 103 ++++++++++++++++++++++++--------------- dependency_cache_test.go | 43 +++++++++++++--- 2 files changed, 100 insertions(+), 46 deletions(-) diff --git a/dependency_cache.go b/dependency_cache.go index 1a54da6..d2c18c1 100644 --- a/dependency_cache.go +++ b/dependency_cache.go @@ -69,28 +69,25 @@ type DependencyCache struct { // httpClientTimeouts contains the timeout values used by HTTP client HttpClientTimeouts HttpClientTimeouts - // Alternative source used for downloading dependencies. - DependencyMirror string + // Alternative sources used for downloading dependencies. + DependencyMirrors map[string]string } // NewDependencyCache creates a new instance setting the default cache path (/dependencies) and user // agent (/). // Mappings will be read from any libcnb.Binding in the context with type "dependency-mappings". // -// In some air-gapped environments, dependencies might not be download directly but need to be pulled from a local mirror registry. -// In such cases, an alternative URI can either be provided as environment variable "BP_DEPENDENCY_MIRROR", or by a binding of type "dependency-mirror" -// where a file named "uri" holds the desired location. -// The two schemes https:// and file:// are supported in mirror URIs where the expected format is (optional parts in "[]"): -// ://[:@][:][/] -// The optional path part of the provided URI is used as a prefix that might be necessary in some setups. -// This (prefix) path may also include a placeholder of "{originalHost}" at any level (in sub-paths or at top-level) and is replaced with the -// hostname of the original download URI at build time. A sample mirror URI might look like this: https://local-mirror.example.com/buildpacks-dependencies/{originalHost} +// In some environments, many dependencies might need to be downloaded from a (local) mirror registry or filesystem. +// Such alternative locations can be configured using bindings of type "dependency-mirror", avoiding too many "dependency-mapping" bindings. +// Environment variables named "BP_DEPENDENCY_MIRROR" (default) or "BP_DEPENDENCY_MIRROR_" (hostname-specific mirror) +// can also be used for the same purpose. func NewDependencyCache(context libcnb.BuildContext) (DependencyCache, error) { cache := DependencyCache{ - CachePath: filepath.Join(context.Buildpack.Path, "dependencies"), - DownloadPath: os.TempDir(), - UserAgent: fmt.Sprintf("%s/%s", context.Buildpack.Info.ID, context.Buildpack.Info.Version), - Mappings: map[string]string{}, + CachePath: filepath.Join(context.Buildpack.Path, "dependencies"), + DownloadPath: os.TempDir(), + UserAgent: fmt.Sprintf("%s/%s", context.Buildpack.Info.ID, context.Buildpack.Info.Version), + Mappings: map[string]string{}, + DependencyMirrors: map[string]string{}, } mappings, err := filterBindingsByType(context.Platform.Bindings, "dependency-mapping") if err != nil { @@ -104,11 +101,11 @@ func NewDependencyCache(context libcnb.BuildContext) (DependencyCache, error) { } cache.HttpClientTimeouts = *clientTimeouts - dependencyMirror, err := getDependencyMirror(context.Platform.Bindings) + dependencyMirrors, err := getDependencyMirrors(context.Platform.Bindings) if err != nil { - return DependencyCache{}, err + return DependencyCache{}, fmt.Errorf("unable to read dependency mirrors\n%w", err) } - cache.DependencyMirror = dependencyMirror + cache.DependencyMirrors = dependencyMirrors return cache, nil } @@ -153,22 +150,46 @@ func customizeHttpClientTimeouts() (*HttpClientTimeouts, error) { }, nil } -// Returns the URI of a dependency mirror (optional). -// Such mirror location can be defined in a binding of type 'dependency-mirror' with filename 'default' -// or using the environment variable 'BP_DEPENDENCY_MIRROR'. The latter takes precedence in case both are found. -func getDependencyMirror(bindings libcnb.Bindings) (string, error) { - dependencyMirror := sherpa.GetEnvWithDefault("BP_DEPENDENCY_MIRROR", "") - // If no mirror was found in environment variables, try to find one in bindings. - if dependencyMirror == "" { - dependencyMirrorBindings, err := filterBindingsByType(bindings, "dependency-mirror") - if err == nil { - // Use the content of the file named "default" as the mirror's URI. - dependencyMirror = dependencyMirrorBindings["default"] - } else { - return "", err +// Returns a key/value map with the URIs of all dependency mirrors. An empty map is returned if no mirrors are set. +// Mirror locations can be defined in bindings of type 'dependency-mirror' or using env variables prefixed with 'BP_DEPENDENCY_MIRROR'. +// Settings provided by env variables override those defined in bindings. +func getDependencyMirrors(bindings libcnb.Bindings) (map[string]string, error) { + dependencyMirrors, err := filterBindingsByType(bindings, "dependency-mirror") + if err != nil { + return nil, err + } + dependencyMirrorsFromEnv := getDependencyMirrorsFromEnv() + for host, uri := range dependencyMirrorsFromEnv { + dependencyMirrors[host] = uri + } + return dependencyMirrors, nil +} + +// Returns a key/value map of all dependency mirrors set in environment variables. +func getDependencyMirrorsFromEnv() map[string]string { + mirrors := map[string]string{} + envs := os.Environ() + for _, env := range envs { + envPair := strings.SplitN(env, "=", 2) + hostnameSuffix, isMirror := strings.CutPrefix(envPair[0], "BP_DEPENDENCY_MIRROR") + hostnameEncoded, _ := strings.CutPrefix(hostnameSuffix, "_") + if isMirror { + mirrors[decodeHostnameEnv(hostnameEncoded)] = envPair[1] } } - return dependencyMirror, nil + return mirrors +} + +// Takes an encoded hostname (from env key) and returns the decoded version in lower case. +// Replaces double underscores (__) with one dash (-) and single underscores (_) with one period (.). +func decodeHostnameEnv(encodedHostname string) string { + var decodedHostname string + if encodedHostname == "" { + decodedHostname = "default" + } else { + decodedHostname = strings.ReplaceAll(strings.ReplaceAll(encodedHostname, "__", "-"), "_", ".") + } + return strings.ToLower(decodedHostname) } // Returns a key/value map with all entries for a given binding type. @@ -181,7 +202,7 @@ func filterBindingsByType(bindings libcnb.Bindings, bindingType string) (map[str if _, ok := filteredBindings[key]; ok { return nil, fmt.Errorf("multiple %s bindings found with duplicate keys %s", binding.Type, key) } - filteredBindings[key] = value + filteredBindings[strings.ToLower(key)] = value } } } @@ -225,11 +246,17 @@ func (d *DependencyCache) Artifact(dependency BuildpackDependency, mods ...Reque return nil, fmt.Errorf("unable to parse URI. see DEBUG log level") } - if isBinding && d.DependencyMirror != "" { + mirror := d.DependencyMirrors["default"] + mirrorHostSpecific := d.DependencyMirrors[urlP.Hostname()] + if mirrorHostSpecific != "" { + mirror = mirrorHostSpecific + } + + if isBinding && mirror != "" { d.Logger.Bodyf("Both dependency mirror and bindings are present. %s Please remove dependency map bindings if you wish to use the mirror.", color.YellowString("Mirror is being ignored.")) } else { - d.setDependencyMirror(urlP) + d.setDependencyMirror(urlP, mirror) } if dependency.SHA256 == "" { @@ -422,10 +449,10 @@ func (DependencyCache) verify(path string, expected string) error { return nil } -func (d DependencyCache) setDependencyMirror(urlD *url.URL) { - if d.DependencyMirror != "" { +func (d DependencyCache) setDependencyMirror(urlD *url.URL, mirror string) { + if mirror != "" { d.Logger.Bodyf("%s Download URIs will be overridden.", color.GreenString("Dependency mirror found.")) - urlOverride, err := url.ParseRequestURI(d.DependencyMirror) + urlOverride, err := url.ParseRequestURI(mirror) if strings.ToLower(urlOverride.Scheme) == "https" || strings.ToLower(urlOverride.Scheme) == "file" { urlD.Scheme = urlOverride.Scheme @@ -433,7 +460,7 @@ func (d DependencyCache) setDependencyMirror(urlD *url.URL) { urlD.Path = strings.Replace(urlOverride.Path, "{originalHost}", urlD.Hostname(), 1) + urlD.Path urlD.Host = urlOverride.Host } else { - d.Logger.Debugf("Dependency mirror URI is invalid: %s\n%w", d.DependencyMirror, err) + d.Logger.Debugf("Dependency mirror URI is invalid: %s\n%w", mirror, err) d.Logger.Bodyf("%s is ignored. Have you used one of the supported schemes https:// or file://?", color.YellowString("Invalid dependency mirror")) } } diff --git a/dependency_cache_test.go b/dependency_cache_test.go index 693471f..37178ac 100644 --- a/dependency_cache_test.go +++ b/dependency_cache_test.go @@ -154,21 +154,25 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { context("dependency mirror from environment variable", func() { it.Before(func() { t.Setenv("BP_DEPENDENCY_MIRROR", "https://env-var-mirror.acme.com") + t.Setenv("BP_DEPENDENCY_MIRROR_EXAMP__LE_COM", "https://examp-le.com") }) it("uses BP_DEPENDENCY_MIRROR environment variable", func() { dependencyCache, err := libpak.NewDependencyCache(ctx) Expect(err).NotTo(HaveOccurred()) - Expect(dependencyCache.DependencyMirror).To(Equal("https://env-var-mirror.acme.com")) + Expect(dependencyCache.DependencyMirrors["default"]).To(Equal("https://env-var-mirror.acme.com")) + Expect(dependencyCache.DependencyMirrors["examp-le.com"]).To(Equal("https://examp-le.com")) }) }) - context("dependency mirror from binding", func() { + context("dependency mirror from binding and environment variable", func() { it.Before(func() { + t.Setenv("BP_DEPENDENCY_MIRROR_EXAMP__LE_COM", "https://examp-le.com") ctx.Platform.Bindings = append(ctx.Platform.Bindings, libcnb.Binding{ Type: "dependency-mirror", Secret: map[string]string{ - "default": "https://bindings-mirror.acme.com", + "default": "https://bindings-mirror.acme.com", + "examp-le.com": "https://invalid.com", }, }) }) @@ -176,7 +180,13 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { it("uses dependency-mirror binding", func() { dependencyCache, err := libpak.NewDependencyCache(ctx) Expect(err).NotTo(HaveOccurred()) - Expect(dependencyCache.DependencyMirror).To(Equal("https://bindings-mirror.acme.com")) + Expect(dependencyCache.DependencyMirrors["default"]).To(Equal("https://bindings-mirror.acme.com")) + }) + + it("environment variable overrides binding", func() { + dependencyCache, err := libpak.NewDependencyCache(ctx) + Expect(err).NotTo(HaveOccurred()) + Expect(dependencyCache.DependencyMirrors["examp-le.com"]).To(Equal("https://examp-le.com")) }) }) }) @@ -332,6 +342,7 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { it.Before(func() { mirrorServer = ghttp.NewTLSServer() + dependencyCache.DependencyMirrors = map[string]string{} }) it.After(func() { @@ -347,7 +358,7 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { ghttp.RespondWith(http.StatusOK, "test-fixture"), )) - dependencyCache.DependencyMirror = url.Scheme + "://" + "username:password@" + url.Host + "/foo/bar" + dependencyCache.DependencyMirrors["default"] = url.Scheme + "://" + "username:password@" + url.Host + "/foo/bar" a, err := dependencyCache.Artifact(dependency) Expect(err).NotTo(HaveOccurred()) @@ -362,7 +373,22 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { ghttp.RespondWith(http.StatusOK, "test-fixture"), )) - dependencyCache.DependencyMirror = url.Scheme + "://" + url.Host + "/{originalHost}" + dependencyCache.DependencyMirrors["default"] = url.Scheme + "://" + url.Host + "/{originalHost}" + a, err := dependencyCache.Artifact(dependency) + Expect(err).NotTo(HaveOccurred()) + + Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture"))) + }) + + it("downloads from https mirror host specific", func() { + url, err := url.Parse(mirrorServer.URL()) + Expect(err).NotTo(HaveOccurred()) + mirrorServer.AppendHandlers(ghttp.CombineHandlers( + ghttp.VerifyRequest(http.MethodGet, "/host-specific/test-path", ""), + ghttp.RespondWith(http.StatusOK, "test-fixture"), + )) + + dependencyCache.DependencyMirrors["127.0.0.1"] = url.Scheme + "://" + url.Host + "/host-specific" a, err := dependencyCache.Artifact(dependency) Expect(err).NotTo(HaveOccurred()) @@ -384,6 +410,7 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { Expect(err).NotTo(HaveOccurred()) mirrorPathPreservedHost = filepath.Join(mirrorPath, originalUrl.Hostname(), "prefix") Expect(os.MkdirAll(mirrorPathPreservedHost, os.ModePerm)).NotTo(HaveOccurred()) + dependencyCache.DependencyMirrors = map[string]string{} }) it.After(func() { @@ -394,7 +421,7 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { mirrorFile := filepath.Join(mirrorPath, "test-path") Expect(os.WriteFile(mirrorFile, []byte("test-fixture"), 0644)).ToNot(HaveOccurred()) - dependencyCache.DependencyMirror = "file://" + mirrorPath + dependencyCache.DependencyMirrors["default"] = "file://" + mirrorPath a, err := dependencyCache.Artifact(dependency) Expect(err).NotTo(HaveOccurred()) @@ -405,7 +432,7 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) { mirrorFilePreservedHost := filepath.Join(mirrorPathPreservedHost, "test-path") Expect(os.WriteFile(mirrorFilePreservedHost, []byte("test-fixture"), 0644)).ToNot(HaveOccurred()) - dependencyCache.DependencyMirror = "file://" + mirrorPath + "/{originalHost}" + "/prefix" + dependencyCache.DependencyMirrors["default"] = "file://" + mirrorPath + "/{originalHost}" + "/prefix" a, err := dependencyCache.Artifact(dependency) Expect(err).NotTo(HaveOccurred()) From 517ce1970588e0d19001976c51c11894c9c83360 Mon Sep 17 00:00:00 2001 From: bitgully <32452884+bitgully@users.noreply.github.com> Date: Sat, 6 Apr 2024 10:35:39 +0200 Subject: [PATCH 3/5] checks for illegal mirror hostname and env variable names added --- dependency_cache.go | 51 ++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/dependency_cache.go b/dependency_cache.go index d2c18c1..5a90b46 100644 --- a/dependency_cache.go +++ b/dependency_cache.go @@ -88,6 +88,7 @@ func NewDependencyCache(context libcnb.BuildContext) (DependencyCache, error) { UserAgent: fmt.Sprintf("%s/%s", context.Buildpack.Info.ID, context.Buildpack.Info.Version), Mappings: map[string]string{}, DependencyMirrors: map[string]string{}, + Logger: bard.NewLogger(os.Stdout), } mappings, err := filterBindingsByType(context.Platform.Bindings, "dependency-mapping") if err != nil { @@ -101,11 +102,11 @@ func NewDependencyCache(context libcnb.BuildContext) (DependencyCache, error) { } cache.HttpClientTimeouts = *clientTimeouts - dependencyMirrors, err := getDependencyMirrors(context.Platform.Bindings) + bindingMirrors, err := filterBindingsByType(context.Platform.Bindings, "dependency-mirror") if err != nil { - return DependencyCache{}, fmt.Errorf("unable to read dependency mirrors\n%w", err) + return DependencyCache{}, fmt.Errorf("unable to process dependency-mirror bindings\n%w", err) } - cache.DependencyMirrors = dependencyMirrors + cache.setDependencyMirrors(bindingMirrors) return cache, nil } @@ -150,39 +151,37 @@ func customizeHttpClientTimeouts() (*HttpClientTimeouts, error) { }, nil } -// Returns a key/value map with the URIs of all dependency mirrors. An empty map is returned if no mirrors are set. -// Mirror locations can be defined in bindings of type 'dependency-mirror' or using env variables prefixed with 'BP_DEPENDENCY_MIRROR'. -// Settings provided by env variables override those defined in bindings. -func getDependencyMirrors(bindings libcnb.Bindings) (map[string]string, error) { - dependencyMirrors, err := filterBindingsByType(bindings, "dependency-mirror") - if err != nil { - return nil, err - } - dependencyMirrorsFromEnv := getDependencyMirrorsFromEnv() - for host, uri := range dependencyMirrorsFromEnv { - dependencyMirrors[host] = uri - } - return dependencyMirrors, nil -} - -// Returns a key/value map of all dependency mirrors set in environment variables. -func getDependencyMirrorsFromEnv() map[string]string { - mirrors := map[string]string{} +func (d *DependencyCache) setDependencyMirrors(bindingMirrors map[string]string) { + // Initialize with mirrors from bindings. + d.DependencyMirrors = bindingMirrors + // Add mirrors from env variables and override duplicate hostnames set in bindings. envs := os.Environ() for _, env := range envs { envPair := strings.SplitN(env, "=", 2) + if len(envPair) != 2 { + continue + } hostnameSuffix, isMirror := strings.CutPrefix(envPair[0], "BP_DEPENDENCY_MIRROR") - hostnameEncoded, _ := strings.CutPrefix(hostnameSuffix, "_") if isMirror { - mirrors[decodeHostnameEnv(hostnameEncoded)] = envPair[1] + hostnameEncoded, _ := strings.CutPrefix(hostnameSuffix, "_") + if hostnameEncoded == "default" { + d.Logger.Bodyf("%s with illegal hostname 'default'. Please use BP_DEPENDENCY_MIRROR to set a default.", + color.YellowString("Ignored dependency mirror")) + continue + } + d.DependencyMirrors[decodeHostnameEnv(hostnameEncoded, d)] = envPair[1] } } - return mirrors } // Takes an encoded hostname (from env key) and returns the decoded version in lower case. // Replaces double underscores (__) with one dash (-) and single underscores (_) with one period (.). -func decodeHostnameEnv(encodedHostname string) string { +func decodeHostnameEnv(encodedHostname string, d *DependencyCache) string { + if strings.ContainsAny(encodedHostname, "-.") || encodedHostname != strings.ToUpper(encodedHostname) { + d.Logger.Bodyf("%s These will be allowed but for best results across different shells, you should replace . characters with _ characters "+ + "and - characters with __, and use all upper case letters. The buildpack will convert these back before using the mirror.", + color.YellowString("You have invalid characters in your mirror host environment variable.")) + } var decodedHostname string if encodedHostname == "" { decodedHostname = "default" @@ -199,7 +198,7 @@ func filterBindingsByType(bindings libcnb.Bindings, bindingType string) (map[str for _, binding := range bindings { if strings.ToLower(binding.Type) == bindingType { for key, value := range binding.Secret { - if _, ok := filteredBindings[key]; ok { + if _, ok := filteredBindings[strings.ToLower(key)]; ok { return nil, fmt.Errorf("multiple %s bindings found with duplicate keys %s", binding.Type, key) } filteredBindings[strings.ToLower(key)] = value From a0b065a6189239534e075e890597898558facf15 Mon Sep 17 00:00:00 2001 From: bitgully <32452884+bitgully@users.noreply.github.com> Date: Sat, 6 Apr 2024 10:57:56 +0200 Subject: [PATCH 4/5] case-insensitive check of 'default' mirror --- dependency_cache.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependency_cache.go b/dependency_cache.go index 5a90b46..c400511 100644 --- a/dependency_cache.go +++ b/dependency_cache.go @@ -164,7 +164,7 @@ func (d *DependencyCache) setDependencyMirrors(bindingMirrors map[string]string) hostnameSuffix, isMirror := strings.CutPrefix(envPair[0], "BP_DEPENDENCY_MIRROR") if isMirror { hostnameEncoded, _ := strings.CutPrefix(hostnameSuffix, "_") - if hostnameEncoded == "default" { + if strings.ToLower(hostnameEncoded) == "default" { d.Logger.Bodyf("%s with illegal hostname 'default'. Please use BP_DEPENDENCY_MIRROR to set a default.", color.YellowString("Ignored dependency mirror")) continue From 73fe19fb8d9f0719bd22f26030a7219fea236dc6 Mon Sep 17 00:00:00 2001 From: Daniel Mikusa Date: Fri, 12 Apr 2024 14:29:15 -0400 Subject: [PATCH 5/5] Update dependency_cache.go --- dependency_cache.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dependency_cache.go b/dependency_cache.go index c400511..b6c702e 100644 --- a/dependency_cache.go +++ b/dependency_cache.go @@ -88,6 +88,9 @@ func NewDependencyCache(context libcnb.BuildContext) (DependencyCache, error) { UserAgent: fmt.Sprintf("%s/%s", context.Buildpack.Info.ID, context.Buildpack.Info.Version), Mappings: map[string]string{}, DependencyMirrors: map[string]string{}, + // We create the logger here because the initialization process may log some warnings that should be visible to users. + // This goes against the usual pattern, which has the user supply the Logger after initialization. + // There's no choice though, if we want the warning messages to be visible to users. We should clean this up in v2. Logger: bard.NewLogger(os.Stdout), } mappings, err := filterBindingsByType(context.Platform.Bindings, "dependency-mapping")