Skip to content

Commit

Permalink
Merge pull request #322 from bitgully/main
Browse files Browse the repository at this point in the history 
Allow hostname specific dependency mirrors
  • Loading branch information
@dmikusa
dmikusa authored Apr 12, 2024
2 parents 92cc31b + 73fe19f commit 99fe914
Show file tree
Hide file tree
Showing 2 changed files with 103 additions and 47 deletions.
107 changes: 68 additions & 39 deletions dependency_cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,28 +69,29 @@ type DependencyCache struct {
// httpClientTimeouts contains the timeout values used by HTTP client
HttpClientTimeouts HttpClientTimeouts

// Alternative source used for downloading dependencies.
DependencyMirror string
// Alternative sources used for downloading dependencies.
DependencyMirrors map[string]string
}

// NewDependencyCache creates a new instance setting the default cache path (<BUILDPACK_PATH>/dependencies) and user
// agent (<BUILDPACK_ID>/<BUILDPACK_VERSION>).
// Mappings will be read from any libcnb.Binding in the context with type "dependency-mappings".
//
// In some air-gapped environments, dependencies might not be download directly but need to be pulled from a local mirror registry.
// In such cases, an alternative URI can either be provided as environment variable "BP_DEPENDENCY_MIRROR", or by a binding of type "dependency-mirror"
// where a file named "uri" holds the desired location.
// The two schemes https:// and file:// are supported in mirror URIs where the expected format is (optional parts in "[]"):
// <scheme>://[<username>:<password>@]<hostname>[:<port>][/<prefix>]
// The optional path part of the provided URI is used as a prefix that might be necessary in some setups.
// This (prefix) path may also include a placeholder of "{originalHost}" at any level (in sub-paths or at top-level) and is replaced with the
// hostname of the original download URI at build time. A sample mirror URI might look like this: https://local-mirror.example.com/buildpacks-dependencies/{originalHost}
// In some environments, many dependencies might need to be downloaded from a (local) mirror registry or filesystem.
// Such alternative locations can be configured using bindings of type "dependency-mirror", avoiding too many "dependency-mapping" bindings.
// Environment variables named "BP_DEPENDENCY_MIRROR" (default) or "BP_DEPENDENCY_MIRROR_<HOSTNAME>" (hostname-specific mirror)
// can also be used for the same purpose.
func NewDependencyCache(context libcnb.BuildContext) (DependencyCache, error) {
cache := DependencyCache{
CachePath: filepath.Join(context.Buildpack.Path, "dependencies"),
DownloadPath: os.TempDir(),
UserAgent: fmt.Sprintf("%s/%s", context.Buildpack.Info.ID, context.Buildpack.Info.Version),
Mappings: map[string]string{},
CachePath: filepath.Join(context.Buildpack.Path, "dependencies"),
DownloadPath: os.TempDir(),
UserAgent: fmt.Sprintf("%s/%s", context.Buildpack.Info.ID, context.Buildpack.Info.Version),
Mappings: map[string]string{},
DependencyMirrors: map[string]string{},
// We create the logger here because the initialization process may log some warnings that should be visible to users.
// This goes against the usual pattern, which has the user supply the Logger after initialization.
// There's no choice though, if we want the warning messages to be visible to users. We should clean this up in v2.
Logger: bard.NewLogger(os.Stdout),
}
mappings, err := filterBindingsByType(context.Platform.Bindings, "dependency-mapping")
if err != nil {
Expand All @@ -104,11 +105,11 @@ func NewDependencyCache(context libcnb.BuildContext) (DependencyCache, error) {
}
cache.HttpClientTimeouts = *clientTimeouts

dependencyMirror, err := getDependencyMirror(context.Platform.Bindings)
bindingMirrors, err := filterBindingsByType(context.Platform.Bindings, "dependency-mirror")
if err != nil {
return DependencyCache{}, err
return DependencyCache{}, fmt.Errorf("unable to process dependency-mirror bindings\n%w", err)
}
cache.DependencyMirror = dependencyMirror
cache.setDependencyMirrors(bindingMirrors)

return cache, nil
}
Expand Down Expand Up @@ -153,22 +154,44 @@ func customizeHttpClientTimeouts() (*HttpClientTimeouts, error) {
}, nil
}

// Returns the URI of a dependency mirror (optional).
// Such mirror location can be defined in a binding of type 'dependency-mirror' with filename 'uri'
// or using the environment variable 'BP_DEPENDENCY_MIRROR'. The latter takes precedence in case both are found.
func getDependencyMirror(bindings libcnb.Bindings) (string, error) {
dependencyMirror := sherpa.GetEnvWithDefault("BP_DEPENDENCY_MIRROR", "")
// If no mirror was found in environment variables, try to find one in bindings.
if dependencyMirror == "" {
dependencyMirrorBindings, err := filterBindingsByType(bindings, "dependency-mirror")
if err == nil {
// Use the content of the file named "uri" as the mirror's URI.
dependencyMirror = dependencyMirrorBindings["uri"]
} else {
return "", err
func (d *DependencyCache) setDependencyMirrors(bindingMirrors map[string]string) {
// Initialize with mirrors from bindings.
d.DependencyMirrors = bindingMirrors
// Add mirrors from env variables and override duplicate hostnames set in bindings.
envs := os.Environ()
for _, env := range envs {
envPair := strings.SplitN(env, "=", 2)
if len(envPair) != 2 {
continue
}
hostnameSuffix, isMirror := strings.CutPrefix(envPair[0], "BP_DEPENDENCY_MIRROR")
if isMirror {
hostnameEncoded, _ := strings.CutPrefix(hostnameSuffix, "_")
if strings.ToLower(hostnameEncoded) == "default" {
d.Logger.Bodyf("%s with illegal hostname 'default'. Please use BP_DEPENDENCY_MIRROR to set a default.",
color.YellowString("Ignored dependency mirror"))
continue
}
d.DependencyMirrors[decodeHostnameEnv(hostnameEncoded, d)] = envPair[1]
}
}
return dependencyMirror, nil
}

// Takes an encoded hostname (from env key) and returns the decoded version in lower case.
// Replaces double underscores (__) with one dash (-) and single underscores (_) with one period (.).
func decodeHostnameEnv(encodedHostname string, d *DependencyCache) string {
if strings.ContainsAny(encodedHostname, "-.") || encodedHostname != strings.ToUpper(encodedHostname) {
d.Logger.Bodyf("%s These will be allowed but for best results across different shells, you should replace . characters with _ characters "+
"and - characters with __, and use all upper case letters. The buildpack will convert these back before using the mirror.",
color.YellowString("You have invalid characters in your mirror host environment variable."))
}
var decodedHostname string
if encodedHostname == "" {
decodedHostname = "default"
} else {
decodedHostname = strings.ReplaceAll(strings.ReplaceAll(encodedHostname, "__", "-"), "_", ".")
}
return strings.ToLower(decodedHostname)
}

// Returns a key/value map with all entries for a given binding type.
Expand All @@ -178,10 +201,10 @@ func filterBindingsByType(bindings libcnb.Bindings, bindingType string) (map[str
for _, binding := range bindings {
if strings.ToLower(binding.Type) == bindingType {
for key, value := range binding.Secret {
if _, ok := filteredBindings[key]; ok {
if _, ok := filteredBindings[strings.ToLower(key)]; ok {
return nil, fmt.Errorf("multiple %s bindings found with duplicate keys %s", binding.Type, key)
}
filteredBindings[key] = value
filteredBindings[strings.ToLower(key)] = value
}
}
}
Expand Down Expand Up @@ -225,11 +248,17 @@ func (d *DependencyCache) Artifact(dependency BuildpackDependency, mods ...Reque
return nil, fmt.Errorf("unable to parse URI. see DEBUG log level")
}

if isBinding && d.DependencyMirror != "" {
mirror := d.DependencyMirrors["default"]
mirrorHostSpecific := d.DependencyMirrors[urlP.Hostname()]
if mirrorHostSpecific != "" {
mirror = mirrorHostSpecific
}

if isBinding && mirror != "" {
d.Logger.Bodyf("Both dependency mirror and bindings are present. %s Please remove dependency map bindings if you wish to use the mirror.",
color.YellowString("Mirror is being ignored."))
} else {
d.setDependencyMirror(urlP)
d.setDependencyMirror(urlP, mirror)
}

if dependency.SHA256 == "" {
Expand Down Expand Up @@ -422,18 +451,18 @@ func (DependencyCache) verify(path string, expected string) error {
return nil
}

func (d DependencyCache) setDependencyMirror(urlD *url.URL) {
if d.DependencyMirror != "" {
func (d DependencyCache) setDependencyMirror(urlD *url.URL, mirror string) {
if mirror != "" {
d.Logger.Bodyf("%s Download URIs will be overridden.", color.GreenString("Dependency mirror found."))
urlOverride, err := url.ParseRequestURI(d.DependencyMirror)
urlOverride, err := url.ParseRequestURI(mirror)

if strings.ToLower(urlOverride.Scheme) == "https" || strings.ToLower(urlOverride.Scheme) == "file" {
urlD.Scheme = urlOverride.Scheme
urlD.User = urlOverride.User
urlD.Path = strings.Replace(urlOverride.Path, "{originalHost}", urlD.Hostname(), 1) + urlD.Path
urlD.Host = urlOverride.Host
} else {
d.Logger.Debugf("Dependency mirror URI is invalid: %s\n%w", d.DependencyMirror, err)
d.Logger.Debugf("Dependency mirror URI is invalid: %s\n%w", mirror, err)
d.Logger.Bodyf("%s is ignored. Have you used one of the supported schemes https:// or file://?", color.YellowString("Invalid dependency mirror"))
}
}
Expand Down
43 changes: 35 additions & 8 deletions dependency_cache_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,29 +154,39 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) {
context("dependency mirror from environment variable", func() {
it.Before(func() {
t.Setenv("BP_DEPENDENCY_MIRROR", "https://env-var-mirror.acme.com")
t.Setenv("BP_DEPENDENCY_MIRROR_EXAMP__LE_COM", "https://examp-le.com")
})

it("uses BP_DEPENDENCY_MIRROR environment variable", func() {
dependencyCache, err := libpak.NewDependencyCache(ctx)
Expect(err).NotTo(HaveOccurred())
Expect(dependencyCache.DependencyMirror).To(Equal("https://env-var-mirror.acme.com"))
Expect(dependencyCache.DependencyMirrors["default"]).To(Equal("https://env-var-mirror.acme.com"))
Expect(dependencyCache.DependencyMirrors["examp-le.com"]).To(Equal("https://examp-le.com"))
})
})

context("dependency mirror from binding", func() {
context("dependency mirror from binding and environment variable", func() {
it.Before(func() {
t.Setenv("BP_DEPENDENCY_MIRROR_EXAMP__LE_COM", "https://examp-le.com")
ctx.Platform.Bindings = append(ctx.Platform.Bindings, libcnb.Binding{
Type: "dependency-mirror",
Secret: map[string]string{
"uri": "https://bindings-mirror.acme.com",
"default": "https://bindings-mirror.acme.com",
"examp-le.com": "https://invalid.com",
},
})
})

it("uses dependency-mirror binding", func() {
dependencyCache, err := libpak.NewDependencyCache(ctx)
Expect(err).NotTo(HaveOccurred())
Expect(dependencyCache.DependencyMirror).To(Equal("https://bindings-mirror.acme.com"))
Expect(dependencyCache.DependencyMirrors["default"]).To(Equal("https://bindings-mirror.acme.com"))
})

it("environment variable overrides binding", func() {
dependencyCache, err := libpak.NewDependencyCache(ctx)
Expect(err).NotTo(HaveOccurred())
Expect(dependencyCache.DependencyMirrors["examp-le.com"]).To(Equal("https://examp-le.com"))
})
})
})
Expand Down Expand Up @@ -332,6 +342,7 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) {

it.Before(func() {
mirrorServer = ghttp.NewTLSServer()
dependencyCache.DependencyMirrors = map[string]string{}
})

it.After(func() {
Expand All @@ -347,7 +358,7 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) {
ghttp.RespondWith(http.StatusOK, "test-fixture"),
))

dependencyCache.DependencyMirror = url.Scheme + "://" + "username:password@" + url.Host + "/foo/bar"
dependencyCache.DependencyMirrors["default"] = url.Scheme + "://" + "username:password@" + url.Host + "/foo/bar"
a, err := dependencyCache.Artifact(dependency)
Expect(err).NotTo(HaveOccurred())

Expand All @@ -362,7 +373,22 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) {
ghttp.RespondWith(http.StatusOK, "test-fixture"),
))

dependencyCache.DependencyMirror = url.Scheme + "://" + url.Host + "/{originalHost}"
dependencyCache.DependencyMirrors["default"] = url.Scheme + "://" + url.Host + "/{originalHost}"
a, err := dependencyCache.Artifact(dependency)
Expect(err).NotTo(HaveOccurred())

Expect(io.ReadAll(a)).To(Equal([]byte("test-fixture")))
})

it("downloads from https mirror host specific", func() {
url, err := url.Parse(mirrorServer.URL())
Expect(err).NotTo(HaveOccurred())
mirrorServer.AppendHandlers(ghttp.CombineHandlers(
ghttp.VerifyRequest(http.MethodGet, "/host-specific/test-path", ""),
ghttp.RespondWith(http.StatusOK, "test-fixture"),
))

dependencyCache.DependencyMirrors["127.0.0.1"] = url.Scheme + "://" + url.Host + "/host-specific"
a, err := dependencyCache.Artifact(dependency)
Expect(err).NotTo(HaveOccurred())

Expand All @@ -384,6 +410,7 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) {
Expect(err).NotTo(HaveOccurred())
mirrorPathPreservedHost = filepath.Join(mirrorPath, originalUrl.Hostname(), "prefix")
Expect(os.MkdirAll(mirrorPathPreservedHost, os.ModePerm)).NotTo(HaveOccurred())
dependencyCache.DependencyMirrors = map[string]string{}
})

it.After(func() {
Expand All @@ -394,7 +421,7 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) {
mirrorFile := filepath.Join(mirrorPath, "test-path")
Expect(os.WriteFile(mirrorFile, []byte("test-fixture"), 0644)).ToNot(HaveOccurred())

dependencyCache.DependencyMirror = "file://" + mirrorPath
dependencyCache.DependencyMirrors["default"] = "file://" + mirrorPath
a, err := dependencyCache.Artifact(dependency)
Expect(err).NotTo(HaveOccurred())

Expand All @@ -405,7 +432,7 @@ func testDependencyCache(t *testing.T, context spec.G, it spec.S) {
mirrorFilePreservedHost := filepath.Join(mirrorPathPreservedHost, "test-path")
Expect(os.WriteFile(mirrorFilePreservedHost, []byte("test-fixture"), 0644)).ToNot(HaveOccurred())

dependencyCache.DependencyMirror = "file://" + mirrorPath + "/{originalHost}" + "/prefix"
dependencyCache.DependencyMirrors["default"] = "file://" + mirrorPath + "/{originalHost}" + "/prefix"
a, err := dependencyCache.Artifact(dependency)
Expect(err).NotTo(HaveOccurred())

Expand Down

0 comments on commit 99fe914

Please sign in to comment.