From a4bb4839fb2ac3062890873ef1cf4f2be4994f8e Mon Sep 17 00:00:00 2001 From: Jon Johnson Date: Tue, 7 Dec 2021 10:14:28 -0800 Subject: [PATCH] Split layerCache into separate file This makes things a little cleaner by having a single place that calls buildLayer and passing a thunk down into the cache logic to call that on a cache miss. Also, remove the debug logging to make the code easier to follow (if you need to recompile anyway, it's easy enough to add log lines). --- pkg/build/cache.go | 210 +++++++++++++++++++++++++++++++++++++++++++ pkg/build/gobuild.go | 195 ++-------------------------------------- 2 files changed, 215 insertions(+), 190 deletions(-) create mode 100644 pkg/build/cache.go diff --git a/pkg/build/cache.go b/pkg/build/cache.go new file mode 100644 index 0000000000..86512e81cc --- /dev/null +++ b/pkg/build/cache.go @@ -0,0 +1,210 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" +) + +type diffIDToDescriptor map[v1.Hash]v1.Descriptor +type buildIDToDiffID map[string]v1.Hash + +type layerCache struct { + buildToDiff map[string]buildIDToDiffID + diffToDesc map[string]diffIDToDescriptor + sync.Mutex +} + +type layerFactory func() (v1.Layer, error) + +func (c *layerCache) get(ctx context.Context, file string, platform *v1.Platform, miss layerFactory) (v1.Layer, error) { + if os.Getenv("KOCACHE") == "" { + return miss() + } + + // Cache hit. + if diffid, desc, err := c.getMeta(ctx, file); err == nil { + return &lazyLayer{ + diffid: *diffid, + desc: *desc, + buildLayer: miss, + }, nil + } + + // Cache miss. + layer, err := miss() + if err != nil { + return nil, err + } + if err := c.put(ctx, file, layer); err != nil { + log.Printf("failed to cache metadata %s: %v", file, err) + } + return layer, nil +} + +func (c *layerCache) getMeta(ctx context.Context, file string) (*v1.Hash, *v1.Descriptor, error) { + buildid, err := getBuildID(ctx, file) + if err != nil { + return nil, nil, err + } + + if buildid == "" { + return nil, nil, fmt.Errorf("no buildid for %s", file) + } + + btod, err := c.readBuildToDiff(file) + if err != nil { + return nil, nil, err + } + dtod, err := c.readDiffToDesc(file) + if err != nil { + return nil, nil, err + } + + diffid, ok := btod[buildid] + if !ok { + return nil, nil, fmt.Errorf("no diffid for %q", buildid) + } + + desc, ok := dtod[diffid] + if !ok { + return nil, nil, fmt.Errorf("no desc for %q", diffid) + } + + return &diffid, &desc, nil +} + +// Compute new layer metadata and cache it in-mem and on-disk. +func (c *layerCache) put(ctx context.Context, file string, layer v1.Layer) error { + buildid, err := getBuildID(ctx, file) + if err != nil { + return err + } + + desc, err := partial.Descriptor(layer) + if err != nil { + return err + } + + diffid, err := layer.DiffID() + if err != nil { + return err + } + + btod, ok := c.buildToDiff[file] + if !ok { + btod = buildIDToDiffID{} + } + btod[buildid] = diffid + + dtod, ok := c.diffToDesc[file] + if !ok { + dtod = diffIDToDescriptor{} + } + dtod[diffid] = *desc + + // TODO: Implement better per-file locking. + c.Lock() + defer c.Unlock() + + btodf, err := os.OpenFile(filepath.Join(filepath.Dir(file), "buildid-to-diffid"), os.O_RDWR|os.O_CREATE, 0755) + if err != nil { + return err + } + defer btodf.Close() + + dtodf, err := os.OpenFile(filepath.Join(filepath.Dir(file), "diffid-to-descriptor"), os.O_RDWR|os.O_CREATE, 0755) + if err != nil { + return err + } + defer dtodf.Close() + + enc := json.NewEncoder(btodf) + enc.SetIndent("", " ") + if err := enc.Encode(&btod); err != nil { + return err + } + + enc = json.NewEncoder(dtodf) + enc.SetIndent("", " ") + if err := enc.Encode(&dtod); err != nil { + return err + } + + return nil +} + +func (c *layerCache) readDiffToDesc(file string) (diffIDToDescriptor, error) { + if dtod, ok := c.diffToDesc[file]; ok { + return dtod, nil + } + + dtodf, err := os.Open(filepath.Join(filepath.Dir(file), "diffid-to-descriptor")) + if err != nil { + return nil, err + } + defer dtodf.Close() + + var dtod diffIDToDescriptor + if err := json.NewDecoder(dtodf).Decode(&dtod); err != nil { + return nil, err + } + c.diffToDesc[file] = dtod + return dtod, nil +} + +func (c *layerCache) readBuildToDiff(file string) (buildIDToDiffID, error) { + if btod, ok := c.buildToDiff[file]; ok { + return btod, nil + } + + btodf, err := os.Open(filepath.Join(filepath.Dir(file), "buildid-to-diffid")) + if err != nil { + return nil, err + } + defer btodf.Close() + + var btod buildIDToDiffID + if err := json.NewDecoder(btodf).Decode(&btod); err != nil { + return nil, err + } + c.buildToDiff[file] = btod + return btod, nil +} + +func getBuildID(ctx context.Context, file string) (string, error) { + cmd := exec.CommandContext(ctx, "go", "tool", "buildid", file) + var output bytes.Buffer + cmd.Stderr = &output + cmd.Stdout = &output + + if err := cmd.Run(); err != nil { + log.Printf("Unexpected error running \"go tool buildid %s\": %v\n%v", err, file, output.String()) + return "", err + } + return strings.TrimSpace(output.String()), nil +} diff --git a/pkg/build/gobuild.go b/pkg/build/gobuild.go index 7946d18867..53593eb973 100644 --- a/pkg/build/gobuild.go +++ b/pkg/build/gobuild.go @@ -20,7 +20,6 @@ import ( "archive/tar" "bytes" "context" - "encoding/json" "errors" "fmt" gb "go/build" @@ -33,7 +32,6 @@ import ( "path/filepath" "strconv" "strings" - "sync" "text/template" "github.com/containerd/stargz-snapshotter/estargz" @@ -41,7 +39,6 @@ import ( v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" - "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/google/go-containerregistry/pkg/v1/types" "github.com/google/ko/internal/sbom" @@ -56,14 +53,8 @@ import ( const ( defaultAppFilename = "ko-app" - - // Currently guards spammy cache logs that compile out of the binary. - debug = false ) -type diffIDToDescriptor map[v1.Hash]v1.Descriptor -type buildIDToDiffID map[string]v1.Hash - // GetBase takes an importpath and returns a base image reference and base image (or index). type GetBase func(context.Context, string) (name.Reference, Result, error) @@ -700,34 +691,13 @@ func (g *gobuild) buildOne(ctx context.Context, refStr string, base v1.Image, pl appDir := "/ko-app" appPath := path.Join(appDir, appFilename(ref.Path())) - var binaryLayer v1.Layer - if os.Getenv("KOCACHE") != "" { - binaryLayer, err = g.cache.get(ctx, appPath, file, platform) - if err != nil { - if debug { - log.Printf("Cache miss: %s for %s: %v", ref.Path(), platformToString(*platform), err) - } - - // Make typecheck below fail - binaryLayer = nil - } else if debug { - log.Printf("Cache hit: %s for %s", ref.Path(), platformToString(*platform)) - } + miss := func() (v1.Layer, error) { + return buildLayer(appPath, file, platform) } - // Cache miss. - if _, ok := binaryLayer.(*lazyLayer); !ok { - binaryLayer, err = buildLayer(appPath, file, platform) - if err != nil { - return nil, err - } - if os.Getenv("KOCACHE") != "" { - if err := g.cache.put(ctx, file, binaryLayer); err != nil { - log.Printf("failed to cache metadata for %s: %v", ref.Path(), err) - } else if debug { - log.Printf("Cached %s for %s under %s", ref.Path(), platformToString(*platform), filepath.Dir(file)) - } - } + binaryLayer, err := g.cache.get(ctx, file, platform, miss) + if err != nil { + return nil, err } layers = append(layers, mutate.Addendum{ @@ -998,158 +968,3 @@ func (pm *platformMatcher) matches(base *v1.Platform) bool { return false } - -type layerCache struct { - buildToDiff map[string]buildIDToDiffID - diffToDesc map[string]diffIDToDescriptor - sync.Mutex -} - -func (c *layerCache) get(ctx context.Context, appPath, file string, platform *v1.Platform) (*lazyLayer, error) { - buildid, err := getBuildID(ctx, file) - if err != nil { - return nil, err - } - - if buildid == "" { - return nil, fmt.Errorf("no buildid for %s", file) - } - - btod, err := c.readBuildToDiff(file) - if err != nil { - return nil, err - } - dtod, err := c.readDiffToDesc(file) - if err != nil { - return nil, err - } - - diffid, ok := btod[buildid] - if !ok { - return nil, fmt.Errorf("no diffid for %q", buildid) - } - - desc, ok := dtod[diffid] - if !ok { - return nil, fmt.Errorf("no desc for %q", diffid) - } - - return &lazyLayer{ - diffid: diffid, - desc: desc, - buildLayer: func() (v1.Layer, error) { - return buildLayer(appPath, file, platform) - }, - }, nil -} - -// Compute new layer metadata and cache it in-mem and on-disk. -func (c *layerCache) put(ctx context.Context, file string, layer v1.Layer) error { - buildid, err := getBuildID(ctx, file) - if err != nil { - return err - } - - desc, err := partial.Descriptor(layer) - if err != nil { - return err - } - - diffid, err := layer.DiffID() - if err != nil { - return err - } - - btod, ok := c.buildToDiff[file] - if !ok { - btod = buildIDToDiffID{} - } - btod[buildid] = diffid - - dtod, ok := c.diffToDesc[file] - if !ok { - dtod = diffIDToDescriptor{} - } - dtod[diffid] = *desc - - // TODO: Implement better per-file locking. - c.Lock() - defer c.Unlock() - - btodf, err := os.OpenFile(filepath.Join(filepath.Dir(file), "buildid-to-diffid"), os.O_RDWR|os.O_CREATE, 0755) - if err != nil { - return err - } - defer btodf.Close() - - dtodf, err := os.OpenFile(filepath.Join(filepath.Dir(file), "diffid-to-descriptor"), os.O_RDWR|os.O_CREATE, 0755) - if err != nil { - return err - } - defer dtodf.Close() - - enc := json.NewEncoder(btodf) - enc.SetIndent("", " ") - if err := enc.Encode(&btod); err != nil { - return err - } - - enc = json.NewEncoder(dtodf) - enc.SetIndent("", " ") - if err := enc.Encode(&dtod); err != nil { - return err - } - - return nil -} - -func (c *layerCache) readDiffToDesc(file string) (diffIDToDescriptor, error) { - if dtod, ok := c.diffToDesc[file]; ok { - return dtod, nil - } - - dtodf, err := os.Open(filepath.Join(filepath.Dir(file), "diffid-to-descriptor")) - if err != nil { - return nil, err - } - defer dtodf.Close() - - var dtod diffIDToDescriptor - if err := json.NewDecoder(dtodf).Decode(&dtod); err != nil { - return nil, err - } - c.diffToDesc[file] = dtod - return dtod, nil -} - -func (c *layerCache) readBuildToDiff(file string) (buildIDToDiffID, error) { - if btod, ok := c.buildToDiff[file]; ok { - return btod, nil - } - - btodf, err := os.Open(filepath.Join(filepath.Dir(file), "buildid-to-diffid")) - if err != nil { - return nil, err - } - defer btodf.Close() - - var btod buildIDToDiffID - if err := json.NewDecoder(btodf).Decode(&btod); err != nil { - return nil, err - } - c.buildToDiff[file] = btod - return btod, nil -} - -func getBuildID(ctx context.Context, file string) (string, error) { - cmd := exec.CommandContext(ctx, "go", "tool", "buildid", file) - var output bytes.Buffer - cmd.Stderr = &output - cmd.Stdout = &output - - if err := cmd.Run(); err != nil { - log.Printf("Unexpected error running \"go tool buildid %s\": %v\n%v", err, file, output.String()) - return "", err - } - return strings.TrimSpace(output.String()), nil -}