diff --git a/README.md b/README.md index 798e9e02997d..0f87a1beb7d7 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ You don't need to read this document unless you want to use the full-featured st - [Local directory](#local-directory-1) - [GitHub Actions cache (experimental)](#github-actions-cache-experimental) - [Consistent hashing](#consistent-hashing) +- [Metadata](#metadata) - [Systemd socket activation](#systemd-socket-activation) - [Expose BuildKit as a TCP service](#expose-buildkit-as-a-tcp-service) - [Load balancing](#load-balancing) @@ -232,6 +233,7 @@ Keys supported by image output: * `name-canonical=true`: add additional canonical name `name@` * `compression=[uncompressed,gzip,estargz,zstd]`: choose compression type for layers newly created and cached, gzip is default value. estargz should be used with `oci-mediatypes=true`. * `force-compression=true`: forcefully apply `compression` option to all layers (including already existing layers). +* `buildinfo=[all,imageconfig,metadata,none]`: choose [build dependency](docs/build-repro.md#build-dependencies) version to export (default `all`). If credentials are required, `buildctl` will attempt to read Docker configuration file `$DOCKER_CONFIG/config.json`. `$DOCKER_CONFIG` defaults to `~/.docker`. @@ -416,7 +418,7 @@ To output build metadata such as the image digest, pass the `--metadata-file` fl The metadata will be written as a JSON object to the specified file. The directory of the specified file must already exist and be writable. -``` +```bash buildctl build ... --metadata-file metadata.json ``` diff --git a/docs/build-repro.md b/docs/build-repro.md new file mode 100644 index 000000000000..b34d9303f419 --- /dev/null +++ b/docs/build-repro.md @@ -0,0 +1,83 @@ +# Build reproducibility + +## Build dependencies + +Build dependencies are generated when your image has been built. These +dependencies include versions of used images, git repositories and HTTP URLs +used by LLB `Source` operation. + +By default, the build dependencies are embedded in the image configuration and +also available in the solver response. The export mode can be refined with +the [`buildinfo` attribute](../README.md#imageregistry). + +### Image config + +A new field similar to the one for inline cache has been added to the image +configuration to embed build dependencies: + +```text +"moby.buildkit.buildinfo.v1": +``` + +The structure is base64 encoded and has the following format when decoded: + +```json +{ + "sources": [ + { + "type": "docker-image", + "ref": "docker.io/docker/buildx-bin:0.6.1@sha256:a652ced4a4141977c7daaed0a074dcd9844a78d7d2615465b12f433ae6dd29f0", + "pin": "sha256:a652ced4a4141977c7daaed0a074dcd9844a78d7d2615465b12f433ae6dd29f0" + }, + { + "type": "docker-image", + "ref": "docker.io/library/alpine:3.13", + "pin": "sha256:1d30d1ba3cb90962067e9b29491fbd56997979d54376f23f01448b5c5cd8b462" + }, + { + "type": "git", + "ref": "https://github.com/crazy-max/buildkit-buildsources-test.git#master", + "pin": "259a5aa5aa5bb3562d12cc631fe399f4788642c1" + }, + { + "type": "http", + "ref": "https://raw.githubusercontent.com/moby/moby/master/README.md", + "pin": "sha256:419455202b0ef97e480d7f8199b26a721a417818bc0e2d106975f74323f25e6c" + } + ] +} +``` + +* `type` defines the source type (`docker-image`, `git` or `http`). +* `ref` is the reference of the source. +* `pin` is the source digest. + +### Exporter response (metadata) + +The solver response (`ExporterResponse`) also contains a new key +`containerimage.buildinfo` with the same structure as image config encoded in +base64: + +```json +{ + "ExporterResponse": { + "containerimage.buildinfo": "", + "containerimage.digest": "sha256:...", + "image.name": "..." + } +} +``` + +If multi-platforms are specified, they will be suffixed with the corresponding +platform: + +```json +{ + "ExporterResponse": { + "containerimage.buildinfo/linux/amd64": "", + "containerimage.buildinfo/linux/arm64": "", + "containerimage.digest": "sha256:...", + "image.name": "..." + } +} +``` diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go index a96bfd1b5f2b..09aec6397e0d 100644 --- a/exporter/containerimage/export.go +++ b/exporter/containerimage/export.go @@ -19,6 +19,7 @@ import ( "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/session" "github.com/moby/buildkit/snapshot" + "github.com/moby/buildkit/util/buildinfo" "github.com/moby/buildkit/util/compression" "github.com/moby/buildkit/util/contentutil" "github.com/moby/buildkit/util/leaseutil" @@ -40,6 +41,7 @@ const ( keyNameCanonical = "name-canonical" keyLayerCompression = "compression" keyForceCompression = "force-compression" + keyBuildInfo = "buildinfo" ociTypes = "oci-mediatypes" ) @@ -68,6 +70,7 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp i := &imageExporterInstance{ imageExporter: e, layerCompression: compression.Default, + buildInfoMode: buildinfo.ExportDefault, } var esgz bool @@ -161,6 +164,15 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp return nil, errors.Wrapf(err, "non-bool value specified for %s", k) } i.forceCompression = b + case keyBuildInfo: + if v == "" { + continue + } + bimode, err := buildinfo.ParseExportMode(v) + if err != nil { + return nil, err + } + i.buildInfoMode = bimode default: if i.meta == nil { i.meta = make(map[string][]byte) @@ -187,6 +199,7 @@ type imageExporterInstance struct { danglingPrefix string layerCompression compression.Type forceCompression bool + buildInfoMode buildinfo.ExportMode meta map[string][]byte } @@ -208,7 +221,7 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source, } defer done(context.TODO()) - desc, err := e.opt.ImageWriter.Commit(ctx, src, e.ociTypes, e.layerCompression, e.forceCompression, sessionID) + desc, err := e.opt.ImageWriter.Commit(ctx, src, e.ociTypes, e.layerCompression, e.buildInfoMode, e.forceCompression, sessionID) if err != nil { return nil, err } @@ -217,6 +230,15 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source, e.opt.ImageWriter.ContentStore().Delete(context.TODO(), desc.Digest) }() + if e.buildInfoMode&buildinfo.ExportMetadata == 0 { + for k := range src.Metadata { + if !strings.HasPrefix(k, exptypes.ExporterBuildInfo) { + continue + } + delete(src.Metadata, k) + } + } + resp := make(map[string]string) if n, ok := src.Metadata["image.name"]; e.targetName == "*" && ok { diff --git a/exporter/containerimage/exptypes/types.go b/exporter/containerimage/exptypes/types.go index 316a9b323fc4..11ff4cec598c 100644 --- a/exporter/containerimage/exptypes/types.go +++ b/exporter/containerimage/exptypes/types.go @@ -1,6 +1,7 @@ package exptypes import ( + srctypes "github.com/moby/buildkit/source/types" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -11,6 +12,7 @@ const ( ExporterImageConfigKey = "containerimage.config" ExporterImageConfigDigestKey = "containerimage.config.digest" ExporterInlineCache = "containerimage.inlinecache" + ExporterBuildInfo = "containerimage.buildinfo" ExporterPlatformsKey = "refs.platforms" ) @@ -24,3 +26,29 @@ type Platform struct { ID string Platform ocispecs.Platform } + +// BuildInfo defines build dependencies that will be added to image config as +// moby.buildkit.buildinfo.v1 key and returned in solver ExporterResponse as +// ExporterBuildInfo key. +type BuildInfo struct { + // Type defines the BuildInfoType source type (docker-image, git, http). + Type BuildInfoType `json:"type,omitempty"` + // Ref is the reference of the source. + Ref string `json:"ref,omitempty"` + // Alias is a special field used to match with the actual source ref + // because frontend might have already transformed a string user typed + // before generating LLB. + Alias string `json:"alias,omitempty"` + // Pin is the source digest. + Pin string `json:"pin,omitempty"` +} + +// BuildInfoType contains source type. +type BuildInfoType string + +// List of source types. +const ( + BuildInfoTypeDockerImage BuildInfoType = srctypes.DockerImageScheme + BuildInfoTypeGit BuildInfoType = srctypes.GitScheme + BuildInfoTypeHTTP BuildInfoType = srctypes.HTTPScheme +) diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index 2d92fadc6de7..35ab23f05fbf 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -8,11 +8,6 @@ import ( "strings" "time" - "github.com/moby/buildkit/util/bklog" - "github.com/moby/buildkit/util/tracing" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - "github.com/containerd/containerd/content" "github.com/containerd/containerd/diff" "github.com/containerd/containerd/images" @@ -23,13 +18,18 @@ import ( "github.com/moby/buildkit/session" "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/util/bklog" + "github.com/moby/buildkit/util/buildinfo" "github.com/moby/buildkit/util/compression" "github.com/moby/buildkit/util/progress" "github.com/moby/buildkit/util/system" + "github.com/moby/buildkit/util/tracing" digest "github.com/opencontainers/go-digest" specs "github.com/opencontainers/image-spec/specs-go" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "golang.org/x/sync/errgroup" ) @@ -48,7 +48,7 @@ type ImageWriter struct { opt WriterOpt } -func (ic *ImageWriter) Commit(ctx context.Context, inp exporter.Source, oci bool, compressionType compression.Type, forceCompression bool, sessionID string) (*ocispecs.Descriptor, error) { +func (ic *ImageWriter) Commit(ctx context.Context, inp exporter.Source, oci bool, compressionType compression.Type, buildInfoMode buildinfo.ExportMode, forceCompression bool, sessionID string) (*ocispecs.Descriptor, error) { platformsBytes, ok := inp.Metadata[exptypes.ExporterPlatformsKey] if len(inp.Refs) > 0 && !ok { @@ -60,7 +60,13 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp exporter.Source, oci bool if err != nil { return nil, err } - mfstDesc, configDesc, err := ic.commitDistributionManifest(ctx, inp.Ref, inp.Metadata[exptypes.ExporterImageConfigKey], &remotes[0], oci, inp.Metadata[exptypes.ExporterInlineCache]) + + var buildInfo []byte + if buildInfoMode&buildinfo.ExportImageConfig > 0 { + buildInfo = inp.Metadata[exptypes.ExporterBuildInfo] + } + + mfstDesc, configDesc, err := ic.commitDistributionManifest(ctx, inp.Ref, inp.Metadata[exptypes.ExporterImageConfigKey], &remotes[0], oci, inp.Metadata[exptypes.ExporterInlineCache], buildInfo) if err != nil { return nil, err } @@ -68,6 +74,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp exporter.Source, oci bool mfstDesc.Annotations = make(map[string]string) } mfstDesc.Annotations[exptypes.ExporterConfigDigestKey] = configDesc.Digest.String() + return mfstDesc, nil } @@ -119,8 +126,14 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp exporter.Source, oci bool return nil, errors.Errorf("failed to find ref for ID %s", p.ID) } config := inp.Metadata[fmt.Sprintf("%s/%s", exptypes.ExporterImageConfigKey, p.ID)] + inlineCache := inp.Metadata[fmt.Sprintf("%s/%s", exptypes.ExporterInlineCache, p.ID)] + + var buildInfo []byte + if buildInfoMode&buildinfo.ExportImageConfig > 0 { + buildInfo = inp.Metadata[fmt.Sprintf("%s/%s", exptypes.ExporterBuildInfo, p.ID)] + } - desc, _, err := ic.commitDistributionManifest(ctx, r, config, &remotes[remotesMap[p.ID]], oci, inp.Metadata[fmt.Sprintf("%s/%s", exptypes.ExporterInlineCache, p.ID)]) + desc, _, err := ic.commitDistributionManifest(ctx, r, config, &remotes[remotesMap[p.ID]], oci, inlineCache, buildInfo) if err != nil { return nil, err } @@ -184,7 +197,7 @@ func (ic *ImageWriter) exportLayers(ctx context.Context, compressionType compres return out, err } -func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, ref cache.ImmutableRef, config []byte, remote *solver.Remote, oci bool, inlineCache []byte) (*ocispecs.Descriptor, *ocispecs.Descriptor, error) { +func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, ref cache.ImmutableRef, config []byte, remote *solver.Remote, oci bool, inlineCache []byte, buildInfo []byte) (*ocispecs.Descriptor, *ocispecs.Descriptor, error) { if len(config) == 0 { var err error config, err = emptyImageConfig() @@ -206,7 +219,7 @@ func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, ref cache remote, history = normalizeLayersAndHistory(ctx, remote, history, ref, oci) - config, err = patchImageConfig(config, remote.Descriptors, history, inlineCache) + config, err = patchImageConfig(config, remote.Descriptors, history, inlineCache, buildInfo) if err != nil { return nil, nil, err } @@ -346,7 +359,7 @@ func parseHistoryFromConfig(dt []byte) ([]ocispecs.History, error) { return config.History, nil } -func patchImageConfig(dt []byte, descs []ocispecs.Descriptor, history []ocispecs.History, cache []byte) ([]byte, error) { +func patchImageConfig(dt []byte, descs []ocispecs.Descriptor, history []ocispecs.History, cache []byte, buildInfo []byte) ([]byte, error) { m := map[string]json.RawMessage{} if err := json.Unmarshal(dt, &m); err != nil { return nil, errors.Wrap(err, "failed to parse image config for patch") @@ -391,6 +404,16 @@ func patchImageConfig(dt []byte, descs []ocispecs.Descriptor, history []ocispecs m["moby.buildkit.cache.v0"] = dt } + if buildInfo != nil { + dt, err := json.Marshal(buildInfo) + if err != nil { + return nil, err + } + m[buildinfo.ImageConfigField] = dt + } else if _, ok := m[buildinfo.ImageConfigField]; ok { + delete(m, buildinfo.ImageConfigField) + } + dt, err = json.Marshal(m) return dt, errors.Wrap(err, "failed to marshal config after patch") } diff --git a/exporter/oci/export.go b/exporter/oci/export.go index d57a37d3032f..0ed60a90c56f 100644 --- a/exporter/oci/export.go +++ b/exporter/oci/export.go @@ -15,6 +15,7 @@ import ( "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/filesync" + "github.com/moby/buildkit/util/buildinfo" "github.com/moby/buildkit/util/compression" "github.com/moby/buildkit/util/contentutil" "github.com/moby/buildkit/util/grpcerrors" @@ -35,6 +36,7 @@ const ( VariantDocker = "docker" ociTypes = "oci-mediatypes" keyForceCompression = "force-compression" + keyBuildInfo = "buildinfo" ) type Opt struct { @@ -58,6 +60,7 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp i := &imageExporterInstance{ imageExporter: e, layerCompression: compression.Default, + buildInfoMode: buildinfo.ExportDefault, } var esgz bool for k, v := range opt { @@ -99,6 +102,15 @@ func (e *imageExporter) Resolve(ctx context.Context, opt map[string]string) (exp return nil, errors.Wrapf(err, "non-bool value specified for %s", k) } *ot = b + case keyBuildInfo: + if v == "" { + continue + } + bimode, err := buildinfo.ParseExportMode(v) + if err != nil { + return nil, err + } + i.buildInfoMode = bimode default: if i.meta == nil { i.meta = make(map[string][]byte) @@ -125,6 +137,7 @@ type imageExporterInstance struct { ociTypes bool layerCompression compression.Type forceCompression bool + buildInfoMode buildinfo.ExportMode } func (e *imageExporterInstance) Name() string { @@ -149,13 +162,23 @@ func (e *imageExporterInstance) Export(ctx context.Context, src exporter.Source, } defer done(context.TODO()) - desc, err := e.opt.ImageWriter.Commit(ctx, src, e.ociTypes, e.layerCompression, e.forceCompression, sessionID) + desc, err := e.opt.ImageWriter.Commit(ctx, src, e.ociTypes, e.layerCompression, e.buildInfoMode, e.forceCompression, sessionID) if err != nil { return nil, err } defer func() { e.opt.ImageWriter.ContentStore().Delete(context.TODO(), desc.Digest) }() + + if e.buildInfoMode&buildinfo.ExportMetadata == 0 { + for k := range src.Metadata { + if !strings.HasPrefix(k, exptypes.ExporterBuildInfo) { + continue + } + delete(src.Metadata, k) + } + } + if desc.Annotations == nil { desc.Annotations = map[string]string{} } diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index f1256b8f10ae..afa04330caa9 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -19,6 +19,7 @@ import ( "github.com/docker/go-connections/nat" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/client/llb/imagemetaresolver" + "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/moby/buildkit/frontend/dockerfile/parser" "github.com/moby/buildkit/frontend/dockerfile/shell" @@ -276,7 +277,6 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, p := autoDetectPlatform(img, *platform, platformOpt.buildPlatforms) platform = &p } - d.image = img if dgst != "" { ref, err = reference.WithDigest(ref, dgst) if err != nil { @@ -294,6 +294,17 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, } } } + if !isScratch { + // image not scratch set original image name as ref + // and actual reference as alias in BuildInfo + d.buildInfo = &exptypes.BuildInfo{ + Type: exptypes.BuildInfoTypeDockerImage, + Ref: origName, + Alias: ref.String(), + Pin: dgst.String(), + } + } + d.image = img } if isScratch { d.state = llb.Scratch() @@ -319,11 +330,18 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, buildContext := &mutableOutput{} ctxPaths := map[string]struct{}{} + var buildInfos []exptypes.BuildInfo for _, d := range allDispatchStates.states { if !isReachable(target, d) { continue } + + // collect build dependencies + if d.buildInfo != nil { + buildInfos = append(buildInfos, *d.buildInfo) + } + if d.base != nil { d.state = d.base.state d.platform = d.base.platform @@ -395,6 +413,18 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, } } + // set target with gathered build dependencies + target.image.BuildInfo = []byte{} + if len(buildInfos) > 0 { + sort.Slice(buildInfos, func(i, j int) bool { + return buildInfos[i].Ref < buildInfos[j].Ref + }) + target.image.BuildInfo, err = json.Marshal(buildInfos) + if err != nil { + return nil, nil, err + } + } + if len(opt.Labels) != 0 && target.image.Config.Labels == nil { target.image.Config.Labels = make(map[string]string, len(opt.Labels)) } @@ -582,6 +612,7 @@ type dispatchState struct { cmdIndex int cmdTotal int prefixPlatform bool + buildInfo *exptypes.BuildInfo } type dispatchStates struct { diff --git a/frontend/dockerfile/dockerfile2llb/convert_test.go b/frontend/dockerfile/dockerfile2llb/convert_test.go index 8fe1aa6e2ad8..8584e4dbb81a 100644 --- a/frontend/dockerfile/dockerfile2llb/convert_test.go +++ b/frontend/dockerfile/dockerfile2llb/convert_test.go @@ -1,12 +1,17 @@ package dockerfile2llb import ( + "encoding/json" + "strings" "testing" + "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/moby/buildkit/frontend/dockerfile/shell" "github.com/moby/buildkit/util/appcontext" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func toEnvMap(args []instructions.KeyValuePairOptional, env []string) map[string]string { @@ -188,3 +193,34 @@ COPY --from=stage1 f2 /sub/ _, _, err = Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{}) assert.EqualError(t, err, "circular dependency detected on stage: stage0") } + +// moby/buildkit#2311 +func TestTargetBuildInfo(t *testing.T) { + df := ` +FROM busybox +ADD https://raw.githubusercontent.com/moby/buildkit/master/README.md / +` + _, image, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{ + TargetPlatform: &ocispecs.Platform{ + Architecture: "amd64", OS: "linux", + }, + BuildPlatforms: []ocispecs.Platform{ + { + Architecture: "amd64", OS: "linux", + }, + }, + }) + + require.NoError(t, err) + require.NotNil(t, image.BuildInfo) + + var bi []exptypes.BuildInfo + err = json.Unmarshal(image.BuildInfo, &bi) + require.NoError(t, err) + require.Equal(t, 1, len(bi)) + + assert.Equal(t, exptypes.BuildInfoTypeDockerImage, bi[0].Type) + assert.Equal(t, "busybox", bi[0].Ref) + assert.True(t, strings.HasPrefix(bi[0].Alias, "docker.io/library/busybox@")) + assert.NotEmpty(t, bi[0].Pin) +} diff --git a/frontend/dockerfile/dockerfile2llb/image.go b/frontend/dockerfile/dockerfile2llb/image.go index d4c82700e3e4..b4fca4a5705e 100644 --- a/frontend/dockerfile/dockerfile2llb/image.go +++ b/frontend/dockerfile/dockerfile2llb/image.go @@ -53,6 +53,9 @@ type Image struct { // Variant defines platform variant. To be added to OCI. Variant string `json:"variant,omitempty"` + + // BuildInfo defines build dependencies. + BuildInfo []byte `json:"moby.buildkit.buildinfo.v1,omitempty"` } func clone(src Image) Image { @@ -61,6 +64,7 @@ func clone(src Image) Image { img.Config.Env = append([]string{}, src.Config.Env...) img.Config.Cmd = append([]string{}, src.Config.Cmd...) img.Config.Entrypoint = append([]string{}, src.Config.Entrypoint...) + img.BuildInfo = src.BuildInfo return img } diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index 23039eacf385..ab054a0cbb98 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -5,6 +5,7 @@ import ( "bytes" "compress/gzip" "context" + "encoding/base64" "encoding/json" "fmt" "io" @@ -29,6 +30,7 @@ import ( "github.com/containerd/continuity/fs/fstest" "github.com/moby/buildkit/client" "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/exporter/containerimage/exptypes" "github.com/moby/buildkit/frontend/dockerfile/builder" "github.com/moby/buildkit/frontend/dockerfile/dockerfile2llb" gateway "github.com/moby/buildkit/frontend/gateway/client" @@ -43,6 +45,7 @@ import ( "github.com/moby/buildkit/util/testutil/integration" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -111,6 +114,7 @@ var allTests = []integration.Test{ testExportCacheLoop, testWildcardRenameCache, testDockerfileInvalidInstruction, + testBuildInfo, } var fileOpTests = []integration.Test{ @@ -5144,6 +5148,95 @@ RUN echo $(hostname) | grep testtest require.NoError(t, err) } +// moby/buildkit#2311 +func testBuildInfo(t *testing.T, sb integration.Sandbox) { + f := getFrontend(t, sb) + + gitDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(gitDir) + + dockerfile := ` +ARG DOCKERFILE_VERSION="1.3.0" +FROM docker/dockerfile-upstream:${DOCKERFILE_VERSION} AS dockerfile +FROM docker.io/docker/buildx-bin:0.6.1@sha256:a652ced4a4141977c7daaed0a074dcd9844a78d7d2615465b12f433ae6dd29f0 AS buildx +FROM busybox:latest +ADD https://raw.githubusercontent.com/moby/moby/master/README.md / +COPY --from=dockerfile /bin/dockerfile-frontend /tmp/ +COPY --from=buildx /buildx /usr/libexec/docker/cli-plugins/docker-buildx +` + + err = ioutil.WriteFile(filepath.Join(gitDir, "Dockerfile"), []byte(dockerfile), 0600) + require.NoError(t, err) + + err = runShell(gitDir, + "git init", + "git config --local user.email test", + "git config --local user.name test", + "git add Dockerfile", + "git commit -m initial", + "git branch buildinfo", + "git update-server-info", + ) + require.NoError(t, err) + + server := httptest.NewServer(http.FileServer(http.Dir(filepath.Join(gitDir)))) + defer server.Close() + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + out := filepath.Join(destDir, "out.tar") + outW, err := os.Create(out) + require.NoError(t, err) + + c, err := client.New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + res, err := f.Solve(sb.Context(), c, client.SolveOpt{ + Exports: []client.ExportEntry{ + { + Type: client.ExporterOCI, + Output: fixedWriteCloser(outW), + }, + }, + FrontendAttrs: map[string]string{ + builder.DefaultLocalNameContext: server.URL + "/.git#buildinfo", + }, + }, nil) + require.NoError(t, err) + + require.Contains(t, res.ExporterResponse, exptypes.ExporterBuildInfo) + dtbi, err := base64.StdEncoding.DecodeString(res.ExporterResponse[exptypes.ExporterBuildInfo]) + require.NoError(t, err) + + var bi map[string][]exptypes.BuildInfo + err = json.Unmarshal(dtbi, &bi) + require.NoError(t, err) + + _, ok := bi["sources"] + require.True(t, ok) + require.Equal(t, 4, len(bi["sources"])) + + assert.Equal(t, exptypes.BuildInfoTypeDockerImage, bi["sources"][0].Type) + assert.Equal(t, "docker.io/docker/buildx-bin:0.6.1@sha256:a652ced4a4141977c7daaed0a074dcd9844a78d7d2615465b12f433ae6dd29f0", bi["sources"][0].Ref) + assert.Equal(t, "sha256:a652ced4a4141977c7daaed0a074dcd9844a78d7d2615465b12f433ae6dd29f0", bi["sources"][0].Pin) + + assert.Equal(t, exptypes.BuildInfoTypeDockerImage, bi["sources"][1].Type) + assert.Equal(t, "docker.io/docker/dockerfile-upstream:1.3.0", bi["sources"][1].Ref) + assert.Equal(t, "sha256:9e2c9eca7367393aecc68795c671f93466818395a2693498debe831fd67f5e89", bi["sources"][1].Pin) + + assert.Equal(t, exptypes.BuildInfoTypeDockerImage, bi["sources"][2].Type) + assert.Equal(t, "docker.io/library/busybox:latest", bi["sources"][2].Ref) + assert.NotEmpty(t, bi["sources"][2].Pin) + + assert.Equal(t, exptypes.BuildInfoTypeHTTP, bi["sources"][3].Type) + assert.Equal(t, "https://raw.githubusercontent.com/moby/moby/master/README.md", bi["sources"][3].Ref) + assert.Equal(t, "sha256:419455202b0ef97e480d7f8199b26a721a417818bc0e2d106975f74323f25e6c", bi["sources"][3].Pin) +} + func tmpdir(appliers ...fstest.Applier) (string, error) { tmpdir, err := ioutil.TempDir("", "buildkit-dockerfile") if err != nil { diff --git a/frontend/gateway/gateway.go b/frontend/gateway/gateway.go index 9a0f20e83aea..ac48a795e043 100644 --- a/frontend/gateway/gateway.go +++ b/frontend/gateway/gateway.go @@ -12,8 +12,6 @@ import ( "sync" "time" - "github.com/moby/buildkit/util/bklog" - "github.com/docker/distribution/reference" "github.com/gogo/googleapis/google/rpc" gogotypes "github.com/gogo/protobuf/types" @@ -35,6 +33,7 @@ import ( llberrdefs "github.com/moby/buildkit/solver/llbsolver/errdefs" opspb "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/apicaps" + "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/grpcerrors" "github.com/moby/buildkit/util/stack" "github.com/moby/buildkit/util/tracing" diff --git a/solver/jobs.go b/solver/jobs.go index 0533366b2248..2bbd1ad5f57e 100644 --- a/solver/jobs.go +++ b/solver/jobs.go @@ -22,7 +22,7 @@ import ( type ResolveOpFunc func(Vertex, Builder) (Op, error) type Builder interface { - Build(ctx context.Context, e Edge) (CachedResult, error) + Build(ctx context.Context, e Edge) (CachedResult, BuildInfo, error) InContext(ctx context.Context, f func(ctx context.Context, g session.Group) error) error EachValue(ctx context.Context, key string, fn func(interface{}) error) error } @@ -197,15 +197,16 @@ type subBuilder struct { exporters []ExportableCacheKey } -func (sb *subBuilder) Build(ctx context.Context, e Edge) (CachedResult, error) { +func (sb *subBuilder) Build(ctx context.Context, e Edge) (CachedResult, BuildInfo, error) { + // TODO(@crazy-max): Handle BuildInfo from subbuild res, err := sb.solver.subBuild(ctx, e, sb.vtx) if err != nil { - return nil, err + return nil, nil, err } sb.mu.Lock() sb.exporters = append(sb.exporters, res.CacheKeys()[0]) // all keys already have full export chain sb.mu.Unlock() - return res, nil + return res, nil, nil } func (sb *subBuilder) InContext(ctx context.Context, f func(context.Context, session.Group) error) error { @@ -495,17 +496,43 @@ func (jl *Solver) deleteIfUnreferenced(k digest.Digest, st *state) { } } -func (j *Job) Build(ctx context.Context, e Edge) (CachedResult, error) { +func (j *Job) Build(ctx context.Context, e Edge) (CachedResult, BuildInfo, error) { if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() { j.span = span } v, err := j.list.load(e.Vertex, nil, j) if err != nil { - return nil, err + return nil, nil, err } e.Vertex = v - return j.list.s.build(ctx, e) + + res, err := j.list.s.build(ctx, e) + if err != nil { + return nil, nil, err + } + + j.list.mu.Lock() + defer j.list.mu.Unlock() + return res, j.walkBuildInfo(ctx, e, make(BuildInfo)), nil +} + +func (j *Job) walkBuildInfo(ctx context.Context, e Edge, bi BuildInfo) BuildInfo { + for _, inp := range e.Vertex.Inputs() { + if st, ok := j.list.actives[inp.Vertex.Digest()]; ok { + st.mu.Lock() + for _, cacheRes := range st.op.cacheRes { + for key, val := range cacheRes.BuildInfo { + if _, ok := bi[key]; !ok { + bi[key] = val + } + } + } + st.mu.Unlock() + bi = j.walkBuildInfo(ctx, inp, bi) + } + } + return bi } func (j *Job) Discard() error { diff --git a/solver/llbsolver/bridge.go b/solver/llbsolver/bridge.go index 4c7adf91e3dd..c14cc99ce0d4 100644 --- a/solver/llbsolver/bridge.go +++ b/solver/llbsolver/bridge.go @@ -36,20 +36,20 @@ type llbBridge struct { sm *session.Manager } -func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImports []gw.CacheOptionsEntry) (solver.CachedResult, error) { +func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImports []gw.CacheOptionsEntry) (solver.CachedResult, solver.BuildInfo, error) { w, err := b.resolveWorker() if err != nil { - return nil, err + return nil, nil, err } ent, err := loadEntitlements(b.builder) if err != nil { - return nil, err + return nil, nil, err } var cms []solver.CacheManager for _, im := range cacheImports { cmID, err := cmKey(im) if err != nil { - return nil, err + return nil, nil, err } b.cmsMu.Lock() var cm solver.CacheManager @@ -86,7 +86,7 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp edge, err := Load(def, dpc.Load, ValidateEntitlements(ent), WithCacheSources(cms), NormalizeRuntimePlatforms(), WithValidateCaps()) if err != nil { - return nil, errors.Wrap(err, "failed to load LLB") + return nil, nil, errors.Wrap(err, "failed to load LLB") } if len(dpc.ids) > 0 { @@ -97,15 +97,15 @@ func (b *llbBridge) loadResult(ctx context.Context, def *pb.Definition, cacheImp if err := b.eachWorker(func(w worker.Worker) error { return w.PruneCacheMounts(ctx, ids) }); err != nil { - return nil, err + return nil, nil, err } } - res, err := b.builder.Build(ctx, edge) + res, bi, err := b.builder.Build(ctx, edge) if err != nil { - return nil, err + return nil, nil, err } - return res, nil + return res, bi, nil } func (b *llbBridge) Solve(ctx context.Context, req frontend.SolveRequest, sid string) (res *frontend.Result, err error) { @@ -136,12 +136,13 @@ func (b *llbBridge) Solve(ctx context.Context, req frontend.SolveRequest, sid st } type resultProxy struct { - cb func(context.Context) (solver.CachedResult, error) + cb func(context.Context) (solver.CachedResult, solver.BuildInfo, error) def *pb.Definition g flightcontrol.Group mu sync.Mutex released bool v solver.CachedResult + bi solver.BuildInfo err error errResults []solver.Result } @@ -150,8 +151,8 @@ func newResultProxy(b *llbBridge, req frontend.SolveRequest) *resultProxy { rp := &resultProxy{ def: req.Definition, } - rp.cb = func(ctx context.Context) (solver.CachedResult, error) { - res, err := b.loadResult(ctx, req.Definition, req.CacheImports) + rp.cb = func(ctx context.Context) (solver.CachedResult, solver.BuildInfo, error) { + res, bi, err := b.loadResult(ctx, req.Definition, req.CacheImports) var ee *llberrdefs.ExecError if errors.As(err, &ee) { ee.EachRef(func(res solver.Result) error { @@ -161,7 +162,7 @@ func newResultProxy(b *llbBridge, req frontend.SolveRequest) *resultProxy { // acquire ownership so ExecError finalizer doesn't attempt to release as well ee.OwnerBorrowed = true } - return res, err + return res, bi, err } return rp } @@ -170,6 +171,10 @@ func (rp *resultProxy) Definition() *pb.Definition { return rp.def } +func (rp *resultProxy) BuildInfo() solver.BuildInfo { + return rp.bi +} + func (rp *resultProxy) Release(ctx context.Context) (err error) { rp.mu.Lock() defer rp.mu.Unlock() @@ -228,7 +233,7 @@ func (rp *resultProxy) Result(ctx context.Context) (res solver.CachedResult, err return rp.v, rp.err } rp.mu.Unlock() - v, err := rp.cb(ctx) + v, bi, err := rp.cb(ctx) if err != nil { select { case <-ctx.Done(): @@ -247,6 +252,7 @@ func (rp *resultProxy) Result(ctx context.Context) (res solver.CachedResult, err return nil, errors.Errorf("evaluating released result") } rp.v = v + rp.bi = bi rp.err = err rp.mu.Unlock() return v, err diff --git a/solver/llbsolver/ops/source.go b/solver/llbsolver/ops/source.go index 6cec6320b58a..ed7be8d695e8 100644 --- a/solver/llbsolver/ops/source.go +++ b/solver/llbsolver/ops/source.go @@ -67,21 +67,27 @@ func (s *sourceOp) CacheMap(ctx context.Context, g session.Group, index int) (*s if err != nil { return nil, false, err } - k, cacheOpts, done, err := src.CacheKey(ctx, g, index) + + k, pin, cacheOpts, done, err := src.CacheKey(ctx, g, index) if err != nil { return nil, false, err } dgst := digest.FromBytes([]byte(sourceCacheType + ":" + k)) - if strings.HasPrefix(k, "session:") { dgst = digest.Digest("random:" + strings.TrimPrefix(dgst.String(), dgst.Algorithm().String()+":")) } + var buildInfo map[string]string + if !strings.HasPrefix(s.op.Source.GetIdentifier(), "local://") { + buildInfo = map[string]string{s.op.Source.GetIdentifier(): pin} + } + return &solver.CacheMap{ // TODO: add os/arch - Digest: dgst, - Opts: cacheOpts, + Digest: dgst, + Opts: cacheOpts, + BuildInfo: buildInfo, }, done, nil } diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index ff38f46f9db3..5c08c0b1759f 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -2,6 +2,7 @@ package llbsolver import ( "context" + "encoding/base64" "fmt" "strings" "time" @@ -16,6 +17,7 @@ import ( "github.com/moby/buildkit/frontend/gateway" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/util/buildinfo" "github.com/moby/buildkit/util/compression" "github.com/moby/buildkit/util/entitlements" "github.com/moby/buildkit/util/progress" @@ -173,12 +175,20 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro } inp.Ref = workerRef.ImmutableRef - dt, err := inlineCache(ctx, exp.CacheExporter, r, session.NewGroup(sessionID)) + dtbi, err := buildinfo.Merge(ctx, res.BuildInfo(), inp.Metadata[exptypes.ExporterImageConfigKey]) if err != nil { return nil, err } - if dt != nil { - inp.Metadata[exptypes.ExporterInlineCache] = dt + if dtbi != nil && len(dtbi) > 0 { + inp.Metadata[exptypes.ExporterBuildInfo] = dtbi + } + + dtic, err := inlineCache(ctx, exp.CacheExporter, r, session.NewGroup(sessionID)) + if err != nil { + return nil, err + } + if dtic != nil { + inp.Metadata[exptypes.ExporterInlineCache] = dtic } } if res.Refs != nil { @@ -197,12 +207,20 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro } m[k] = workerRef.ImmutableRef - dt, err := inlineCache(ctx, exp.CacheExporter, r, session.NewGroup(sessionID)) + dtbi, err := buildinfo.Merge(ctx, res.BuildInfo(), inp.Metadata[fmt.Sprintf("%s/%s", exptypes.ExporterImageConfigKey, k)]) if err != nil { return nil, err } - if dt != nil { - inp.Metadata[fmt.Sprintf("%s/%s", exptypes.ExporterInlineCache, k)] = dt + if dtbi != nil && len(dtbi) > 0 { + inp.Metadata[fmt.Sprintf("%s/%s", exptypes.ExporterBuildInfo, k)] = dtbi + } + + dtic, err := inlineCache(ctx, exp.CacheExporter, r, session.NewGroup(sessionID)) + if err != nil { + return nil, err + } + if dtic != nil { + inp.Metadata[fmt.Sprintf("%s/%s", exptypes.ExporterInlineCache, k)] = dtic } } } @@ -253,6 +271,9 @@ func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req fro if strings.HasPrefix(k, "frontend.") { exporterResponse[k] = string(v) } + if strings.HasPrefix(k, exptypes.ExporterBuildInfo) { + exporterResponse[k] = base64.StdEncoding.EncodeToString(v) + } } for k, v := range cacheExporterResponse { if strings.HasPrefix(k, "cache.") { diff --git a/solver/scheduler_test.go b/solver/scheduler_test.go index b077df2c508a..80ecce54156e 100644 --- a/solver/scheduler_test.go +++ b/solver/scheduler_test.go @@ -54,10 +54,11 @@ func TestSingleLevelActiveGraph(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.NotNil(t, res) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.Equal(t, *g0.Vertex.(*vertex).cacheCallCount, int64(1)) require.Equal(t, *g0.Vertex.(*vertex).execCallCount, int64(1)) @@ -80,9 +81,10 @@ func TestSingleLevelActiveGraph(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - res, err = j1.Build(ctx, g1) + res, bi, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.Equal(t, *g0.Vertex.(*vertex).cacheCallCount, int64(1)) require.Equal(t, *g0.Vertex.(*vertex).execCallCount, int64(1)) @@ -111,9 +113,10 @@ func TestSingleLevelActiveGraph(t *testing.T) { } g2.Vertex.(*vertex).setupCallCounters() - res, err = j2.Build(ctx, g2) + res, bi, err = j2.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.Equal(t, *g0.Vertex.(*vertex).cacheCallCount, int64(1)) require.Equal(t, *g0.Vertex.(*vertex).execCallCount, int64(1)) @@ -146,9 +149,10 @@ func TestSingleLevelActiveGraph(t *testing.T) { } g3.Vertex.(*vertex).setupCallCounters() - res, err = j3.Build(ctx, g3) + res, bi, err = j3.Build(ctx, g3) require.NoError(t, err) require.Equal(t, unwrap(res), "result3") + require.Equal(t, len(bi), 0) require.Equal(t, *g3.Vertex.(*vertex).cacheCallCount, int64(1)) require.Equal(t, *g3.Vertex.(*vertex).execCallCount, int64(1)) @@ -188,16 +192,18 @@ func TestSingleLevelActiveGraph(t *testing.T) { eg, _ := errgroup.WithContext(ctx) eg.Go(func() error { - res, err := j4.Build(ctx, g4) + res, bi, err := j4.Build(ctx, g4) require.NoError(t, err) require.Equal(t, unwrap(res), "result4") + require.Equal(t, len(bi), 0) return err }) eg.Go(func() error { - res, err := j5.Build(ctx, g4) + res, bi, err := j5.Build(ctx, g4) require.NoError(t, err) require.Equal(t, unwrap(res), "result4") + require.Equal(t, len(bi), 0) return err }) @@ -234,9 +240,10 @@ func TestSingleLevelCache(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -260,9 +267,10 @@ func TestSingleLevelCache(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - res, err = j1.Build(ctx, g1) + res, bi, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result1") + require.Equal(t, len(bi), 0) require.Equal(t, *g1.Vertex.(*vertex).cacheCallCount, int64(1)) require.Equal(t, *g1.Vertex.(*vertex).execCallCount, int64(1)) @@ -290,9 +298,10 @@ func TestSingleLevelCache(t *testing.T) { } g2.Vertex.(*vertex).setupCallCounters() - res, err = j2.Build(ctx, g2) + res, bi, err = j2.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.Equal(t, *g0.Vertex.(*vertex).cacheCallCount, int64(1)) require.Equal(t, *g0.Vertex.(*vertex).execCallCount, int64(1)) @@ -358,16 +367,18 @@ func TestSingleLevelCacheParallel(t *testing.T) { eg, _ := errgroup.WithContext(ctx) eg.Go(func() error { - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) return err }) eg.Go(func() error { - res, err := j1.Build(ctx, g1) + res, bi, err := j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) return err }) @@ -451,16 +462,18 @@ func TestMultiLevelCacheParallel(t *testing.T) { eg, _ := errgroup.WithContext(ctx) eg.Go(func() error { - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) return err }) eg.Go(func() error { - res, err := j1.Build(ctx, g1) + res, bi, err := j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) return err }) @@ -503,7 +516,7 @@ func TestSingleCancelCache(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - _, err = j0.Build(ctx, g0) + _, _, err = j0.Build(ctx, g0) require.Error(t, err) require.Equal(t, true, errors.Is(err, context.Canceled)) @@ -546,7 +559,7 @@ func TestSingleCancelExec(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - _, err = j1.Build(ctx, g1) + _, _, err = j1.Build(ctx, g1) require.Error(t, err) require.Equal(t, true, errors.Is(err, context.Canceled)) @@ -599,7 +612,7 @@ func TestSingleCancelParallel(t *testing.T) { }), } - _, err = j.Build(ctx, g) + _, _, err = j.Build(ctx, g) close(firstErrored) require.Error(t, err) require.Equal(t, true, errors.Is(err, context.Canceled)) @@ -623,9 +636,10 @@ func TestSingleCancelParallel(t *testing.T) { } <-firstReady - res, err := j.Build(ctx, g) + res, bi, err := j.Build(ctx, g) require.NoError(t, err) require.Equal(t, unwrap(res), "result2") + require.Equal(t, len(bi), 0) return err }) @@ -672,9 +686,10 @@ func TestMultiLevelCalculation(t *testing.T) { }), } - res, err := j0.Build(ctx, g) + res, bi, err := j0.Build(ctx, g) require.NoError(t, err) require.Equal(t, unwrapInt(res), 42) // 1 + 2*(7 + 2) + 2 + 2 + 19 + require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -710,9 +725,10 @@ func TestMultiLevelCalculation(t *testing.T) { }, }), } - res, err = j1.Build(ctx, g2) + res, bi, err = j1.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrapInt(res), 42) + require.Equal(t, len(bi), 0) } @@ -745,9 +761,10 @@ func TestHugeGraph(t *testing.T) { // printGraph(g, "") g.Vertex.(*vertexSum).setupCallCounters() - res, err := j0.Build(ctx, g) + res, bi, err := j0.Build(ctx, g) require.NoError(t, err) require.Equal(t, unwrapInt(res), v) + require.Equal(t, len(bi), 0) require.Equal(t, int64(nodes), *g.Vertex.(*vertexSum).cacheCallCount) // execCount := *g.Vertex.(*vertexSum).execCallCount // require.True(t, execCount < 1000) @@ -767,9 +784,10 @@ func TestHugeGraph(t *testing.T) { }() g.Vertex.(*vertexSum).setupCallCounters() - res, err = j1.Build(ctx, g) + res, bi, err = j1.Build(ctx, g) require.NoError(t, err) require.Equal(t, unwrapInt(res), v) + require.Equal(t, len(bi), 0) require.Equal(t, int64(nodes), *g.Vertex.(*vertexSum).cacheCallCount) require.Equal(t, int64(0), *g.Vertex.(*vertexSum).execCallCount) @@ -823,9 +841,10 @@ func TestOptimizedCacheAccess(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.Equal(t, int64(3), *g0.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(3), *g0.Vertex.(*vertex).execCallCount) @@ -869,9 +888,10 @@ func TestOptimizedCacheAccess(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - res, err = j1.Build(ctx, g1) + res, bi, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.Equal(t, int64(3), *g1.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(1), *g1.Vertex.(*vertex).execCallCount) @@ -931,9 +951,10 @@ func TestOptimizedCacheAccess2(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.Equal(t, int64(3), *g0.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(3), *g0.Vertex.(*vertex).execCallCount) @@ -978,9 +999,10 @@ func TestOptimizedCacheAccess2(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - res, err = j1.Build(ctx, g1) + res, bi, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.Equal(t, int64(3), *g1.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(1), *g1.Vertex.(*vertex).execCallCount) @@ -1024,9 +1046,10 @@ func TestOptimizedCacheAccess2(t *testing.T) { } g2.Vertex.(*vertex).setupCallCounters() - res, err = j2.Build(ctx, g2) + res, bi, err = j2.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.Equal(t, int64(3), *g2.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(2), *g2.Vertex.(*vertex).execCallCount) @@ -1074,9 +1097,10 @@ func TestSlowCache(t *testing.T) { }), } - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -1108,9 +1132,10 @@ func TestSlowCache(t *testing.T) { }), } - res, err = j1.Build(ctx, g1) + res, bi, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.NoError(t, j1.Discard()) j1 = nil @@ -1166,9 +1191,10 @@ func TestParallelInputs(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -1220,7 +1246,7 @@ func TestErrorReturns(t *testing.T) { }), } - _, err = j0.Build(ctx, g0) + _, _, err = j0.Build(ctx, g0) require.Error(t, err) require.Contains(t, err.Error(), "error-from-test") @@ -1261,7 +1287,7 @@ func TestErrorReturns(t *testing.T) { }), } - _, err = j1.Build(ctx, g1) + _, _, err = j1.Build(ctx, g1) require.Error(t, err) require.Equal(t, true, errors.Is(err, context.Canceled)) @@ -1302,7 +1328,7 @@ func TestErrorReturns(t *testing.T) { }), } - _, err = j2.Build(ctx, g2) + _, _, err = j2.Build(ctx, g2) require.Error(t, err) require.Contains(t, err.Error(), "exec-error-from-test") @@ -1347,9 +1373,10 @@ func TestMultipleCacheSources(t *testing.T) { }), } - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.Equal(t, int64(0), cacheManager.loadCounter) require.NoError(t, j0.Discard()) @@ -1389,9 +1416,10 @@ func TestMultipleCacheSources(t *testing.T) { }), } - res, err = j1.Build(ctx, g1) + res, bi, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.Equal(t, int64(1), cacheManager.loadCounter) require.Equal(t, int64(0), cacheManager2.loadCounter) @@ -1417,9 +1445,10 @@ func TestMultipleCacheSources(t *testing.T) { }), } - res, err = j1.Build(ctx, g2) + res, bi, err = j1.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrap(res), "result2") + require.Equal(t, len(bi), 0) require.Equal(t, int64(2), cacheManager.loadCounter) require.Equal(t, int64(0), cacheManager2.loadCounter) @@ -1461,9 +1490,10 @@ func TestRepeatBuildWithIgnoreCache(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.Equal(t, int64(2), *g0.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(2), *g0.Vertex.(*vertex).execCallCount) @@ -1499,9 +1529,10 @@ func TestRepeatBuildWithIgnoreCache(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - res, err = j1.Build(ctx, g1) + res, bi, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0-1") + require.Equal(t, len(bi), 0) require.Equal(t, int64(2), *g1.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(2), *g1.Vertex.(*vertex).execCallCount) @@ -1536,9 +1567,10 @@ func TestRepeatBuildWithIgnoreCache(t *testing.T) { } g2.Vertex.(*vertex).setupCallCounters() - res, err = j2.Build(ctx, g2) + res, bi, err = j2.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrap(res), "result0-2") + require.Equal(t, len(bi), 0) require.Equal(t, int64(2), *g2.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(2), *g2.Vertex.(*vertex).execCallCount) @@ -1585,9 +1617,10 @@ func TestIgnoreCacheResumeFromSlowCache(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.Equal(t, int64(2), *g0.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(2), *g0.Vertex.(*vertex).execCallCount) @@ -1625,9 +1658,10 @@ func TestIgnoreCacheResumeFromSlowCache(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - res, err = j1.Build(ctx, g1) + res, bi, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.Equal(t, int64(2), *g1.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(1), *g1.Vertex.(*vertex).execCallCount) @@ -1662,10 +1696,10 @@ func TestParallelBuildsIgnoreCache(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) - require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) // match by vertex digest j1, err := l.NewJob("j1") @@ -1687,10 +1721,10 @@ func TestParallelBuildsIgnoreCache(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - res, err = j1.Build(ctx, g1) + res, bi, err = j1.Build(ctx, g1) require.NoError(t, err) - require.Equal(t, unwrap(res), "result1") + require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -1716,10 +1750,10 @@ func TestParallelBuildsIgnoreCache(t *testing.T) { } g2.Vertex.(*vertex).setupCallCounters() - res, err = j2.Build(ctx, g2) + res, bi, err = j2.Build(ctx, g2) require.NoError(t, err) - require.Equal(t, unwrap(res), "result2") + require.Equal(t, len(bi), 0) // match by cache key j3, err := l.NewJob("j3") @@ -1741,10 +1775,10 @@ func TestParallelBuildsIgnoreCache(t *testing.T) { } g3.Vertex.(*vertex).setupCallCounters() - res, err = j3.Build(ctx, g3) + res, bi, err = j3.Build(ctx, g3) require.NoError(t, err) - require.Equal(t, unwrap(res), "result3") + require.Equal(t, len(bi), 0) // add another ignorecache merges now @@ -1767,10 +1801,10 @@ func TestParallelBuildsIgnoreCache(t *testing.T) { } g4.Vertex.(*vertex).setupCallCounters() - res, err = j4.Build(ctx, g4) + res, bi, err = j4.Build(ctx, g4) require.NoError(t, err) - require.Equal(t, unwrap(res), "result3") + require.Equal(t, len(bi), 0) // add another !ignorecache merges now @@ -1792,10 +1826,10 @@ func TestParallelBuildsIgnoreCache(t *testing.T) { } g5.Vertex.(*vertex).setupCallCounters() - res, err = j5.Build(ctx, g5) + res, bi, err = j5.Build(ctx, g5) require.NoError(t, err) - require.Equal(t, unwrap(res), "result3") + require.Equal(t, len(bi), 0) } func TestSubbuild(t *testing.T) { @@ -1827,9 +1861,10 @@ func TestSubbuild(t *testing.T) { } g0.Vertex.(*vertexSum).setupCallCounters() - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrapInt(res), 8) + require.Equal(t, len(bi), 0) require.Equal(t, int64(2), *g0.Vertex.(*vertexSum).cacheCallCount) require.Equal(t, int64(2), *g0.Vertex.(*vertexSum).execCallCount) @@ -1848,9 +1883,10 @@ func TestSubbuild(t *testing.T) { g0.Vertex.(*vertexSum).setupCallCounters() - res, err = j1.Build(ctx, g0) + res, bi, err = j1.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrapInt(res), 8) + require.Equal(t, len(bi), 0) require.Equal(t, int64(2), *g0.Vertex.(*vertexSum).cacheCallCount) require.Equal(t, int64(0), *g0.Vertex.(*vertexSum).execCallCount) @@ -1900,9 +1936,10 @@ func TestCacheWithSelector(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.Equal(t, int64(2), *g0.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(2), *g0.Vertex.(*vertex).execCallCount) @@ -1941,9 +1978,10 @@ func TestCacheWithSelector(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - res, err = j1.Build(ctx, g1) + res, bi, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.Equal(t, int64(2), *g1.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(0), *g1.Vertex.(*vertex).execCallCount) @@ -1982,9 +2020,10 @@ func TestCacheWithSelector(t *testing.T) { } g2.Vertex.(*vertex).setupCallCounters() - res, err = j2.Build(ctx, g2) + res, bi, err = j2.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrap(res), "result0-1") + require.Equal(t, len(bi), 0) require.Equal(t, int64(2), *g2.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(1), *g2.Vertex.(*vertex).execCallCount) @@ -2037,9 +2076,10 @@ func TestCacheSlowWithSelector(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.Equal(t, int64(2), *g0.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(2), *g0.Vertex.(*vertex).execCallCount) @@ -2081,9 +2121,10 @@ func TestCacheSlowWithSelector(t *testing.T) { } g1.Vertex.(*vertex).setupCallCounters() - res, err = j1.Build(ctx, g1) + res, bi, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.Equal(t, int64(2), *g1.Vertex.(*vertex).cacheCallCount) require.Equal(t, int64(0), *g1.Vertex.(*vertex).execCallCount) @@ -2123,9 +2164,10 @@ func TestCacheExporting(t *testing.T) { }), } - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrapInt(res), 6) + require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -2154,9 +2196,10 @@ func TestCacheExporting(t *testing.T) { } }() - res, err = j1.Build(ctx, g0) + res, bi, err = j1.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrapInt(res), 6) + require.Equal(t, len(bi), 0) require.NoError(t, j1.Discard()) j1 = nil @@ -2211,9 +2254,10 @@ func TestCacheExportingModeMin(t *testing.T) { }), } - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrapInt(res), 11) + require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -2244,9 +2288,10 @@ func TestCacheExportingModeMin(t *testing.T) { } }() - res, err = j1.Build(ctx, g0) + res, bi, err = j1.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrapInt(res), 11) + require.Equal(t, len(bi), 0) require.NoError(t, j1.Discard()) j1 = nil @@ -2278,9 +2323,10 @@ func TestCacheExportingModeMin(t *testing.T) { } }() - res, err = j2.Build(ctx, g0) + res, bi, err = j2.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrapInt(res), 11) + require.Equal(t, len(bi), 0) require.NoError(t, j2.Discard()) j2 = nil @@ -2360,9 +2406,10 @@ func TestSlowCacheAvoidAccess(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.Equal(t, int64(0), cacheManager.loadCounter) require.NoError(t, j0.Discard()) @@ -2379,9 +2426,10 @@ func TestSlowCacheAvoidAccess(t *testing.T) { } }() - res, err = j1.Build(ctx, g0) + res, bi, err = j1.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.NoError(t, j1.Discard()) j1 = nil @@ -2461,9 +2509,10 @@ func TestSlowCacheAvoidLoadOnCache(t *testing.T) { } g0.Vertex.(*vertex).setupCallCounters() - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "resultmain") + require.Equal(t, len(bi), 0) require.Equal(t, int64(0), cacheManager.loadCounter) require.NoError(t, j0.Discard()) @@ -2534,9 +2583,10 @@ func TestSlowCacheAvoidLoadOnCache(t *testing.T) { } }() - res, err = j1.Build(ctx, g0) + res, bi, err = j1.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "resultmain") + require.Equal(t, len(bi), 0) require.NoError(t, j1.Discard()) j1 = nil @@ -2578,9 +2628,10 @@ func TestCacheMultipleMaps(t *testing.T) { value: "result0", }), } - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -2614,9 +2665,10 @@ func TestCacheMultipleMaps(t *testing.T) { }), } - res, err = j1.Build(ctx, g1) + res, bi, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.NoError(t, j1.Discard()) j1 = nil @@ -2649,9 +2701,10 @@ func TestCacheMultipleMaps(t *testing.T) { }), } - res, err = j2.Build(ctx, g2) + res, bi, err = j2.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.NoError(t, j2.Discard()) j2 = nil @@ -2703,9 +2756,10 @@ func TestCacheInputMultipleMaps(t *testing.T) { }}, }), } - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) expTarget := newTestExporterTarget() @@ -2744,9 +2798,10 @@ func TestCacheInputMultipleMaps(t *testing.T) { }}, }), } - res, err = j1.Build(ctx, g1) + res, bi, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) _, err = res.CacheKeys()[0].Exporter.ExportTo(ctx, expTarget, testExporterOpts(true)) require.NoError(t, err) @@ -2800,9 +2855,10 @@ func TestCacheExportingPartialSelector(t *testing.T) { }), } - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -2833,9 +2889,10 @@ func TestCacheExportingPartialSelector(t *testing.T) { g1 := g0 - res, err = j1.Build(ctx, g1) + res, bi, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.NoError(t, j1.Discard()) j1 = nil @@ -2887,9 +2944,10 @@ func TestCacheExportingPartialSelector(t *testing.T) { }), } - res, err = j2.Build(ctx, g2) + res, bi, err = j2.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.NoError(t, j2.Discard()) j2 = nil @@ -2933,9 +2991,10 @@ func TestCacheExportingPartialSelector(t *testing.T) { ), } - res, err = j3.Build(ctx, g3) + res, bi, err = j3.Build(ctx, g3) require.NoError(t, err) require.Equal(t, unwrap(res), "result2") + require.Equal(t, len(bi), 0) require.NoError(t, j3.Discard()) j3 = nil @@ -3031,9 +3090,10 @@ func TestCacheExportingMergedKey(t *testing.T) { }), } - res, err := j0.Build(ctx, g0) + res, bi, err := j0.Build(ctx, g0) require.NoError(t, err) require.Equal(t, unwrap(res), "result0") + require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -3091,9 +3151,10 @@ func TestMergedEdgesLookup(t *testing.T) { } g.Vertex.(*vertexSum).setupCallCounters() - res, err := j0.Build(ctx, g) + res, bi, err := j0.Build(ctx, g) require.NoError(t, err) require.Equal(t, unwrapInt(res), 11) + require.Equal(t, len(bi), 0) require.Equal(t, int64(7), *g.Vertex.(*vertexSum).cacheCallCount) require.Equal(t, int64(0), cacheManager.loadCounter) @@ -3142,12 +3203,13 @@ func TestCacheLoadError(t *testing.T) { } g.Vertex.(*vertexSum).setupCallCounters() - res, err := j0.Build(ctx, g) + res, bi, err := j0.Build(ctx, g) require.NoError(t, err) require.Equal(t, unwrapInt(res), 11) require.Equal(t, int64(7), *g.Vertex.(*vertexSum).cacheCallCount) require.Equal(t, int64(5), *g.Vertex.(*vertexSum).execCallCount) require.Equal(t, int64(0), cacheManager.loadCounter) + require.Equal(t, len(bi), 0) require.NoError(t, j0.Discard()) j0 = nil @@ -3166,12 +3228,13 @@ func TestCacheLoadError(t *testing.T) { g1.Vertex.(*vertexSum).setupCallCounters() - res, err = j1.Build(ctx, g1) + res, bi, err = j1.Build(ctx, g1) require.NoError(t, err) require.Equal(t, unwrapInt(res), 11) require.Equal(t, int64(7), *g.Vertex.(*vertexSum).cacheCallCount) require.Equal(t, int64(0), *g.Vertex.(*vertexSum).execCallCount) require.Equal(t, int64(1), cacheManager.loadCounter) + require.Equal(t, len(bi), 0) require.NoError(t, j1.Discard()) j1 = nil @@ -3192,12 +3255,13 @@ func TestCacheLoadError(t *testing.T) { cacheManager.forceFail = true - res, err = j2.Build(ctx, g2) + res, bi, err = j2.Build(ctx, g2) require.NoError(t, err) require.Equal(t, unwrapInt(res), 11) require.Equal(t, int64(7), *g.Vertex.(*vertexSum).cacheCallCount) require.Equal(t, int64(5), *g.Vertex.(*vertexSum).execCallCount) require.Equal(t, int64(6), cacheManager.loadCounter) + require.Equal(t, len(bi), 0) require.NoError(t, j2.Discard()) j2 = nil @@ -3245,7 +3309,7 @@ func TestInputRequestDeadlock(t *testing.T) { }), } - _, err = j0.Build(ctx, g0) + _, _, err = j0.Build(ctx, g0) require.NoError(t, err) require.NoError(t, j0.Discard()) j0 = nil @@ -3283,7 +3347,7 @@ func TestInputRequestDeadlock(t *testing.T) { }), } - _, err = j1.Build(ctx, g1) + _, _, err = j1.Build(ctx, g1) require.NoError(t, err) require.NoError(t, j1.Discard()) j1 = nil @@ -3324,7 +3388,7 @@ func TestInputRequestDeadlock(t *testing.T) { }), } - _, err = j2.Build(ctx, g2) + _, _, err = j2.Build(ctx, g2) require.NoError(t, err) require.NoError(t, j2.Discard()) j2 = nil @@ -3627,7 +3691,7 @@ func (v *vertexSubBuild) Exec(ctx context.Context, g session.Group, inputs []Res if err := v.exec(ctx, inputs); err != nil { return nil, err } - res, err := v.b.Build(ctx, v.g) + res, _, err := v.b.Build(ctx, v.g) if err != nil { return nil, err } diff --git a/solver/types.go b/solver/types.go index 45f8524680d0..929609e6b4d1 100644 --- a/solver/types.go +++ b/solver/types.go @@ -74,6 +74,7 @@ type ResultProxy interface { Result(context.Context) (CachedResult, error) Release(context.Context) error Definition() *pb.Definition + BuildInfo() BuildInfo } // CacheExportMode is the type for setting cache exporting modes @@ -190,8 +191,15 @@ type CacheMap struct { // such as oci descriptor content providers and progress writers to be passed to // the cache. Opts should not have any impact on the computed cache key. Opts CacheOpts + + // BuildInfo contains build dependencies that will be set from source + // operation. + BuildInfo map[string]string } +// BuildInfo contains solved build dependencies. +type BuildInfo map[string]string + // ExportableCacheKey is a cache key connected with an exporter that can export // a chain of cacherecords pointing to that key type ExportableCacheKey struct { diff --git a/source/containerimage/pull.go b/source/containerimage/pull.go index 5773c8ae4805..902f8f547ea1 100644 --- a/source/containerimage/pull.go +++ b/source/containerimage/pull.go @@ -25,6 +25,7 @@ import ( "github.com/moby/buildkit/solver" "github.com/moby/buildkit/solver/errdefs" "github.com/moby/buildkit/source" + srctypes "github.com/moby/buildkit/source/types" "github.com/moby/buildkit/util/flightcontrol" "github.com/moby/buildkit/util/imageutil" "github.com/moby/buildkit/util/leaseutil" @@ -67,7 +68,7 @@ func NewSource(opt SourceOpt) (*Source, error) { } func (is *Source) ID() string { - return source.DockerImageScheme + return srctypes.DockerImageScheme } func (is *Source) ResolveImageConfig(ctx context.Context, ref string, opt llb.ResolveImageConfigOpt, sm *session.Manager, g session.Group) (digest.Digest, []byte, error) { @@ -172,7 +173,7 @@ func mainManifestKey(ctx context.Context, desc ocispecs.Descriptor, platform oci return digest.FromBytes(dt), nil } -func (p *puller) CacheKey(ctx context.Context, g session.Group, index int) (cacheKey string, cacheOpts solver.CacheOpts, cacheDone bool, err error) { +func (p *puller) CacheKey(ctx context.Context, g session.Group, index int) (cacheKey string, imgDigest string, cacheOpts solver.CacheOpts, cacheDone bool, err error) { p.Puller.Resolver = resolver.DefaultPool.GetResolver(p.RegistryHosts, p.Ref, "pull", p.SessionManager, g).WithImageStore(p.ImageStore, p.id.ResolveMode) // progressFactory needs the outer context, the context in `p.g.Do` will @@ -268,7 +269,7 @@ func (p *puller) CacheKey(ctx context.Context, g session.Group, index int) (cach return nil, nil }) if err != nil { - return "", nil, false, err + return "", "", nil, false, err } cacheOpts = solver.CacheOpts(make(map[interface{}]interface{})) @@ -278,9 +279,9 @@ func (p *puller) CacheKey(ctx context.Context, g session.Group, index int) (cach cacheDone = index > 0 if index == 0 || p.configKey == "" { - return p.manifestKey, cacheOpts, cacheDone, nil + return p.manifestKey, p.manifest.MainManifestDesc.Digest.String(), cacheOpts, cacheDone, nil } - return p.configKey, cacheOpts, cacheDone, nil + return p.configKey, p.manifest.MainManifestDesc.Digest.String(), cacheOpts, cacheDone, nil } func (p *puller) Snapshot(ctx context.Context, g session.Group) (ir cache.ImmutableRef, err error) { diff --git a/source/git/gitsource.go b/source/git/gitsource.go index 6498a8e49057..914793ffc095 100644 --- a/source/git/gitsource.go +++ b/source/git/gitsource.go @@ -26,6 +26,7 @@ import ( "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/source" + srctypes "github.com/moby/buildkit/source/types" "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/progress/logs" "github.com/moby/buildkit/util/urlutil" @@ -64,7 +65,7 @@ func NewSource(opt Opt) (source.Source, error) { } func (gs *gitSource) ID() string { - return source.GitScheme + return srctypes.GitScheme } // needs to be called with repo lock @@ -292,22 +293,22 @@ func (gs *gitSourceHandler) mountKnownHosts(ctx context.Context) (string, func() return knownHosts.Name(), cleanup, nil } -func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index int) (string, solver.CacheOpts, bool, error) { +func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index int) (string, string, solver.CacheOpts, bool, error) { remote := gs.src.Remote gs.locker.Lock(remote) defer gs.locker.Unlock(remote) if ref := gs.src.Ref; ref != "" && isCommitSHA(ref) { - ref = gs.shaToCacheKey(ref) - gs.cacheKey = ref - return ref, nil, true, nil + cacheKey := gs.shaToCacheKey(ref) + gs.cacheKey = cacheKey + return cacheKey, ref, nil, true, nil } gs.getAuthToken(ctx, g) gitDir, unmountGitDir, err := gs.mountRemote(ctx, remote, gs.auth, g) if err != nil { - return "", nil, false, err + return "", "", nil, false, err } defer unmountGitDir() @@ -316,7 +317,7 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index var unmountSock func() error sock, unmountSock, err = gs.mountSSHAuthSock(ctx, gs.src.MountSSHSock, g) if err != nil { - return "", nil, false, err + return "", "", nil, false, err } defer unmountSock() } @@ -326,7 +327,7 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index var unmountKnownHosts func() error knownHosts, unmountKnownHosts, err = gs.mountKnownHosts(ctx) if err != nil { - return "", nil, false, err + return "", "", nil, false, err } defer unmountKnownHosts() } @@ -335,7 +336,7 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index if ref == "" { ref, err = getDefaultBranch(ctx, gitDir, "", sock, knownHosts, gs.auth, gs.src.Remote) if err != nil { - return "", nil, false, err + return "", "", nil, false, err } } @@ -343,28 +344,28 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index buf, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, gs.auth, "ls-remote", "origin", ref) if err != nil { - return "", nil, false, errors.Wrapf(err, "failed to fetch remote %s", urlutil.RedactCredentials(remote)) + return "", "", nil, false, errors.Wrapf(err, "failed to fetch remote %s", urlutil.RedactCredentials(remote)) } out := buf.String() idx := strings.Index(out, "\t") if idx == -1 { - return "", nil, false, errors.Errorf("repository does not contain ref %s, output: %q", ref, string(out)) + return "", "", nil, false, errors.Errorf("repository does not contain ref %s, output: %q", ref, string(out)) } sha := string(out[:idx]) if !isCommitSHA(sha) { - return "", nil, false, errors.Errorf("invalid commit sha %q", sha) + return "", "", nil, false, errors.Errorf("invalid commit sha %q", sha) } - sha = gs.shaToCacheKey(sha) - gs.cacheKey = sha - return sha, nil, true, nil + cacheKey := gs.shaToCacheKey(sha) + gs.cacheKey = cacheKey + return cacheKey, sha, nil, true, nil } func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out cache.ImmutableRef, retErr error) { cacheKey := gs.cacheKey if cacheKey == "" { var err error - cacheKey, _, _, err = gs.CacheKey(ctx, g, 0) + cacheKey, _, _, _, err = gs.CacheKey(ctx, g, 0) if err != nil { return nil, err } diff --git a/source/git/gitsource_test.go b/source/git/gitsource_test.go index 116eac540381..27ea300d220f 100644 --- a/source/git/gitsource_test.go +++ b/source/git/gitsource_test.go @@ -60,7 +60,7 @@ func testRepeatedFetch(t *testing.T, keepGitDir bool) { g, err := gs.Resolve(ctx, id, nil, nil) require.NoError(t, err) - key1, _, done, err := g.CacheKey(ctx, nil, 0) + key1, pin1, _, done, err := g.CacheKey(ctx, nil, 0) require.NoError(t, err) require.True(t, done) @@ -70,6 +70,7 @@ func testRepeatedFetch(t *testing.T, keepGitDir bool) { } require.Equal(t, expLen, len(key1)) + require.Equal(t, 40, len(pin1)) ref1, err := g.Snapshot(ctx, nil) require.NoError(t, err) @@ -102,10 +103,11 @@ func testRepeatedFetch(t *testing.T, keepGitDir bool) { g, err = gs.Resolve(ctx, id, nil, nil) require.NoError(t, err) - key2, _, _, err := g.CacheKey(ctx, nil, 0) + key2, pin2, _, _, err := g.CacheKey(ctx, nil, 0) require.NoError(t, err) require.Equal(t, key1, key2) + require.Equal(t, pin1, pin2) ref2, err := g.Snapshot(ctx, nil) require.NoError(t, err) @@ -118,9 +120,10 @@ func testRepeatedFetch(t *testing.T, keepGitDir bool) { g, err = gs.Resolve(ctx, id, nil, nil) require.NoError(t, err) - key3, _, _, err := g.CacheKey(ctx, nil, 0) + key3, pin3, _, _, err := g.CacheKey(ctx, nil, 0) require.NoError(t, err) require.NotEqual(t, key1, key3) + require.NotEqual(t, pin1, pin3) ref3, err := g.Snapshot(ctx, nil) require.NoError(t, err) @@ -187,7 +190,7 @@ func testFetchBySHA(t *testing.T, keepGitDir bool) { g, err := gs.Resolve(ctx, id, nil, nil) require.NoError(t, err) - key1, _, done, err := g.CacheKey(ctx, nil, 0) + key1, pin1, _, done, err := g.CacheKey(ctx, nil, 0) require.NoError(t, err) require.True(t, done) @@ -197,6 +200,7 @@ func testFetchBySHA(t *testing.T, keepGitDir bool) { } require.Equal(t, expLen, len(key1)) + require.Equal(t, 40, len(pin1)) ref1, err := g.Snapshot(ctx, nil) require.NoError(t, err) @@ -278,15 +282,18 @@ func testMultipleRepos(t *testing.T, keepGitDir bool) { expLen += 4 } - key1, _, _, err := g.CacheKey(ctx, nil, 0) + key1, pin1, _, _, err := g.CacheKey(ctx, nil, 0) require.NoError(t, err) require.Equal(t, expLen, len(key1)) + require.Equal(t, 40, len(pin1)) - key2, _, _, err := g2.CacheKey(ctx, nil, 0) + key2, pin2, _, _, err := g2.CacheKey(ctx, nil, 0) require.NoError(t, err) require.Equal(t, expLen, len(key2)) + require.Equal(t, 40, len(pin2)) require.NotEqual(t, key1, key2) + require.NotEqual(t, pin1, pin2) ref1, err := g.Snapshot(ctx, nil) require.NoError(t, err) @@ -343,7 +350,7 @@ func TestCredentialRedaction(t *testing.T) { g, err := gs.Resolve(ctx, id, nil, nil) require.NoError(t, err) - _, _, _, err = g.CacheKey(ctx, nil, 0) + _, _, _, _, err = g.CacheKey(ctx, nil, 0) require.Error(t, err) require.False(t, strings.Contains(err.Error(), "keepthissecret")) } @@ -390,7 +397,7 @@ func testSubdir(t *testing.T, keepGitDir bool) { g, err := gs.Resolve(ctx, id, nil, nil) require.NoError(t, err) - key1, _, done, err := g.CacheKey(ctx, nil, 0) + key1, pin1, _, done, err := g.CacheKey(ctx, nil, 0) require.NoError(t, err) require.True(t, done) @@ -400,6 +407,7 @@ func testSubdir(t *testing.T, keepGitDir bool) { } require.Equal(t, expLen, len(key1)) + require.Equal(t, 40, len(pin1)) ref1, err := g.Snapshot(ctx, nil) require.NoError(t, err) diff --git a/source/gitidentifier.go b/source/gitidentifier.go index 24c4779370bb..6055c94e2f9b 100644 --- a/source/gitidentifier.go +++ b/source/gitidentifier.go @@ -5,6 +5,7 @@ import ( "path" "strings" + srctypes "github.com/moby/buildkit/source/types" "github.com/moby/buildkit/util/sshutil" ) @@ -53,7 +54,7 @@ func NewGitIdentifier(remoteURL string) (*GitIdentifier, error) { } func (i *GitIdentifier) ID() string { - return "git" + return srctypes.GitScheme } // isGitTransport returns true if the provided str is a git transport by inspecting diff --git a/source/http/httpsource.go b/source/http/httpsource.go index a2d4157ba269..7139af35b6f2 100644 --- a/source/http/httpsource.go +++ b/source/http/httpsource.go @@ -21,6 +21,7 @@ import ( "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/solver" "github.com/moby/buildkit/source" + srctypes "github.com/moby/buildkit/source/types" "github.com/moby/buildkit/util/tracing" "github.com/moby/locker" digest "github.com/opencontainers/go-digest" @@ -52,7 +53,7 @@ func NewSource(opt Opt) (source.Source, error) { } func (hs *httpSource) ID() string { - return source.HTTPSScheme + return srctypes.HTTPSScheme } type httpSourceHandler struct { @@ -118,26 +119,26 @@ func (hs *httpSourceHandler) formatCacheKey(filename string, dgst digest.Digest, return digest.FromBytes(dt) } -func (hs *httpSourceHandler) CacheKey(ctx context.Context, g session.Group, index int) (string, solver.CacheOpts, bool, error) { +func (hs *httpSourceHandler) CacheKey(ctx context.Context, g session.Group, index int) (string, string, solver.CacheOpts, bool, error) { if hs.src.Checksum != "" { hs.cacheKey = hs.src.Checksum - return hs.formatCacheKey(getFileName(hs.src.URL, hs.src.Filename, nil), hs.src.Checksum, "").String(), nil, true, nil + return hs.formatCacheKey(getFileName(hs.src.URL, hs.src.Filename, nil), hs.src.Checksum, "").String(), hs.src.Checksum.String(), nil, true, nil } uh, err := hs.urlHash() if err != nil { - return "", nil, false, nil + return "", "", nil, false, nil } // look up metadata(previously stored headers) for that URL mds, err := searchHTTPURLDigest(ctx, hs.cache, uh) if err != nil { - return "", nil, false, errors.Wrapf(err, "failed to search metadata for %s", uh) + return "", "", nil, false, errors.Wrapf(err, "failed to search metadata for %s", uh) } req, err := http.NewRequest("GET", hs.src.URL, nil) if err != nil { - return "", nil, false, err + return "", "", nil, false, err } req = req.WithContext(ctx) m := map[string]cacheRefMetadata{} @@ -194,7 +195,7 @@ func (hs *httpSourceHandler) CacheKey(ctx context.Context, g session.Group, inde if dgst != "" { modTime := md.getHTTPModTime() resp.Body.Close() - return hs.formatCacheKey(getFileName(hs.src.URL, hs.src.Filename, resp), dgst, modTime).String(), nil, true, nil + return hs.formatCacheKey(getFileName(hs.src.URL, hs.src.Filename, resp), dgst, modTime).String(), dgst.String(), nil, true, nil } } } @@ -205,10 +206,10 @@ func (hs *httpSourceHandler) CacheKey(ctx context.Context, g session.Group, inde resp, err := client.Do(req) if err != nil { - return "", nil, false, err + return "", "", nil, false, err } if resp.StatusCode < 200 || resp.StatusCode >= 400 { - return "", nil, false, errors.Errorf("invalid response status %d", resp.StatusCode) + return "", "", nil, false, errors.Errorf("invalid response status %d", resp.StatusCode) } if resp.StatusCode == http.StatusNotModified { respETag := resp.Header.Get("ETag") @@ -221,27 +222,27 @@ func (hs *httpSourceHandler) CacheKey(ctx context.Context, g session.Group, inde } md, ok := m[respETag] if !ok { - return "", nil, false, errors.Errorf("invalid not-modified ETag: %v", respETag) + return "", "", nil, false, errors.Errorf("invalid not-modified ETag: %v", respETag) } hs.refID = md.ID() dgst := md.getHTTPChecksum() if dgst == "" { - return "", nil, false, errors.Errorf("invalid metadata change") + return "", "", nil, false, errors.Errorf("invalid metadata change") } modTime := md.getHTTPModTime() resp.Body.Close() - return hs.formatCacheKey(getFileName(hs.src.URL, hs.src.Filename, resp), dgst, modTime).String(), nil, true, nil + return hs.formatCacheKey(getFileName(hs.src.URL, hs.src.Filename, resp), dgst, modTime).String(), dgst.String(), nil, true, nil } ref, dgst, err := hs.save(ctx, resp, g) if err != nil { - return "", nil, false, err + return "", "", nil, false, err } ref.Release(context.TODO()) hs.cacheKey = dgst - return hs.formatCacheKey(getFileName(hs.src.URL, hs.src.Filename, resp), dgst, resp.Header.Get("Last-Modified")).String(), nil, true, nil + return hs.formatCacheKey(getFileName(hs.src.URL, hs.src.Filename, resp), dgst, resp.Header.Get("Last-Modified")).String(), dgst.String(), nil, true, nil } func (hs *httpSourceHandler) save(ctx context.Context, resp *http.Response, s session.Group) (ref cache.ImmutableRef, dgst digest.Digest, retErr error) { diff --git a/source/http/httpsource_test.go b/source/http/httpsource_test.go index 898968d34e40..127793042c58 100644 --- a/source/http/httpsource_test.go +++ b/source/http/httpsource_test.go @@ -54,12 +54,14 @@ func TestHTTPSource(t *testing.T) { h, err := hs.Resolve(ctx, id, nil, nil) require.NoError(t, err) - k, _, _, err := h.CacheKey(ctx, nil, 0) + k, p, _, _, err := h.CacheKey(ctx, nil, 0) require.NoError(t, err) expectedContent1 := "sha256:0b1a154faa3003c1fbe7fda9c8a42d55fde2df2a2c405c32038f8ac7ed6b044a" + expectedPin1 := "sha256:d0b425e00e15a0d36b9b361f02bab63563aed6cb4665083905386c55d5b679fa" require.Equal(t, expectedContent1, k) + require.Equal(t, expectedPin1, p) require.Equal(t, server.Stats("/foo").AllRequests, 1) require.Equal(t, server.Stats("/foo").CachedRequests, 0) @@ -83,10 +85,11 @@ func TestHTTPSource(t *testing.T) { h, err = hs.Resolve(ctx, id, nil, nil) require.NoError(t, err) - k, _, _, err = h.CacheKey(ctx, nil, 0) + k, p, _, _, err = h.CacheKey(ctx, nil, 0) require.NoError(t, err) require.Equal(t, expectedContent1, k) + require.Equal(t, expectedPin1, p) require.Equal(t, server.Stats("/foo").AllRequests, 2) require.Equal(t, server.Stats("/foo").CachedRequests, 1) @@ -112,6 +115,7 @@ func TestHTTPSource(t *testing.T) { } expectedContent2 := "sha256:888722f299c02bfae173a747a0345bb2291cf6a076c36d8eb6fab442a8adddfa" + expectedPin2 := "sha256:dab741b6289e7dccc1ed42330cae1accc2b755ce8079c2cd5d4b5366c9f769a6" // update etag, downloads again server.SetRoute("/foo", resp2) @@ -119,10 +123,11 @@ func TestHTTPSource(t *testing.T) { h, err = hs.Resolve(ctx, id, nil, nil) require.NoError(t, err) - k, _, _, err = h.CacheKey(ctx, nil, 0) + k, p, _, _, err = h.CacheKey(ctx, nil, 0) require.NoError(t, err) require.Equal(t, expectedContent2, k) + require.Equal(t, expectedPin2, p) require.Equal(t, server.Stats("/foo").AllRequests, 4) require.Equal(t, server.Stats("/foo").CachedRequests, 1) @@ -172,10 +177,11 @@ func TestHTTPDefaultName(t *testing.T) { h, err := hs.Resolve(ctx, id, nil, nil) require.NoError(t, err) - k, _, _, err := h.CacheKey(ctx, nil, 0) + k, p, _, _, err := h.CacheKey(ctx, nil, 0) require.NoError(t, err) require.Equal(t, "sha256:146f16ec8810a62a57ce314aba391f95f7eaaf41b8b1ebaf2ab65fd63b1ad437", k) + require.Equal(t, "sha256:d0b425e00e15a0d36b9b361f02bab63563aed6cb4665083905386c55d5b679fa", p) require.Equal(t, server.Stats("/").AllRequests, 1) require.Equal(t, server.Stats("/").CachedRequests, 0) @@ -215,7 +221,7 @@ func TestHTTPInvalidURL(t *testing.T) { h, err := hs.Resolve(ctx, id, nil, nil) require.NoError(t, err) - _, _, _, err = h.CacheKey(ctx, nil, 0) + _, _, _, _, err = h.CacheKey(ctx, nil, 0) require.Error(t, err) require.Contains(t, err.Error(), "invalid response") } @@ -249,13 +255,16 @@ func TestHTTPChecksum(t *testing.T) { h, err := hs.Resolve(ctx, id, nil, nil) require.NoError(t, err) - k, _, _, err := h.CacheKey(ctx, nil, 0) + k, p, _, _, err := h.CacheKey(ctx, nil, 0) require.NoError(t, err) expectedContentDifferent := "sha256:f25996f463dca69cffb580f8273ffacdda43332b5f0a8bea2ead33900616d44b" expectedContentCorrect := "sha256:c6a440110a7757b9e1e47b52e413cba96c62377c37a474714b6b3c4f8b74e536" + expectedPinDifferent := "sha256:ab0d5a7aa55c1c95d59c302eb12c55368940e6f0a257646afd455cabe248edc4" + expectedPinCorrect := "sha256:f5fa14774044d2ec428ffe7efbfaa0a439db7bc8127d6b71aea21e1cd558d0f0" require.Equal(t, expectedContentDifferent, k) + require.Equal(t, expectedPinDifferent, p) require.Equal(t, server.Stats("/foo").AllRequests, 0) require.Equal(t, server.Stats("/foo").CachedRequests, 0) @@ -263,6 +272,7 @@ func TestHTTPChecksum(t *testing.T) { require.Error(t, err) require.Equal(t, expectedContentDifferent, k) + require.Equal(t, expectedPinDifferent, p) require.Equal(t, server.Stats("/foo").AllRequests, 1) require.Equal(t, server.Stats("/foo").CachedRequests, 0) @@ -271,10 +281,11 @@ func TestHTTPChecksum(t *testing.T) { h, err = hs.Resolve(ctx, id, nil, nil) require.NoError(t, err) - k, _, _, err = h.CacheKey(ctx, nil, 0) + k, p, _, _, err = h.CacheKey(ctx, nil, 0) require.NoError(t, err) require.Equal(t, expectedContentCorrect, k) + require.Equal(t, expectedPinCorrect, p) require.Equal(t, server.Stats("/foo").AllRequests, 1) require.Equal(t, server.Stats("/foo").CachedRequests, 0) @@ -292,6 +303,7 @@ func TestHTTPChecksum(t *testing.T) { require.Equal(t, dt, []byte("content-correct")) require.Equal(t, expectedContentCorrect, k) + require.Equal(t, expectedPinCorrect, p) require.Equal(t, server.Stats("/foo").AllRequests, 2) require.Equal(t, server.Stats("/foo").CachedRequests, 0) diff --git a/source/identifier.go b/source/identifier.go index 859e693a0fa2..1032399e112d 100644 --- a/source/identifier.go +++ b/source/identifier.go @@ -8,6 +8,7 @@ import ( "github.com/containerd/containerd/reference" "github.com/moby/buildkit/client" "github.com/moby/buildkit/solver/pb" + srctypes "github.com/moby/buildkit/source/types" digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -27,14 +28,6 @@ const ( ResolveModePreferLocal ) -const ( - DockerImageScheme = "docker-image" - GitScheme = "git" - LocalScheme = "local" - HTTPScheme = "http" - HTTPSScheme = "https" -) - type Identifier interface { ID() string // until sources are in process this string comparison could be avoided } @@ -47,15 +40,15 @@ func FromString(s string) (Identifier, error) { } switch parts[0] { - case DockerImageScheme: + case srctypes.DockerImageScheme: return NewImageIdentifier(parts[1]) - case GitScheme: + case srctypes.GitScheme: return NewGitIdentifier(parts[1]) - case LocalScheme: + case srctypes.LocalScheme: return NewLocalIdentifier(parts[1]) - case HTTPSScheme: + case srctypes.HTTPSScheme: return NewHTTPIdentifier(parts[1], true) - case HTTPScheme: + case srctypes.HTTPScheme: return NewHTTPIdentifier(parts[1], false) default: return nil, errors.Wrapf(errNotFound, "unknown schema %s", parts[0]) @@ -212,7 +205,7 @@ func NewImageIdentifier(str string) (*ImageIdentifier, error) { } func (*ImageIdentifier) ID() string { - return DockerImageScheme + return srctypes.DockerImageScheme } type LocalIdentifier struct { @@ -230,7 +223,7 @@ func NewLocalIdentifier(str string) (*LocalIdentifier, error) { } func (*LocalIdentifier) ID() string { - return LocalScheme + return srctypes.LocalScheme } func NewHTTPIdentifier(str string, tls bool) (*HTTPIdentifier, error) { @@ -252,7 +245,7 @@ type HTTPIdentifier struct { } func (*HTTPIdentifier) ID() string { - return HTTPSScheme + return srctypes.HTTPSScheme } func (r ResolveMode) String() string { diff --git a/source/local/local.go b/source/local/local.go index 1eb3ab9a3c28..89f49fa373d1 100644 --- a/source/local/local.go +++ b/source/local/local.go @@ -6,6 +6,7 @@ import ( "fmt" "time" + srctypes "github.com/moby/buildkit/source/types" "github.com/moby/buildkit/util/bklog" "github.com/docker/docker/pkg/idtools" @@ -43,7 +44,7 @@ type localSource struct { } func (ls *localSource) ID() string { - return source.LocalScheme + return srctypes.LocalScheme } func (ls *localSource) Resolve(ctx context.Context, id source.Identifier, sm *session.Manager, _ solver.Vertex) (source.SourceInstance, error) { @@ -65,13 +66,13 @@ type localSourceHandler struct { *localSource } -func (ls *localSourceHandler) CacheKey(ctx context.Context, g session.Group, index int) (string, solver.CacheOpts, bool, error) { +func (ls *localSourceHandler) CacheKey(ctx context.Context, g session.Group, index int) (string, string, solver.CacheOpts, bool, error) { sessionID := ls.src.SessionID if sessionID == "" { id := g.SessionIterator().NextSession() if id == "" { - return "", nil, false, errors.New("could not access local files without session") + return "", "", nil, false, errors.New("could not access local files without session") } sessionID = id } @@ -82,9 +83,9 @@ func (ls *localSourceHandler) CacheKey(ctx context.Context, g session.Group, ind FollowPaths []string }{SessionID: sessionID, IncludePatterns: ls.src.IncludePatterns, ExcludePatterns: ls.src.ExcludePatterns, FollowPaths: ls.src.FollowPaths}) if err != nil { - return "", nil, false, err + return "", "", nil, false, err } - return "session:" + ls.src.Name + ":" + digest.FromBytes(dt).String(), nil, true, nil + return "session:" + ls.src.Name + ":" + digest.FromBytes(dt).String(), digest.FromBytes(dt).String(), nil, true, nil } func (ls *localSourceHandler) Snapshot(ctx context.Context, g session.Group) (cache.ImmutableRef, error) { diff --git a/source/manager.go b/source/manager.go index aba45bffe13e..3f4a0cb4783d 100644 --- a/source/manager.go +++ b/source/manager.go @@ -16,7 +16,7 @@ type Source interface { } type SourceInstance interface { - CacheKey(ctx context.Context, g session.Group, index int) (string, solver.CacheOpts, bool, error) + CacheKey(ctx context.Context, g session.Group, index int) (string, string, solver.CacheOpts, bool, error) Snapshot(ctx context.Context, g session.Group) (cache.ImmutableRef, error) } diff --git a/source/types/types.go b/source/types/types.go new file mode 100644 index 000000000000..b96eac2333e6 --- /dev/null +++ b/source/types/types.go @@ -0,0 +1,9 @@ +package srctypes + +const ( + DockerImageScheme = "docker-image" + GitScheme = "git" + LocalScheme = "local" + HTTPScheme = "http" + HTTPSScheme = "https" +) diff --git a/util/buildinfo/buildinfo.go b/util/buildinfo/buildinfo.go new file mode 100644 index 000000000000..b43f1c0666a0 --- /dev/null +++ b/util/buildinfo/buildinfo.go @@ -0,0 +1,137 @@ +package buildinfo + +import ( + "context" + "encoding/json" + "sort" + + "github.com/docker/distribution/reference" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/moby/buildkit/source" + "github.com/moby/buildkit/util/urlutil" + "github.com/pkg/errors" +) + +const ImageConfigField = "moby.buildkit.buildinfo.v1" + +// Merge combines and fixes build info from image config +// key moby.buildkit.buildinfo.v1. +func Merge(ctx context.Context, buildInfo map[string]string, imageConfig []byte) ([]byte, error) { + icbi, err := imageConfigBuildInfo(imageConfig) + if err != nil { + return nil, err + } + + // Iterate and combine build sources + mbis := map[string]exptypes.BuildInfo{} + for srcs, di := range buildInfo { + src, err := source.FromString(srcs) + if err != nil { + return nil, err + } + switch sid := src.(type) { + case *source.ImageIdentifier: + for idx, bi := range icbi { + // Use original user input from image config + if bi.Type == exptypes.BuildInfoTypeDockerImage && bi.Alias == sid.Reference.String() { + if _, ok := mbis[bi.Alias]; !ok { + parsed, err := reference.ParseNormalizedNamed(bi.Ref) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse %s", bi.Ref) + } + mbis[bi.Alias] = exptypes.BuildInfo{ + Type: exptypes.BuildInfoTypeDockerImage, + Ref: reference.TagNameOnly(parsed).String(), + Pin: di, + } + icbi = append(icbi[:idx], icbi[idx+1:]...) + } + break + } + } + if _, ok := mbis[sid.Reference.String()]; !ok { + mbis[sid.Reference.String()] = exptypes.BuildInfo{ + Type: exptypes.BuildInfoTypeDockerImage, + Ref: sid.Reference.String(), + Pin: di, + } + } + case *source.GitIdentifier: + sref := sid.Remote + if len(sid.Ref) > 0 { + sref += "#" + sid.Ref + } + if len(sid.Subdir) > 0 { + sref += ":" + sid.Subdir + } + if _, ok := mbis[sref]; !ok { + mbis[sref] = exptypes.BuildInfo{ + Type: exptypes.BuildInfoTypeGit, + Ref: urlutil.RedactCredentials(sref), + Pin: di, + } + } + case *source.HTTPIdentifier: + if _, ok := mbis[sid.URL]; !ok { + mbis[sid.URL] = exptypes.BuildInfo{ + Type: exptypes.BuildInfoTypeHTTP, + Ref: urlutil.RedactCredentials(sid.URL), + Pin: di, + } + } + } + } + + // Leftovers build deps in image config. Mostly duplicated ones we + // don't need but there is an edge case if no instruction except source's + // one is defined (eg. FROM ...) that can be valid so take it into account. + for _, bi := range icbi { + if bi.Type != exptypes.BuildInfoTypeDockerImage { + continue + } + if _, ok := mbis[bi.Alias]; !ok { + parsed, err := reference.ParseNormalizedNamed(bi.Ref) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse %s", bi.Ref) + } + mbis[bi.Alias] = exptypes.BuildInfo{ + Type: exptypes.BuildInfoTypeDockerImage, + Ref: reference.TagNameOnly(parsed).String(), + Pin: bi.Pin, + } + } + } + + bis := make([]exptypes.BuildInfo, 0, len(mbis)) + for _, bi := range mbis { + bis = append(bis, bi) + } + sort.Slice(bis, func(i, j int) bool { + return bis[i].Ref < bis[j].Ref + }) + + return json.Marshal(map[string][]exptypes.BuildInfo{ + "sources": bis, + }) +} + +// imageConfigBuildInfo returns build dependencies from image config +func imageConfigBuildInfo(imageConfig []byte) ([]exptypes.BuildInfo, error) { + if len(imageConfig) == 0 { + return nil, nil + } + var config struct { + BuildInfo []byte `json:"moby.buildkit.buildinfo.v1,omitempty"` + } + if err := json.Unmarshal(imageConfig, &config); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal buildinfo from config") + } + if len(config.BuildInfo) == 0 { + return nil, nil + } + var bi []exptypes.BuildInfo + if err := json.Unmarshal(config.BuildInfo, &bi); err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal %s", ImageConfigField) + } + return bi, nil +} diff --git a/util/buildinfo/export.go b/util/buildinfo/export.go new file mode 100644 index 000000000000..485ccf7d82f0 --- /dev/null +++ b/util/buildinfo/export.go @@ -0,0 +1,39 @@ +package buildinfo + +import "github.com/pkg/errors" + +// ExportMode represents the export mode for buildinfo opt. +type ExportMode int + +const ( + // ExportNone doesn't export build dependencies. + ExportNone ExportMode = 0 + // ExportImageConfig exports build dependencies to + // the image config. + ExportImageConfig = 1 << iota + // ExportMetadata exports build dependencies as metadata to + // the exporter response. + ExportMetadata + // ExportAll exports build dependencies as metadata and + // image config. + ExportAll = -1 +) + +// ExportDefault is the default export mode for buildinfo opt. +const ExportDefault = ExportAll + +// ParseExportMode returns the export mode matching a string. +func ParseExportMode(s string) (ExportMode, error) { + switch s { + case "none": + return ExportNone, nil + case "imageconfig": + return ExportImageConfig, nil + case "metadata": + return ExportMetadata, nil + case "all": + return ExportAll, nil + default: + return 0, errors.Errorf("unknown buildinfo export mode: %s", s) + } +} diff --git a/worker/tests/common.go b/worker/tests/common.go index 6160523df58b..67e362283c61 100644 --- a/worker/tests/common.go +++ b/worker/tests/common.go @@ -26,7 +26,7 @@ func NewBusyboxSourceSnapshot(ctx context.Context, t *testing.T, w *base.Worker, require.NoError(t, err) src, err := w.SourceManager.Resolve(ctx, img, sm, nil) require.NoError(t, err) - _, _, _, err = src.CacheKey(ctx, nil, 0) + _, _, _, _, err = src.CacheKey(ctx, nil, 0) require.NoError(t, err) snap, err := src.Snapshot(ctx, nil) require.NoError(t, err)