Skip to content

Commit

Permalink
Merge pull request #2607 from tonistiigi/locals-git-refactor
Browse files Browse the repository at this point in the history 
build: refactor setting git info to local mounts
  • Loading branch information
@tonistiigi
tonistiigi authored Jul 24, 2024
2 parents 1651809 + d8f26f7 commit 30dbdcf
Show file tree
Hide file tree
Showing 4 changed files with 107 additions and 74 deletions.
8 changes: 3 additions & 5 deletions build/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opt map[s
for k, opt := range opt {
multiDriver := len(drivers[k]) > 1
hasMobyDriver := false
gitattrs, addVCSLocalDir, err := getGitAttributes(ctx, opt.Inputs.ContextPath, opt.Inputs.DockerfilePath)
addGitAttrs, err := getGitAttributes(ctx, opt.Inputs.ContextPath, opt.Inputs.DockerfilePath)
if err != nil {
logrus.WithError(err).Warn("current commit information was not captured by the build")
}
Expand All @@ -230,16 +230,14 @@ func BuildWithResultHandler(ctx context.Context, nodes []builder.Node, opt map[s
if err != nil {
return nil, err
}
so, release, err := toSolveOpt(ctx, np.Node(), multiDriver, opt, gatewayOpts, configDir, addVCSLocalDir, w, docker)
so, release, err := toSolveOpt(ctx, np.Node(), multiDriver, opt, gatewayOpts, configDir, w, docker)
if err != nil {
return nil, err
}
if err := saveLocalState(so, k, opt, np.Node(), configDir); err != nil {
return nil, err
}
for k, v := range gitattrs {
so.FrontendAttrs[k] = v
}
addGitAttrs(so)
defers = append(defers, release)
reqn = append(reqn, &reqForNode{
resolvedNode: np,
Expand Down
65 changes: 44 additions & 21 deletions build/git.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,13 @@ import (

const DockerfileLabel = "com.docker.image.source.entrypoint"

func getGitAttributes(ctx context.Context, contextPath string, dockerfilePath string) (map[string]string, func(key, dir string, so *client.SolveOpt), error) {
res := make(map[string]string)
type gitAttrsAppendFunc func(so *client.SolveOpt)

func gitAppendNoneFunc(_ *client.SolveOpt) {}

func getGitAttributes(ctx context.Context, contextPath, dockerfilePath string) (gitAttrsAppendFunc, error) {
if contextPath == "" {
return nil, nil, nil
return gitAppendNoneFunc, nil
}

setGitLabels := false
Expand All @@ -39,7 +42,7 @@ func getGitAttributes(ctx context.Context, contextPath string, dockerfilePath st
}

if !setGitLabels && !setGitInfo {
return nil, nil, nil
return gitAppendNoneFunc, nil
}

// figure out in which directory the git command needs to run in
Expand All @@ -54,25 +57,27 @@ func getGitAttributes(ctx context.Context, contextPath string, dockerfilePath st
gitc, err := gitutil.New(gitutil.WithContext(ctx), gitutil.WithWorkingDir(wd))
if err != nil {
if st, err1 := os.Stat(path.Join(wd, ".git")); err1 == nil && st.IsDir() {
return res, nil, errors.Wrap(err, "git was not found in the system")
return nil, errors.Wrap(err, "git was not found in the system")
}
return nil, nil, nil
return gitAppendNoneFunc, nil
}

if !gitc.IsInsideWorkTree() {
if st, err := os.Stat(path.Join(wd, ".git")); err == nil && st.IsDir() {
return res, nil, errors.New("failed to read current commit information with git rev-parse --is-inside-work-tree")
return nil, errors.New("failed to read current commit information with git rev-parse --is-inside-work-tree")
}
return nil, nil, nil
return gitAppendNoneFunc, nil
}

root, err := gitc.RootDir()
if err != nil {
return res, nil, errors.Wrap(err, "failed to get git root dir")
return nil, errors.Wrap(err, "failed to get git root dir")
}

res := make(map[string]string)

if sha, err := gitc.FullCommit(); err != nil && !gitutil.IsUnknownRevision(err) {
return res, nil, errors.Wrap(err, "failed to get git commit")
return nil, errors.Wrap(err, "failed to get git commit")
} else if sha != "" {
checkDirty := false
if v, ok := os.LookupEnv("BUILDX_GIT_CHECK_DIRTY"); ok {
Expand Down Expand Up @@ -112,20 +117,38 @@ func getGitAttributes(ctx context.Context, contextPath string, dockerfilePath st
}
}

return res, func(key, dir string, so *client.SolveOpt) {
if !setGitInfo || root == "" {
return
return func(so *client.SolveOpt) {
if so.FrontendAttrs == nil {
so.FrontendAttrs = make(map[string]string)
}
dir, err := filepath.Abs(dir)
if err != nil {
return
for k, v := range res {
so.FrontendAttrs[k] = v
}
if lp, err := osutil.GetLongPathName(dir); err == nil {
dir = lp

if !setGitInfo || root == "" {
return
}
dir = osutil.SanitizePath(dir)
if r, err := filepath.Rel(root, dir); err == nil && !strings.HasPrefix(r, "..") {
so.FrontendAttrs["vcs:localdir:"+key] = r

for key, mount := range so.LocalMounts {
fs, ok := mount.(*fs)
if !ok {
continue
}
dir, err := filepath.EvalSymlinks(fs.dir) // keep same behavior as fsutil.NewFS
if err != nil {
continue
}
dir, err = filepath.Abs(dir)
if err != nil {
continue
}
if lp, err := osutil.GetLongPathName(dir); err == nil {
dir = lp
}
dir = osutil.SanitizePath(dir)
if r, err := filepath.Rel(root, dir); err == nil && !strings.HasPrefix(r, "..") {
so.FrontendAttrs["vcs:localdir:"+key] = r
}
}
}, nil
}
74 changes: 43 additions & 31 deletions build/git_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,24 +31,26 @@ func setupTest(tb testing.TB) {
}

func TestGetGitAttributesNotGitRepo(t *testing.T) {
_, _, err := getGitAttributes(context.Background(), t.TempDir(), "Dockerfile")
_, err := getGitAttributes(context.Background(), t.TempDir(), "Dockerfile")
assert.NoError(t, err)
}

func TestGetGitAttributesBadGitRepo(t *testing.T) {
tmp := t.TempDir()
require.NoError(t, os.MkdirAll(path.Join(tmp, ".git"), 0755))

_, _, err := getGitAttributes(context.Background(), tmp, "Dockerfile")
_, err := getGitAttributes(context.Background(), tmp, "Dockerfile")
assert.Error(t, err)
}

func TestGetGitAttributesNoContext(t *testing.T) {
setupTest(t)

gitattrs, _, err := getGitAttributes(context.Background(), "", "Dockerfile")
addGitAttrs, err := getGitAttributes(context.Background(), "", "Dockerfile")
assert.NoError(t, err)
assert.Empty(t, gitattrs)
var so client.SolveOpt
addGitAttrs(&so)
assert.Empty(t, so.FrontendAttrs)
}

func TestGetGitAttributes(t *testing.T) {
Expand Down Expand Up @@ -115,15 +117,17 @@ func TestGetGitAttributes(t *testing.T) {
if tt.envGitInfo != "" {
t.Setenv("BUILDX_GIT_INFO", tt.envGitInfo)
}
gitattrs, _, err := getGitAttributes(context.Background(), ".", "Dockerfile")
addGitAttrs, err := getGitAttributes(context.Background(), ".", "Dockerfile")
require.NoError(t, err)
var so client.SolveOpt
addGitAttrs(&so)
for _, e := range tt.expected {
assert.Contains(t, gitattrs, e)
assert.NotEmpty(t, gitattrs[e])
assert.Contains(t, so.FrontendAttrs, e)
assert.NotEmpty(t, so.FrontendAttrs[e])
if e == "label:"+DockerfileLabel {
assert.Equal(t, "Dockerfile", gitattrs[e])
assert.Equal(t, "Dockerfile", so.FrontendAttrs[e])
} else if e == "label:"+specs.AnnotationSource || e == "vcs:source" {
assert.Equal(t, "[email protected]:docker/buildx.git", gitattrs[e])
assert.Equal(t, "[email protected]:docker/buildx.git", so.FrontendAttrs[e])
}
}
})
Expand All @@ -140,20 +144,25 @@ func TestGetGitAttributesDirty(t *testing.T) {
require.NoError(t, os.WriteFile(filepath.Join("dir", "Dockerfile"), df, 0644))

t.Setenv("BUILDX_GIT_LABELS", "true")
gitattrs, _, _ := getGitAttributes(context.Background(), ".", "Dockerfile")
assert.Equal(t, 5, len(gitattrs))

assert.Contains(t, gitattrs, "label:"+DockerfileLabel)
assert.Equal(t, "Dockerfile", gitattrs["label:"+DockerfileLabel])
assert.Contains(t, gitattrs, "label:"+specs.AnnotationSource)
assert.Equal(t, "[email protected]:docker/buildx.git", gitattrs["label:"+specs.AnnotationSource])
assert.Contains(t, gitattrs, "label:"+specs.AnnotationRevision)
assert.True(t, strings.HasSuffix(gitattrs["label:"+specs.AnnotationRevision], "-dirty"))

assert.Contains(t, gitattrs, "vcs:source")
assert.Equal(t, "[email protected]:docker/buildx.git", gitattrs["vcs:source"])
assert.Contains(t, gitattrs, "vcs:revision")
assert.True(t, strings.HasSuffix(gitattrs["vcs:revision"], "-dirty"))
addGitAttrs, err := getGitAttributes(context.Background(), ".", "Dockerfile")
require.NoError(t, err)

var so client.SolveOpt
addGitAttrs(&so)

assert.Equal(t, 5, len(so.FrontendAttrs))

assert.Contains(t, so.FrontendAttrs, "label:"+DockerfileLabel)
assert.Equal(t, "Dockerfile", so.FrontendAttrs["label:"+DockerfileLabel])
assert.Contains(t, so.FrontendAttrs, "label:"+specs.AnnotationSource)
assert.Equal(t, "[email protected]:docker/buildx.git", so.FrontendAttrs["label:"+specs.AnnotationSource])
assert.Contains(t, so.FrontendAttrs, "label:"+specs.AnnotationRevision)
assert.True(t, strings.HasSuffix(so.FrontendAttrs["label:"+specs.AnnotationRevision], "-dirty"))

assert.Contains(t, so.FrontendAttrs, "vcs:source")
assert.Equal(t, "[email protected]:docker/buildx.git", so.FrontendAttrs["vcs:source"])
assert.Contains(t, so.FrontendAttrs, "vcs:revision")
assert.True(t, strings.HasSuffix(so.FrontendAttrs["vcs:revision"], "-dirty"))
}

func TestLocalDirs(t *testing.T) {
Expand All @@ -163,15 +172,17 @@ func TestLocalDirs(t *testing.T) {
FrontendAttrs: map[string]string{},
}

_, addVCSLocalDir, err := getGitAttributes(context.Background(), ".", "Dockerfile")
addGitAttrs, err := getGitAttributes(context.Background(), ".", "Dockerfile")
require.NoError(t, err)
require.NotNil(t, addVCSLocalDir)

require.NoError(t, setLocalMount("context", ".", so, addVCSLocalDir))
require.NoError(t, setLocalMount("context", ".", so))
require.NoError(t, setLocalMount("dockerfile", ".", so))

addGitAttrs(so)

require.Contains(t, so.FrontendAttrs, "vcs:localdir:context")
assert.Equal(t, ".", so.FrontendAttrs["vcs:localdir:context"])

require.NoError(t, setLocalMount("dockerfile", ".", so, addVCSLocalDir))
require.Contains(t, so.FrontendAttrs, "vcs:localdir:dockerfile")
assert.Equal(t, ".", so.FrontendAttrs["vcs:localdir:dockerfile"])
}
Expand All @@ -194,16 +205,17 @@ func TestLocalDirsSub(t *testing.T) {
so := &client.SolveOpt{
FrontendAttrs: map[string]string{},
}
require.NoError(t, setLocalMount("context", ".", so))
require.NoError(t, setLocalMount("dockerfile", "app", so))

_, addVCSLocalDir, err := getGitAttributes(context.Background(), ".", "app/Dockerfile")
addGitAttrs, err := getGitAttributes(context.Background(), ".", "app/Dockerfile")
require.NoError(t, err)
require.NotNil(t, addVCSLocalDir)

require.NoError(t, setLocalMount("context", ".", so, addVCSLocalDir))
addGitAttrs(so)

require.Contains(t, so.FrontendAttrs, "vcs:localdir:context")
assert.Equal(t, ".", so.FrontendAttrs["vcs:localdir:context"])

require.NoError(t, setLocalMount("dockerfile", "app", so, addVCSLocalDir))
require.Contains(t, so.FrontendAttrs, "vcs:localdir:dockerfile")
assert.Equal(t, "app", so.FrontendAttrs["vcs:localdir:dockerfile"])
}
34 changes: 17 additions & 17 deletions build/opt.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import (
"github.com/tonistiigi/fsutil"
)

func toSolveOpt(ctx context.Context, node builder.Node, multiDriver bool, opt Options, bopts gateway.BuildOpts, configDir string, addVCSLocalDir func(key, dir string, so *client.SolveOpt), pw progress.Writer, docker *dockerutil.Client) (_ *client.SolveOpt, release func(), err error) {
func toSolveOpt(ctx context.Context, node builder.Node, multiDriver bool, opt Options, bopts gateway.BuildOpts, configDir string, pw progress.Writer, docker *dockerutil.Client) (_ *client.SolveOpt, release func(), err error) {
nodeDriver := node.Driver
defers := make([]func(), 0, 2)
releaseF := func() {
Expand Down Expand Up @@ -262,7 +262,7 @@ func toSolveOpt(ctx context.Context, node builder.Node, multiDriver bool, opt Op
so.Exports = opt.Exports
so.Session = opt.Session

releaseLoad, err := loadInputs(ctx, nodeDriver, opt.Inputs, addVCSLocalDir, pw, &so)
releaseLoad, err := loadInputs(ctx, nodeDriver, opt.Inputs, pw, &so)
if err != nil {
return nil, nil, err
}
Expand Down Expand Up @@ -355,7 +355,7 @@ func toSolveOpt(ctx context.Context, node builder.Node, multiDriver bool, opt Op
return &so, releaseF, nil
}

func loadInputs(ctx context.Context, d *driver.DriverHandle, inp Inputs, addVCSLocalDir func(key, dir string, so *client.SolveOpt), pw progress.Writer, target *client.SolveOpt) (func(), error) {
func loadInputs(ctx context.Context, d *driver.DriverHandle, inp Inputs, pw progress.Writer, target *client.SolveOpt) (func(), error) {
if inp.ContextPath == "" {
return nil, errors.New("please specify build context (e.g. \".\" for the current directory)")
}
Expand Down Expand Up @@ -401,13 +401,13 @@ func loadInputs(ctx context.Context, d *driver.DriverHandle, inp Inputs, addVCSL
dockerfileReader = buf
inp.ContextPath, _ = os.MkdirTemp("", "empty-dir")
toRemove = append(toRemove, inp.ContextPath)
if err := setLocalMount("context", inp.ContextPath, target, addVCSLocalDir); err != nil {
if err := setLocalMount("context", inp.ContextPath, target); err != nil {
return nil, err
}
}
}
case osutil.IsLocalDir(inp.ContextPath):
if err := setLocalMount("context", inp.ContextPath, target, addVCSLocalDir); err != nil {
if err := setLocalMount("context", inp.ContextPath, target); err != nil {
return nil, err
}
sharedKey := inp.ContextPath
Expand Down Expand Up @@ -466,7 +466,7 @@ func loadInputs(ctx context.Context, d *driver.DriverHandle, inp Inputs, addVCSL
}

if dockerfileDir != "" {
if err := setLocalMount("dockerfile", dockerfileDir, target, addVCSLocalDir); err != nil {
if err := setLocalMount("dockerfile", dockerfileDir, target); err != nil {
return nil, err
}
dockerfileName = handleLowercaseDockerfile(dockerfileDir, dockerfileName)
Expand Down Expand Up @@ -528,7 +528,7 @@ func loadInputs(ctx context.Context, d *driver.DriverHandle, inp Inputs, addVCSL
if k == "context" || k == "dockerfile" {
localName = "_" + k // underscore to avoid collisions
}
if err := setLocalMount(localName, v.Path, target, addVCSLocalDir); err != nil {
if err := setLocalMount(localName, v.Path, target); err != nil {
return nil, err
}
target.FrontendAttrs["context:"+k] = "local:" + localName
Expand Down Expand Up @@ -570,22 +570,15 @@ func resolveDigest(localPath, tag string) (dig string, _ error) {
return dig, nil
}

func setLocalMount(name, root string, so *client.SolveOpt, addVCSLocalDir func(key, dir string, so *client.SolveOpt)) error {
lm, err := fsutil.NewFS(root)
if err != nil {
return err
}
root, err = filepath.EvalSymlinks(root) // keep same behavior as fsutil.NewFS
func setLocalMount(name, dir string, so *client.SolveOpt) error {
lm, err := fsutil.NewFS(dir)
if err != nil {
return err
}
if so.LocalMounts == nil {
so.LocalMounts = map[string]fsutil.FS{}
}
so.LocalMounts[name] = lm
if addVCSLocalDir != nil {
addVCSLocalDir(name, root, so)
}
so.LocalMounts[name] = &fs{FS: lm, dir: dir}
return nil
}

Expand Down Expand Up @@ -635,3 +628,10 @@ func handleLowercaseDockerfile(dir, p string) string {
}
return p
}

type fs struct {
fsutil.FS
dir string
}

var _ fsutil.FS = &fs{}

0 comments on commit 30dbdcf

Please sign in to comment.