diff --git a/build/localstate.go b/build/localstate.go
index ad8c869b0671..5d902e440efc 100644
--- a/build/localstate.go
+++ b/build/localstate.go
@@ -15,29 +15,29 @@ func saveLocalState(so *client.SolveOpt, target string, opts Options, node build
 	}
 	lp := opts.Inputs.ContextPath
 	dp := opts.Inputs.DockerfilePath
-	if lp != "" || dp != "" {
-		if lp != "" {
-			lp, err = filepath.Abs(lp)
-			if err != nil {
-				return err
-			}
-		}
-		if dp != "" {
-			dp, err = filepath.Abs(dp)
-			if err != nil {
-				return err
-			}
+	if dp != "" && !IsRemoteURL(lp) && lp != "-" && dp != "-" {
+		dp, err = filepath.Abs(dp)
+		if err != nil {
+			return err
 		}
-		l, err := localstate.New(configDir)
+	}
+	if lp != "" && !IsRemoteURL(lp) && lp != "-" {
+		lp, err = filepath.Abs(lp)
 		if err != nil {
 			return err
 		}
-		return l.SaveRef(node.Builder, node.Name, so.Ref, localstate.State{
-			Target:         target,
-			LocalPath:      lp,
-			DockerfilePath: dp,
-			GroupRef:       opts.GroupRef,
-		})
 	}
-	return nil
+	if lp == "" && dp == "" {
+		return nil
+	}
+	l, err := localstate.New(configDir)
+	if err != nil {
+		return err
+	}
+	return l.SaveRef(node.Builder, node.Name, so.Ref, localstate.State{
+		Target:         target,
+		LocalPath:      lp,
+		DockerfilePath: dp,
+		GroupRef:       opts.GroupRef,
+	})
 }
diff --git a/localstate/localstate.go b/localstate/localstate.go
index 54ec9f39efe7..943494c79d0b 100644
--- a/localstate/localstate.go
+++ b/localstate/localstate.go
@@ -21,9 +21,10 @@ const (
 type State struct {
 	// Target is the name of the invoked target (default if empty)
 	Target string
-	// LocalPath is the absolute path to the context
+	// LocalPath is the absolute path to the context or remote context
 	LocalPath string
-	// DockerfilePath is the absolute path to the Dockerfile
+	// DockerfilePath is the absolute path to the Dockerfile or relative if
+	// context is remote
 	DockerfilePath string
 	// GroupRef is the ref of the state group that this ref belongs to
 	GroupRef string `json:",omitempty"`
diff --git a/tests/build.go b/tests/build.go
index cd4af2aec916..737a0124f16b 100644
--- a/tests/build.go
+++ b/tests/build.go
@@ -16,6 +16,7 @@ import (
 	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/continuity/fs/fstest"
 	"github.com/creack/pty"
+	"github.com/docker/buildx/localstate"
 	"github.com/docker/buildx/util/gitutil"
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/frontend/subrequests/lint"
@@ -44,6 +45,9 @@ var buildTests = []func(t *testing.T, sb integration.Sandbox){
 	testBuild,
 	testBuildStdin,
 	testBuildRemote,
+	testBuildLocalState,
+	testBuildLocalStateStdin,
+	testBuildLocalStateRemote,
 	testImageIDOutput,
 	testBuildLocalExport,
 	testBuildRegistryExport,
@@ -122,6 +126,145 @@ COPY foo /foo
 	require.FileExists(t, filepath.Join(dirDest, "foo"))
 }
 
+func testBuildLocalState(t *testing.T, sb integration.Sandbox) {
+	dockerfile := []byte(`
+FROM busybox:latest AS base
+COPY foo /etc/foo
+RUN cp /etc/foo /etc/bar
+
+FROM scratch
+COPY --from=base /etc/bar /bar
+`)
+	dir := tmpdir(
+		t,
+		fstest.CreateFile("build.Dockerfile", dockerfile, 0600),
+		fstest.CreateFile("foo", []byte("foo"), 0600),
+	)
+
+	out, err := buildCmd(sb, withDir(dir), withArgs(
+		"-f", "build.Dockerfile",
+		"--metadata-file", filepath.Join(dir, "md.json"),
+		".",
+	))
+	require.NoError(t, err, out)
+
+	dt, err := os.ReadFile(filepath.Join(dir, "md.json"))
+	require.NoError(t, err)
+
+	type mdT struct {
+		BuildRef string `json:"buildx.build.ref"`
+	}
+	var md mdT
+	err = json.Unmarshal(dt, &md)
+	require.NoError(t, err)
+
+	ls, err := localstate.New(buildxConfig(sb))
+	require.NoError(t, err)
+
+	refParts := strings.Split(md.BuildRef, "/")
+	require.Len(t, refParts, 3)
+
+	ref, err := ls.ReadRef(refParts[0], refParts[1], refParts[2])
+	require.NoError(t, err)
+	require.NotNil(t, ref)
+	require.DirExists(t, ref.LocalPath)
+	require.FileExists(t, ref.DockerfilePath)
+}
+
+func testBuildLocalStateStdin(t *testing.T, sb integration.Sandbox) {
+	dockerfile := []byte(`
+FROM busybox:latest AS base
+COPY foo /etc/foo
+RUN cp /etc/foo /etc/bar
+
+FROM scratch
+COPY --from=base /etc/bar /bar
+`)
+	dir := tmpdir(
+		t,
+		fstest.CreateFile("foo", []byte("foo"), 0600),
+	)
+
+	cmd := buildxCmd(sb, withDir(dir), withArgs("build", "--progress=quiet", "--metadata-file", filepath.Join(dir, "md.json"), "-f-", dir))
+	cmd.Stdin = bytes.NewReader(dockerfile)
+	out, err := cmd.CombinedOutput()
+	require.NoError(t, err, string(out))
+
+	dt, err := os.ReadFile(filepath.Join(dir, "md.json"))
+	require.NoError(t, err)
+
+	type mdT struct {
+		BuildRef string `json:"buildx.build.ref"`
+	}
+	var md mdT
+	err = json.Unmarshal(dt, &md)
+	require.NoError(t, err)
+
+	ls, err := localstate.New(buildxConfig(sb))
+	require.NoError(t, err)
+
+	refParts := strings.Split(md.BuildRef, "/")
+	require.Len(t, refParts, 3)
+
+	ref, err := ls.ReadRef(refParts[0], refParts[1], refParts[2])
+	require.NoError(t, err)
+	require.NotNil(t, ref)
+	require.DirExists(t, ref.LocalPath)
+	require.Equal(t, "-", ref.DockerfilePath)
+}
+
+func testBuildLocalStateRemote(t *testing.T, sb integration.Sandbox) {
+	dockerfile := []byte(`
+FROM busybox:latest
+COPY foo /foo
+`)
+	dir := tmpdir(
+		t,
+		fstest.CreateFile("build.Dockerfile", dockerfile, 0600),
+		fstest.CreateFile("foo", []byte("foo"), 0600),
+	)
+	dirDest := t.TempDir()
+
+	git, err := gitutil.New(gitutil.WithWorkingDir(dir))
+	require.NoError(t, err)
+
+	gitutil.GitInit(git, t)
+	gitutil.GitAdd(git, t, "build.Dockerfile", "foo")
+	gitutil.GitCommit(git, t, "initial commit")
+	addr := gitutil.GitServeHTTP(git, t)
+
+	out, err := buildCmd(sb, withDir(dir), withArgs(
+		"-f", "build.Dockerfile",
+		"--metadata-file", filepath.Join(dirDest, "md.json"),
+		"--output", "type=local,dest="+dirDest,
+		addr,
+	))
+	require.NoError(t, err, out)
+	require.FileExists(t, filepath.Join(dirDest, "foo"))
+
+	dt, err := os.ReadFile(filepath.Join(dirDest, "md.json"))
+	require.NoError(t, err)
+
+	type mdT struct {
+		BuildRef string `json:"buildx.build.ref"`
+	}
+	var md mdT
+	err = json.Unmarshal(dt, &md)
+	require.NoError(t, err)
+
+	ls, err := localstate.New(buildxConfig(sb))
+	require.NoError(t, err)
+
+	refParts := strings.Split(md.BuildRef, "/")
+	require.Len(t, refParts, 3)
+
+	ref, err := ls.ReadRef(refParts[0], refParts[1], refParts[2])
+	require.NoError(t, err)
+	require.NotNil(t, ref)
+	require.Empty(t, addr)
+	require.Equal(t, "build.Dockerfile", ref.DockerfilePath)
+}
+
 func testBuildLocalExport(t *testing.T, sb integration.Sandbox) {
 	dir := createTestProject(t)
 	out, err := buildCmd(sb, withArgs(fmt.Sprintf("--output=type=local,dest=%s/result", dir), dir))
diff --git a/tests/integration.go b/tests/integration.go
index a17b9569e1ec..920d1dce110b 100644
--- a/tests/integration.go
+++ b/tests/integration.go
@@ -56,7 +56,7 @@ func buildxCmd(sb integration.Sandbox, opts ...cmdOpt) *exec.Cmd {
 
 	if builder := sb.Address(); builder != "" {
 		cmd.Env = append(cmd.Env,
-			"BUILDX_CONFIG=/tmp/buildx-"+builder,
+			"BUILDX_CONFIG="+buildxConfig(sb),
 			"BUILDX_BUILDER="+builder,
 		)
 	}
@@ -86,6 +86,13 @@ func dockerCmd(sb integration.Sandbox, opts ...cmdOpt) *exec.Cmd {
 	return cmd
 }
 
+func buildxConfig(sb integration.Sandbox) string {
+	if builder := sb.Address(); builder != "" {
+		return "/tmp/buildx-" + builder
+	}
+	return ""
+}
+
 func isMobyWorker(sb integration.Sandbox) bool {
 	name, hasFeature := driverName(sb.Name())
 	return name == "docker" && !hasFeature