fix: clean up containerd state on installer run/validate

There's a low chance of a failure to clean up the containerd state from
previous installer pull/validate attempts which might render next
attempts unsuccessful as the snapshot already exists.

Sample error:

```
error validating installer image "foo": snapshot "validate": already exists
```

Signed-off-by: Andrey Smirnov <[email protected]>

internal/app/machined/internal/install/install.go

@@ -36,6 +36,8 @@ import (
 //
 //nolint:gocyclo,cyclop
 func RunInstallerContainer(disk, platform, ref string, cfg config.Provider, opts ...Option) error {
+	const containerID = "upgrade"
+
 	options := DefaultInstallOptions()
 
 	for _, opt := range opts {
@@ -97,6 +99,19 @@ func RunInstallerContainer(disk, platform, ref string, cfg config.Provider, opts
 		}
 	}()
 
+	// See if there's previous container/snapshot to clean up
+	var oldcontainer containerd.Container
+
+	if oldcontainer, err = client.LoadContainer(ctx, containerID); err == nil {
+		if err = oldcontainer.Delete(ctx, containerd.WithSnapshotCleanup); err != nil {
+			return fmt.Errorf("error deleting old container instance: %w", err)
+		}
+	}
+
+	if err = client.SnapshotService("").Remove(ctx, containerID); err != nil && !errdefs.IsNotFound(err) {
+		return fmt.Errorf("error cleaning up stale snapshot: %w", err)
+	}
+
 	mounts := []specs.Mount{
 		{Type: "bind", Destination: "/dev", Source: "/dev", Options: []string{"rbind", "rshared", "rw"}},
 		{Type: "bind", Destination: constants.SystemExtensionsPath, Source: constants.SystemExtensionsPath, Options: []string{"rbind", "rshared", "ro"}},
@@ -166,11 +181,11 @@ func RunInstallerContainer(disk, platform, ref string, cfg config.Provider, opts
 
 	containerOpts := []containerd.NewContainerOpts{
 		containerd.WithImage(img),
-		containerd.WithNewSnapshot("upgrade", img),
+		containerd.WithNewSnapshot(containerID, img),
 		containerd.WithNewSpec(specOpts...),
 	}
 
-	container, err := client.NewContainer(ctx, "upgrade", containerOpts...)
+	container, err := client.NewContainer(ctx, containerID, containerOpts...)
 	if err != nil {
 		return err
 	}

internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go

@@ -25,6 +25,7 @@ import (
 	"github.com/AlekSi/pointer"
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/cio"
+	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/namespaces"
 	"github.com/containerd/containerd/oci"
 	criconstants "github.com/containerd/cri/pkg/constants"
@@ -1320,20 +1321,38 @@ func (s *Server) Events(req *machine.EventsRequest, l machine.MachineService_Eve
 	return <-errCh
 }
 
+//nolint:gocyclo
 func pullAndValidateInstallerImage(ctx context.Context, reg config.Registries, ref string) error {
 	// Pull down specified installer image early so we can bail if it doesn't exist in the upstream registry
 	containerdctx := namespaces.WithNamespace(ctx, constants.SystemContainerdNamespace)
 
+	const containerID = "validate"
+
 	client, err := containerd.New(constants.SystemContainerdAddress)
 	if err != nil {
 		return err
 	}
 
-	img, err := image.Pull(containerdctx, reg, client, ref)
+	defer client.Close() //nolint:errcheck
+
+	img, err := image.Pull(containerdctx, reg, client, ref, image.WithSkipIfAlreadyPulled())
 	if err != nil {
 		return err
 	}
 
+	// See if there's previous container/snapshot to clean up
+	var oldcontainer containerd.Container
+
+	if oldcontainer, err = client.LoadContainer(containerdctx, containerID); err == nil {
+		if err = oldcontainer.Delete(containerdctx, containerd.WithSnapshotCleanup); err != nil {
+			return fmt.Errorf("error deleting old container instance: %w", err)
+		}
+	}
+
+	if err = client.SnapshotService("").Remove(containerdctx, containerID); err != nil && !errdefs.IsNotFound(err) {
+		return fmt.Errorf("error cleaning up stale snapshot: %w", err)
+	}
+
 	// Launch the container with a known help command for a simple check to make sure the image is valid
 	args := []string{
 		"/bin/installer",
@@ -1347,11 +1366,11 @@ func pullAndValidateInstallerImage(ctx context.Context, reg config.Registries, r
 
 	containerOpts := []containerd.NewContainerOpts{
 		containerd.WithImage(img),
-		containerd.WithNewSnapshot("validate", img),
+		containerd.WithNewSnapshot(containerID, img),
 		containerd.WithNewSpec(specOpts...),
 	}
 
-	container, err := client.NewContainer(containerdctx, "validate", containerOpts...)
+	container, err := client.NewContainer(containerdctx, containerID, containerOpts...)
 	if err != nil {
 		return err
 	}