Add verifyTrace buildImage option

This option can to be set to debug digest mismatch issue. It generates
a trace a build time and the script `image.verifyTrace`. These traces
contains all file attribute written to the tar stream and the chechsum
of all file writter to the tar stream.

This could allow to identify differences because of the sandbox used
at build time while the Nix build sandbox is not used to run time,
when the image is pushed to a destination.

README.md

@@ -366,3 +366,25 @@ in [this branch](https://github.com/nlewo/image/tree/nix).
 
 For more information, refer to [the Go
 documentation](https://pkg.go.dev/github.com/nlewo/nix2container).
+
+## How to debug the "Digest did not match" issue
+
+nix2container generates the digest of layers at build time, in the Nix
+sandbox. At runtime, this digest is announced to the destination and
+when it doesn't exist on this destination, the missing layer is
+created by reading all required store paths (outside of the Nix sandbox).
+
+Theorically, we should not observe any differences when reading store
+paths at build time or at runtime. But, in practice, bugs exist and it
+can be really hard to identify where the differences are.
+
+The `buildImage.verifyTrace` option allows you to easily identify
+these differences. A trace is generated at build time and compared to
+a trace generated at runtime. These traces contains the attributes and
+checksum of all files written to the tar stream.
+
+When this options is set to `true`, it generates the script
+`image.verifyTrace` which has to be run to compare traces.
+
+Note you also need to ensure `buildLayer.trace` is set to `true` to
+all your layers.

cmd/image.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"runtime"
 
@@ -14,13 +15,15 @@ import (
 )
 
 var fromImageFilename string
+var traces []string
+var traceOutput string
 
 var imageCmd = &cobra.Command{
 	Use:   "image OUTPUT-FILENAME CONFIG.JSON LAYERS-1.JSON LAYERS-2.JSON ...",
 	Short: "Generate an image.json file from a image configuration and layers",
 	Args:  cobra.MinimumNArgs(3),
 	Run: func(cmd *cobra.Command, args []string) {
-		err := image(args[0], args[1], fromImageFilename, args[2:])
+		err := image(args[0], args[1], fromImageFilename, args[2:], traces, traceOutput)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "%s", err)
 			os.Exit(1)
@@ -88,7 +91,7 @@ func imageFromManifest(outputFilename, manifestFilename string, blobsFilename st
 	return nil
 }
 
-func image(outputFilename, imageConfigPath string, fromImageFilename string, layerPaths []string) error {
+func image(outputFilename, imageConfigPath string, fromImageFilename string, layerPaths []string, tracePaths []string, traceOutput string) error {
 	var imageConfig v1.ImageConfig
 	var image types.Image
 
@@ -139,12 +142,33 @@ func image(outputFilename, imageConfigPath string, fromImageFilename string, lay
 		return err
 	}
 	logrus.Infof("Image has been written to %s", outputFilename)
+
+	if len(tracePaths) > 0 {
+		destination, err := os.Create(traceOutput)
+		if err != nil {
+			return err
+		}
+		for _, path := range tracePaths {
+			source, err := os.Open(path)
+			if err != nil {
+				return err
+			}
+			defer source.Close()
+			_, err = io.Copy(destination, source)
+			if err != nil {
+				return err
+			}
+		}
+		logrus.Infof("Image trace has been written to %s", traceOutput)
+	}
 	return nil
 }
 
 func init() {
 	rootCmd.AddCommand(imageCmd)
 	imageCmd.Flags().StringVarP(&fromImageFilename, "from-image", "", "", "A JSON file describing the base image")
+	imageCmd.Flags().StringSliceVar(&traces, "traces", traces, "The list of trace files")
+	imageCmd.Flags().StringVar(&traceOutput, "trace-output", "trace", "The path of the trace output")
 	rootCmd.AddCommand(imageFromDirCmd)
 	rootCmd.AddCommand(imageFromManifestCmd)
 }

cmd/layers.go

@@ -24,13 +24,16 @@ var tarDirectory string
 var permsFilepath string
 var rewritesFilepath string
 var maxLayers int
+var traceFilename string
 
 // layerCmd represents the layer command
 var layersReproducibleCmd = &cobra.Command{
 	Use:   "layers-from-reproducible-storepaths OUTPUT-FILENAME.JSON CLOSURE-GRAPH.JSON LAYER-1.JSON LAYER-2.json ...",
 	Short: "Generate a layers.json file from a list of reproducible paths",
 	Args:  cobra.MinimumNArgs(2),
 	Run: func(cmd *cobra.Command, args []string) {
+		var err error
+		var layers []types.Layer
 		closureGraph, err := closure.ReadClosureGraphFile(args[1])
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "%s", err)
@@ -62,7 +65,11 @@ var layersReproducibleCmd = &cobra.Command{
 				os.Exit(1)
 			}
 		}
-		layers, err := nix.NewLayers(storepaths, maxLayers, parents, rewrites, ignore, perms)
+		if traceFilename == "" {
+			layers, err = nix.NewLayers(storepaths, maxLayers, parents, rewrites, ignore, perms)
+		} else {
+			layers, err = nix.NewLayersWithTrace(storepaths, maxLayers, parents, rewrites, ignore, perms, traceFilename)
+		}
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "%s", err)
 			os.Exit(1)
@@ -164,5 +171,6 @@ func init() {
 	layersReproducibleCmd.Flags().StringVarP(&rewritesFilepath, "rewrites", "", "", "A JSON file containing path rewrites")
 	layersReproducibleCmd.Flags().StringVarP(&permsFilepath, "perms", "", "", "A JSON file containing file permissions")
 	layersReproducibleCmd.Flags().IntVarP(&maxLayers, "max-layers", "", 1, "The maximum number of layers")
+	layersReproducibleCmd.Flags().StringVarP(&traceFilename, "trace-filename", "", "", "When set, generates a trace (be careful, it slows down the process)")
 
 }

cmd/trace.go

@@ -0,0 +1,30 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/nlewo/nix2container/nix"
+	"github.com/spf13/cobra"
+)
+
+var traceCmd = &cobra.Command{
+	Use:   "trace IMAGE.JSON",
+	Short: "Generate a trace based on the image.json",
+	Args:  cobra.MinimumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		image, err := nix.NewImageFromFile(args[0])
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s", err)
+			os.Exit(1)
+		}
+
+		for _, l := range image.Layers {
+			nix.TarPathsTrace(l.Paths, os.Stdout)
+		}
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(traceCmd)
+}

default.nix

@@ -75,6 +75,17 @@ let
     skopeo --insecure-policy inspect containers-storage:${image.imageName}:${image.imageTag}
   '';
 
+  verifyTraceScript = image: pkgs.writers.writeBashBin "verify-trace" ''
+       echo "Generating the trace for image ${image}/image.json"
+       ${nix2container-bin}/bin/nix2container trace ${image}/image.json > trace
+       if cmp -s ${image}/trace trace ; then
+         printf  'The build time trace "%s" and run time trace "%s" are identical' "${image}/trace" "${nix2container-bin}/bin/nix2container trace ${image}/image.json"
+       else
+         printf 'error: The build time trace "%s" and runtime trace "%s" are different' "${image}/trace" "${nix2container-bin}/bin/nix2container trace ${image}/image.json"
+         printf 'Hints: you need to manually compare these files to know identify different store paths'
+       fi
+   '';
+
   # Pull an image from a registry with Skopeo and translate it to a
   # nix2container image.json file.
   # This mainly comes from nixpkgs/build-support/docker/default.nix.
@@ -247,6 +258,9 @@ let
     maxLayers ? 1,
     # Deprecated: will be removed on v1
     contents ? null,
+    # Whether to generate a trace. Be careful, this slows down the
+    # process. (This only works when reproducible is true.)
+    trace ? false,  
   }: let
     subcommand = if reproducible
               then "layers-from-reproducible-storepaths"
@@ -271,6 +285,7 @@ let
     permsFlag = l.optionalString (perms != []) "--perms ${permsFile}";
     allDeps = deps ++ copyToRootList;
     tarDirectory = l.optionalString (! reproducible) "--tar-directory $out";
+    traceFilename = l.optionalString trace "--trace-filename $out/trace";
     layersJSON = pkgs.runCommand "layers.json" {} ''
       mkdir $out
       ${nix2container-bin}/bin/nix2container ${subcommand} \
@@ -280,6 +295,7 @@ let
         ${rewritesFlag} \
         ${permsFlag} \
         ${tarDirectory} \
+        ${traceFilename} \
         ${l.concatMapStringsSep " "  (l: l + "/layers.json") layers} \
       '';
   in checked { inherit copyToRoot contents; } layersJSON;
@@ -394,6 +410,11 @@ let
     # Deprecated: will be removed
     contents ? null,
     meta ? {},
+    # Whether to verify the buildtime trace and runtime trace are
+    # identical. This is only a debugging option which slows down the
+    # image construction process.
+    # This is especially useful to debug the famous "digest mismatch" issue.
+    verifyTrace ? false
   }:
     let
       configFile = pkgs.writeText "config.json" (l.toJSON config);
@@ -438,9 +459,14 @@ let
         deps = [configFile];
         ignore = configFile;
         layers = layers;
+        trace = verifyTrace;
       };
       fromImageFlag = l.optionalString (fromImage != "") "--from-image ${fromImage}";
       layerPaths = l.concatMapStringsSep " " (l: l + "/layers.json") (layers ++ [customizationLayer]);
+      tracePaths = l.optionalString verifyTrace (l.concatMapStringsSep " " (l: "--traces " + l + "/trace") (layers ++ [customizationLayer]));
+      traceOutput = l.optionalString verifyTrace "--trace-output $out/trace";
+      traceCheck = l.optionalString verifyTrace ''
+      '';
       image = let
         imageName = l.toLower name;
         imageTag =
@@ -461,15 +487,18 @@ let
           copyToRegistry = copyToRegistry image;
           copyToPodman = copyToPodman image;
           copyTo = copyTo image;
+          verifyTrace = verifyTraceScript image;
         };
       }
-      ''
+        ''
         mkdir $out
         ${nix2container-bin}/bin/nix2container image \
         $out/image.json \
         ${fromImageFlag} \
         ${configFile} \
-        ${layerPaths}
+        ${layerPaths} \
+        ${tracePaths} \
+        ${traceOutput}
       '';
     in checked { inherit copyToRoot contents; } image;
 

examples/basic.nix

@@ -1,6 +1,7 @@
 { pkgs, nix2container }:
 nix2container.buildImage {
   name = "basic";
+  verifyTrace = true;
   config = {
     entrypoint = ["${pkgs.hello}/bin/hello"];
   };

nix/layers.go

@@ -3,6 +3,7 @@ package nix
 import (
 	_ "crypto/sha256"
 	_ "crypto/sha512"
+	"os"
 	"reflect"
 
 	"github.com/nlewo/nix2container/types"
@@ -64,7 +65,7 @@ func getPaths(storePaths []string, parents []types.Layer, rewrites []types.Rewri
 // If tarDirectory is not an empty string, the tar layer is written to
 // the disk. This is useful for layer containing non reproducible
 // store paths.
-func newLayers(paths types.Paths, tarDirectory string, maxLayers int) (layers []types.Layer, err error) {
+func newLayers(paths types.Paths, tarDirectory string, traceFilename string, maxLayers int) (layers []types.Layer, err error) {
 	offset := 0
 	for offset < len(paths) {
 		max := offset + 1
@@ -80,6 +81,14 @@ func newLayers(paths types.Paths, tarDirectory string, maxLayers int) (layers []
 		} else {
 			layerPath, digest, size, err = TarPathsWrite(paths, tarDirectory)
 		}
+		if traceFilename != "" {
+			file, err := os.Create(traceFilename)
+			if err != nil {
+				return layers, err
+			}
+			defer file.Close()
+			TarPathsTrace(layerPaths, file)
+		}
 		if err != nil {
 			return layers, err
 		}
@@ -104,14 +113,19 @@ func newLayers(paths types.Paths, tarDirectory string, maxLayers int) (layers []
 	return layers, nil
 }
 
+func NewLayersWithTrace(storePaths []string, maxLayers int, parents []types.Layer, rewrites []types.RewritePath, exclude string, perms []types.PermPath, traceFilename string) ([]types.Layer, error) {
+	paths := getPaths(storePaths, parents, rewrites, exclude, perms)
+	return newLayers(paths, "", traceFilename, maxLayers)
+}
+
 func NewLayers(storePaths []string, maxLayers int, parents []types.Layer, rewrites []types.RewritePath, exclude string, perms []types.PermPath) ([]types.Layer, error) {
 	paths := getPaths(storePaths, parents, rewrites, exclude, perms)
-	return newLayers(paths, "", maxLayers)
+	return newLayers(paths, "", "", maxLayers)
 }
 
 func NewLayersNonReproducible(storePaths []string, maxLayers int, tarDirectory string, parents []types.Layer, rewrites []types.RewritePath, exclude string, perms []types.PermPath) (layers []types.Layer, err error) {
 	paths := getPaths(storePaths, parents, rewrites, exclude, perms)
-	return newLayers(paths, tarDirectory, maxLayers)
+	return newLayers(paths, tarDirectory, "", maxLayers)
 }
 
 func isPathInLayers(layers []types.Layer, path types.Path) bool {