Generate and verify trace to debug the "digest did not match" issue

README.md

@@ -366,3 +366,25 @@ in [this branch](https://github.com/nlewo/image/tree/nix).
 
 For more information, refer to [the Go
 documentation](https://pkg.go.dev/github.com/nlewo/nix2container).
+
+## How to debug the "Digest did not match" issue
+
+nix2container generates the digest of layers at build time, in the Nix
+sandbox. At runtime, this digest is announced to the destination and
+when it doesn't exist on this destination, the missing layer is
+created by reading all required store paths (outside of the Nix sandbox).
+
+Theorically, we should not observe any differences when reading store
+paths at build time or at runtime. But, in practice, bugs exist and it
+can be really hard to identify where the differences are.
+
+The `buildImage.verifyTrace` option allows you to easily identify
+these differences. A trace is generated at build time and compared to
+a trace generated at runtime. These traces contains the attributes and
+checksum of all files written to the tar stream.
+
+When this options is set to `true`, it generates the script
+`image.verifyTrace` which has to be run to compare traces.
+
+Note you also need to ensure `buildLayer.trace` is set to `true` to
+all your layers.

cmd/image.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"runtime"
 
@@ -14,13 +15,15 @@ import (
 )
 
 var fromImageFilename string
+var traces []string
+var traceOutput string
 
 var imageCmd = &cobra.Command{
 	Use:   "image OUTPUT-FILENAME CONFIG.JSON LAYERS-1.JSON LAYERS-2.JSON ...",
 	Short: "Generate an image.json file from a image configuration and layers",
 	Args:  cobra.MinimumNArgs(3),
 	Run: func(cmd *cobra.Command, args []string) {
-		err := image(args[0], args[1], fromImageFilename, args[2:])
+		err := image(args[0], args[1], fromImageFilename, args[2:], traces, traceOutput)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "%s", err)
 			os.Exit(1)
@@ -88,7 +91,7 @@ func imageFromManifest(outputFilename, manifestFilename string, blobsFilename st
 	return nil
 }
 
-func image(outputFilename, imageConfigPath string, fromImageFilename string, layerPaths []string) error {
+func image(outputFilename, imageConfigPath string, fromImageFilename string, layerPaths []string, tracePaths []string, traceOutput string) error {
 	var imageConfig v1.ImageConfig
 	var image types.Image
 
@@ -139,12 +142,33 @@ func image(outputFilename, imageConfigPath string, fromImageFilename string, lay
 		return err
 	}
 	logrus.Infof("Image has been written to %s", outputFilename)
+
+	if len(tracePaths) > 0 {
+		destination, err := os.Create(traceOutput)
+		if err != nil {
+			return err
+		}
+		for _, path := range tracePaths {
+			source, err := os.Open(path)
+			if err != nil {
+				return err
+			}
+			defer source.Close()
+			_, err = io.Copy(destination, source)
+			if err != nil {
+				return err
+			}
+		}
+		logrus.Infof("Image trace has been written to %s", traceOutput)
+	}
 	return nil
 }
 
 func init() {
 	rootCmd.AddCommand(imageCmd)
 	imageCmd.Flags().StringVarP(&fromImageFilename, "from-image", "", "", "A JSON file describing the base image")
+	imageCmd.Flags().StringSliceVar(&traces, "traces", traces, "The list of trace files")
+	imageCmd.Flags().StringVar(&traceOutput, "trace-output", "trace", "The path of the trace output")
 	rootCmd.AddCommand(imageFromDirCmd)
 	rootCmd.AddCommand(imageFromManifestCmd)
 }

cmd/layers.go

@@ -24,13 +24,16 @@ var tarDirectory string
 var permsFilepath string
 var rewritesFilepath string
 var maxLayers int
+var traceFilename string
 
 // layerCmd represents the layer command
 var layersReproducibleCmd = &cobra.Command{
-	Use:   "layers-from-reproducible-storepaths OUTPUT-FILENAME.JSON CLOSURE-GRAPH.JSON",
+	Use:   "layers-from-reproducible-storepaths OUTPUT-FILENAME.JSON CLOSURE-GRAPH.JSON LAYER-1.JSON LAYER-2.json ...",
 	Short: "Generate a layers.json file from a list of reproducible paths",
 	Args:  cobra.MinimumNArgs(2),
 	Run: func(cmd *cobra.Command, args []string) {
+		var err error
+		var layers []types.Layer
 		closureGraph, err := closure.ReadClosureGraphFile(args[1])
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "%s", err)
@@ -62,7 +65,11 @@ var layersReproducibleCmd = &cobra.Command{
 				os.Exit(1)
 			}
 		}
-		layers, err := nix.NewLayers(storepaths, maxLayers, parents, rewrites, ignore, perms)
+		if traceFilename == "" {
+			layers, err = nix.NewLayers(storepaths, maxLayers, parents, rewrites, ignore, perms)
+		} else {
+			layers, err = nix.NewLayersWithTrace(storepaths, maxLayers, parents, rewrites, ignore, perms, traceFilename)
+		}
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "%s", err)
 			os.Exit(1)
@@ -77,7 +84,7 @@ var layersReproducibleCmd = &cobra.Command{
 
 // layerCmd represents the layer command
 var layersNonReproducibleCmd = &cobra.Command{
-	Use:   "layers-from-non-reproducible-storepaths OUTPUT-FILENAME.JSON CLOSURE-GRAPH.JSON",
+	Use:   "layers-from-non-reproducible-storepaths OUTPUT-FILENAME.JSON CLOSURE-GRAPH.JSON LAYER-1.JSON LAYER-2.json ...",
 	Short: "Generate a layers.json file from a list of paths",
 	Args:  cobra.MinimumNArgs(2),
 	Run: func(cmd *cobra.Command, args []string) {
@@ -164,5 +171,6 @@ func init() {
 	layersReproducibleCmd.Flags().StringVarP(&rewritesFilepath, "rewrites", "", "", "A JSON file containing path rewrites")
 	layersReproducibleCmd.Flags().StringVarP(&permsFilepath, "perms", "", "", "A JSON file containing file permissions")
 	layersReproducibleCmd.Flags().IntVarP(&maxLayers, "max-layers", "", 1, "The maximum number of layers")
+	layersReproducibleCmd.Flags().StringVarP(&traceFilename, "trace-filename", "", "", "When set, generates a trace (be careful, it slows down the process)")
 
 }

cmd/trace.go

@@ -0,0 +1,30 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/nlewo/nix2container/nix"
+	"github.com/spf13/cobra"
+)
+
+var traceCmd = &cobra.Command{
+	Use:   "trace IMAGE.JSON",
+	Short: "Generate a trace based on the image.json",
+	Args:  cobra.MinimumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		image, err := nix.NewImageFromFile(args[0])
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s", err)
+			os.Exit(1)
+		}
+
+		for _, l := range image.Layers {
+			nix.TarPathsTrace(l.Paths, os.Stdout)
+		}
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(traceCmd)
+}

default.nix

@@ -56,25 +56,36 @@ let
 
   copyToDockerDaemon = image: writeSkopeoApplication "copy-to-docker-daemon" ''
     echo "Copy to Docker daemon image ${image.imageName}:${image.imageTag}"
-    skopeo --insecure-policy copy nix:${image} docker-daemon:${image.imageName}:${image.imageTag} $@
+    skopeo --insecure-policy copy nix:${image}/image.json docker-daemon:${image.imageName}:${image.imageTag} $@
   '';
 
   copyToRegistry = image: writeSkopeoApplication "copy-to-registry" ''
     echo "Copy to Docker registry image ${image.imageName}:${image.imageTag}"
-    skopeo --insecure-policy copy nix:${image} docker://${image.imageName}:${image.imageTag} $@
+    skopeo --insecure-policy copy nix:${image}/image.json docker://${image.imageName}:${image.imageTag} $@
   '';
 
   copyTo = image: writeSkopeoApplication "copy-to" ''
     echo Running skopeo --insecure-policy copy nix:${image} $@
-    skopeo --insecure-policy copy nix:${image} $@
+    skopeo --insecure-policy copy nix:${image}/image.json $@
   '';
 
   copyToPodman = image: writeSkopeoApplication "copy-to-podman" ''
     echo "Copy to podman image ${image.imageName}:${image.imageTag}"
-    skopeo --insecure-policy copy nix:${image} containers-storage:${image.imageName}:${image.imageTag}
+    skopeo --insecure-policy copy nix:${image}/image.json containers-storage:${image.imageName}:${image.imageTag}
     skopeo --insecure-policy inspect containers-storage:${image.imageName}:${image.imageTag}
   '';
 
+  verifyTraceScript = image: pkgs.writers.writeBashBin "verify-trace" ''
+       echo "Generating the trace for image ${image}/image.json"
+       ${nix2container-bin}/bin/nix2container trace ${image}/image.json > trace
+       if cmp -s ${image}/trace trace ; then
+         printf  'The build time trace "%s" and run time trace "%s" are identical' "${image}/trace" "${nix2container-bin}/bin/nix2container trace ${image}/image.json"
+       else
+         printf 'error: The build time trace "%s" and runtime trace "%s" are different' "${image}/trace" "${nix2container-bin}/bin/nix2container trace ${image}/image.json"
+         printf 'Hints: you need to manually compare these files to know identify different store paths'
+       fi
+   '';
+
   # Pull an image from a registry with Skopeo and translate it to a
   # nix2container image.json file.
   # This mainly comes from nixpkgs/build-support/docker/default.nix.
@@ -247,6 +258,9 @@ let
     maxLayers ? 1,
     # Deprecated: will be removed on v1
     contents ? null,
+    # Whether to generate a trace. Be careful, this slows down the
+    # process. (This only works when reproducible is true.)
+    trace ? false,  
   }: let
     subcommand = if reproducible
               then "layers-from-reproducible-storepaths"
@@ -271,6 +285,7 @@ let
     permsFlag = l.optionalString (perms != []) "--perms ${permsFile}";
     allDeps = deps ++ copyToRootList;
     tarDirectory = l.optionalString (! reproducible) "--tar-directory $out";
+    traceFilename = l.optionalString trace "--trace-filename $out/trace";
     layersJSON = pkgs.runCommand "layers.json" {} ''
       mkdir $out
       ${nix2container-bin}/bin/nix2container ${subcommand} \
@@ -280,6 +295,7 @@ let
         ${rewritesFlag} \
         ${permsFlag} \
         ${tarDirectory} \
+        ${traceFilename} \
         ${l.concatMapStringsSep " "  (l: l + "/layers.json") layers} \
       '';
   in checked { inherit copyToRoot contents; } layersJSON;
@@ -394,6 +410,11 @@ let
     # Deprecated: will be removed
     contents ? null,
     meta ? {},
+    # Whether to verify the buildtime trace and runtime trace are
+    # identical. This is only a debugging option which slows down the
+    # image construction process.
+    # This is especially useful to debug the famous "digest mismatch" issue.
+    verifyTrace ? false
   }:
     let
       configFile = pkgs.writeText "config.json" (l.toJSON config);
@@ -438,17 +459,22 @@ let
         deps = [configFile];
         ignore = configFile;
         layers = layers;
+        trace = verifyTrace;
       };
       fromImageFlag = l.optionalString (fromImage != "") "--from-image ${fromImage}";
       layerPaths = l.concatMapStringsSep " " (l: l + "/layers.json") (layers ++ [customizationLayer]);
+      tracePaths = l.optionalString verifyTrace (l.concatMapStringsSep " " (l: "--traces " + l + "/trace") (layers ++ [customizationLayer]));
+      traceOutput = l.optionalString verifyTrace "--trace-output $out/trace";
+      traceCheck = l.optionalString verifyTrace ''
+      '';
       image = let
         imageName = l.toLower name;
         imageTag =
           if tag != null
           then tag
           else
           l.head (l.strings.splitString "-" (baseNameOf image.outPath));
-      in pkgs.runCommand "image-${baseNameOf name}.json"
+      in pkgs.runCommand "image-${baseNameOf name}"
       {
         inherit imageName meta;
         passthru = {
@@ -461,14 +487,18 @@ let
           copyToRegistry = copyToRegistry image;
           copyToPodman = copyToPodman image;
           copyTo = copyTo image;
+          verifyTrace = verifyTraceScript image;
         };
       }
-      ''
+        ''
+        mkdir $out
         ${nix2container-bin}/bin/nix2container image \
-        $out \
+        $out/image.json \
         ${fromImageFlag} \
         ${configFile} \
-        ${layerPaths}
+        ${layerPaths} \
+        ${tracePaths} \
+        ${traceOutput}
       '';
     in checked { inherit copyToRoot contents; } image;
 

examples/basic.nix

@@ -1,6 +1,7 @@
 { pkgs, nix2container }:
 nix2container.buildImage {
   name = "basic";
+  verifyTrace = true;
   config = {
     entrypoint = ["${pkgs.hello}/bin/hello"];
   };

flake.nix

@@ -26,5 +26,12 @@
             inherit examples tests;
           };
           defaultPackage = packages.nix2container-bin;
+          devShells.default = let
+            pkgs = nixpkgs.legacyPackages.x86_64-linux;
+          in pkgs.mkShell {
+            buildInputs = [
+              pkgs.go pkgs.godef pkgs.gopls
+            ];
+          };
         });
 }

nix/layers.go

@@ -3,6 +3,7 @@
 import (
 	_ "crypto/sha256"
 	_ "crypto/sha512"
+	"os"
 	"reflect"
 
 	"github.com/nlewo/nix2container/types"
@@ -64,7 +65,7 @@
 // If tarDirectory is not an empty string, the tar layer is written to
 // the disk. This is useful for layer containing non reproducible
 // store paths.
-func newLayers(paths types.Paths, tarDirectory string, maxLayers int) (layers []types.Layer, err error) {
+func newLayers(paths types.Paths, tarDirectory string, traceFilename string, maxLayers int) (layers []types.Layer, err error) {
 	offset := 0
 	for offset < len(paths) {
 		max := offset + 1
@@ -80,6 +81,14 @@
 		} else {
 			layerPath, digest, size, err = TarPathsWrite(paths, tarDirectory)
 		}
+		if traceFilename != "" {
+			file, err := os.Create(traceFilename)
+			if err != nil {
+				return layers, err
+			}
+			defer file.Close()
+			TarPathsTrace(layerPaths, file)
+		}
 		if err != nil {
 			return layers, err
 		}
@@ -104,14 +113,19 @@
 	return layers, nil
 }
 
+func NewLayersWithTrace(storePaths []string, maxLayers int, parents []types.Layer, rewrites []types.RewritePath, exclude string, perms []types.PermPath, traceFilename string) ([]types.Layer, error) {
+	paths := getPaths(storePaths, parents, rewrites, exclude, perms)
+	return newLayers(paths, "", traceFilename, maxLayers)
+}
+
 func NewLayers(storePaths []string, maxLayers int, parents []types.Layer, rewrites []types.RewritePath, exclude string, perms []types.PermPath) ([]types.Layer, error) {
 	paths := getPaths(storePaths, parents, rewrites, exclude, perms)
-	return newLayers(paths, "", maxLayers)
+	return newLayers(paths, "", "", maxLayers)
 }
 
 func NewLayersNonReproducible(storePaths []string, maxLayers int, tarDirectory string, parents []types.Layer, rewrites []types.RewritePath, exclude string, perms []types.PermPath) (layers []types.Layer, err error) {
 	paths := getPaths(storePaths, parents, rewrites, exclude, perms)
-	return newLayers(paths, tarDirectory, maxLayers)
+	return newLayers(paths, tarDirectory, "", maxLayers)
 }
 
 func isPathInLayers(layers []types.Layer, path types.Path) bool {