Merge pull request #77 from clearpathrobotics/pull-image-by-manifest

Pull image layers from a supplied manifest.

README.md

@@ -49,6 +49,7 @@ $ podman run -it bash
 
 - [`bash`](./examples/bash.nix): Bash in `/bin/`
 - [`fromImage`](./examples/from-image.nix): Alpine as base image
+- [`fromImageManifest`](./examples/from-image-manifest.nix): Alpine as base image, from a stored `manifest.json`.
 - [`nginx`](./examples/nginx.nix)
 - [`nonReproducible`](./examples/non-reproducible.nix): with a non reproducible store path :/
 - [`openbar`](./examples/openbar.nix): set permissions on files (without root nor VM)
@@ -86,7 +87,8 @@ Function arguments are:
     ```
 
 - **`fromImage`** (defaults to `null`): an image that is used as base
-    image of this image.
+    image of this image; use `pullImage` or `pullImageFromManifest` to
+    supply this.
 
 - **`maxLayers`** (defaults to `1`): the maximum number of layers to
     create. This is based on the store path "popularity" as described
@@ -123,6 +125,10 @@ Function arguments are:
 
 ### `nix2container.pullImage`
 
+Pull an image from a container registry by name and tag/digest, storing the
+entirety of the image (manifest and layer tarballs) in a single store path.
+The supplied `sha256` is the narhash of that store path.
+
 Function arguments are:
 
 - **`imageName`** (required): the name of the image to pull.
@@ -138,6 +144,46 @@ Function arguments are:
 - **`tlsVerify`** (defaults to `true`)
 
 
+### `nix2container.pullImageFromManifest`
+
+Pull a base image from a container registry using a supplied manifest file, and the
+hashes contained within it. The advantages of this over the basic `pullImage`:
+
+- Each layer archive is in its own store path, which means each will download just once
+  and naturally deduplicate for multiple base images that share layers.
+- There is no Nix-specific hash, so it's possible update the base image by simply
+  re-fetching the `manifest.json` from the registry; no need to actually pull the whole
+  image just to compute a new narhash for it.
+
+With this function the `manifest.json` acts as a lockfile meant to be stored in
+source control alongside the Nix container definitions. As a convenience, the manifest
+can be fetched/updated using the supplied passthru script, eg:
+
+```
+nix run .#examples.fromImageManifest.fromImage.getManifest > examples/alpine-manifest.json
+```
+
+Function arguments are:
+
+- **`imageName`** (required): the name of the image to pull.
+
+- **`imageManifest`** (required): the manifest file of the image to pull.
+
+- **`imageTag`** (defaults to `latest`)
+
+- **`os`** (defaults to `linux`)
+
+- **`arch`** (defaults to `x86_64`)
+
+- **`tlsVerify`** (defaults to `true`)
+
+- **`registryUrl`** (defaults to `registry.hub.docker.com`)
+
+Note that `imageTag`, `os`, and `arch` do not affect the pulled image; that is
+governed entirely by the supplied `manifest.json` file. These arguments are
+used for the manifest-selection logic in the included `getManifest` script.
+
+
 #### Authentication
 
 If the Nix daemon is used for building, here is how to set up registry

cmd/image.go

@@ -58,6 +58,36 @@ func imageFromDir(outputFilename, directory string) error {
 	return nil
 }
 
+var imageFromManifestCmd = &cobra.Command{
+	Use:   "image-from-manifest OUTPUT-FILENAME MANIFEST.JSON BLOBS.JSON",
+	Short: "Write an image.json file to OUTPUT-FILENAME from a skopeo raw manifest and blobs JSON.",
+	Args:  cobra.MinimumNArgs(3),
+	Run: func(cmd *cobra.Command, args []string) {
+		err := imageFromManifest(args[0], args[1], args[2])
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s", err)
+			os.Exit(1)
+		}
+	},
+}
+
+func imageFromManifest(outputFilename, manifestFilename string, blobsFilename string) error {
+	image, err := nix.NewImageFromManifest(manifestFilename, blobsFilename)
+	if err != nil {
+		return err
+	}
+	res, err := json.MarshalIndent(image, "", "\t")
+	if err != nil {
+		return err
+	}
+	err = os.WriteFile(outputFilename, []byte(res), 0666)
+	if err != nil {
+		return err
+	}
+	logrus.Infof("Image has been written to %s", outputFilename)
+	return nil
+}
+
 func image(outputFilename, imageConfigPath string, fromImageFilename string, layerPaths []string) error {
 	var imageConfig v1.ImageConfig
 	var image types.Image
@@ -116,4 +146,5 @@ func init() {
 	rootCmd.AddCommand(imageCmd)
 	imageCmd.Flags().StringVarP(&fromImageFilename, "from-image", "", "", "A JSON file describing the base image")
 	rootCmd.AddCommand(imageFromDirCmd)
+	rootCmd.AddCommand(imageFromManifestCmd)
 }

default.nix

@@ -129,6 +129,75 @@ let
       ${nix2container-bin}/bin/nix2container image-from-dir $out ${dir}
     '';
 
+  pullImageFromManifest =
+    { imageName
+    , imageManifest ? null
+    # The manifest dictates what is pulled; these three are only used for
+    # the supplied manifest-pulling script.
+    , imageTag ? "latest"
+    , os ? "linux"
+    , arch ? pkgs.go.GOARCH
+    , tlsVerify ? true
+    , registryUrl ? "registry.hub.docker.com"
+    , meta ? {}
+    }: let
+      manifest = l.fromJSON (l.readFile imageManifest);
+
+      buildImageBlob = digest:
+        let
+          blobUrl = "https://${registryUrl}/v2/${imageName}/blobs/${digest}";
+          plainDigest = l.replaceStrings ["sha256:"] [""] digest;
+          insecureFlag = l.strings.optionalString (!tlsVerify) "--insecure";
+        in pkgs.runCommand plainDigest {
+          outputHash = plainDigest;
+          outputHashMode = "flat";
+          outputHashAlgo = "sha256";
+        } ''
+          SSL_CERT_FILE="${pkgs.cacert.out}/etc/ssl/certs/ca-bundle.crt";
+
+          # This initial access is expected to fail as we don't have a token.
+          ${pkgs.curl}/bin/curl ${insecureFlag} "${blobUrl}" --head --silent --write-out '%header{www-authenticate}' --output /dev/null > bearer.txt
+          tokenUrl=$(sed -n 's/Bearer realm="\(.*\)",service="\(.*\)",scope="\(.*\)"/\1?service=\2\&scope=\3/p' bearer.txt)
+
+          echo "Token URL: $tokenUrl"
+          ${pkgs.curl}/bin/curl ${insecureFlag} --fail --silent "$tokenUrl" --output token.json
+          token="$(${pkgs.jq}/bin/jq --raw-output .token token.json)"
+
+          echo "Blob URL: ${blobUrl}"
+          ${pkgs.curl}/bin/curl ${insecureFlag} --fail -H "Authorization: Bearer $token" "${blobUrl}" --location --output $out
+        '';
+
+      # Pull the blobs (archives) for all layers, as well as the one for the image's config JSON.
+      layerBlobs = map (layerManifest: buildImageBlob layerManifest.digest) manifest.layers;
+      configBlob = buildImageBlob manifest.config.digest;
+
+      # Write the blob map out to a JSON file for the GO executable to consume.
+      blobMap = l.listToAttrs(map (drv: { name = drv.name; value = drv; }) (layerBlobs ++ [configBlob]));
+      blobMapFile = pkgs.writeText "${imageName}-blobs.json" (l.toJSON blobMap);
+
+      # Convenience scripts for manifest-updating.
+      filter = ''.manifests[] | select((.platform.os=="${os}") and (.platform.architecture=="${arch}")) | .digest'';
+      getManifest = pkgs.writeShellApplication {
+        name = "get-manifest";
+        runtimeInputs = [ pkgs.jq skopeo-nix2container ];
+        text = ''
+          set -e
+          manifest=$(skopeo inspect docker://${registryUrl}/${imageName}:${imageTag} --raw | jq)
+          if echo "$manifest" | jq -e .manifests >/dev/null; then
+            # Multi-arch image, pick the one that matches the supplied platform details.
+            hash=$(echo "$manifest" | jq -r '${filter}')
+            skopeo inspect "docker://${registryUrl}/${imageName}@$hash" --raw | jq
+          else
+            # Single-arch image, return the initial response.
+            echo "$manifest"
+          fi
+        '';
+      };
+
+    in pkgs.runCommand "nix2container-${imageName}.json" { passthru = { inherit getManifest; }; } ''
+      ${nix2container-bin}/bin/nix2container image-from-manifest $out ${imageManifest} ${blobMapFile}
+    '';
+
   buildLayer = {
     # A list of store paths to include in the layer.
     deps ? [],
@@ -330,7 +399,7 @@ let
       {
         inherit imageName meta;
         passthru = {
-          inherit imageTag;
+          inherit fromImage imageTag;
           # provide a cheap to evaluate image reference for use with external tools like docker
           # DO NOT use as an input to other derivations, as there is no guarantee that the image
           # reference will exist in the store.
@@ -359,5 +428,5 @@ let
 in
 {
   inherit nix2container-bin skopeo-nix2container;
-  nix2container = { inherit buildImage buildLayer pullImage; };
+  nix2container = { inherit buildImage buildLayer pullImage pullImageFromManifest; };
 }

examples/alpine-manifest.json

@@ -0,0 +1,16 @@
+{
+  "schemaVersion": 2,
+  "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+  "config": {
+    "mediaType": "application/vnd.docker.container.image.v1+json",
+    "size": 1472,
+    "digest": "sha256:c1aabb73d2339c5ebaa3681de2e9d9c18d57485045a4e311d9f8004bec208d67"
+  },
+  "layers": [
+    {
+      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+      "size": 3397879,
+      "digest": "sha256:31e352740f534f9ad170f75378a84fe453d6156e40700b882d737a8f4a6988a3"
+    }
+  ]
+}

examples/default.nix

@@ -5,6 +5,8 @@
   basic = pkgs.callPackage ./basic.nix { inherit nix2container; };
   nonReproducible = pkgs.callPackage ./non-reproducible.nix { inherit nix2container; };
   fromImage = pkgs.callPackage ./from-image.nix { inherit nix2container; };
+  fromImageManifest = pkgs.callPackage ./from-image-manifest.nix { inherit nix2container; };
+  getManifest = pkgs.callPackage ./get-manifest.nix { inherit nix2container; };
   uwsgi = pkgs.callPackage ./uwsgi { inherit nix2container; };
   openbar = pkgs.callPackage ./openbar.nix { inherit nix2container; };
   layered = pkgs.callPackage ./layered.nix { inherit nix2container; };

examples/from-image-manifest.nix

@@ -0,0 +1,20 @@
+{ pkgs, nix2container }: let
+  # nix run .#examples.fromImageManifest.fromImage.getManifest > examples/alpine-manifest.json
+  alpine = nix2container.pullImageFromManifest {
+    imageName = "library/alpine";
+    imageManifest = ./alpine-manifest.json;
+
+    # These attributes aren't checked against the manifest; they are only
+    # used to populate the supplied getManifest script.
+    imageTag = "latest";
+    os = "linux";
+    arch = "amd64";
+  };
+in
+nix2container.buildImage {
+  name = "from-image-manifest";
+  fromImage = alpine;
+  config = {
+    entrypoint = [ "${pkgs.coreutils}/bin/ls" "-l" "/etc/alpine-release"];
+  };
+}

examples/get-manifest.nix

@@ -0,0 +1,21 @@
+{ nix2container }:
+{
+  images = {
+    multiArch = nix2container.pullImageFromManifest {
+      imageName = "library/alpine";
+      os = "linux";
+      arch = "amd64";
+    };
+
+    singleArch = nix2container.pullImageFromManifest {
+      imageName = "rancher/systemd-node";
+      imageTag = "v0.0.4";
+    };
+
+    quayio = nix2container.pullImageFromManifest {
+      imageName = "containers/podman";
+      imageTag = "v4.5";
+      registryUrl = "quay.io";
+    };
+  };
+}

nix/image.go

@@ -96,7 +96,7 @@ func getV1Image(image types.Image) (imageV1 v1.Image, err error) {
 	return
 }
 
-// NewImageFromDir creates an Image from a JSON file describing an
+// NewImageFromFile creates an Image from a JSON file describing an
 // image. This file has usually been created by Nix through the
 // nix2container binary.
 func NewImageFromFile(filename string) (image types.Image, err error) {
@@ -177,6 +177,70 @@ func NewImageFromDir(directory string) (image types.Image, err error) {
 	return image, nil
 }
 
+// NewImageFromManifest builds an Image based on a registry manifest
+// and a separate JSON mapping pointing to the locations of the
+// associated blobs (layer archives).
+func NewImageFromManifest(manifestFilename string, blobMapFilename string) (image types.Image, err error) {
+	image.Version = types.ImageVersion
+
+	content, err := os.ReadFile(manifestFilename)
+	if err != nil {
+		return image, err
+	}
+	var v1Manifest v1.Manifest
+	err = json.Unmarshal(content, &v1Manifest)
+	if err != nil {
+		return image, err
+	}
+
+	var blobMap map[string]string
+	content, err = os.ReadFile(blobMapFilename)
+	if err != nil {
+		return image, err
+	}
+	err = json.Unmarshal(content, &blobMap)
+	if err != nil {
+		return image, err
+	}
+
+	var configFilename = blobMap[v1Manifest.Config.Digest.Encoded()]
+	content, err = os.ReadFile(configFilename)
+	if err != nil {
+		return image, err
+	}
+	var v1ImageConfig manifest.Schema2Image
+	err = json.Unmarshal(content, &v1ImageConfig)
+	if err != nil {
+		return image, err
+	}
+
+	for i, l := range v1Manifest.Layers {
+		layerFilename := blobMap[l.Digest.Encoded()]
+		logrus.Infof("Adding tar file '%s' as image layer", layerFilename)
+		layer := types.Layer{
+			LayerPath: layerFilename,
+			Digest:    l.Digest.String(),
+			DiffIDs:   v1ImageConfig.RootFS.DiffIDs[i].String(),
+		}
+		switch l.MediaType {
+		case "application/vnd.docker.image.rootfs.diff.tar":
+			layer.MediaType = v1.MediaTypeImageLayer
+		case "application/vnd.docker.image.rootfs.diff.tar.gzip":
+			layer.MediaType = v1.MediaTypeImageLayerGzip
+		case "application/vnd.oci.image.layer.v1.tar":
+			layer.MediaType = l.MediaType
+		case "application/vnd.oci.image.layer.v1.tar+gzip":
+			layer.MediaType = l.MediaType
+		case "application/vnd.oci.image.layer.v1.tar+zstd":
+			layer.MediaType = l.MediaType
+		default:
+			return image, fmt.Errorf("Unsupported media type: %q", l.MediaType)
+		}
+		image.Layers = append(image.Layers, layer)
+	}
+	return image, nil
+}
+
 type nopCloser struct {
 	io.Reader
 }

tests/default.nix

@@ -54,6 +54,10 @@ let
       image = examples.fromImage;
       pattern = "/etc/alpine-release$";
     };
+    fromImageManifest = testScript {
+      image = examples.fromImageManifest;
+      pattern = "/etc/alpine-release$";
+    };
     layered = testScript {
       image = examples.layered;
       pattern = "Hello, world";
@@ -125,7 +129,6 @@ let
       # The file test2.txt should not have 777 perms
       pattern = "^-r--r--r-- 1 0 0 0 Jan  1  1970 test2.txt";
     };
-
     copyToRoot = testScript {
       image = nix2container.buildImage {
         name = "copy-to-root";
@@ -134,7 +137,18 @@ let
       };
       pattern = "Hello, world!";
     };
-  };
+  } //
+  (pkgs.lib.mapAttrs' (name: drv: {
+    name = "${name}GetManifest";
+    value = pkgs.writeScriptBin "test-script" ''
+      set -e
+      # Don't pipe directly here, as we don't want to swallow a return code.
+      manifest=$(${drv.getManifest}/bin/get-manifest)
+      echo "$manifest" | ${pkgs.jq}/bin/jq -e 'has("layers")' > /dev/null
+      echo "Test Passed"
+    '';
+  }) examples.getManifest.images);
+
   all =
     let scripts =
       pkgs.lib.concatStringsSep