diff --git a/README.md b/README.md index 4b98516..f21eb5b 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ $ podman run -it bash - [`bash`](./examples/bash.nix): Bash in `/bin/` - [`fromImage`](./examples/from-image.nix): Alpine as base image +- [`fromImageManifest`](./examples/from-image-manifest.nix): Alpine as base image, from a stored `manifest.json`. - [`nginx`](./examples/nginx.nix) - [`nonReproducible`](./examples/non-reproducible.nix): with a non reproducible store path :/ - [`openbar`](./examples/openbar.nix): set permissions on files (without root nor VM) @@ -86,7 +87,8 @@ Function arguments are: ``` - **`fromImage`** (defaults to `null`): an image that is used as base - image of this image. + image of this image; use `pullImage` or `pullImageFromManifest` to + supply this. - **`maxLayers`** (defaults to `1`): the maximum number of layers to create. This is based on the store path "popularity" as described @@ -123,6 +125,10 @@ Function arguments are: ### `nix2container.pullImage` +Pull an image from a container registry by name and tag/digest, storing the +entirety of the image (manifest and layer tarballs) in a single store path. +The supplied `sha256` is the narhash of that store path. + Function arguments are: - **`imageName`** (required): the name of the image to pull. @@ -138,6 +144,46 @@ Function arguments are: - **`tlsVerify`** (defaults to `true`) +### `nix2container.pullImageFromManifest` + +Pull a base image from a container registry using a supplied manifest file, and the +hashes contained within it. The advantages of this over the basic `pullImage`: + +- Each layer archive is in its own store path, which means each will download just once + and naturally deduplicate for multiple base images that share layers. +- There is no Nix-specific hash, so it's possible update the base image by simply + re-fetching the `manifest.json` from the registry; no need to actually pull the whole + image just to compute a new narhash for it. + +With this function the `manifest.json` acts as a lockfile meant to be stored in +source control alongside the Nix container definitions. As a convenience, the manifest +can be fetched/updated using the supplied passthru script, eg: + +``` +nix run .#examples.fromImageManifest.fromImage.getManifest > examples/alpine-manifest.json +``` + +Function arguments are: + +- **`imageName`** (required): the name of the image to pull. + +- **`imageManifest`** (required): the manifest file of the image to pull. + +- **`imageTag`** (defaults to `latest`) + +- **`os`** (defaults to `linux`) + +- **`arch`** (defaults to `x86_64`) + +- **`tlsVerify`** (defaults to `true`) + +- **`registryUrl`** (defaults to `registry.hub.docker.com`) + +Note that `imageTag`, `os`, and `arch` do not affect the pulled image; that is +governed entirely by the supplied `manifest.json` file. These arguments are +used for the manifest-selection logic in the included `getManifest` script. + + #### Authentication If the Nix daemon is used for building, here is how to set up registry diff --git a/cmd/image.go b/cmd/image.go index 68e7abc..b0ae74d 100644 --- a/cmd/image.go +++ b/cmd/image.go @@ -58,6 +58,36 @@ func imageFromDir(outputFilename, directory string) error { return nil } +var imageFromManifestCmd = &cobra.Command{ + Use: "image-from-manifest OUTPUT-FILENAME MANIFEST.JSON BLOBS.JSON", + Short: "Write an image.json file to OUTPUT-FILENAME from a skopeo raw manifest and blobs JSON.", + Args: cobra.MinimumNArgs(3), + Run: func(cmd *cobra.Command, args []string) { + err := imageFromManifest(args[0], args[1], args[2]) + if err != nil { + fmt.Fprintf(os.Stderr, "%s", err) + os.Exit(1) + } + }, +} + +func imageFromManifest(outputFilename, manifestFilename string, blobsFilename string) error { + image, err := nix.NewImageFromManifest(manifestFilename, blobsFilename) + if err != nil { + return err + } + res, err := json.MarshalIndent(image, "", "\t") + if err != nil { + return err + } + err = os.WriteFile(outputFilename, []byte(res), 0666) + if err != nil { + return err + } + logrus.Infof("Image has been written to %s", outputFilename) + return nil +} + func image(outputFilename, imageConfigPath string, fromImageFilename string, layerPaths []string) error { var imageConfig v1.ImageConfig var image types.Image @@ -116,4 +146,5 @@ func init() { rootCmd.AddCommand(imageCmd) imageCmd.Flags().StringVarP(&fromImageFilename, "from-image", "", "", "A JSON file describing the base image") rootCmd.AddCommand(imageFromDirCmd) + rootCmd.AddCommand(imageFromManifestCmd) } diff --git a/default.nix b/default.nix index 2406b69..0ae5d1a 100644 --- a/default.nix +++ b/default.nix @@ -129,6 +129,75 @@ let ${nix2container-bin}/bin/nix2container image-from-dir $out ${dir} ''; + pullImageFromManifest = + { imageName + , imageManifest ? null + # The manifest dictates what is pulled; these three are only used for + # the supplied manifest-pulling script. + , imageTag ? "latest" + , os ? "linux" + , arch ? pkgs.go.GOARCH + , tlsVerify ? true + , registryUrl ? "registry.hub.docker.com" + , meta ? {} + }: let + manifest = l.fromJSON (l.readFile imageManifest); + + buildImageBlob = digest: + let + blobUrl = "https://${registryUrl}/v2/${imageName}/blobs/${digest}"; + plainDigest = l.replaceStrings ["sha256:"] [""] digest; + insecureFlag = l.strings.optionalString (!tlsVerify) "--insecure"; + in pkgs.runCommand plainDigest { + outputHash = plainDigest; + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + } '' + SSL_CERT_FILE="${pkgs.cacert.out}/etc/ssl/certs/ca-bundle.crt"; + + # This initial access is expected to fail as we don't have a token. + ${pkgs.curl}/bin/curl ${insecureFlag} "${blobUrl}" --head --silent --write-out '%header{www-authenticate}' --output /dev/null > bearer.txt + tokenUrl=$(sed -n 's/Bearer realm="\(.*\)",service="\(.*\)",scope="\(.*\)"/\1?service=\2\&scope=\3/p' bearer.txt) + + echo "Token URL: $tokenUrl" + ${pkgs.curl}/bin/curl ${insecureFlag} --fail --silent "$tokenUrl" --output token.json + token="$(${pkgs.jq}/bin/jq --raw-output .token token.json)" + + echo "Blob URL: ${blobUrl}" + ${pkgs.curl}/bin/curl ${insecureFlag} --fail -H "Authorization: Bearer $token" "${blobUrl}" --location --output $out + ''; + + # Pull the blobs (archives) for all layers, as well as the one for the image's config JSON. + layerBlobs = map (layerManifest: buildImageBlob layerManifest.digest) manifest.layers; + configBlob = buildImageBlob manifest.config.digest; + + # Write the blob map out to a JSON file for the GO executable to consume. + blobMap = l.listToAttrs(map (drv: { name = drv.name; value = drv; }) (layerBlobs ++ [configBlob])); + blobMapFile = pkgs.writeText "${imageName}-blobs.json" (l.toJSON blobMap); + + # Convenience scripts for manifest-updating. + filter = ''.manifests[] | select((.platform.os=="${os}") and (.platform.architecture=="${arch}")) | .digest''; + getManifest = pkgs.writeShellApplication { + name = "get-manifest"; + runtimeInputs = [ pkgs.jq skopeo-nix2container ]; + text = '' + set -e + manifest=$(skopeo inspect docker://${registryUrl}/${imageName}:${imageTag} --raw | jq) + if echo "$manifest" | jq -e .manifests >/dev/null; then + # Multi-arch image, pick the one that matches the supplied platform details. + hash=$(echo "$manifest" | jq -r '${filter}') + skopeo inspect "docker://${registryUrl}/${imageName}@$hash" --raw | jq + else + # Single-arch image, return the initial response. + echo "$manifest" + fi + ''; + }; + + in pkgs.runCommand "nix2container-${imageName}.json" { passthru = { inherit getManifest; }; } '' + ${nix2container-bin}/bin/nix2container image-from-manifest $out ${imageManifest} ${blobMapFile} + ''; + buildLayer = { # A list of store paths to include in the layer. deps ? [], @@ -330,7 +399,7 @@ let { inherit imageName meta; passthru = { - inherit imageTag; + inherit fromImage imageTag; # provide a cheap to evaluate image reference for use with external tools like docker # DO NOT use as an input to other derivations, as there is no guarantee that the image # reference will exist in the store. @@ -359,5 +428,5 @@ let in { inherit nix2container-bin skopeo-nix2container; - nix2container = { inherit buildImage buildLayer pullImage; }; + nix2container = { inherit buildImage buildLayer pullImage pullImageFromManifest; }; } diff --git a/examples/alpine-manifest.json b/examples/alpine-manifest.json new file mode 100644 index 0000000..bc27bc7 --- /dev/null +++ b/examples/alpine-manifest.json @@ -0,0 +1,16 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 1472, + "digest": "sha256:c1aabb73d2339c5ebaa3681de2e9d9c18d57485045a4e311d9f8004bec208d67" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 3397879, + "digest": "sha256:31e352740f534f9ad170f75378a84fe453d6156e40700b882d737a8f4a6988a3" + } + ] +} diff --git a/examples/default.nix b/examples/default.nix index 60f80a8..79f8336 100644 --- a/examples/default.nix +++ b/examples/default.nix @@ -5,6 +5,8 @@ basic = pkgs.callPackage ./basic.nix { inherit nix2container; }; nonReproducible = pkgs.callPackage ./non-reproducible.nix { inherit nix2container; }; fromImage = pkgs.callPackage ./from-image.nix { inherit nix2container; }; + fromImageManifest = pkgs.callPackage ./from-image-manifest.nix { inherit nix2container; }; + getManifest = pkgs.callPackage ./get-manifest.nix { inherit nix2container; }; uwsgi = pkgs.callPackage ./uwsgi { inherit nix2container; }; openbar = pkgs.callPackage ./openbar.nix { inherit nix2container; }; layered = pkgs.callPackage ./layered.nix { inherit nix2container; }; diff --git a/examples/from-image-manifest.nix b/examples/from-image-manifest.nix new file mode 100644 index 0000000..5fb20ed --- /dev/null +++ b/examples/from-image-manifest.nix @@ -0,0 +1,20 @@ +{ pkgs, nix2container }: let + # nix run .#examples.fromImageManifest.fromImage.getManifest > examples/alpine-manifest.json + alpine = nix2container.pullImageFromManifest { + imageName = "library/alpine"; + imageManifest = ./alpine-manifest.json; + + # These attributes aren't checked against the manifest; they are only + # used to populate the supplied getManifest script. + imageTag = "latest"; + os = "linux"; + arch = "amd64"; + }; +in +nix2container.buildImage { + name = "from-image-manifest"; + fromImage = alpine; + config = { + entrypoint = [ "${pkgs.coreutils}/bin/ls" "-l" "/etc/alpine-release"]; + }; +} diff --git a/examples/get-manifest.nix b/examples/get-manifest.nix new file mode 100644 index 0000000..28088ff --- /dev/null +++ b/examples/get-manifest.nix @@ -0,0 +1,21 @@ +{ nix2container }: +{ + images = { + multiArch = nix2container.pullImageFromManifest { + imageName = "library/alpine"; + os = "linux"; + arch = "amd64"; + }; + + singleArch = nix2container.pullImageFromManifest { + imageName = "rancher/systemd-node"; + imageTag = "v0.0.4"; + }; + + quayio = nix2container.pullImageFromManifest { + imageName = "containers/podman"; + imageTag = "v4.5"; + registryUrl = "quay.io"; + }; + }; +} diff --git a/nix/image.go b/nix/image.go index a1a2a59..07dbd52 100644 --- a/nix/image.go +++ b/nix/image.go @@ -96,7 +96,7 @@ func getV1Image(image types.Image) (imageV1 v1.Image, err error) { return } -// NewImageFromDir creates an Image from a JSON file describing an +// NewImageFromFile creates an Image from a JSON file describing an // image. This file has usually been created by Nix through the // nix2container binary. func NewImageFromFile(filename string) (image types.Image, err error) { @@ -177,6 +177,70 @@ func NewImageFromDir(directory string) (image types.Image, err error) { return image, nil } +// NewImageFromManifest builds an Image based on a registry manifest +// and a separate JSON mapping pointing to the locations of the +// associated blobs (layer archives). +func NewImageFromManifest(manifestFilename string, blobMapFilename string) (image types.Image, err error) { + image.Version = types.ImageVersion + + content, err := os.ReadFile(manifestFilename) + if err != nil { + return image, err + } + var v1Manifest v1.Manifest + err = json.Unmarshal(content, &v1Manifest) + if err != nil { + return image, err + } + + var blobMap map[string]string + content, err = os.ReadFile(blobMapFilename) + if err != nil { + return image, err + } + err = json.Unmarshal(content, &blobMap) + if err != nil { + return image, err + } + + var configFilename = blobMap[v1Manifest.Config.Digest.Encoded()] + content, err = os.ReadFile(configFilename) + if err != nil { + return image, err + } + var v1ImageConfig manifest.Schema2Image + err = json.Unmarshal(content, &v1ImageConfig) + if err != nil { + return image, err + } + + for i, l := range v1Manifest.Layers { + layerFilename := blobMap[l.Digest.Encoded()] + logrus.Infof("Adding tar file '%s' as image layer", layerFilename) + layer := types.Layer{ + LayerPath: layerFilename, + Digest: l.Digest.String(), + DiffIDs: v1ImageConfig.RootFS.DiffIDs[i].String(), + } + switch l.MediaType { + case "application/vnd.docker.image.rootfs.diff.tar": + layer.MediaType = v1.MediaTypeImageLayer + case "application/vnd.docker.image.rootfs.diff.tar.gzip": + layer.MediaType = v1.MediaTypeImageLayerGzip + case "application/vnd.oci.image.layer.v1.tar": + layer.MediaType = l.MediaType + case "application/vnd.oci.image.layer.v1.tar+gzip": + layer.MediaType = l.MediaType + case "application/vnd.oci.image.layer.v1.tar+zstd": + layer.MediaType = l.MediaType + default: + return image, fmt.Errorf("Unsupported media type: %q", l.MediaType) + } + image.Layers = append(image.Layers, layer) + } + return image, nil +} + type nopCloser struct { io.Reader } diff --git a/tests/default.nix b/tests/default.nix index d526b87..2897dff 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -54,6 +54,10 @@ let image = examples.fromImage; pattern = "/etc/alpine-release$"; }; + fromImageManifest = testScript { + image = examples.fromImageManifest; + pattern = "/etc/alpine-release$"; + }; layered = testScript { image = examples.layered; pattern = "Hello, world"; @@ -125,7 +129,6 @@ let # The file test2.txt should not have 777 perms pattern = "^-r--r--r-- 1 0 0 0 Jan 1 1970 test2.txt"; }; - copyToRoot = testScript { image = nix2container.buildImage { name = "copy-to-root"; @@ -134,7 +137,18 @@ let }; pattern = "Hello, world!"; }; - }; + } // + (pkgs.lib.mapAttrs' (name: drv: { + name = "${name}GetManifest"; + value = pkgs.writeScriptBin "test-script" '' + set -e + # Don't pipe directly here, as we don't want to swallow a return code. + manifest=$(${drv.getManifest}/bin/get-manifest) + echo "$manifest" | ${pkgs.jq}/bin/jq -e 'has("layers")' > /dev/null + echo "Test Passed" + ''; + }) examples.getManifest.images); + all = let scripts = pkgs.lib.concatStringsSep