diff --git a/cmd/image.go b/cmd/image.go index b0ae74d..794bb20 100644 --- a/cmd/image.go +++ b/cmd/image.go @@ -14,13 +14,14 @@ import ( ) var fromImageFilename string +var imageArch string var imageCmd = &cobra.Command{ Use: "image OUTPUT-FILENAME CONFIG.JSON LAYERS-1.JSON LAYERS-2.JSON ...", Short: "Generate an image.json file from a image configuration and layers", Args: cobra.MinimumNArgs(3), Run: func(cmd *cobra.Command, args []string) { - err := image(args[0], args[1], fromImageFilename, args[2:]) + err := image(args[0], args[1], fromImageFilename, imageArch, args[2:]) if err != nil { fmt.Fprintf(os.Stderr, "%s", err) os.Exit(1) @@ -88,7 +89,7 @@ func imageFromManifest(outputFilename, manifestFilename string, blobsFilename st return nil } -func image(outputFilename, imageConfigPath string, fromImageFilename string, layerPaths []string) error { +func image(outputFilename, imageConfigPath string, fromImageFilename string, arch string, layerPaths []string) error { var imageConfig v1.ImageConfig var image types.Image @@ -114,7 +115,7 @@ func image(outputFilename, imageConfigPath string, fromImageFilename string, lay logrus.Infof("Using base image %s containing %d layers", fromImageFilename, len(fromImage.Layers)) } - image.Arch = runtime.GOARCH + image.Arch = arch image.ImageConfig = imageConfig for _, path := range layerPaths { @@ -145,6 +146,7 @@ func image(outputFilename, imageConfigPath string, fromImageFilename string, lay func init() { rootCmd.AddCommand(imageCmd) imageCmd.Flags().StringVarP(&fromImageFilename, "from-image", "", "", "A JSON file describing the base image") + imageCmd.Flags().StringVarP(&imageArch, "arch", "", runtime.GOARCH, "Target CPU architecture of the image") rootCmd.AddCommand(imageFromDirCmd) rootCmd.AddCommand(imageFromManifestCmd) } diff --git a/default.nix b/default.nix index 99b02a7..71d4022 100644 --- a/default.nix +++ b/default.nix @@ -1,485 +1,42 @@ -{ pkgs ? import { }, system }: +{ pkgs ? import { }, system ? pkgs.system }: let - l = pkgs.lib // builtins; - - nix2container-bin = pkgs.buildGoModule rec { - pname = "nix2container"; - version = "1.0.0"; - src = l.cleanSourceWith { - src = ./.; - filter = path: type: - let - p = baseNameOf path; - in !( - p == "flake.nix" || - p == "flake.lock" || - p == "examples" || - p == "tests" || - p == "README.md" || - p == "default.nix" - ); - }; - vendorHash = "sha256-/j4ZHOwU5Xi8CE/fHha+2iZhsLd/y2ovzVhvg8HDV78="; - ldflags = pkgs.lib.optionals pkgs.stdenv.isDarwin [ - "-X github.com/nlewo/nix2container/nix.useNixCaseHack=true" - ]; + lib = pkgs.lib; + + scopeFn = self: { + nix2container-bin = self.callPackage ./lib/nix2container-bin.nix { }; + skopeo-nix2container = self.callPackage ./lib/skopeo-nix2container.nix { }; + + buildImage = self.callPackage ./lib/buildImage.nix { }; + buildLayer = self.callPackage ./lib/buildLayer.nix { }; + checkedParams = self.callPackage ./lib/checkedParams.nix { }; + closureGraph = self.callPackage ./lib/closureGraph.nix { }; + copyTo = self.callPackage ./lib/copyTo.nix { }; + copyToDockerDaemon = self.callPackage ./lib/copyToDockerDaemon.nix { }; + copyToPodman = self.callPackage ./lib/copyToPodman.nix { }; + copyToRegistry = self.callPackage ./lib/copyToRegistry.nix { }; + makeNixDatabase = self.callPackage ./lib/makeNixDatabase.nix { }; + pullImage = self.callPackage ./lib/pullImage.nix { }; + pullImageFromManifest = self.callPackage ./lib/pullImageFromManifest.nix { }; + writeSkopeoApplication = self.callPackage ./lib/writeSkopeoApplication.nix { }; + + inherit (otherSplices) selfBuildBuild selfBuildHost selfBuildTarget selfHostHost selfHostTarget selfTargetTarget; }; - skopeo-nix2container = pkgs.skopeo.overrideAttrs (old: { - EXTRA_LDFLAGS = pkgs.lib.optionalString pkgs.stdenv.isDarwin "-X github.com/nlewo/nix2container/nix.useNixCaseHack=true"; - nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.patchutils ]; - preBuild = let - patch = pkgs.fetchurl { - url = "https://github.com/nlewo/image/commit/c2254c998433cf02af60bf0292042bd80b96a77e.patch"; - sha256 = "sha256-dKEObfZY2fdsza/kObCLhv4l2snuzAbpDi4fGmtTPUQ="; - - }; - in '' - mkdir -p vendor/github.com/nlewo/nix2container/ - cp -r ${nix2container-bin.src}/* vendor/github.com/nlewo/nix2container/ - cd vendor/github.com/containers/image/v5 - mkdir nix/ - touch nix/transport.go - # The patch for alltransports.go does not apply cleanly to skopeo > 1.14, - # filter the patch and insert the import manually here instead. - filterdiff -x '*/alltransports.go' ${patch} | patch -p1 - sed -i '\#_ "github.com/containers/image/v5/tarball"#a _ "github.com/containers/image/v5/nix"' transports/alltransports/alltransports.go - cd - - ''; - }); - - writeSkopeoApplication = name: text: pkgs.writeShellApplication { - inherit name text; - runtimeInputs = [ pkgs.jq skopeo-nix2container ]; - excludeShellChecks = [ "SC2068" ]; + otherSplices = { + selfBuildBuild = lib.makeScope pkgs.pkgsBuildBuild.newScope scopeFn; + selfBuildHost = lib.makeScope pkgs.pkgsBuildHost.newScope scopeFn; + selfBuildTarget = lib.makeScope pkgs.pkgsBuildTarget.newScope scopeFn; + selfHostHost = lib.makeScope pkgs.pkgsHostHost.newScope scopeFn; + selfHostTarget = lib.makeScope pkgs.pkgsHostTarget.newScope scopeFn; + selfTargetTarget = lib.optionalAttrs (pkgs.pkgsTargetTarget?newScope) (lib.makeScope pkgs.pkgsTargetTarget.newScope scopeFn); }; - copyToDockerDaemon = image: writeSkopeoApplication "copy-to-docker-daemon" '' - echo "Copy to Docker daemon image ${image.imageName}:${image.imageTag}" - skopeo --insecure-policy copy nix:${image} docker-daemon:${image.imageName}:${image.imageTag} $@ - ''; - - copyToRegistry = image: writeSkopeoApplication "copy-to-registry" '' - echo "Copy to Docker registry image ${image.imageName}:${image.imageTag}" - skopeo --insecure-policy copy nix:${image} docker://${image.imageName}:${image.imageTag} $@ - ''; - - copyTo = image: writeSkopeoApplication "copy-to" '' - echo Running skopeo --insecure-policy copy nix:${image} $@ - skopeo --insecure-policy copy nix:${image} $@ - ''; - - copyToPodman = image: writeSkopeoApplication "copy-to-podman" '' - echo "Copy to podman image ${image.imageName}:${image.imageTag}" - skopeo --insecure-policy copy nix:${image} containers-storage:${image.imageName}:${image.imageTag} - skopeo --insecure-policy inspect containers-storage:${image.imageName}:${image.imageTag} - ''; - - # Pull an image from a registry with Skopeo and translate it to a - # nix2container image.json file. - # This mainly comes from nixpkgs/build-support/docker/default.nix. - # - # Credentials: - # If you use the nix daemon for building, here is how you set up creds: - # docker login URL to whatever it is - # copy ~/.docker/config.json to /etc/nix/skopeo/auth.json - # Make the directory and all the files readable to the nixbld group - # sudo chmod -R g+rx /etc/nix/skopeo - # sudo chgrp -R nixbld /etc/nix/skopeo - # Now, bind mount the file into the nix build sandbox - # extra-sandbox-paths = /etc/skopeo/auth.json=/etc/nix/skopeo/auth.json - # update /etc/nix/skopeo/auth.json every time you add a new registry auth - pullImage = - let - fixName = name: l.replaceStrings [ "/" ":" ] [ "-" "-" ] name; - in - { imageName - # To find the digest of an image, you can use skopeo: - # see doc/functions.xml - , imageDigest - , sha256 - , os ? "linux" - , arch ? pkgs.go.GOARCH - , tlsVerify ? true - , name ? fixName "docker-image-${imageName}" - }: let - authFile = "/etc/skopeo/auth.json"; - dir = pkgs.runCommand name - { - inherit imageDigest; - impureEnvVars = l.fetchers.proxyImpureEnvVars; - outputHashMode = "recursive"; - outputHashAlgo = "sha256"; - outputHash = sha256; - - nativeBuildInputs = l.singleton pkgs.skopeo; - SSL_CERT_FILE = "${pkgs.cacert.out}/etc/ssl/certs/ca-bundle.crt"; - - sourceURL = "docker://${imageName}@${imageDigest}"; - } '' - skopeo \ - --insecure-policy \ - --tmpdir=$TMPDIR \ - --override-os ${os} \ - --override-arch ${arch} \ - copy \ - --src-tls-verify=${l.boolToString tlsVerify} \ - $( - if test -f "${authFile}" - then - echo "--authfile=${authFile} $sourceURL" - else - echo "$sourceURL" - fi - ) \ - "dir://$out" \ - | cat # pipe through cat to force-disable progress bar - ''; - in pkgs.runCommand "nix2container-${imageName}.json" { } '' - ${nix2container-bin}/bin/nix2container image-from-dir $out ${dir} - ''; - - pullImageFromManifest = - { imageName - , imageManifest ? null - # The manifest dictates what is pulled; these three are only used for - # the supplied manifest-pulling script. - , imageTag ? "latest" - , os ? "linux" - , arch ? pkgs.go.GOARCH - , tlsVerify ? true - , registryUrl ? "registry.hub.docker.com" - , meta ? {} - }: let - manifest = l.fromJSON (l.readFile imageManifest); - - buildImageBlob = digest: - let - blobUrl = "https://${registryUrl}/v2/${imageName}/blobs/${digest}"; - plainDigest = l.replaceStrings ["sha256:"] [""] digest; - insecureFlag = l.strings.optionalString (!tlsVerify) "--insecure"; - in pkgs.runCommand plainDigest { - outputHash = plainDigest; - outputHashMode = "flat"; - outputHashAlgo = "sha256"; - } '' - SSL_CERT_FILE="${pkgs.cacert.out}/etc/ssl/certs/ca-bundle.crt"; - - # This initial access is expected to fail as we don't have a token. - ${pkgs.curl}/bin/curl --location ${insecureFlag} "${blobUrl}" --head --silent --write-out '%header{www-authenticate}' --output /dev/null > bearer.txt - tokenUrl=$(sed -n 's/Bearer realm="\(.*\)",service="\(.*\)",scope="\(.*\)"/\1?service=\2\&scope=\3/p' bearer.txt) - - declare -a auth_args - if [ -n "$tokenUrl" ]; then - echo "Token URL: $tokenUrl" - ${pkgs.curl}/bin/curl --location ${insecureFlag} --fail --silent "$tokenUrl" --output token.json - token="$(${pkgs.jq}/bin/jq --raw-output .token token.json)" - auth_args=(-H "Authorization: Bearer $token") - else - echo "No token URL found, trying without authentication" - auth_args=() - fi - - echo "Blob URL: ${blobUrl}" - ${pkgs.curl}/bin/curl ${insecureFlag} --fail "''${auth_args[@]}" "${blobUrl}" --location --output $out - ''; - - # Pull the blobs (archives) for all layers, as well as the one for the image's config JSON. - layerBlobs = map (layerManifest: buildImageBlob layerManifest.digest) manifest.layers; - configBlob = buildImageBlob manifest.config.digest; - - # Write the blob map out to a JSON file for the GO executable to consume. - blobMap = l.listToAttrs(map (drv: { name = drv.name; value = drv; }) (layerBlobs ++ [configBlob])); - blobMapFile = pkgs.writeText "${imageName}-blobs.json" (l.toJSON blobMap); - - # Convenience scripts for manifest-updating. - filter = ''.manifests[] | select((.platform.os=="${os}") and (.platform.architecture=="${arch}")) | .digest''; - getManifest = writeSkopeoApplication "get-manifest" '' - set -e - manifest=$(skopeo inspect docker://${registryUrl}/${imageName}:${imageTag} --raw | jq) - if echo "$manifest" | jq -e .manifests >/dev/null; then - # Multi-arch image, pick the one that matches the supplied platform details. - hash=$(echo "$manifest" | jq -r '${filter}') - skopeo inspect "docker://${registryUrl}/${imageName}@$hash" --raw | jq - else - # Single-arch image, return the initial response. - echo "$manifest" - fi - ''; - - in pkgs.runCommand "nix2container-${imageName}.json" { passthru = { inherit getManifest; }; } '' - ${nix2container-bin}/bin/nix2container image-from-manifest $out ${imageManifest} ${blobMapFile} - ''; - - buildLayer = { - # A list of store paths to include in the layer. - deps ? [], - # A derivation (or list of derivations) to include in the layer - # root directory. The store path prefix /nix/store/hash-path is - # removed. The store path content is then located at the layer /. - copyToRoot ? null, - # A store path to ignore. This is mainly useful to ignore the - # configuration file from the container layer. - ignore ? null, - # A list of layers built with the buildLayer function: if a store - # path in deps or copyToRoot belongs to one of these layers, this - # store path is skipped. This is pretty useful to - # isolate store paths that are often updated from more stable - # store paths, to speed up build and push time. - layers ? [], - # Store the layer tar in the derivation. This is useful when the - # layer dependencies are not bit reproducible. - reproducible ? true, - # A list of file permisssions which are set when the tar layer is - # created: these permissions are not written to the Nix store. - # - # Each element of this permission list is a dict such as - # { path = "a store path"; - # regex = ".*"; - # mode = "0664"; - # } - # The mode is applied on a specific path. In this path subtree, - # the mode is then applied on all files matching the regex. - perms ? [], - # The maximun number of layer to create. This is based on the - # store path "popularity" as described in - # https://grahamc.com/blog/nix-and-layered-docker-images - maxLayers ? 1, - # Deprecated: will be removed on v1 - contents ? null, - }: let - subcommand = if reproducible - then "layers-from-reproducible-storepaths" - else "layers-from-non-reproducible-storepaths"; - copyToRootList = - let derivations = if !isNull contents then contents else copyToRoot; - in if isNull derivations - then [] - else if !builtins.isList derivations - then [derivations] - else derivations; - # This is to move all storepaths in the copyToRoot attribute to the - # image root. - rewrites = l.map (p: { - path = p; - regex = "^${p}"; - repl = ""; - }) copyToRootList; - rewritesFile = pkgs.writeText "rewrites.json" (l.toJSON rewrites); - rewritesFlag = "--rewrites ${rewritesFile}"; - permsFile = pkgs.writeText "perms.json" (l.toJSON perms); - permsFlag = l.optionalString (perms != []) "--perms ${permsFile}"; - allDeps = deps ++ copyToRootList; - tarDirectory = l.optionalString (! reproducible) "--tar-directory $out"; - layersJSON = pkgs.runCommand "layers.json" {} '' - mkdir $out - ${nix2container-bin}/bin/nix2container ${subcommand} \ - $out/layers.json \ - ${closureGraph allDeps ignore} \ - --max-layers ${toString maxLayers} \ - ${rewritesFlag} \ - ${permsFlag} \ - ${tarDirectory} \ - ${l.concatMapStringsSep " " (l: l + "/layers.json") layers} \ - ''; - in checked { inherit copyToRoot contents; } layersJSON; - - # Create a nix database from all paths contained in the given closureGraphJson. - # Also makes all these paths store roots to prevent them from being garbage collected. - makeNixDatabase = closureGraphJson: - assert l.isDerivation closureGraphJson; - pkgs.runCommand "nix-database" {}'' - mkdir $out - echo "Generating the nix database from ${closureGraphJson}..." - export NIX_REMOTE=local?root=$PWD - # A user is required by nix - # https://github.com/NixOS/nix/blob/9348f9291e5d9e4ba3c4347ea1b235640f54fd79/src/libutil/util.cc#L478 - export USER=nobody - export PATH=${pkgs.jq.bin}/bin:${pkgs.sqlite}/bin:"$PATH" - # Avoid including the closureGraph derivation itself. - # Transformation taken from https://github.com/NixOS/nixpkgs/blob/e7f49215422317c96445e0263f21e26e0180517e/pkgs/build-support/closure-info.nix#L33 - jq -r 'map([.path, .narHash, .narSize, "", (.references | length)] + .references) | add | map("\(.)\n") | add' ${closureGraphJson} \ - | head -n -1 \ - | ${pkgs.nix}/bin/nix-store --load-db -j 1 - - # Sanitize time stamps - sqlite3 $PWD/nix/var/nix/db/db.sqlite \ - 'UPDATE ValidPaths SET registrationTime = 0;'; - - # Dump and reimport to ensure that the update order doesn't somehow change the DB. - sqlite3 $PWD/nix/var/nix/db/db.sqlite '.dump' > db.dump - mkdir -p $out/nix/var/nix/db/ - sqlite3 $out/nix/var/nix/db/db.sqlite '.read db.dump' - mkdir -p $out/nix/store/.links - - mkdir -p $out/nix/var/nix/gcroots/docker/ - for i in $(jq -r 'map("\(.path)\n") | add' ${closureGraphJson}); do - ln -s $i $out/nix/var/nix/gcroots/docker/$(basename $i) - done; - ''; - - # Write the references of `path' to a file but do not include `ignore' itself if non-null. - closureGraph = paths: ignore: - let ignoreList = - if ignore == null - then [] - else if !(builtins.isList ignore) - then [ignore] - else ignore; - in pkgs.runCommand "closure-graph.json" - { - exportReferencesGraph.graph = paths; - __structuredAttrs = true; - PATH = "${pkgs.jq}/bin"; - ignoreListJson = builtins.toJSON (builtins.map builtins.toString ignoreList); - outputChecks.out = { - disallowedReferences = ignoreList; - }; - builder = l.toFile "builder" - '' - . .attrs.sh - jq --argjson ignore "$ignoreListJson" \ - '.graph|map(select(.path as $p | $ignore | index($p) | not))|map(.references|=sort_by(.))|sort_by(.path)' \ - .attrs.json \ - > ''${outputs[out]} - ''; - } - ""; - - buildImage = { - name, - # Image tag, when null then the nix output hash will be used. - tag ? null, - # An attribute set describing an image configuration as defined in - # https://github.com/opencontainers/image-spec/blob/8b9d41f48198a7d6d0a5c1a12dc2d1f7f47fc97f/specs-go/v1/config.go#L23 - config ? {}, - # A list of layers built with the buildLayer function: if a store - # path in deps or copyToRoot belongs to one of these layers, this - # store path is skipped. This is pretty useful to - # isolate store paths that are often updated from more stable - # store paths, to speed up build and push time. - layers ? [], - # A derivation (or list of derivation) to include in the layer - # root. The store path prefix /nix/store/hash-path is removed. The - # store path content is then located at the image /. - copyToRoot ? null, - # An image that is used as base image of this image. - fromImage ? "", - # A list of file permisssions which are set when the tar layer is - # created: these permissions are not written to the Nix store. - # - # Each element of this permission list is a dict such as - # { path = "a store path"; - # regex = ".*"; - # mode = "0664"; - # } - # The mode is applied on a specific path. In this path subtree, - # the mode is then applied on all files matching the regex. - perms ? [], - # The maximun number of layer to create. This is based on the - # store path "popularity" as described in - # https://grahamc.com/blog/nix-and-layered-docker-images - # Note this is applied on the image layers and not on layers added - # with the buildImage.layers attribute - maxLayers ? 1, - # If set to true, the Nix database is initialized with all store - # paths added into the image. Note this is only useful to run nix - # commands from the image, for instance to build an image used by - # a CI to run Nix builds. - initializeNixDatabase ? false, - # If initializeNixDatabase is set to true, the uid/gid of /nix can be - # controlled using nixUid/nixGid. - nixUid ? 0, - nixGid ? 0, - # Deprecated: will be removed - contents ? null, - meta ? {}, - }: - let - configFile = pkgs.writeText "config.json" (l.toJSON config); - copyToRootList = - let derivations = if !isNull contents then contents else copyToRoot; - in if isNull derivations - then [] - else if !builtins.isList derivations - then [derivations] - else derivations; - - # Expand the given list of layers to include all their transitive layer dependencies. - layersWithNested = layers: - let layerWithNested = layer: [layer] ++ (builtins.concatMap layerWithNested (layer.layers or [])); - in builtins.concatMap layerWithNested layers; - explodedLayers = layersWithNested layers; - ignore = [configFile]++explodedLayers; - - closureGraphForAllLayers = closureGraph ([configFile] ++ copyToRootList ++ layers) ignore; - nixDatabase = makeNixDatabase closureGraphForAllLayers; - # This layer contains all config dependencies. We ignore the - # configFile because it is already part of the image, as a - # specific blob. - - perms' = perms ++ l.optionals initializeNixDatabase - [ - { - path = nixDatabase; - regex = ".*"; - mode = "0755"; - uid = nixUid; - gid = nixGid; - } - ]; - - customizationLayer = buildLayer { - inherit maxLayers; - perms = perms'; - copyToRoot = if initializeNixDatabase - then copyToRootList ++ [nixDatabase] - else copyToRootList; - deps = [configFile]; - ignore = configFile; - layers = layers; - }; - fromImageFlag = l.optionalString (fromImage != "") "--from-image ${fromImage}"; - layerPaths = l.concatMapStringsSep " " (l: l + "/layers.json") (layers ++ [customizationLayer]); - image = let - imageName = l.toLower name; - imageTag = - if tag != null - then tag - else - l.head (l.strings.splitString "-" (baseNameOf image.outPath)); - in pkgs.runCommand "image-${baseNameOf name}.json" - { - inherit imageName meta; - passthru = { - inherit fromImage imageTag; - # provide a cheap to evaluate image reference for use with external tools like docker - # DO NOT use as an input to other derivations, as there is no guarantee that the image - # reference will exist in the store. - imageRefUnsafe = builtins.unsafeDiscardStringContext "${imageName}:${imageTag}"; - copyToDockerDaemon = copyToDockerDaemon image; - copyToRegistry = copyToRegistry image; - copyToPodman = copyToPodman image; - copyTo = copyTo image; - }; - } - '' - ${nix2container-bin}/bin/nix2container image \ - $out \ - ${fromImageFlag} \ - ${configFile} \ - ${layerPaths} - ''; - in checked { inherit copyToRoot contents; } image; - - checked = { copyToRoot, contents }: - pkgs.lib.warnIf (contents != null) - "The contents parameter is deprecated. Change to copyToRoot if the contents are designed to be copied to the root filesystem, such as when you use `buildEnv` or similar between contents and your packages. Use copyToRoot = buildEnv { ... }; or similar if you intend to add packages to /bin." - pkgs.lib.throwIf (contents != null && copyToRoot != null) - "You can not specify both contents and copyToRoot." - ; + scope = pkgs.makeScopeWithSplicing' { + f = scopeFn; + inherit otherSplices; + }; in { - inherit nix2container-bin skopeo-nix2container; - nix2container = { inherit buildImage buildLayer pullImage pullImageFromManifest; }; + inherit (scope) nix2container-bin skopeo-nix2container; + nix2container = { inherit (scope) buildImage buildLayer pullImage pullImageFromManifest; }; } diff --git a/flake.nix b/flake.nix index b961414..2e3d785 100644 --- a/flake.nix +++ b/flake.nix @@ -20,11 +20,14 @@ inherit (nix2container) nix2container; }; in - rec { - packages = { - inherit (nix2container) nix2container-bin skopeo-nix2container nix2container; - inherit examples tests; - }; - defaultPackage = packages.nix2container-bin; - }); + rec { + packages = { + inherit (nix2container) nix2container-bin skopeo-nix2container; + default = packages.nix2container-bin; + }; + legacyPackages = { + inherit (nix2container) nix2container; + inherit examples tests; + }; + }); } diff --git a/lib/buildImage.nix b/lib/buildImage.nix new file mode 100644 index 0000000..b02fa1f --- /dev/null +++ b/lib/buildImage.nix @@ -0,0 +1,133 @@ +{ lib, runCommand, writeText, go, buildLayer, closureGraph, makeNixDatabase, checkedParams, nix2container-bin, selfBuildHost }: + +{ name +, # Image tag, when null then the nix output hash will be used. + tag ? null +, # An attribute set describing an image configuration as defined in + # https://github.com/opencontainers/image-spec/blob/8b9d41f48198a7d6d0a5c1a12dc2d1f7f47fc97f/specs-go/v1/config.go#L23 + config ? { } +, # A list of layers built with the buildLayer function: if a store + # path in deps or copyToRoot belongs to one of these layers, this + # store path is skipped. This is pretty useful to + # isolate store paths that are often updated from more stable + # store paths, to speed up build and push time. + layers ? [ ] +, # A derivation (or list of derivation) to include in the layer + # root. The store path prefix /nix/store/hash-path is removed. The + # store path content is then located at the image /. + copyToRoot ? null +, # An image that is used as base image of this image. + fromImage ? "" +, # A list of file permisssions which are set when the tar layer is + # created: these permissions are not written to the Nix store. + # + # Each element of this permission list is a dict such as + # { path = "a store path"; + # regex = ".*"; + # mode = "0664"; + # } + # The mode is applied on a specific path. In this path subtree, + # the mode is then applied on all files matching the regex. + perms ? [ ] +, # The maximun number of layer to create. This is based on the + # store path "popularity" as described in + # https://grahamc.com/blog/nix-and-layered-docker-images + # Note this is applied on the image layers and not on layers added + # with the buildImage.layers attribute + maxLayers ? 1 +, # If set to true, the Nix database is initialized with all store + # paths added into the image. Note this is only useful to run nix + # commands from the image, for instance to build an image used by + # a CI to run Nix builds. + initializeNixDatabase ? false +, # If initializeNixDatabase is set to true, the uid/gid of /nix can be + # controlled using nixUid/nixGid. + nixUid ? 0 +, nixGid ? 0 +, # Deprecated: will be removed + contents ? null +, meta ? { } +, +}: +let + configFile = writeText "config.json" (builtins.toJSON config); + copyToRootList = + let derivations = if !isNull contents then contents else copyToRoot; + in if isNull derivations + then [ ] + else if !builtins.isList derivations + then [ derivations ] + else derivations; + + # Expand the given list of layers to include all their transitive layer dependencies. + layersWithNested = layers: + let layerWithNested = layer: [ layer ] ++ (builtins.concatMap layerWithNested (layer.layers or [ ])); + in builtins.concatMap layerWithNested layers; + explodedLayers = layersWithNested layers; + ignore = [ configFile ] ++ explodedLayers; + + closureGraphForAllLayers = closureGraph ([ configFile ] ++ copyToRootList ++ layers) ignore; + nixDatabase = makeNixDatabase closureGraphForAllLayers; + # This layer contains all config dependencies. We ignore the + # configFile because it is already part of the image, as a + # specific blob. + + perms' = perms ++ lib.optionals initializeNixDatabase + [ + { + path = nixDatabase; + regex = ".*"; + mode = "0755"; + uid = nixUid; + gid = nixGid; + } + ]; + + customizationLayer = buildLayer { + inherit maxLayers; + perms = perms'; + copyToRoot = + if initializeNixDatabase + then copyToRootList ++ [ nixDatabase ] + else copyToRootList; + deps = [ configFile ]; + ignore = configFile; + layers = layers; + }; + fromImageFlag = lib.optionalString (fromImage != "") "--from-image ${fromImage}"; + layerPaths = lib.concatMapStringsSep " " (l: l + "/layers.json") (layers ++ [ customizationLayer ]); + image = + let + imageName = lib.toLower name; + imageTag = + if tag != null + then tag + else + lib.head (lib.strings.splitString "-" (baseNameOf image.outPath)); + in + runCommand "image-${baseNameOf name}.json" + { + inherit imageName meta; + nativeBuildInputs = [ nix2container-bin ]; + passthru = { + inherit fromImage imageTag; + # provide a cheap to evaluate image reference for use with external tools like docker + # DO NOT use as an input to other derivations, as there is no guarantee that the image + # reference will exist in the store. + imageRefUnsafe = builtins.unsafeDiscardStringContext "${imageName}:${imageTag}"; + copyToDockerDaemon = selfBuildHost.copyToDockerDaemon image; + copyToRegistry = selfBuildHost.copyToRegistry image; + copyToPodman = selfBuildHost.copyToPodman image; + copyTo = selfBuildHost.copyTo image; + }; + } + '' + nix2container image \ + --arch ${go.GOARCH} \ + $out \ + ${fromImageFlag} \ + ${configFile} \ + ${layerPaths} + ''; +in +checkedParams { inherit copyToRoot contents; } image diff --git a/lib/buildLayer.nix b/lib/buildLayer.nix new file mode 100644 index 0000000..4a55879 --- /dev/null +++ b/lib/buildLayer.nix @@ -0,0 +1,83 @@ +{ lib, runCommand, writeText, checkedParams, closureGraph, nix2container-bin }: + +{ + # A list of store paths to include in the layer. + deps ? [ ] +, # A derivation (or list of derivations) to include in the layer + # root directory. The store path prefix /nix/store/hash-path is + # removed. The store path content is then located at the layer /. + copyToRoot ? null +, # A store path to ignore. This is mainly useful to ignore the + # configuration file from the container layer. + ignore ? null +, # A list of layers built with the buildLayer function: if a store + # path in deps or copyToRoot belongs to one of these layers, this + # store path is skipped. This is pretty useful to + # isolate store paths that are often updated from more stable + # store paths, to speed up build and push time. + layers ? [ ] +, # Store the layer tar in the derivation. This is useful when the + # layer dependencies are not bit reproducible. + reproducible ? true +, # A list of file permisssions which are set when the tar layer is + # created: these permissions are not written to the Nix store. + # + # Each element of this permission list is a dict such as + # { path = "a store path"; + # regex = ".*"; + # mode = "0664"; + # } + # The mode is applied on a specific path. In this path subtree, + # the mode is then applied on all files matching the regex. + perms ? [ ] +, # The maximun number of layer to create. This is based on the + # store path "popularity" as described in + # https://grahamc.com/blog/nix-and-layered-docker-images + maxLayers ? 1 +, # Deprecated: will be removed on v1 + contents ? null +, +}: +let + subcommand = + if reproducible + then "layers-from-reproducible-storepaths" + else "layers-from-non-reproducible-storepaths"; + copyToRootList = + let derivations = if !isNull contents then contents else copyToRoot; + in if isNull derivations + then [ ] + else if !builtins.isList derivations + then [ derivations ] + else derivations; + # This is to move all storepaths in the copyToRoot attribute to the + # image root. + rewrites = map + (p: { + path = p; + regex = "^${p}"; + repl = ""; + }) + copyToRootList; + rewritesFile = writeText "rewrites.json" (builtins.toJSON rewrites); + rewritesFlag = "--rewrites ${rewritesFile}"; + permsFile = writeText "perms.json" (builtins.toJSON perms); + permsFlag = lib.optionalString (perms != [ ]) "--perms ${permsFile}"; + allDeps = deps ++ copyToRootList; + tarDirectory = lib.optionalString (! reproducible) "--tar-directory $out"; + layersJSON = runCommand "layers.json" + { + nativeBuildInputs = [ nix2container-bin ]; + } '' + mkdir $out + nix2container ${subcommand} \ + $out/layers.json \ + ${closureGraph allDeps ignore} \ + --max-layers ${toString maxLayers} \ + ${rewritesFlag} \ + ${permsFlag} \ + ${tarDirectory} \ + ${lib.concatMapStringsSep " " (l: l + "/layers.json") layers} \ + ''; +in +checkedParams { inherit copyToRoot contents; } layersJSON diff --git a/lib/checkedParams.nix b/lib/checkedParams.nix new file mode 100644 index 0000000..834197b --- /dev/null +++ b/lib/checkedParams.nix @@ -0,0 +1,9 @@ +{ lib }: + +{ copyToRoot, contents }: + +lib.warnIf (contents != null) + "The contents parameter is deprecated. Change to copyToRoot if the contents are designed to be copied to the root filesystem, such as when you use `buildEnv` or similar between contents and your packages. Use copyToRoot = buildEnv { ... }; or similar if you intend to add packages to /bin." + lib.throwIf + (contents != null && copyToRoot != null) + "You can not specify both contents and copyToRoot." diff --git a/lib/closureGraph.nix b/lib/closureGraph.nix new file mode 100644 index 0000000..dd893b2 --- /dev/null +++ b/lib/closureGraph.nix @@ -0,0 +1,32 @@ +{ lib, runCommand, pkgsBuildHost }: + +# Write the references of `path' to a file but do not include `ignore' itself if non-null. +paths: ignore: + +let + ignoreList = + if ignore == null + then [ ] + else if !(builtins.isList ignore) + then [ ignore ] + else ignore; +in +runCommand "closure-graph.json" +{ + exportReferencesGraph.graph = paths; + __structuredAttrs = true; + PATH = "${pkgsBuildHost.jq}/bin"; + ignoreListJson = builtins.toJSON (builtins.map builtins.toString ignoreList); + outputChecks.out = { + disallowedReferences = ignoreList; + }; + builder = builtins.toFile "builder" + '' + . .attrs.sh + jq --argjson ignore "$ignoreListJson" \ + '.graph|map(select(.path as $p | $ignore | index($p) | not))|map(.references|=sort_by(.))|sort_by(.path)' \ + .attrs.json \ + > ''${outputs[out]} + ''; +} + "" diff --git a/lib/copyTo.nix b/lib/copyTo.nix new file mode 100644 index 0000000..61c4aa5 --- /dev/null +++ b/lib/copyTo.nix @@ -0,0 +1,8 @@ +{ writeSkopeoApplication }: + +image: + +writeSkopeoApplication "copy-to" '' + echo Running skopeo --insecure-policy copy nix:${image} $@ + skopeo --insecure-policy copy nix:${image} $@ +'' diff --git a/lib/copyToDockerDaemon.nix b/lib/copyToDockerDaemon.nix new file mode 100644 index 0000000..da3d4fa --- /dev/null +++ b/lib/copyToDockerDaemon.nix @@ -0,0 +1,8 @@ +{ writeSkopeoApplication }: + +image: + +writeSkopeoApplication "copy-to-docker-daemon" '' + echo "Copy to Docker daemon image ${image.imageName}:${image.imageTag}" + skopeo --insecure-policy copy nix:${image} docker-daemon:${image.imageName}:${image.imageTag} $@ +'' diff --git a/lib/copyToPodman.nix b/lib/copyToPodman.nix new file mode 100644 index 0000000..09f76c9 --- /dev/null +++ b/lib/copyToPodman.nix @@ -0,0 +1,9 @@ +{ writeSkopeoApplication }: + +image: + +writeSkopeoApplication "copy-to-podman" '' + echo "Copy to podman image ${image.imageName}:${image.imageTag}" + skopeo --insecure-policy copy nix:${image} containers-storage:${image.imageName}:${image.imageTag} + skopeo --insecure-policy inspect containers-storage:${image.imageName}:${image.imageTag} +'' diff --git a/lib/copyToRegistry.nix b/lib/copyToRegistry.nix new file mode 100644 index 0000000..724980f --- /dev/null +++ b/lib/copyToRegistry.nix @@ -0,0 +1,8 @@ +{ writeSkopeoApplication }: + +image: + +writeSkopeoApplication "copy-to-registry" '' + echo "Copy to Docker registry image ${image.imageName}:${image.imageTag}" + skopeo --insecure-policy copy nix:${image} docker://${image.imageName}:${image.imageTag} $@ +'' diff --git a/lib/makeNixDatabase.nix b/lib/makeNixDatabase.nix new file mode 100644 index 0000000..d784e98 --- /dev/null +++ b/lib/makeNixDatabase.nix @@ -0,0 +1,35 @@ +{ lib, runCommand, jq, nix, sqlite }: + +# Create a nix database from all paths contained in the given closureGraphJson. +# Also makes all these paths store roots to prevent them from being garbage collected. +closureGraphJson: +assert lib.isDerivation closureGraphJson; +runCommand "nix-database" +{ nativeBuildInputs = [ jq nix sqlite ]; } '' + mkdir $out + echo "Generating the nix database from ${closureGraphJson}..." + export NIX_REMOTE=local?root=$PWD + # A user is required by nix + # https://github.com/NixOS/nix/blob/9348f9291e5d9e4ba3c4347ea1b235640f54fd79/src/libutil/util.cc#L478 + export USER=nobody + # Avoid including the closureGraph derivation itself. + # Transformation taken from https://github.com/NixOS/nixpkgs/blob/e7f49215422317c96445e0263f21e26e0180517e/pkgs/build-support/closure-info.nix#L33 + jq -r 'map([.path, .narHash, .narSize, "", (.references | length)] + .references) | add | map("\(.)\n") | add' ${closureGraphJson} \ + | head -n -1 \ + | nix-store --load-db -j 1 + + # Sanitize time stamps + sqlite3 $PWD/nix/var/nix/db/db.sqlite \ + 'UPDATE ValidPaths SET registrationTime = 0;'; + + # Dump and reimport to ensure that the update order doesn't somehow change the DB. + sqlite3 $PWD/nix/var/nix/db/db.sqlite '.dump' > db.dump + mkdir -p $out/nix/var/nix/db/ + sqlite3 $out/nix/var/nix/db/db.sqlite '.read db.dump' + mkdir -p $out/nix/store/.links + + mkdir -p $out/nix/var/nix/gcroots/docker/ + for i in $(jq -r 'map("\(.path)\n") | add' ${closureGraphJson}); do + ln -s $i $out/nix/var/nix/gcroots/docker/$(basename $i) + done; +'' diff --git a/lib/nix2container-bin.nix b/lib/nix2container-bin.nix new file mode 100644 index 0000000..3374c05 --- /dev/null +++ b/lib/nix2container-bin.nix @@ -0,0 +1,25 @@ +{ lib, stdenv, buildGoModule }: + +buildGoModule { + pname = "nix2container"; + version = "1.0.0"; + src = lib.cleanSourceWith { + src = ../.; + filter = path: type: + let + p = baseNameOf path; + in + !( + p == "flake.nix" || + p == "flake.lock" || + p == "examples" || + p == "tests" || + p == "README.md" || + p == "default.nix" + ); + }; + vendorHash = "sha256-/j4ZHOwU5Xi8CE/fHha+2iZhsLd/y2ovzVhvg8HDV78="; + ldflags = lib.optionals stdenv.isDarwin [ + "-X github.com/nlewo/nix2container/nix.useNixCaseHack=true" + ]; +} diff --git a/lib/pullImage.nix b/lib/pullImage.nix new file mode 100644 index 0000000..f9a92ba --- /dev/null +++ b/lib/pullImage.nix @@ -0,0 +1,67 @@ +{ lib, runCommand, go, skopeo, cacert, nix2container-bin }: + +# Pull an image from a registry with Skopeo and translate it to a +# nix2container image.json file. +# This mainly comes from nixpkgs/build-support/docker/default.nix. +# +# Credentials: +# If you use the nix daemon for building, here is how you set up creds: +# docker login URL to whatever it is +# copy ~/.docker/config.json to /etc/nix/skopeo/auth.json +# Make the directory and all the files readable to the nixbld group +# sudo chmod -R g+rx /etc/nix/skopeo +# sudo chgrp -R nixbld /etc/nix/skopeo +# Now, bind mount the file into the nix build sandbox +# extra-sandbox-paths = /etc/skopeo/auth.json=/etc/nix/skopeo/auth.json +# update /etc/nix/skopeo/auth.json every time you add a new registry auth +let + fixName = name: lib.replaceStrings [ "/" ":" ] [ "-" "-" ] name; +in +{ imageName + # To find the digest of an image, you can use skopeo: + # see doc/functions.xml +, imageDigest +, sha256 +, os ? "linux" +, arch ? go.GOARCH +, tlsVerify ? true +, name ? fixName "docker-image-${imageName}" +}: +let + authFile = "/etc/skopeo/auth.json"; + dir = runCommand name + { + inherit imageDigest; + impureEnvVars = lib.fetchers.proxyImpureEnvVars; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + outputHash = sha256; + + nativeBuildInputs = [ skopeo ]; + SSL_CERT_FILE = "${cacert.out}/etc/ssl/certs/ca-bundle.crt"; + + sourceURL = "docker://${imageName}@${imageDigest}"; + } '' + skopeo \ + --insecure-policy \ + --tmpdir=$TMPDIR \ + --override-os ${os} \ + --override-arch ${arch} \ + copy \ + --src-tls-verify=${lib.boolToString tlsVerify} \ + $( + if test -f "${authFile}" + then + echo "--authfile=${authFile} $sourceURL" + else + echo "$sourceURL" + fi + ) \ + "dir://$out" \ + | cat # pipe through cat to force-disable progress bar + ''; +in +runCommand "nix2container-${imageName}.json" +{ nativeBuildInputs = [ nix2container-bin ]; } '' + nix2container image-from-dir $out ${dir} +'' diff --git a/lib/pullImageFromManifest.nix b/lib/pullImageFromManifest.nix new file mode 100644 index 0000000..2fe45fa --- /dev/null +++ b/lib/pullImageFromManifest.nix @@ -0,0 +1,81 @@ +{ lib, runCommand, writeText, go, cacert, curl, jq, nix2container-bin, selfBuildHost }: + +{ imageName +, imageManifest ? null + # The manifest dictates what is pulled; these three are only used for + # the supplied manifest-pulling script. +, imageTag ? "latest" +, os ? "linux" +, arch ? go.GOARCH +, tlsVerify ? true +, registryUrl ? "registry.hub.docker.com" +, meta ? { } +}: +let + manifest = builtins.fromJSON (lib.readFile imageManifest); + + buildImageBlob = digest: + let + blobUrl = "https://${registryUrl}/v2/${imageName}/blobs/${digest}"; + plainDigest = lib.replaceStrings [ "sha256:" ] [ "" ] digest; + insecureFlag = lib.strings.optionalString (!tlsVerify) "--insecure"; + in + runCommand plainDigest + { + nativeBuildInputs = [ curl jq ]; + outputHash = plainDigest; + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + } '' + SSL_CERT_FILE="${cacert.out}/etc/ssl/certs/ca-bundle.crt"; + + # This initial access is expected to fail as we don't have a token. + curl --location ${insecureFlag} "${blobUrl}" --head --silent --write-out '%header{www-authenticate}' --output /dev/null > bearer.txt + tokenUrl=$(sed -n 's/Bearer realm="\(.*\)",service="\(.*\)",scope="\(.*\)"/\1?service=\2\&scope=\3/p' bearer.txt) + + declare -a auth_args + if [ -n "$tokenUrl" ]; then + echo "Token URL: $tokenUrl" + curl --location ${insecureFlag} --fail --silent "$tokenUrl" --output token.json + token="$(jq --raw-output .token token.json)" + auth_args=(-H "Authorization: Bearer $token") + else + echo "No token URL found, trying without authentication" + auth_args=() + fi + + echo "Blob URL: ${blobUrl}" + curl ${insecureFlag} --fail "''${auth_args[@]}" "${blobUrl}" --location --output $out + ''; + + # Pull the blobs (archives) for all layers, as well as the one for the image's config JSON. + layerBlobs = map (layerManifest: buildImageBlob layerManifest.digest) manifest.layers; + configBlob = buildImageBlob manifest.config.digest; + + # Write the blob map out to a JSON file for the GO executable to consume. + blobMap = lib.listToAttrs (map (drv: { name = drv.name; value = drv; }) (layerBlobs ++ [ configBlob ])); + blobMapFile = writeText "${imageName}-blobs.json" (builtins.toJSON blobMap); + + # Convenience scripts for manifest-updating. + filter = ''.manifests[] | select((.platform.os=="${os}") and (.platform.architecture=="${arch}")) | .digest''; + getManifest = selfBuildHost.writeSkopeoApplication "get-manifest" '' + set -e + manifest=$(skopeo inspect docker://${registryUrl}/${imageName}:${imageTag} --raw | jq) + if echo "$manifest" | jq -e .manifests >/dev/null; then + # Multi-arch image, pick the one that matches the supplied platform details. + hash=$(echo "$manifest" | jq -r '${filter}') + skopeo inspect "docker://${registryUrl}/${imageName}@$hash" --raw | jq + else + # Single-arch image, return the initial response. + echo "$manifest" + fi + ''; + +in +runCommand "nix2container-${imageName}.json" +{ + nativeBuildInputs = [ nix2container-bin ]; + passthru = { inherit getManifest; }; +} '' + nix2container image-from-manifest $out ${imageManifest} ${blobMapFile} +'' diff --git a/lib/skopeo-nix2container.nix b/lib/skopeo-nix2container.nix new file mode 100644 index 0000000..61b81f3 --- /dev/null +++ b/lib/skopeo-nix2container.nix @@ -0,0 +1,24 @@ +{ lib, stdenv, skopeo, patchutils, fetchurl, nix2container-bin }: + +let + patch = fetchurl { + url = "https://github.com/nlewo/image/commit/c2254c998433cf02af60bf0292042bd80b96a77e.patch"; + sha256 = "sha256-dKEObfZY2fdsza/kObCLhv4l2snuzAbpDi4fGmtTPUQ="; + }; +in +skopeo.overrideAttrs (old: { + EXTRA_LDFLAGS = lib.optionalString stdenv.isDarwin "-X github.com/nlewo/nix2container/nix.useNixCaseHack=true"; + nativeBuildInputs = old.nativeBuildInputs ++ [ patchutils ]; + preBuild = '' + mkdir -p vendor/github.com/nlewo/nix2container/ + cp -r ${nix2container-bin.src}/* vendor/github.com/nlewo/nix2container/ + cd vendor/github.com/containers/image/v5 + mkdir nix/ + touch nix/transport.go + # The patch for alltransports.go does not apply cleanly to skopeo > 1.14, + # filter the patch and insert the import manually here instead. + filterdiff -x '*/alltransports.go' ${patch} | patch -p1 + sed -i '\#_ "github.com/containers/image/v5/tarball"#a _ "github.com/containers/image/v5/nix"' transports/alltransports/alltransports.go + cd - + ''; +}) diff --git a/lib/writeSkopeoApplication.nix b/lib/writeSkopeoApplication.nix new file mode 100644 index 0000000..70d5dfa --- /dev/null +++ b/lib/writeSkopeoApplication.nix @@ -0,0 +1,9 @@ +{ writeShellApplication, jq, skopeo-nix2container }: + +name: text: + +writeShellApplication { + inherit name text; + runtimeInputs = [ jq skopeo-nix2container ]; + excludeShellChecks = [ "SC2068" ]; +}