From 76c16843a30c0ff6305399328a1faeed1f191d37 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Fri, 28 Jun 2024 18:01:24 +0000 Subject: [PATCH] feat: [NODE-1380] Switch IC-OS to newer FS build tools --- .devcontainer/devcontainer.json | 4 +- Cargo.lock | 2 + gitlab-ci/config/common.yml | 2 +- gitlab-ci/config/zz-generated-gitlab-ci.yaml | 2 +- gitlab-ci/container/Dockerfile | 20 ++++ gitlab-ci/container/TAG | 2 +- gitlab-ci/container/files/packages.common | 1 - ic-os/defs.bzl | 6 +- rs/ic_os/diroid/BUILD.bazel | 2 + rs/ic_os/diroid/Cargo.toml | 2 + rs/ic_os/diroid/src/main.rs | 86 ++++++++++++--- toolchains/sysimage/build_ext4_image.py | 104 ++++++------------- toolchains/sysimage/toolchain.bzl | 9 +- 13 files changed, 148 insertions(+), 94 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b4612fa64c6..626347c9ddd 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,5 +1,5 @@ { - "image": "dfinity/ic-build:24bbf7e3a37a1de06f55b44b3419a92a929f919ba8e1dacb540255a17de36ad7", + "image": "dfinity/ic-build:f3527253faea5555085c508e10e98ed276988b1b3cd6e63c249e03318cecb596", "remoteUser": "ubuntu", "privileged": true, "runArgs": [ @@ -58,4 +58,4 @@ ] } } -} \ No newline at end of file +} diff --git a/Cargo.lock b/Cargo.lock index 042b9380f93..666e3a35dca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3267,6 +3267,8 @@ dependencies = [ name = "diroid" version = "0.1.0" dependencies = [ + "anyhow", + "clap 4.4.6", "walkdir", ] diff --git a/gitlab-ci/config/common.yml b/gitlab-ci/config/common.yml index a338efb6eda..20b48f80b1a 100644 --- a/gitlab-ci/config/common.yml +++ b/gitlab-ci/config/common.yml @@ -22,7 +22,7 @@ default: expire_in: 3 days when: always image: - name: "registry.gitlab.com/dfinity-lab/core/docker/ic-build:24bbf7e3a37a1de06f55b44b3419a92a929f919ba8e1dacb540255a17de36ad7" + name: "registry.gitlab.com/dfinity-lab/core/docker/ic-build:f3527253faea5555085c508e10e98ed276988b1b3cd6e63c249e03318cecb596" tags: - dfinity-ic diff --git a/gitlab-ci/config/zz-generated-gitlab-ci.yaml b/gitlab-ci/config/zz-generated-gitlab-ci.yaml index dac7ffce5b1..8c158cbd6b8 100644 --- a/gitlab-ci/config/zz-generated-gitlab-ci.yaml +++ b/gitlab-ci/config/zz-generated-gitlab-ci.yaml @@ -1975,7 +1975,7 @@ default: expire_in: 3 days when: always image: - name: registry.gitlab.com/dfinity-lab/core/docker/ic-build:24bbf7e3a37a1de06f55b44b3419a92a929f919ba8e1dacb540255a17de36ad7 + name: registry.gitlab.com/dfinity-lab/core/docker/ic-build:f3527253faea5555085c508e10e98ed276988b1b3cd6e63c249e03318cecb596 interruptible: true retry: max: 2 diff --git a/gitlab-ci/container/Dockerfile b/gitlab-ci/container/Dockerfile index 5e3ee22aa9c..38b7df5b260 100644 --- a/gitlab-ci/container/Dockerfile +++ b/gitlab-ci/container/Dockerfile @@ -35,6 +35,26 @@ RUN curl -L "https://apt.llvm.org/llvm-snapshot.gpg.key" | apt-key add - && \ apt -yqq install --no-install-recommends lld-18 llvm-18 llvm-18-dev clang-18 libclang-rt-18-dev google-cloud-cli \ gcc-$(gcc --version|head -n1|sed 's/\..*//'|sed 's/.* //')-plugin-dev libstdc++-$(gcc --version|head -n1|sed 's/\..*//'|sed 's/.* //')-dev +# Install a version of google-android-platform-tools-installer with e2fsdroid +RUN export DEBIAN_FRONTEND=noninteractive && \ + mkdir e2fsdroid && \ + cd e2fsdroid && \ + curl -fsSLO http://mirrors.kernel.org/ubuntu/pool/multiverse/g/google-android-installers/google-android-platform-tools-installer_35.0.0+1710437545-3build2_amd64.deb && \ + curl -fsSLO http://mirrors.kernel.org/ubuntu/pool/multiverse/g/google-android-installers/google-android-licenses_1710437545-3build2_all.deb && \ + dpkg-deb -R google-android-platform-tools-installer_35.0.0+1710437545-3build2_amd64.deb contents && \ + cd contents && \ + sed -i 's/35.0.0/33.0.2/' DEBIAN/control DEBIAN/postinst && \ + sed -i '/Version/ s/$/+downgraded/' DEBIAN/control && \ + rm usr/share/google-android-platform-tools-installer/platform-tools_r35.0.0-linux.zip.sha1 && \ + echo "6bf4f747ad929b02378b44ce083b4502d26109c7 platform-tools_r33.0.2-linux.zip" > usr/share/google-android-platform-tools-installer/platform-tools_r33.0.2-linux.zip.sha1 && \ + find . -type f -not -path "./DEBIAN/*" -exec md5sum {} + | sort -k 2 | sed 's/\.\/\(.*\)/\1/' > DEBIAN/md5sums && \ + cd .. && \ + dpkg-deb -b contents/ downgraded.deb && \ + apt -yqq install ./downgraded.deb ./google-android-licenses_1710437545-3build2_all.deb && \ + apt-mark hold google-android-platform-tools-installer && \ + cd .. && \ + rm -rf e2fsdroid + ARG sdk_version=0.12.0 ARG sdk_sha=40da56ad27774d5e1b2cbc35f94c17368be8c8da557aca19878940264bd82a0a RUN mkdir -p /tmp/sdk && curl -fsSL https://github.com/dfinity/sdk/releases/download/${sdk_version}/dfx-${sdk_version}-x86_64-linux.tar.gz -o /tmp/sdk/dfx.tar.gz && \ diff --git a/gitlab-ci/container/TAG b/gitlab-ci/container/TAG index 994ea02cb11..ca22c397a0e 100644 --- a/gitlab-ci/container/TAG +++ b/gitlab-ci/container/TAG @@ -1 +1 @@ -24bbf7e3a37a1de06f55b44b3419a92a929f919ba8e1dacb540255a17de36ad7 +f3527253faea5555085c508e10e98ed276988b1b3cd6e63c249e03318cecb596 diff --git a/gitlab-ci/container/files/packages.common b/gitlab-ci/container/files/packages.common index 695e8bad7a1..cf1e0fec848 100644 --- a/gitlab-ci/container/files/packages.common +++ b/gitlab-ci/container/files/packages.common @@ -53,7 +53,6 @@ libffi-dev libgmp-dev # IC-OS -android-sdk-ext4-utils cryptsetup-bin dosfstools fakeroot diff --git a/ic-os/defs.bzl b/ic-os/defs.bzl index 4ba8a8a53d1..ab84e9f39b8 100644 --- a/ic-os/defs.bzl +++ b/ic-os/defs.bzl @@ -128,10 +128,14 @@ def icos_build( src = ":rootfs-tree.tar", file_contexts = ":file_contexts", partition_size = image_deps["rootfs_size"], + # NOTE: e2fsdroid does not support filenames with spaces, fortunately, + # there are only two in our build. strip_paths = [ "/run", "/boot", "/var", + "/usr/lib/firmware/brcm/brcmfmac43430a0-sdio.ONDA-V80 PLUS.txt", + "/usr/lib/firmware/brcm/brcmfmac43455-sdio.MINIX-NEO Z83-4.txt", ], target_compatible_with = [ "@platforms//os:linux", @@ -146,7 +150,7 @@ def icos_build( src = ":rootfs-tree.tar", file_contexts = ":file_contexts", partition_size = image_deps["bootfs_size"], - subdir = "boot/", + subdir = "boot", target_compatible_with = [ "@platforms//os:linux", ], diff --git a/rs/ic_os/diroid/BUILD.bazel b/rs/ic_os/diroid/BUILD.bazel index 37abccaebdb..094632617aa 100644 --- a/rs/ic_os/diroid/BUILD.bazel +++ b/rs/ic_os/diroid/BUILD.bazel @@ -3,6 +3,8 @@ load("@rules_rust//rust:defs.bzl", "rust_binary") package(default_visibility = ["//visibility:public"]) DEPENDENCIES = [ + "@crate_index//:anyhow", + "@crate_index//:clap", "@crate_index//:walkdir", ] diff --git a/rs/ic_os/diroid/Cargo.toml b/rs/ic_os/diroid/Cargo.toml index 5af776e7ea6..1c2ff150bd5 100644 --- a/rs/ic_os/diroid/Cargo.toml +++ b/rs/ic_os/diroid/Cargo.toml @@ -6,4 +6,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +anyhow = { workspace = true } +clap = { workspace = true, features = ["derive"] } walkdir = "2.5.0" diff --git a/rs/ic_os/diroid/src/main.rs b/rs/ic_os/diroid/src/main.rs index 9a64eda5d20..2ce54adb438 100644 --- a/rs/ic_os/diroid/src/main.rs +++ b/rs/ic_os/diroid/src/main.rs @@ -1,25 +1,81 @@ -use std::env; +use std::collections::HashMap; +use std::fs::File; +use std::io::{self, BufRead, Write}; use std::os::unix::fs::MetadataExt; +use std::path::PathBuf; -use walkdir::WalkDir; +use anyhow::{Context, Result}; +use clap::Parser; +use walkdir::{DirEntryExt, WalkDir}; -fn main() { - let args: Vec = env::args().collect(); +/// Generate an e2fsdroid fs_config file for a given directory tree +#[derive(Parser)] +struct Args { + /// Fakeroot statefile, to pull ownership from + #[arg(long, required = true)] + fakeroot: PathBuf, + /// Base directory to scan + #[arg(long, required = true)] + input_dir: PathBuf, + /// Output file + #[arg(long, required = true)] + output: PathBuf, +} - if args.len() != 2 { - panic!("Incorrect arguments!"); - } +fn main() -> Result<()> { + let args = Args::parse(); + + let fakeroot_map = read_fakeroot_state(&args.fakeroot)?; + let mut output = File::create(args.output)?; + + for entry in WalkDir::new(&args.input_dir) { + let entry = entry?; - for entry in WalkDir::new(&args[1]) { - let entry = entry.unwrap(); + let metadata = entry.metadata()?; - let metadata = entry.metadata().unwrap(); - println!( + // fakeroot does not track /, so special case this ownership + let (uid, gid) = if entry.path() == args.input_dir { + &(0, 0) + } else { + fakeroot_map.get(&entry.ino()).context(format!( + "fakeroot map does not contain inode: '{}'", + entry.ino() + ))? + }; + + writeln!( + &mut output, "{} {} {} {:o}", - entry.path().strip_prefix(&args[1]).unwrap().display(), - metadata.uid(), - metadata.gid(), + entry.path().strip_prefix(&args.input_dir)?.display(), + uid, + gid, metadata.mode() - ); + )?; } + + Ok(()) +} + +/// Parse a fakeroot statefile, building a map of inode to uid/gid +fn read_fakeroot_state(path: &PathBuf) -> Result> { + let state = File::open(path)?; + + let mut fakeroot_map = HashMap::new(); + for line in io::BufReader::new(state).lines() { + let line = line?; + + let fields: Vec<_> = line.split(',').collect(); + + let ino = fields.iter().find_map(|v| v.strip_prefix("ino=")); + let uid = fields.iter().find_map(|v| v.strip_prefix("uid=")); + let gid = fields.iter().find_map(|v| v.strip_prefix("gid=")); + + let ino = ino.context("fakeroot map invalid, 'ino' not found")?; + let uid = uid.context("fakeroot map invalid, 'uid' not found")?; + let gid = gid.context("fakeroot map invalid, 'gid' not found")?; + + fakeroot_map.insert(ino.parse()?, (uid.parse()?, gid.parse()?)); + } + + Ok(fakeroot_map) } diff --git a/toolchains/sysimage/build_ext4_image.py b/toolchains/sysimage/build_ext4_image.py index a4713ca88dd..356490efecd 100755 --- a/toolchains/sysimage/build_ext4_image.py +++ b/toolchains/sysimage/build_ext4_image.py @@ -15,17 +15,6 @@ import tempfile -def parse_size(s): - if s[-1] == "k" or s[-1] == "K": - return 1024 * int(s[:-1]) - elif s[-1] == "m" or s[-1] == "M": - return 1024 * 1024 * int(s[:-1]) - elif s[-1] == "g" or s[-1] == "G": - return 1024 * 1024 * 1024 * int(s[:-1]) - else: - return int(s) - - def limit_file_contexts(file_contexts, base_path): r""" Projects file contexts to given base path. @@ -56,8 +45,11 @@ def limit_file_contexts(file_contexts, base_path): # Drop all statements assigning no label at all if line.find("<>") != -1: continue - if line.startswith(base_path): - lines.append(line[len(base_path) :]) + if base_path: + if line.startswith(base_path): + lines.append(line[len(base_path) :]) + else: + lines.append(line) return "\n".join(lines) + "\n" @@ -95,10 +87,13 @@ def strip_files(fs_basedir, fakeroot_statefile, strip_paths): if path[0] == "/": path = path[1:] - target_dir = os.path.join(fs_basedir, path) - for entry in os.listdir(target_dir): - del_path = os.path.join(target_dir, entry) - subprocess.run(["fakeroot", "-s", fakeroot_statefile, "-i", fakeroot_statefile, "rm", "-rf", del_path]) + target_path = os.path.join(fs_basedir, path) + if os.path.isdir(target_path): + for entry in os.listdir(target_path): + del_path = os.path.join(target_path, entry) + subprocess.run(["fakeroot", "-s", fakeroot_statefile, "-i", fakeroot_statefile, "rm", "-rf", del_path]) + else: + subprocess.run(["fakeroot", "-s", fakeroot_statefile, "-i", fakeroot_statefile, "rm", "-rf", target_path]) def prepare_tree_from_tar(in_file, fakeroot_statefile, fs_basedir, dir_to_extract): @@ -125,49 +120,13 @@ def prepare_tree_from_tar(in_file, fakeroot_statefile, fs_basedir, dir_to_extrac "-s", fakeroot_statefile, "chown", - "root.root", + "root:root", fs_basedir, ], check=True, ) -def fixup_selinux_root_context(file_contexts, limit_prefix, image_file): - root_context = get_root_context(file_contexts, "/" + limit_prefix[:-1]) - with subprocess.Popen( - ["/usr/sbin/debugfs", "-w", image_file], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL - ) as proc: - proc.stdin.write(('ea_set / security.selinux "%s\\000"\n' % root_context).encode("utf-8")) - proc.stdin.close() - proc.wait() - if proc.returncode != 0: - raise RuntimeError("SELinux root context fixup failed") - - -def fixup_permissions(fs_rootdir, fakeroot_statefile, image_file): - fakeroot_state = read_fakeroot_state(fakeroot_statefile) - for path, subdirs, files in os.walk(fs_rootdir, followlinks=False): - for entry in subdirs + files: - realpath = os.path.join(path, entry) - imgpath = os.path.join(path[len(fs_rootdir) :], entry) - ino = os.lstat(realpath).st_ino - entry = fakeroot_state[ino] - uid = entry["uid"] - gid = entry["gid"] - if uid != "0" or gid != "0": - print("Fix ownership of %s to %s:%s" % (imgpath, uid, gid)) - with subprocess.Popen( - ["/usr/sbin/debugfs", "-w", image_file, "-R", "modify_inode " + imgpath], - stdin=subprocess.PIPE, - stdout=subprocess.DEVNULL, - ) as proc: - proc.stdin.write(("\n%s\n%s\n" % (uid, gid)).encode("utf-8")) - proc.stdin.close() - proc.wait() - if proc.returncode != 0: - raise RuntimeError("Permission fixup failed") - - def make_argparser(): parser = argparse.ArgumentParser() parser.add_argument("-s", "--size", help="Size of image to build", type=str) @@ -199,7 +158,8 @@ def make_argparser(): default=[], help="Directories to be cleared from the tree; expects a list of full paths", ) - parser.add_argument("-d", "--dflate", help="Path to dflate", type=str) + parser.add_argument("-d", "--dflate", help="Path to dflate tool", type=str, required=True) + parser.add_argument("--diroid", help="Path to our diroid tool", type=str, required=True) return parser @@ -208,7 +168,7 @@ def main(): in_file = args.input out_file = args.output - image_size = parse_size(args.size) + image_size = args.size limit_prefix = args.path file_contexts_file = args.file_contexts strip_paths = args.strip_paths @@ -220,7 +180,11 @@ def main(): if file_contexts_file: original_file_contexts = open(file_contexts_file, "r").read() - file_contexts = limit_file_contexts(original_file_contexts, "/" + limit_prefix) + if limit_prefix: + prefix = "/" + limit_prefix + else: + prefix = "" + file_contexts = limit_file_contexts(original_file_contexts, prefix) file_contexts_file = os.path.join(tmpdir, "file_contexts") open(file_contexts_file, "w").write(file_contexts) @@ -238,23 +202,21 @@ def main(): # Now build the basic filesystem image. Wrap again in fakeroot # so correct permissions are read for all files etc. - make_ext4fs_args = ["fakeroot", "-i", fakeroot_statefile, "make_ext4fs", "-T", "0", "-l", str(image_size)] - make_ext4fs_args += [image_file, os.path.join(fs_basedir, limit_prefix)] - if file_contexts_file: - make_ext4fs_args += ["-S", file_contexts_file] - subprocess.run(make_ext4fs_args, check=True) + mke2fs_args = ["faketime", "-f", "1970-1-1 0:0:0", "/usr/sbin/mkfs.ext4", "-E", "hash_seed=c61251eb-100b-48fe-b089-57dea7368612", "-U", "clear", "-F", image_file, str(image_size)] + subprocess.run(mke2fs_args, check=True, env={"E2FSPROGS_FAKE_TIME": "0"}) - # make_ext4fs has two quirks/bugs that will be fixed up now. + # Use our tool, diroid, to create an fs_config file to be used by e2fsdroid. + # This file is a simple list of files with their desired uid, gid, and mode. + fs_config_path = os.path.join(tmpdir, "fs_config") + diroid_args=[args.diroid, "--fakeroot", fakeroot_statefile, "--input-dir", os.path.join(fs_basedir, limit_prefix), "--output", fs_config_path] + subprocess.run(diroid_args, check=True) - # 1. SELinux context of the root inode does not get set correctly. + e2fsdroid_args= ["faketime", "-f", "1970-1-1 0:0:0", "fakeroot", "-i", fakeroot_statefile, "e2fsdroid", "-e", "-a", "/", "-T", "0"] + e2fsdroid_args += ["-C", fs_config_path] if file_contexts_file: - subprocess.run(['sync'], check=True) - fixup_selinux_root_context(original_file_contexts, limit_prefix, image_file) - - subprocess.run(['sync'], check=True) - # 2. Ownership of all inodes is root.root, but that is not what it is - # supposed to be in the final image - fixup_permissions(os.path.join(fs_basedir, limit_prefix), fakeroot_statefile, image_file) + e2fsdroid_args += ["-S", file_contexts_file] + e2fsdroid_args += ["-f", os.path.join(fs_basedir, limit_prefix), image_file] + subprocess.run(e2fsdroid_args, check=True, env={"E2FSPROGS_FAKE_TIME": "0"}) subprocess.run(['sync'], check=True) diff --git a/toolchains/sysimage/toolchain.bzl b/toolchains/sysimage/toolchain.bzl index fcf857d4b90..3d3a689c346 100644 --- a/toolchains/sysimage/toolchain.bzl +++ b/toolchains/sysimage/toolchain.bzl @@ -277,6 +277,7 @@ fat32_image = rule( def _ext4_image_impl(ctx): tool = ctx.files._build_ext4_image[0] + diroid = ctx.files._diroid[0] dflate = ctx.files._dflate[0] out = ctx.actions.declare_file(ctx.label.name) @@ -294,6 +295,8 @@ def _ext4_image_impl(ctx): ctx.attr.partition_size, "-p", ctx.attr.subdir, + "--diroid", + diroid.path, "-d", dflate.path, ] @@ -309,7 +312,7 @@ def _ext4_image_impl(ctx): arguments = args, inputs = inputs, outputs = [out], - tools = [tool, dflate], + tools = [tool, diroid, dflate], ) return [DefaultInfo(files = depset([out]))] @@ -335,6 +338,10 @@ ext4_image = rule( allow_files = True, default = ":build_ext4_image.py", ), + "_diroid": attr.label( + allow_files = True, + default = "//rs/ic_os/diroid", + ), "_dflate": attr.label( allow_files = True, default = "//rs/ic_os/dflate",