Merge branch 'eero/upgrade-diroid' into 'master'

feat: [NODE-1380] Switch IC-OS to newer FS build tools

 

See merge request dfinity-lab/public/ic!19351

.devcontainer/devcontainer.json

@@ -1,5 +1,5 @@
 {
-  "image": "dfinity/ic-build:24bbf7e3a37a1de06f55b44b3419a92a929f919ba8e1dacb540255a17de36ad7",
+  "image": "dfinity/ic-build:f3527253faea5555085c508e10e98ed276988b1b3cd6e63c249e03318cecb596",
   "remoteUser": "ubuntu",
   "privileged": true,
   "runArgs": [
@@ -58,4 +58,4 @@
       ]
     }
   }
-}
+}

gitlab-ci/config/common.yml

@@ -22,7 +22,7 @@ default:
     expire_in: 3 days
     when: always
   image:
-    name: "registry.gitlab.com/dfinity-lab/core/docker/ic-build:24bbf7e3a37a1de06f55b44b3419a92a929f919ba8e1dacb540255a17de36ad7"
+    name: "registry.gitlab.com/dfinity-lab/core/docker/ic-build:f3527253faea5555085c508e10e98ed276988b1b3cd6e63c249e03318cecb596"
   tags:
     - dfinity-ic
 

gitlab-ci/config/zz-generated-gitlab-ci.yaml

@@ -1975,7 +1975,7 @@ default:
     expire_in: 3 days
     when: always
   image:
-    name: registry.gitlab.com/dfinity-lab/core/docker/ic-build:24bbf7e3a37a1de06f55b44b3419a92a929f919ba8e1dacb540255a17de36ad7
+    name: registry.gitlab.com/dfinity-lab/core/docker/ic-build:f3527253faea5555085c508e10e98ed276988b1b3cd6e63c249e03318cecb596
   interruptible: true
   retry:
     max: 2

gitlab-ci/container/Dockerfile

@@ -35,6 +35,26 @@ RUN curl -L "https://apt.llvm.org/llvm-snapshot.gpg.key" | apt-key add - && \
     apt -yqq install --no-install-recommends lld-18 llvm-18 llvm-18-dev clang-18 libclang-rt-18-dev google-cloud-cli \
     gcc-$(gcc --version|head -n1|sed 's/\..*//'|sed 's/.* //')-plugin-dev libstdc++-$(gcc --version|head -n1|sed 's/\..*//'|sed 's/.* //')-dev
 
+# Install a version of google-android-platform-tools-installer with e2fsdroid
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    mkdir e2fsdroid && \
+    cd e2fsdroid && \
+    curl -fsSLO http://mirrors.kernel.org/ubuntu/pool/multiverse/g/google-android-installers/google-android-platform-tools-installer_35.0.0+1710437545-3build2_amd64.deb && \
+    curl -fsSLO http://mirrors.kernel.org/ubuntu/pool/multiverse/g/google-android-installers/google-android-licenses_1710437545-3build2_all.deb && \
+    dpkg-deb -R google-android-platform-tools-installer_35.0.0+1710437545-3build2_amd64.deb contents && \
+    cd contents && \
+    sed -i 's/35.0.0/33.0.2/' DEBIAN/control DEBIAN/postinst && \
+    sed -i '/Version/ s/$/+downgraded/' DEBIAN/control && \
+    rm usr/share/google-android-platform-tools-installer/platform-tools_r35.0.0-linux.zip.sha1 && \
+    echo "6bf4f747ad929b02378b44ce083b4502d26109c7  platform-tools_r33.0.2-linux.zip" > usr/share/google-android-platform-tools-installer/platform-tools_r33.0.2-linux.zip.sha1 && \
+    find . -type f -not -path "./DEBIAN/*" -exec md5sum {} + | sort -k 2 | sed 's/\.\/\(.*\)/\1/' > DEBIAN/md5sums && \
+    cd .. && \
+    dpkg-deb -b contents/ downgraded.deb && \
+    apt -yqq install ./downgraded.deb ./google-android-licenses_1710437545-3build2_all.deb && \
+    apt-mark hold google-android-platform-tools-installer && \
+    cd .. && \
+    rm -rf e2fsdroid
+
 ARG sdk_version=0.12.0
 ARG sdk_sha=40da56ad27774d5e1b2cbc35f94c17368be8c8da557aca19878940264bd82a0a
 RUN mkdir -p /tmp/sdk && curl -fsSL https://github.com/dfinity/sdk/releases/download/${sdk_version}/dfx-${sdk_version}-x86_64-linux.tar.gz -o /tmp/sdk/dfx.tar.gz && \

gitlab-ci/container/TAG

@@ -1 +1 @@
-24bbf7e3a37a1de06f55b44b3419a92a929f919ba8e1dacb540255a17de36ad7
+f3527253faea5555085c508e10e98ed276988b1b3cd6e63c249e03318cecb596

gitlab-ci/container/files/packages.common

@@ -53,7 +53,6 @@ libffi-dev
 libgmp-dev
 
 # IC-OS
-android-sdk-ext4-utils
 cryptsetup-bin
 dosfstools
 fakeroot

ic-os/defs.bzl

@@ -128,10 +128,14 @@ def icos_build(
         src = ":rootfs-tree.tar",
         file_contexts = ":file_contexts",
         partition_size = image_deps["rootfs_size"],
+        # NOTE: e2fsdroid does not support filenames with spaces, fortunately,
+        # there are only two in our build.
         strip_paths = [
             "/run",
             "/boot",
             "/var",
+            "/usr/lib/firmware/brcm/brcmfmac43430a0-sdio.ONDA-V80 PLUS.txt",
+            "/usr/lib/firmware/brcm/brcmfmac43455-sdio.MINIX-NEO Z83-4.txt",
         ],
         target_compatible_with = [
             "@platforms//os:linux",
@@ -146,7 +150,7 @@ def icos_build(
         src = ":rootfs-tree.tar",
         file_contexts = ":file_contexts",
         partition_size = image_deps["bootfs_size"],
-        subdir = "boot/",
+        subdir = "boot",
         target_compatible_with = [
             "@platforms//os:linux",
         ],

rs/ic_os/diroid/BUILD.bazel

@@ -3,6 +3,8 @@ load("@rules_rust//rust:defs.bzl", "rust_binary")
 package(default_visibility = ["//visibility:public"])
 
 DEPENDENCIES = [
+    "@crate_index//:anyhow",
+    "@crate_index//:clap",
     "@crate_index//:walkdir",
 ]
 

rs/ic_os/diroid/Cargo.toml

@@ -6,4 +6,6 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+anyhow = { workspace = true }
+clap = { workspace = true, features = ["derive"] }
 walkdir = "2.5.0"

rs/ic_os/diroid/src/main.rs

@@ -1,25 +1,81 @@
-use std::env;
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::{self, BufRead, Write};
 use std::os::unix::fs::MetadataExt;
+use std::path::PathBuf;
 
-use walkdir::WalkDir;
+use anyhow::{Context, Result};
+use clap::Parser;
+use walkdir::{DirEntryExt, WalkDir};
 
-fn main() {
-    let args: Vec<String> = env::args().collect();
+/// Generate an e2fsdroid fs_config file for a given directory tree
+#[derive(Parser)]
+struct Args {
+    /// Fakeroot statefile, to pull ownership from
+    #[arg(long, required = true)]
+    fakeroot: PathBuf,
+    /// Base directory to scan
+    #[arg(long, required = true)]
+    input_dir: PathBuf,
+    /// Output file
+    #[arg(long, required = true)]
+    output: PathBuf,
+}
 
-    if args.len() != 2 {
-        panic!("Incorrect arguments!");
-    }
+fn main() -> Result<()> {
+    let args = Args::parse();
+
+    let fakeroot_map = read_fakeroot_state(&args.fakeroot)?;
+    let mut output = File::create(args.output)?;
+
+    for entry in WalkDir::new(&args.input_dir) {
+        let entry = entry?;
 
-    for entry in WalkDir::new(&args[1]) {
-        let entry = entry.unwrap();
+        let metadata = entry.metadata()?;
 
-        let metadata = entry.metadata().unwrap();
-        println!(
+        // fakeroot does not track /, so special case this ownership
+        let (uid, gid) = if entry.path() == args.input_dir {
+            &(0, 0)
+        } else {
+            fakeroot_map.get(&entry.ino()).context(format!(
+                "fakeroot map does not contain inode: '{}'",
+                entry.ino()
+            ))?
+        };
+
+        writeln!(
+            &mut output,
             "{} {} {} {:o}",
-            entry.path().strip_prefix(&args[1]).unwrap().display(),
-            metadata.uid(),
-            metadata.gid(),
+            entry.path().strip_prefix(&args.input_dir)?.display(),
+            uid,
+            gid,
             metadata.mode()
-        );
+        )?;
     }
+
+    Ok(())
+}
+
+/// Parse a fakeroot statefile, building a map of inode to uid/gid
+fn read_fakeroot_state(path: &PathBuf) -> Result<HashMap<u64, (u32, u32)>> {
+    let state = File::open(path)?;
+
+    let mut fakeroot_map = HashMap::new();
+    for line in io::BufReader::new(state).lines() {
+        let line = line?;
+
+        let fields: Vec<_> = line.split(',').collect();
+
+        let ino = fields.iter().find_map(|v| v.strip_prefix("ino="));
+        let uid = fields.iter().find_map(|v| v.strip_prefix("uid="));
+        let gid = fields.iter().find_map(|v| v.strip_prefix("gid="));
+
+        let ino = ino.context("fakeroot map invalid, 'ino' not found")?;
+        let uid = uid.context("fakeroot map invalid, 'uid' not found")?;
+        let gid = gid.context("fakeroot map invalid, 'gid' not found")?;
+
+        fakeroot_map.insert(ino.parse()?, (uid.parse()?, gid.parse()?));
+    }
+
+    Ok(fakeroot_map)
 }

toolchains/sysimage/build_ext4_image.py

@@ -15,17 +15,6 @@
 import tempfile
 
 
-def parse_size(s):
-    if s[-1] == "k" or s[-1] == "K":
-        return 1024 * int(s[:-1])
-    elif s[-1] == "m" or s[-1] == "M":
-        return 1024 * 1024 * int(s[:-1])
-    elif s[-1] == "g" or s[-1] == "G":
-        return 1024 * 1024 * 1024 * int(s[:-1])
-    else:
-        return int(s)
-
-
 def limit_file_contexts(file_contexts, base_path):
     r"""
     Projects file contexts to given base path.
@@ -56,8 +45,11 @@ def limit_file_contexts(file_contexts, base_path):
         # Drop all statements assigning no label at all
         if line.find("<<none>>") != -1:
             continue
-        if line.startswith(base_path):
-            lines.append(line[len(base_path) :])
+        if base_path:
+            if line.startswith(base_path):
+                lines.append(line[len(base_path) :])
+        else:
+            lines.append(line)
     return "\n".join(lines) + "\n"
 
 
@@ -95,10 +87,13 @@ def strip_files(fs_basedir, fakeroot_statefile, strip_paths):
         if path[0] == "/":
             path = path[1:]
 
-        target_dir = os.path.join(fs_basedir, path)
-        for entry in os.listdir(target_dir):
-            del_path = os.path.join(target_dir, entry)
-            subprocess.run(["fakeroot", "-s", fakeroot_statefile, "-i", fakeroot_statefile, "rm", "-rf", del_path])
+        target_path = os.path.join(fs_basedir, path)
+        if os.path.isdir(target_path):
+            for entry in os.listdir(target_path):
+                del_path = os.path.join(target_path, entry)
+                subprocess.run(["fakeroot", "-s", fakeroot_statefile, "-i", fakeroot_statefile, "rm", "-rf", del_path])
+        else:
+            subprocess.run(["fakeroot", "-s", fakeroot_statefile, "-i", fakeroot_statefile, "rm", "-rf", target_path])
 
 
 def prepare_tree_from_tar(in_file, fakeroot_statefile, fs_basedir, dir_to_extract):
@@ -125,49 +120,13 @@ def prepare_tree_from_tar(in_file, fakeroot_statefile, fs_basedir, dir_to_extrac
                 "-s",
                 fakeroot_statefile,
                 "chown",
-                "root.root",
+                "root:root",
                 fs_basedir,
             ],
             check=True,
         )
 
 
-def fixup_selinux_root_context(file_contexts, limit_prefix, image_file):
-    root_context = get_root_context(file_contexts, "/" + limit_prefix[:-1])
-    with subprocess.Popen(
-        ["/usr/sbin/debugfs", "-w", image_file], stdin=subprocess.PIPE, stdout=subprocess.DEVNULL
-    ) as proc:
-        proc.stdin.write(('ea_set / security.selinux "%s\\000"\n' % root_context).encode("utf-8"))
-        proc.stdin.close()
-        proc.wait()
-        if proc.returncode != 0:
-            raise RuntimeError("SELinux root context fixup failed")
-
-
-def fixup_permissions(fs_rootdir, fakeroot_statefile, image_file):
-    fakeroot_state = read_fakeroot_state(fakeroot_statefile)
-    for path, subdirs, files in os.walk(fs_rootdir, followlinks=False):
-        for entry in subdirs + files:
-            realpath = os.path.join(path, entry)
-            imgpath = os.path.join(path[len(fs_rootdir) :], entry)
-            ino = os.lstat(realpath).st_ino
-            entry = fakeroot_state[ino]
-            uid = entry["uid"]
-            gid = entry["gid"]
-            if uid != "0" or gid != "0":
-                print("Fix ownership of %s to %s:%s" % (imgpath, uid, gid))
-                with subprocess.Popen(
-                    ["/usr/sbin/debugfs", "-w", image_file, "-R", "modify_inode " + imgpath],
-                    stdin=subprocess.PIPE,
-                    stdout=subprocess.DEVNULL,
-                ) as proc:
-                    proc.stdin.write(("\n%s\n%s\n" % (uid, gid)).encode("utf-8"))
-                    proc.stdin.close()
-                    proc.wait()
-                    if proc.returncode != 0:
-                        raise RuntimeError("Permission fixup failed")
-
-
 def make_argparser():
     parser = argparse.ArgumentParser()
     parser.add_argument("-s", "--size", help="Size of image to build", type=str)
@@ -199,7 +158,8 @@ def make_argparser():
         default=[],
         help="Directories to be cleared from the tree; expects a list of full paths",
     )
-    parser.add_argument("-d", "--dflate", help="Path to dflate", type=str)
+    parser.add_argument("-d", "--dflate", help="Path to dflate tool", type=str, required=True)
+    parser.add_argument("--diroid", help="Path to our diroid tool", type=str, required=True)
     return parser
 
 
@@ -208,7 +168,7 @@ def main():
 
     in_file = args.input
     out_file = args.output
-    image_size = parse_size(args.size)
+    image_size = args.size
     limit_prefix = args.path
     file_contexts_file = args.file_contexts
     strip_paths = args.strip_paths
@@ -220,7 +180,11 @@ def main():
 
     if file_contexts_file:
         original_file_contexts = open(file_contexts_file, "r").read()
-        file_contexts = limit_file_contexts(original_file_contexts, "/" + limit_prefix)
+        if limit_prefix:
+            prefix = "/" + limit_prefix
+        else:
+            prefix = ""
+        file_contexts = limit_file_contexts(original_file_contexts, prefix)
         file_contexts_file = os.path.join(tmpdir, "file_contexts")
         open(file_contexts_file, "w").write(file_contexts)
 
@@ -238,23 +202,21 @@ def main():
 
     # Now build the basic filesystem image. Wrap again in fakeroot
     # so correct permissions are read for all files etc.
-    make_ext4fs_args = ["fakeroot", "-i", fakeroot_statefile, "make_ext4fs", "-T", "0", "-l", str(image_size)]
-    make_ext4fs_args += [image_file, os.path.join(fs_basedir, limit_prefix)]
-    if file_contexts_file:
-        make_ext4fs_args += ["-S", file_contexts_file]
-    subprocess.run(make_ext4fs_args, check=True)
+    mke2fs_args = ["faketime", "-f", "1970-1-1 0:0:0", "/usr/sbin/mkfs.ext4", "-E", "hash_seed=c61251eb-100b-48fe-b089-57dea7368612", "-U", "clear", "-F", image_file, str(image_size)]
+    subprocess.run(mke2fs_args, check=True, env={"E2FSPROGS_FAKE_TIME": "0"})
 
-    # make_ext4fs has two quirks/bugs that will be fixed up now.
+    # Use our tool, diroid, to create an fs_config file to be used by e2fsdroid.
+    # This file is a simple list of files with their desired uid, gid, and mode.
+    fs_config_path = os.path.join(tmpdir, "fs_config")
+    diroid_args=[args.diroid, "--fakeroot", fakeroot_statefile, "--input-dir", os.path.join(fs_basedir, limit_prefix), "--output", fs_config_path]
+    subprocess.run(diroid_args, check=True)
 
-    # 1. SELinux context of the root inode does not get set correctly.
+    e2fsdroid_args= ["faketime", "-f", "1970-1-1 0:0:0", "fakeroot", "-i", fakeroot_statefile, "e2fsdroid", "-e", "-a", "/", "-T", "0"]
+    e2fsdroid_args += ["-C", fs_config_path]
     if file_contexts_file:
-        subprocess.run(['sync'], check=True)
-        fixup_selinux_root_context(original_file_contexts, limit_prefix, image_file)
-
-    subprocess.run(['sync'], check=True)
-    # 2. Ownership of all inodes is root.root, but that is not what it is
-    # supposed to be in the final image
-    fixup_permissions(os.path.join(fs_basedir, limit_prefix), fakeroot_statefile, image_file)
+        e2fsdroid_args += ["-S", file_contexts_file]
+    e2fsdroid_args += ["-f", os.path.join(fs_basedir, limit_prefix), image_file]
+    subprocess.run(e2fsdroid_args, check=True, env={"E2FSPROGS_FAKE_TIME": "0"})
 
     subprocess.run(['sync'], check=True)