From aad6e31448647943f7f36701a0298ae554157298 Mon Sep 17 00:00:00 2001 From: Nicolas Mattia Date: Mon, 3 Feb 2025 16:43:30 +0100 Subject: [PATCH] wip --- bazel/conf/.bazelrc.build | 14 +- ci/container/build-ic.sh | 204 ++++++++++++++-------------- ic-os/guestos/envs/prod/BUILD.bazel | 7 + ic-os/hostos/envs/prod/BUILD.bazel | 8 ++ ic-os/setupos/envs/prod/BUILD.bazel | 7 + publish/binaries/BUILD.bazel | 6 + publish/canisters/BUILD.bazel | 8 ++ publish/defs.bzl | 44 ++++++ 8 files changed, 189 insertions(+), 109 deletions(-) diff --git a/bazel/conf/.bazelrc.build b/bazel/conf/.bazelrc.build index 998576ff8204..4bad56e69af4 100644 --- a/bazel/conf/.bazelrc.build +++ b/bazel/conf/.bazelrc.build @@ -43,13 +43,13 @@ build --workspace_status_command=$(pwd)/bazel/workspace_status.sh build --experimental_repository_downloader_retries=3 # https://bazel.build/reference/command-line-reference#flag--experimental_repository_downloader_retries -build --flag_alias=ic_version=//bazel:ic_version -build --flag_alias=ic_version_rc_only=//bazel:ic_version_rc_only -build --flag_alias=release_build=//bazel:release_build -build --flag_alias=s3_endpoint=//ci/src/artifacts:s3_endpoint -build --flag_alias=s3_upload=//ci/src/artifacts:s3_upload -build --flag_alias=k8s=//rs/tests:k8s -build --flag_alias=timeout_value=//bazel:timeout_value +common --flag_alias=ic_version=//bazel:ic_version +common --flag_alias=ic_version_rc_only=//bazel:ic_version_rc_only +common --flag_alias=release_build=//bazel:release_build +common --flag_alias=s3_endpoint=//ci/src/artifacts:s3_endpoint +common --flag_alias=s3_upload=//ci/src/artifacts:s3_upload +common --flag_alias=k8s=//rs/tests:k8s +common --flag_alias=timeout_value=//bazel:timeout_value # Exclude system tests by default # https://github.com/bazelbuild/bazel/issues/8439 diff --git a/ci/container/build-ic.sh b/ci/container/build-ic.sh index 6e2e385c325e..2bbc1aef12ac 100755 --- a/ci/container/build-ic.sh +++ b/ci/container/build-ic.sh @@ -1,6 +1,23 @@ #!/usr/bin/env bash set -euo pipefail +# TODO: rename "IN_FOO" +if [ "${IN_FOO:-}" == 1 ]; then + echo "$0 nested, aborting" + exit 1 +fi +export IN_FOO=1 + +export ROOT_DIR="$(git rev-parse --show-toplevel)" + +if ! ([ -e /home/ubuntu/.DFINITY-TAG ] && ([ -e /.dockerenv ] || [ -e /run/.containerenv ] || [ -n "${CI_JOB_NAME:-}" ])); then + echo not inside container + echo starting container + exec "$ROOT_DIR"/ci/container/container-run.sh bash "$0" "$@" +else + echo inside container +fi + [ -n "${DEBUG:-}" ] && set -x usage() { @@ -20,14 +37,11 @@ Non-Release Build, is for non-protected branches (revision is not in rc--* or ma EOF } -RED='\033[0;31m' -BLUE='\033[0;34m' -GREEN='\033[0;32m' -NOCOLOR='\033[0m' +export TPUTTERM="${TERM:-xterm}" -echo_red() { echo -e "${RED}${1}${NOCOLOR}"; } -echo_blue() { echo -e "${BLUE}${1}${NOCOLOR}"; } -echo_green() { echo -e "${GREEN}${1}${NOCOLOR}"; } +echo_red() { tput setaf 1; echo "$1"; tput sgr0; } +echo_green() { tput setaf 2; echo "$1"; tput sgr0; } +echo_blue() { tput setaf 4; echo "$1"; tput sgr0; } export BUILD_BIN=false export BUILD_CAN=false @@ -53,7 +67,7 @@ while getopts ':bcinh-:' OPT; do i | icos) BUILD_IMG=true ;; n | non-release | no-release | norelease) RELEASE=false ;; ??*) echo_red "Invalid option --$OPT" && usage && exit 1 ;; - ?) echo_red "Invalid command option.\n" && usage && exit 1 ;; + ?) echo_red "Invalid command option." && usage && exit 1 ;; esac done shift "$(($OPTIND - 1))" @@ -64,15 +78,14 @@ if ! "$BUILD_BIN" && ! "$BUILD_CAN" && ! "$BUILD_IMG"; then usage && exit 1 fi -export ROOT_DIR="$(git rev-parse --show-toplevel)" export VERSION="$(git rev-parse HEAD)" if "$RELEASE"; then export IC_VERSION_RC_ONLY="$VERSION" - echo_red "\nBuilding release revision (master or rc--*)! Use '--no-release' for non-release revision!\n" && sleep 2 + echo_red "Building release revision (master or rc--*)! Use '--no-release' for non-release revision!" && sleep 2 else export IC_VERSION_RC_ONLY="0000000000000000000000000000000000000000" - echo_red "\nBuilding non-release revision!\n" && sleep 2 + echo_red "Building non-release revision!" && sleep 2 fi export BINARIES_DIR=artifacts/release @@ -82,14 +95,10 @@ export BINARIES_DIR_FULL="$ROOT_DIR/$BINARIES_DIR" export CANISTERS_DIR_FULL="$ROOT_DIR/$CANISTERS_DIR" export DISK_DIR_FULL="$ROOT_DIR/$DISK_DIR" -is_inside_DFINITY_container() { - [ -e /home/ubuntu/.DFINITY-TAG ] && ([ -e /.dockerenv ] || [ -e /run/.containerenv ] || [ -n "${CI_JOB_NAME:-}" ]) -} - validate_build_env() { function not_supported_prompt() { echo_red "$1" - read -t 7 -r -s -p $'Press ENTER to continue the build anyway...\n' + read -t 7 -r -s -p $'Press ENTER to continue the build anyway...' } if [ -n "$(git status --porcelain)" ]; then @@ -97,6 +106,7 @@ validate_build_env() { exit 1 fi + # does this make sense? it's always run inside the container if [ "$(uname)" != "Linux" ]; then not_supported_prompt "This script is only supported on Linux!" elif ! grep -q 'Ubuntu' /etc/os-release; then @@ -104,6 +114,14 @@ validate_build_env() { fi } +# TODO: doc +function join_by { + local d=${1-} f=${2-} + if shift 2; then + printf %s "$f" "${@/#/$d}" + fi +} + echo_green "Validating build environment" validate_build_env @@ -112,106 +130,88 @@ rm -rf "$BINARIES_DIR_FULL" rm -rf "$CANISTERS_DIR_FULL" rm -rf "$DISK_DIR_FULL" -echo_green "Building selected IC artifacts" -BAZEL_CMD="bazel build --config=local --ic_version='$VERSION' --ic_version_rc_only='$IC_VERSION_RC_ONLY' \ - --release_build=$RELEASE" - -BAZEL_CLEAN_CMD=$( - cat <<-END - # clear bazel cache - bazel clean -END -) - -BUILD_BINARIES_CMD=$( - cat <<-END - # build binaries - mkdir -p "$BINARIES_DIR" - $BAZEL_CMD //publish/binaries - bazel cquery --config=local --output=files //publish/binaries | xargs -I {} cp {} "$BINARIES_DIR" -END -) - -BUILD_CANISTERS_CMD=$( - cat <<-END - # build canisters - mkdir -p "$CANISTERS_DIR" - $BAZEL_CMD //publish/canisters - bazel cquery --config=local --output=files //publish/canisters | xargs -I {} cp {} "$CANISTERS_DIR" -END -) - -BUILD_IMAGES_CMD=$( - cat <<-END - # build guestos images - mkdir -p "${DISK_DIR}/guestos" - $BAZEL_CMD //ic-os/guestos/envs/prod - bazel cquery --config=local --output=files //ic-os/guestos/envs/prod | xargs -I {} cp {} "${DISK_DIR}/guestos" - # build hostos images - mkdir -p "${DISK_DIR}/hostos" - $BAZEL_CMD //ic-os/hostos/envs/prod - bazel cquery --config=local --output=files //ic-os/hostos/envs/prod | xargs -I {} cp {} "${DISK_DIR}/hostos" - # build setupos images - mkdir -p "${DISK_DIR}/setupos" - $BAZEL_CMD //ic-os/setupos/envs/prod - bazel cquery --config=local --output=files //ic-os/setupos/envs/prod | xargs -I {} cp {} "${DISK_DIR}/setupos" -END -) -BUILD_CMD="${BAZEL_CLEAN_CMD}" - -if "$BUILD_BIN"; then BUILD_CMD="${BUILD_CMD}${BUILD_BINARIES_CMD}"; fi -if "$BUILD_CAN"; then BUILD_CMD="${BUILD_CMD}${BUILD_CANISTERS_CMD}"; fi -if "$BUILD_IMG"; then BUILD_CMD="${BUILD_CMD}${BUILD_IMAGES_CMD}"; fi - -if is_inside_DFINITY_container; then - echo_blue "Building already inside a DFINITY container or CI" - eval "$BUILD_CMD" -else - echo_blue "Building by using a new DFINITY container" - "$ROOT_DIR"/ci/container/container-run.sh bash -c "$BUILD_CMD" -fi +BAZEL_TARGETS=( ) + +BAZEL_COMMON_ARGS=( + --config=local # TODO: explain all args + --ic_version="$VERSION" + --ic_version_rc_only="$IC_VERSION_RC_ONLY" + --release_build="$RELEASE" + ) + +echo_green "Building selected IC artifacts: ${BAZEL_TARGETS[*]}" + +# TODO: why does building this build sns-governance-canister-test as well? +if "$BUILD_BIN"; then BAZEL_TARGETS+=( "//publish/binaries:compute_checksums" ); fi +if "$BUILD_CAN"; then BAZEL_TARGETS+=( "//publish/canisters:compute_checksums" ); fi +if "$BUILD_IMG"; then BAZEL_TARGETS+=( + "//ic-os/guestos/envs/prod:compute_checksums" + "//ic-os/hostos/envs/prod:compute_checksums" + "//ic-os/setupos/envs/prod:compute_checksums" +); fi + + +echo_blue "Bazel targets: ${BAZEL_TARGETS[*]}" + +bazel build "${BAZEL_COMMON_ARGS[@]}" "${BAZEL_TARGETS[@]}" + +query="$(join_by "+" "${BAZEL_TARGETS[@]}")" + +for artifact in $(bazel cquery "${BAZEL_COMMON_ARGS[@]}" --output=files "$query"); do + target_dir= + case "$artifact" in + *guestos*) + target_dir="icos/guestos" + ;; + *hostos*) + target_dir="icos/hostos" + ;; + *setupos*) + target_dir="icos/setupos" + ;; + *binaries*) + target_dir="release" + ;; + *canisters*) + target_dir="canisters" + ;; + *) + echo "don't know where to put artifact '$artifact'" + exit 1 + ;; + esac + + mkdir -p "./artifacts/$target_dir" + ln -s "$(realpath "$artifact")" "./artifacts/$target_dir" +done if "$BUILD_BIN"; then echo_green "##### Binaries SHA256SUMS #####" - pushd "$BINARIES_DIR_FULL" - GLOBIGNORE="SHA256SUMS" - # shellcheck disable=SC2035 - sha256sum -b *.gz | tee SHA256SUMS - popd + pushd "$BINARIES_DIR_FULL" >/dev/null + cat SHA256SUMS + popd >/dev/null fi if "$BUILD_CAN"; then echo_green "##### Canisters SHA256SUMS #####" pushd "$CANISTERS_DIR_FULL" - # shellcheck disable=SC2035 - sha256sum -b *.gz | tee SHA256SUMS - # neuron voters need to verify against the unzipped SHA256SUM - TMP="$(mktemp -d)" - cp *.gz "$TMP/" - cd "$TMP" - gunzip * - # shellcheck disable=SC2035 - sha256sum * + cat SHA256SUMS popd - rm -fr "$TMP" fi if "$BUILD_IMG"; then echo_green "##### GUESTOS SHA256SUMS #####" - pushd "$DISK_DIR_FULL/guestos" - # shellcheck disable=SC2035 - sha256sum -b *.tar.* | tee SHA256SUMS - popd + pushd "$DISK_DIR_FULL/guestos" >/dev/null + cat SHA256SUMS + popd >/dev/null echo_green "##### HOSTOS SHA256SUMS #####" - pushd "$DISK_DIR_FULL/hostos" - # shellcheck disable=SC2035 - sha256sum -b *.tar.* | tee SHA256SUMS - popd + pushd "$DISK_DIR_FULL/hostos" >/dev/null + cat SHA256SUMS + popd >/dev/null echo_green "##### SETUPOS SHA256SUMS #####" - pushd "$DISK_DIR_FULL/setupos" - # shellcheck disable=SC2035 - sha256sum -b *.tar.* | tee SHA256SUMS - popd + pushd "$DISK_DIR_FULL/setupos" >/dev/null + cat SHA256SUMS + popd >/dev/null fi -echo_green "Build complete for revision $(git rev-parse HEAD)" +echo_green "Build complete for revision $VERSION" diff --git a/ic-os/guestos/envs/prod/BUILD.bazel b/ic-os/guestos/envs/prod/BUILD.bazel index 077dd9f155fb..1c70a1dd4b38 100644 --- a/ic-os/guestos/envs/prod/BUILD.bazel +++ b/ic-os/guestos/envs/prod/BUILD.bazel @@ -1,5 +1,6 @@ load("//ic-os:defs.bzl", "icos_build") load("//ic-os/guestos:defs.bzl", "image_deps") +load("//publish:defs.bzl", "checksum_rule") # The macro contains several targets. # Check @@ -11,3 +12,9 @@ icos_build( upload_prefix = "guest-os", visibility = ["//rs:ic-os-pkg"], ) + +# Use the checksum rule with a filegroup +checksum_rule( + name = "compute_checksums", + inputs = [":prod"], # Pass the filegroup +) diff --git a/ic-os/hostos/envs/prod/BUILD.bazel b/ic-os/hostos/envs/prod/BUILD.bazel index d75dcb344180..24bcfae20cc6 100644 --- a/ic-os/hostos/envs/prod/BUILD.bazel +++ b/ic-os/hostos/envs/prod/BUILD.bazel @@ -1,5 +1,6 @@ load("//ic-os:defs.bzl", "icos_build") load("//ic-os/hostos:defs.bzl", "image_deps") +load("//publish:defs.bzl", "checksum_rule") # The macro contains several targets. # Check @@ -12,3 +13,10 @@ icos_build( visibility = ["//rs:ic-os-pkg"], vuln_scan = False, ) + + +# Use the checksum rule with a filegroup +checksum_rule( + name = "compute_checksums", + inputs = [":prod"], # Pass the filegroup +) diff --git a/ic-os/setupos/envs/prod/BUILD.bazel b/ic-os/setupos/envs/prod/BUILD.bazel index 53f67a4d529f..2255ad02db8a 100644 --- a/ic-os/setupos/envs/prod/BUILD.bazel +++ b/ic-os/setupos/envs/prod/BUILD.bazel @@ -1,6 +1,7 @@ load("//ic-os:defs.bzl", "icos_build") load("//ic-os/dev-tools/bare_metal_deployment:tools.bzl", "launch_bare_metal") load("//ic-os/setupos:defs.bzl", "image_deps") +load("//publish:defs.bzl", "checksum_rule") # The macro contains several targets. # Check @@ -18,3 +19,9 @@ launch_bare_metal( name = "launch_bare_metal", image_zst_file = ":disk-img.tar.zst", ) + +# Use the checksum rule with a filegroup +checksum_rule( + name = "compute_checksums", + inputs = [":prod"], # Pass the filegroup +) diff --git a/publish/binaries/BUILD.bazel b/publish/binaries/BUILD.bazel index d383c4e05ed5..dcbfcd84a99c 100644 --- a/publish/binaries/BUILD.bazel +++ b/publish/binaries/BUILD.bazel @@ -1,6 +1,7 @@ load("//bazel:defs.bzl", "gzip_compress") load("//ci/src/artifacts:upload.bzl", "upload_artifacts") load("//publish:defs.bzl", "release_nostrip_binary", "release_strip_binary", "release_strip_binary_test") +load("//publish:defs.bzl", "checksum_rule") package(default_visibility = ["//rs:ic-os-pkg"]) @@ -119,6 +120,11 @@ filegroup( srcs = [name + ".gz" for name in ALL_BINARIES], ) +checksum_rule( + name = "compute_checksums", + inputs = [":binaries"], # Pass the filegroup +) + upload_artifacts( name = "upload", testonly = True, diff --git a/publish/canisters/BUILD.bazel b/publish/canisters/BUILD.bazel index c3c75e835c25..2249405e40f5 100644 --- a/publish/canisters/BUILD.bazel +++ b/publish/canisters/BUILD.bazel @@ -3,6 +3,8 @@ load("//ci/src/artifacts:upload.bzl", "upload_artifacts") load("//rs/nns:nns.bzl", NNS_CANISTER_NAME_TO_MAX_COMPRESSED_WASM_SIZE_E5_BYTES = "CANISTER_NAME_TO_MAX_COMPRESSED_WASM_SIZE_E5_BYTES") load("//rs/sns:sns.bzl", SNS_CANISTER_NAME_TO_MAX_COMPRESSED_WASM_SIZE_E5_BYTES = "CANISTER_NAME_TO_MAX_COMPRESSED_WASM_SIZE_E5_BYTES") +load("//publish:defs.bzl", "checksum_rule") + # The list of canisters that are being published/uploaded as artifacts to our CDN (download.dfinity.systems) include # 1. Officially released canisters that can be downloaded/verified by 3rd parties - https://github.com/dfinity/ic/blob/master/.github/workflows/tag-release.yml # 2. Mainnet canisters - https://github.com/dfinity/ic/blob/master/mainnet-canisters.bzl @@ -125,6 +127,12 @@ filegroup( [name for name in COMPRESSED_CANISTERS], ) +# Use the checksum rule with a filegroup +checksum_rule( + name = "compute_checksums", + inputs = [":canisters"], # Pass the filegroup +) + # the further targets are all testonly for simplicity because if a target was # violating the testonly flag, it would error at this point diff --git a/publish/defs.bzl b/publish/defs.bzl index 7121e0682cbe..5350eb014ea6 100644 --- a/publish/defs.bzl +++ b/publish/defs.bzl @@ -120,3 +120,47 @@ malicious_binary = rule( "binary": attr.label(mandatory = True, cfg = malicious_code_enabled_transition, allow_single_file = True), }, ) + +def _checksum_rule_impl(ctx): + + # List of input files + input_files = ctx.files.inputs + + # Declare output file + out_checksums = ctx.actions.declare_file("_checksums/SHA256SUMS") + + def make_symlink(target): + symlink = ctx.actions.declare_file("_checksums/"+target.basename) + ctx.actions.symlink(output=symlink, target_file=target) + return symlink + + symlinks = [ make_symlink(file) for file in input_files ] + + # Run the shell command + # TODO: strip dirname from SHA256SUMS content? + ctx.actions.run_shell( + inputs = input_files, + arguments = [file.path for file in input_files], + outputs = [out_checksums], + command = """ + set -euo pipefail + + out_checksums="{}" + sha256sum "$@" > "$out_checksums" + sort -o "$out_checksums" -k 2 "$out_checksums" + """.format(out_checksums.path) + ) + + # Return the output file + return [DefaultInfo(files = depset([out_checksums] + symlinks))] + +# Define the rule +checksum_rule = rule( + implementation = _checksum_rule_impl, + attrs = { + "inputs": attr.label_list( + allow_files = True, + mandatory = True, + ), + }, +)