wip

bazel/conf/.bazelrc.build

@@ -43,13 +43,13 @@ build --workspace_status_command=$(pwd)/bazel/workspace_status.sh
 
 build --experimental_repository_downloader_retries=3 # https://bazel.build/reference/command-line-reference#flag--experimental_repository_downloader_retries
 
-build --flag_alias=ic_version=//bazel:ic_version
-build --flag_alias=ic_version_rc_only=//bazel:ic_version_rc_only
-build --flag_alias=release_build=//bazel:release_build
-build --flag_alias=s3_endpoint=//ci/src/artifacts:s3_endpoint
-build --flag_alias=s3_upload=//ci/src/artifacts:s3_upload
-build --flag_alias=k8s=//rs/tests:k8s
-build --flag_alias=timeout_value=//bazel:timeout_value
+common --flag_alias=ic_version=//bazel:ic_version
+common --flag_alias=ic_version_rc_only=//bazel:ic_version_rc_only
+common --flag_alias=release_build=//bazel:release_build
+common --flag_alias=s3_endpoint=//ci/src/artifacts:s3_endpoint
+common --flag_alias=s3_upload=//ci/src/artifacts:s3_upload
+common --flag_alias=k8s=//rs/tests:k8s
+common --flag_alias=timeout_value=//bazel:timeout_value
 
 # Exclude system tests by default
 # https://github.com/bazelbuild/bazel/issues/8439

ci/container/build-ic.sh

@@ -1,6 +1,20 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+# TODO: rename "IN_FOO"
+if [ "${IN_FOO:-}" == 1 ]; then
+    echo "$0 nested, aborting"
+    exit 1
+fi
+export IN_FOO=1
+
+export ROOT_DIR="$(git rev-parse --show-toplevel)"
+
+if ! ([ -e /home/ubuntu/.DFINITY-TAG ] && ([ -e /.dockerenv ] || [ -e /run/.containerenv ] || [ -n "${CI_JOB_NAME:-}" ])); then
+    echo dropping into container
+    exec "$ROOT_DIR"/ci/container/container-run.sh bash "$0" "$@"
+fi
+
 [ -n "${DEBUG:-}" ] && set -x
 
 usage() {
@@ -20,14 +34,11 @@ Non-Release Build, is for non-protected branches (revision is not in rc--* or ma
 EOF
 }
 
-RED='\033[0;31m'
-BLUE='\033[0;34m'
-GREEN='\033[0;32m'
-NOCOLOR='\033[0m'
 
-echo_red() { echo -e "${RED}${1}${NOCOLOR}"; }
-echo_blue() { echo -e "${BLUE}${1}${NOCOLOR}"; }
-echo_green() { echo -e "${GREEN}${1}${NOCOLOR}"; }
+tp() { tput -T xterm "$@"; }
+echo_red() { tp setaf 1; echo "$1"; tp sgr0; }
+echo_green() { tp setaf 2; echo "$1"; tp sgr0; }
+echo_blue() { tp setaf 4; echo "$1"; tp sgr0; }
 
 export BUILD_BIN=false
 export BUILD_CAN=false
@@ -53,7 +64,7 @@ while getopts ':bcinh-:' OPT; do
         i | icos) BUILD_IMG=true ;;
         n | non-release | no-release | norelease) RELEASE=false ;;
         ??*) echo_red "Invalid option --$OPT" && usage && exit 1 ;;
-        ?) echo_red "Invalid command option.\n" && usage && exit 1 ;;
+        ?) echo_red "Invalid command option." && usage && exit 1 ;;
     esac
 done
 shift "$(($OPTIND - 1))"
@@ -64,15 +75,14 @@ if ! "$BUILD_BIN" && ! "$BUILD_CAN" && ! "$BUILD_IMG"; then
     usage && exit 1
 fi
 
-export ROOT_DIR="$(git rev-parse --show-toplevel)"
 export VERSION="$(git rev-parse HEAD)"
 
 if "$RELEASE"; then
     export IC_VERSION_RC_ONLY="$VERSION"
-    echo_red "\nBuilding release revision (master or rc--*)! Use '--no-release' for non-release revision!\n" && sleep 2
+    echo_red "Building release revision (master or rc--*)! Use '--no-release' for non-release revision!" && sleep 2
 else
     export IC_VERSION_RC_ONLY="0000000000000000000000000000000000000000"
-    echo_red "\nBuilding non-release revision!\n" && sleep 2
+    echo before && echo_red "Building non-release revision!" && echo after && sleep 2
 fi
 
 export BINARIES_DIR=artifacts/release
@@ -82,28 +92,34 @@ export BINARIES_DIR_FULL="$ROOT_DIR/$BINARIES_DIR"
 export CANISTERS_DIR_FULL="$ROOT_DIR/$CANISTERS_DIR"
 export DISK_DIR_FULL="$ROOT_DIR/$DISK_DIR"
 
-is_inside_DFINITY_container() {
-    [ -e /home/ubuntu/.DFINITY-TAG ] && ([ -e /.dockerenv ] || [ -e /run/.containerenv ] || [ -n "${CI_JOB_NAME:-}" ])
-}
 
 validate_build_env() {
     function not_supported_prompt() {
         echo_red "$1"
-        read -t 7 -r -s -p $'Press ENTER to continue the build anyway...\n'
+        read -t 7 -r -s -p $'Press ENTER to continue the build anyway...'
     }
 
     if [ -n "$(git status --porcelain)" ]; then
         echo_red "Git working directory is not clean! Clean it and retry."
         exit 1
     fi
 
+    # does this make sense? it's always run inside the container
     if [ "$(uname)" != "Linux" ]; then
         not_supported_prompt "This script is only supported on Linux!"
     elif ! grep -q 'Ubuntu' /etc/os-release; then
         not_supported_prompt "Build reproducibility is only supported on Ubuntu!"
     fi
 }
 
+# TODO: doc
+function join_by {
+  local d=${1-} f=${2-}
+  if shift 2; then
+    printf %s "$f" "${@/#/$d}"
+  fi
+}
+
 echo_green "Validating build environment"
 validate_build_env
 
@@ -112,106 +128,88 @@ rm -rf "$BINARIES_DIR_FULL"
 rm -rf "$CANISTERS_DIR_FULL"
 rm -rf "$DISK_DIR_FULL"
 
-echo_green "Building selected IC artifacts"
-BAZEL_CMD="bazel build --config=local --ic_version='$VERSION' --ic_version_rc_only='$IC_VERSION_RC_ONLY' \
-           --release_build=$RELEASE"
-
-BAZEL_CLEAN_CMD=$(
-    cat <<-END
-    # clear bazel cache
-    bazel clean
-END
-)
-
-BUILD_BINARIES_CMD=$(
-    cat <<-END
-    # build binaries
-    mkdir -p "$BINARIES_DIR"
-    $BAZEL_CMD //publish/binaries
-    bazel cquery --config=local --output=files //publish/binaries | xargs -I {} cp {} "$BINARIES_DIR"
-END
-)
-
-BUILD_CANISTERS_CMD=$(
-    cat <<-END
-    # build canisters
-    mkdir -p "$CANISTERS_DIR"
-    $BAZEL_CMD //publish/canisters
-    bazel cquery --config=local --output=files //publish/canisters | xargs -I {} cp {} "$CANISTERS_DIR"
-END
-)
-
-BUILD_IMAGES_CMD=$(
-    cat <<-END
-    # build guestos images
-    mkdir -p "${DISK_DIR}/guestos"
-    $BAZEL_CMD //ic-os/guestos/envs/prod
-    bazel cquery --config=local --output=files //ic-os/guestos/envs/prod | xargs -I {} cp {} "${DISK_DIR}/guestos"
-    # build hostos images
-    mkdir -p "${DISK_DIR}/hostos"
-    $BAZEL_CMD //ic-os/hostos/envs/prod
-    bazel cquery --config=local --output=files //ic-os/hostos/envs/prod | xargs -I {} cp {} "${DISK_DIR}/hostos"
-    # build setupos images
-    mkdir -p "${DISK_DIR}/setupos"
-    $BAZEL_CMD //ic-os/setupos/envs/prod
-    bazel cquery --config=local --output=files //ic-os/setupos/envs/prod | xargs -I {} cp {} "${DISK_DIR}/setupos"
-END
-)
-BUILD_CMD="${BAZEL_CLEAN_CMD}"
-
-if "$BUILD_BIN"; then BUILD_CMD="${BUILD_CMD}${BUILD_BINARIES_CMD}"; fi
-if "$BUILD_CAN"; then BUILD_CMD="${BUILD_CMD}${BUILD_CANISTERS_CMD}"; fi
-if "$BUILD_IMG"; then BUILD_CMD="${BUILD_CMD}${BUILD_IMAGES_CMD}"; fi
-
-if is_inside_DFINITY_container; then
-    echo_blue "Building already inside a DFINITY container or CI"
-    eval "$BUILD_CMD"
-else
-    echo_blue "Building by using a new DFINITY container"
-    "$ROOT_DIR"/ci/container/container-run.sh bash -c "$BUILD_CMD"
-fi
+BAZEL_TARGETS=(  )
+
+BAZEL_COMMON_ARGS=(
+    --config=local # TODO: explain all args
+    --ic_version="$VERSION"
+    --ic_version_rc_only="$IC_VERSION_RC_ONLY"
+    --release_build="$RELEASE"
+    )
+
+echo_green "Building selected IC artifacts: ${BAZEL_TARGETS[*]}"
+
+# TODO: why does building this build sns-governance-canister-test as well?
+if "$BUILD_BIN"; then BAZEL_TARGETS+=( "//publish/binaries:compute_checksums" ); fi
+if "$BUILD_CAN"; then BAZEL_TARGETS+=( "//publish/canisters:compute_checksums" ); fi
+if "$BUILD_IMG"; then BAZEL_TARGETS+=(
+    "//ic-os/guestos/envs/prod:compute_checksums"
+    "//ic-os/hostos/envs/prod:compute_checksums"
+    "//ic-os/setupos/envs/prod:compute_checksums"
+); fi
+
+
+echo_blue "Bazel targets: ${BAZEL_TARGETS[*]}"
+
+bazel build "${BAZEL_COMMON_ARGS[@]}" "${BAZEL_TARGETS[@]}"
+
+query="$(join_by "+" "${BAZEL_TARGETS[@]}")"
+
+for artifact in $(bazel cquery "${BAZEL_COMMON_ARGS[@]}" --output=files "$query"); do
+    target_dir=
+    case "$artifact" in
+        *guestos*)
+            target_dir="icos/guestos"
+            ;;
+        *hostos*)
+            target_dir="icos/hostos"
+            ;;
+        *setupos*)
+            target_dir="icos/setupos"
+            ;;
+        *binaries*)
+            target_dir="release"
+            ;;
+        *canisters*)
+            target_dir="canisters"
+            ;;
+        *)
+            echo "don't know where to put artifact '$artifact'"
+            exit 1
+            ;;
+    esac
+
+    mkdir -p "./artifacts/$target_dir"
+    ln -s "$(realpath "$artifact")" "./artifacts/$target_dir"
+done
 
 if "$BUILD_BIN"; then
     echo_green "##### Binaries SHA256SUMS #####"
-    pushd "$BINARIES_DIR_FULL"
-    GLOBIGNORE="SHA256SUMS"
-    # shellcheck disable=SC2035
-    sha256sum -b *.gz | tee SHA256SUMS
-    popd
+    pushd "$BINARIES_DIR_FULL" >/dev/null
+    cat SHA256SUMS
+    popd >/dev/null
 fi
 
 if "$BUILD_CAN"; then
     echo_green "##### Canisters SHA256SUMS #####"
     pushd "$CANISTERS_DIR_FULL"
-    # shellcheck disable=SC2035
-    sha256sum -b *.gz | tee SHA256SUMS
-    # neuron voters need to verify against the unzipped SHA256SUM
-    TMP="$(mktemp -d)"
-    cp *.gz "$TMP/"
-    cd "$TMP"
-    gunzip *
-    # shellcheck disable=SC2035
-    sha256sum *
+    cat SHA256SUMS
     popd
-    rm -fr "$TMP"
 fi
 
 if "$BUILD_IMG"; then
     echo_green "##### GUESTOS SHA256SUMS #####"
-    pushd "$DISK_DIR_FULL/guestos"
-    # shellcheck disable=SC2035
-    sha256sum -b *.tar.* | tee SHA256SUMS
-    popd
+    pushd "$DISK_DIR_FULL/guestos" >/dev/null
+    cat SHA256SUMS
+    popd >/dev/null
     echo_green "##### HOSTOS SHA256SUMS #####"
-    pushd "$DISK_DIR_FULL/hostos"
-    # shellcheck disable=SC2035
-    sha256sum -b *.tar.* | tee SHA256SUMS
-    popd
+    pushd "$DISK_DIR_FULL/hostos" >/dev/null
+    cat SHA256SUMS
+    popd >/dev/null
     echo_green "##### SETUPOS SHA256SUMS #####"
-    pushd "$DISK_DIR_FULL/setupos"
-    # shellcheck disable=SC2035
-    sha256sum -b *.tar.* | tee SHA256SUMS
-    popd
+    pushd "$DISK_DIR_FULL/setupos" >/dev/null
+    cat SHA256SUMS
+    popd >/dev/null
 fi
 
-echo_green "Build complete for revision $(git rev-parse HEAD)"
+echo_green "Build complete for revision $VERSION"

ic-os/guestos/envs/prod/BUILD.bazel

@@ -1,5 +1,6 @@
 load("//ic-os:defs.bzl", "icos_build")
 load("//ic-os/guestos:defs.bzl", "image_deps")
+load("//publish:defs.bzl", "checksum_rule")
 
 # The macro contains several targets.
 # Check
@@ -11,3 +12,9 @@ icos_build(
     upload_prefix = "guest-os",
     visibility = ["//rs:ic-os-pkg"],
 )
+
+# Use the checksum rule with a filegroup
+checksum_rule(
+    name = "compute_checksums",
+    inputs = [":prod"],  # Pass the filegroup
+)

ic-os/hostos/envs/prod/BUILD.bazel

@@ -1,5 +1,6 @@
 load("//ic-os:defs.bzl", "icos_build")
 load("//ic-os/hostos:defs.bzl", "image_deps")
+load("//publish:defs.bzl", "checksum_rule")
 
 # The macro contains several targets.
 # Check
@@ -12,3 +13,10 @@ icos_build(
     visibility = ["//rs:ic-os-pkg"],
     vuln_scan = False,
 )
+
+
+# Use the checksum rule with a filegroup
+checksum_rule(
+    name = "compute_checksums",
+    inputs = [":prod"],  # Pass the filegroup
+)

ic-os/setupos/envs/prod/BUILD.bazel

@@ -1,6 +1,7 @@
 load("//ic-os:defs.bzl", "icos_build")
 load("//ic-os/dev-tools/bare_metal_deployment:tools.bzl", "launch_bare_metal")
 load("//ic-os/setupos:defs.bzl", "image_deps")
+load("//publish:defs.bzl", "checksum_rule")
 
 # The macro contains several targets.
 # Check
@@ -18,3 +19,9 @@ launch_bare_metal(
     name = "launch_bare_metal",
     image_zst_file = ":disk-img.tar.zst",
 )
+
+# Use the checksum rule with a filegroup
+checksum_rule(
+    name = "compute_checksums",
+    inputs = [":prod"],  # Pass the filegroup
+)

publish/binaries/BUILD.bazel

@@ -1,6 +1,7 @@
 load("//bazel:defs.bzl", "gzip_compress")
 load("//ci/src/artifacts:upload.bzl", "upload_artifacts")
 load("//publish:defs.bzl", "release_nostrip_binary", "release_strip_binary", "release_strip_binary_test")
+load("//publish:defs.bzl", "checksum_rule")
 
 package(default_visibility = ["//rs:ic-os-pkg"])
 
@@ -119,6 +120,11 @@ filegroup(
     srcs = [name + ".gz" for name in ALL_BINARIES],
 )
 
+checksum_rule(
+    name = "compute_checksums",
+    inputs = [":binaries"],  # Pass the filegroup
+)
+
 upload_artifacts(
     name = "upload",
     testonly = True,

publish/canisters/BUILD.bazel

@@ -3,6 +3,8 @@ load("//ci/src/artifacts:upload.bzl", "upload_artifacts")
 load("//rs/nns:nns.bzl", NNS_CANISTER_NAME_TO_MAX_COMPRESSED_WASM_SIZE_E5_BYTES = "CANISTER_NAME_TO_MAX_COMPRESSED_WASM_SIZE_E5_BYTES")
 load("//rs/sns:sns.bzl", SNS_CANISTER_NAME_TO_MAX_COMPRESSED_WASM_SIZE_E5_BYTES = "CANISTER_NAME_TO_MAX_COMPRESSED_WASM_SIZE_E5_BYTES")
 
+load("//publish:defs.bzl", "checksum_rule")
+
 # The list of canisters that are being published/uploaded as artifacts to our CDN (download.dfinity.systems) include
 # 1. Officially released canisters that can be downloaded/verified by 3rd parties - https://github.com/dfinity/ic/blob/master/.github/workflows/tag-release.yml
 # 2. Mainnet canisters - https://github.com/dfinity/ic/blob/master/mainnet-canisters.bzl
@@ -125,6 +127,12 @@ filegroup(
            [name for name in COMPRESSED_CANISTERS],
 )
 
+# Use the checksum rule with a filegroup
+checksum_rule(
+    name = "compute_checksums",
+    inputs = [":canisters"],  # Pass the filegroup
+)
+
 # the further targets are all testonly for simplicity because if a target was
 # violating the testonly flag, it would error at this point